From 770356711f30f76599b653da5dd587601a60cb1c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 00:50:50 -0400 Subject: [PATCH 001/163] Prepare Dander 0.6.0rc1 --- .github/workflows/ci.yml | 4 ++-- CHANGELOG.md | 8 ++++++++ HANDOFF.md | 3 +-- pyproject.toml | 2 +- scripts/check_distribution.py | 2 +- tests/pipeline/test_graph_deployment.py | 2 +- tests/project/test_scaffold.py | 2 +- tests/test_distribution_contract.py | 2 +- uv.lock | 2 +- 9 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98d33c65..087a5f15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,11 +93,11 @@ jobs: uv pip install --python "$ENV_DIR/bin/python" "$ARTIFACT" ( cd "$TEMP_DIR" - "$ENV_DIR/bin/dander" --version | grep -F "dander 0.5.1" + "$ENV_DIR/bin/dander" --version | grep -F "dander 0.6.0rc1" "$ENV_DIR/bin/dander" new "$PROJECT_DIR" cd "$PROJECT_DIR" "$ENV_DIR/bin/dander" validate - grep -F "ARG DANDER_VERSION=0.5.1" Dockerfile + grep -F "ARG DANDER_VERSION=0.6.0rc1" Dockerfile "$ENV_DIR/bin/python" -c 'import dander; assert "site-packages" in dander.__file__' ) INDEX=$((INDEX + 1)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca15738..1b8f6ada 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ capabilities enter through the next minor release. ## Unreleased +## 0.6.0rc1 — 2026-08-06 (alpha) + ### Added - Add sanitized failure codes and summaries to local and hosted run history, and reconcile stale @@ -17,6 +19,12 @@ capabilities enter through the next minor release. - Expand the Salesforce project contract to four independently watermarked CRM endpoints, four governed staging models, and an Opportunity fact with Account and historical owner dimensions. +### Compatibility + +- Existing manifests, pipeline IDs, built-in connectors, and plugin API version 1 remain valid. +- Salesforce remains read-only; hard-delete discovery, custom fields without declared schemas, + additional clouds, and non-BigQuery destinations remain outside this candidate. + ## 0.5.1 — 2026-08-05 (alpha) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 40a7ae1b..4da601a7 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -6,7 +6,7 @@ - Added five governed staging/fact models with tests, metrics, relationships, and Dataplex publication. - Packaged the Salesforce project overlay in every generated source-free Dander project. - Updated Salesforce documentation for personal data, soft deletion, replay, and custom fields. -- Corrected staging TTL publication and legacy run-history reads found by final review. +- Corrected final review blockers and prepared the `0.6.0rc1` package metadata. ## Try It @@ -31,7 +31,6 @@ example manifest, validate, and run `dander run salesforce_crm --dry-run --proje ## Remaining -- Protected CI and review must pass before this PR merges. - Publish the Dander and Salesforce candidates only after explicit approval. - Run the fresh-project live proof before changing Alpha to Beta or publishing stable releases. - Retained-project deployment, schedule changes, and the seven-day soak remain separately gated. diff --git a/pyproject.toml b/pyproject.toml index 588116f4..e770b298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.5.1" +version = "0.6.0rc1" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index e9bccae2..622a2f97 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -10,7 +10,7 @@ from pathlib import Path, PurePosixPath EXPECTED_NAME = "dander-platform" -EXPECTED_VERSION = "0.5.1" +EXPECTED_VERSION = "0.6.0rc1" WHEEL_REQUIRED = { "dander/templates/project/.dockerignore", "dander/templates/project/.gitignore", diff --git a/tests/pipeline/test_graph_deployment.py b/tests/pipeline/test_graph_deployment.py index 9a69c663..206bbc06 100644 --- a/tests/pipeline/test_graph_deployment.py +++ b/tests/pipeline/test_graph_deployment.py @@ -161,7 +161,7 @@ def test_preview_builds_source_free_snapshot_and_isolates_exact_plan(tmp_path: P ).preview(store, expected_revision=store.load().revision) assert observations["source_free"] is True - assert "ARG DANDER_VERSION=0.5.1" in observations["dockerfile"] + assert "ARG DANDER_VERSION=0.6.0rc1" in observations["dockerfile"] assert "RUN dander plugins install --config dander.yaml" in observations["dockerfile"] assert observations["graph"] == _GRAPH assert observations["publish"] == ("proof-project", "us-central1", "candidate") diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index b70c6cbc..73eacca8 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -20,7 +20,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: assert manifest.pipelines["greenhouse_jobs"].paused assert manifest.platform.safety.require_guarded_free_tier is False dockerfile = (project / "Dockerfile").read_text(encoding="utf-8") - assert "ARG DANDER_VERSION=0.5.1" in dockerfile + assert "ARG DANDER_VERSION=0.6.0rc1" in dockerfile assert "RUN dander plugins install --config dander.yaml" in dockerfile assert "COPY --chown=65532:65532 dander.yaml ./dander.yaml" in dockerfile assert "COPY --chown=65532:65532 connectors ./connectors" in dockerfile diff --git a/tests/test_distribution_contract.py b/tests/test_distribution_contract.py index 7d36ed96..1e41385a 100644 --- a/tests/test_distribution_contract.py +++ b/tests/test_distribution_contract.py @@ -12,7 +12,7 @@ def test_distribution_identity_and_version_are_public_release() -> None: project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] assert project["name"] == "dander-platform" - assert project["version"] == "0.5.1" + assert project["version"] == "0.6.0rc1" assert project["scripts"]["dander"] == "dander.cli.main:app" diff --git a/uv.lock b/uv.lock index ba2e4f39..8dfa346b 100644 --- a/uv.lock +++ b/uv.lock @@ -520,7 +520,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.5.1" +version = "0.6.0rc1" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 99b5b826ef860a63f27a02bfdc8b311b3617045d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 05:30:50 -0400 Subject: [PATCH 002/163] Add deleted-record capability command --- src/dander/cli/main.py | 38 ++++++++++++++++ tests/cli/test_connector_cli.py | 77 +++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 7885285e..df2fbdb5 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import re import shlex @@ -357,6 +358,43 @@ def check_connector( console.print(f"[green]Connector {config.name!r} connection check passed.[/green]") +@connector_app.command("get-deleted") +def get_deleted_records( + source_or_pipeline: str = typer.Argument( # noqa: B008 + ..., + help="Connector source name or pipeline name from dander.yaml.", + ), + endpoint: str = typer.Argument(..., help="Configured connector endpoint name."), # noqa: B008 + since: str | None = typer.Option( + None, + "--since", + help="Optional provider cursor; interpretation is connector-specific.", + ), + project_config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + connectors_dir: Path = typer.Option( # noqa: B008 + _DEFAULT_CONNECTORS_DIR, + "--connectors-dir", + ), +) -> None: + """Stream a connector's optional deleted-record feed as JSON Lines.""" + _, capabilities = _load_connector_capabilities( + source_or_pipeline, + project_config=project_config, + connectors_dir=connectors_dir, + ) + try: + for record in capabilities.get_deleted(endpoint, since=since): + typer.echo(json.dumps(dict(record), sort_keys=True, separators=(",", ":"))) + except ( + EnterpriseSourceError, + InvalidConnectorCapabilityResultError, + OAuthTokenError, + SecretResolutionError, + UnsupportedConnectorOperationError, + ) as error: + raise ClickException(str(error)) from error + + @graph_app.command("serve") def serve_graph( graph_file: Path = typer.Option(..., "--file", help="Existing PipelineGraph YAML/JSON file."), # noqa: B008 diff --git a/tests/cli/test_connector_cli.py b/tests/cli/test_connector_cli.py index f4146dc7..a42cada9 100644 --- a/tests/cli/test_connector_cli.py +++ b/tests/cli/test_connector_cli.py @@ -43,6 +43,22 @@ def test_connection(self) -> ConnectionStatus: return self._status +class _DeletedFeedSource(_CheckableSource): + def __init__(self, config: SourceConfig) -> None: + super().__init__(config, ConnectionStatus(ok=True)) + self.calls: list[tuple[str, str | None]] = [] + + def get_deleted( + self, + endpoint: str, + *, + since: str | None = None, + ) -> Iterator[Mapping[str, Any]]: + self.calls.append((endpoint, since)) + yield {"Id": "001000000000001AAA"} + yield {"Id": "001000000000002AAA"} + + def _config() -> SourceConfig: return SourceConfig( name="example", @@ -175,3 +191,64 @@ def fail_authentication() -> ConnectionStatus: assert result.exit_code == 1 assert result.exception is not None assert "OAuth token request failed" in str(result.exception) + + +def test_get_deleted_streams_json_lines_and_forwards_cursor( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + source = _DeletedFeedSource(_config()) + monkeypatch.setattr( + cli_module, + "_load_connector_capabilities", + lambda *_args, **_kwargs: (source.config, SourceCapabilities(source)), + ) + + result = CliRunner().invoke( + app, + [ + "connector", + "get-deleted", + "example", + "accounts", + "--since", + "2026-08-01T00:00:00Z", + "--config", + str(tmp_path / "missing.yaml"), + ], + ) + + assert result.exit_code == 0, result.output + assert result.output.splitlines() == [ + '{"Id":"001000000000001AAA"}', + '{"Id":"001000000000002AAA"}', + ] + assert source.calls == [("accounts", "2026-08-01T00:00:00Z")] + + +def test_get_deleted_fails_clearly_when_capability_is_absent( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + source = _CheckableSource(_config(), ConnectionStatus(ok=True)) + monkeypatch.setattr( + cli_module, + "_load_connector_capabilities", + lambda *_args, **_kwargs: (source.config, SourceCapabilities(source)), + ) + + result = CliRunner().invoke( + app, + [ + "connector", + "get-deleted", + "example", + "accounts", + "--config", + str(tmp_path / "missing.yaml"), + ], + ) + + assert result.exit_code == 1 + assert result.exception is not None + assert "does not support operation 'get_deleted'" in str(result.exception) From 090c81b668de3c554393c352757a82a73044991d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 05:33:50 -0400 Subject: [PATCH 003/163] Add confirmed connector write-back command --- src/dander/cli/main.py | 117 +++++++++++++++++++++++++++++++ tests/cli/test_connector_cli.py | 119 +++++++++++++++++++++++++++++++- 2 files changed, 235 insertions(+), 1 deletion(-) diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index df2fbdb5..9f47a5c6 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -395,6 +395,102 @@ def get_deleted_records( raise ClickException(str(error)) from error +@connector_app.command("write") +def write_connector_record( + source_or_pipeline: str = typer.Argument( # noqa: B008 + ..., + help="Connector source name or pipeline name from dander.yaml.", + ), + endpoint: str = typer.Argument(..., help="Configured connector endpoint name."), # noqa: B008 + operation: str = typer.Argument( # noqa: B008 + ..., + help="One of: create, update, upsert, delete.", + ), + record: Path | None = typer.Option( # noqa: B008 + None, "--record", help="JSON object for create/upsert." + ), + identity: Path | None = typer.Option( # noqa: B008 + None, + "--identity", + help="JSON business-key object for update/delete.", + ), + changes: Path | None = typer.Option( # noqa: B008 + None, "--changes", help="JSON changes for update." + ), + confirm_write: bool = typer.Option( + False, + "--confirm-write", + help="Required acknowledgement that this command mutates the source system.", + ), + project_config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + connectors_dir: Path = typer.Option( # noqa: B008 + _DEFAULT_CONNECTORS_DIR, + "--connectors-dir", + ), +) -> None: + """Invoke one explicitly confirmed connector write-back operation.""" + if not confirm_write: + raise ClickException( + "Connector write-back changes the source system; rerun with --confirm-write" + ) + try: + selected = ConnectorOperation(operation) + except ValueError as error: + raise ClickException("Write operation must be create, update, upsert, or delete") from error + write_operations = { + ConnectorOperation.CREATE, + ConnectorOperation.UPDATE, + ConnectorOperation.UPSERT, + ConnectorOperation.DELETE, + } + if selected not in write_operations: + raise ClickException("Write operation must be create, update, upsert, or delete") + + record_payload: dict[str, object] | None = None + identity_payload: dict[str, str] | None = None + changes_payload: dict[str, object] | None = None + if selected in {ConnectorOperation.CREATE, ConnectorOperation.UPSERT}: + if identity is not None or changes is not None: + raise ClickException(f"Unexpected JSON input for {selected.value}") + record_payload = _read_json_object(record, "record") + elif selected is ConnectorOperation.UPDATE: + if record is not None: + raise ClickException("Unexpected JSON input for update") + identity_payload = _read_json_identity(identity) + changes_payload = _read_json_object(changes, "changes") + else: + if record is not None or changes is not None: + raise ClickException("Unexpected JSON input for delete") + identity_payload = _read_json_identity(identity) + _, capabilities = _load_connector_capabilities( + source_or_pipeline, + project_config=project_config, + connectors_dir=connectors_dir, + ) + try: + if selected is ConnectorOperation.CREATE: + assert record_payload is not None + result: object = dict(capabilities.create(endpoint, record_payload)) + elif selected is ConnectorOperation.UPDATE: + assert identity_payload is not None and changes_payload is not None + result = dict(capabilities.update(endpoint, identity_payload, changes_payload)) + elif selected is ConnectorOperation.UPSERT: + assert record_payload is not None + result = dict(capabilities.upsert(endpoint, record_payload)) + else: + assert identity_payload is not None + result = {"outcome": capabilities.delete(endpoint, identity_payload).value} + except ( + EnterpriseSourceError, + InvalidConnectorCapabilityResultError, + OAuthTokenError, + SecretResolutionError, + UnsupportedConnectorOperationError, + ) as error: + raise ClickException(str(error)) from error + typer.echo(json.dumps(result, sort_keys=True, separators=(",", ":"))) + + @graph_app.command("serve") def serve_graph( graph_file: Path = typer.Option(..., "--file", help="Existing PipelineGraph YAML/JSON file."), # noqa: B008 @@ -1709,5 +1805,26 @@ def _load_connector_capabilities( raise ClickException(str(error)) from error +def _read_json_object(path: Path | None, label: str) -> dict[str, object]: + """Read one CLI write-back input without echoing record contents in an error.""" + if path is None: + raise ClickException(f"--{label} is required") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ClickException(f"--{label} must name a readable JSON object") from error + if not isinstance(payload, dict) or any(not isinstance(key, str) for key in payload): + raise ClickException(f"--{label} must contain one JSON object") + return {key: value for key, value in payload.items()} + + +def _read_json_identity(path: Path | None) -> dict[str, str]: + """Read a write-back business key, whose protocol requires string values.""" + payload = _read_json_object(path, "identity") + if any(not isinstance(value, str) for value in payload.values()): + raise ClickException("--identity values must be strings") + return {key: value for key, value in payload.items() if isinstance(value, str)} + + if __name__ == "__main__": app() diff --git a/tests/cli/test_connector_cli.py b/tests/cli/test_connector_cli.py index a42cada9..9f15f0d8 100644 --- a/tests/cli/test_connector_cli.py +++ b/tests/cli/test_connector_cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from typing import TYPE_CHECKING, Any @@ -9,7 +10,13 @@ import dander.cli.main as cli_module from dander.cli.main import app -from dander.ingestion import ConnectionStatus, Source, SourceCapabilities, SourceConfig +from dander.ingestion import ( + ConnectionStatus, + DeleteOutcome, + Source, + SourceCapabilities, + SourceConfig, +) from dander.security import OAuthTokenError if TYPE_CHECKING: @@ -59,6 +66,41 @@ def get_deleted( yield {"Id": "001000000000002AAA"} +class _WritableSource(_CheckableSource): + def __init__(self, config: SourceConfig) -> None: + super().__init__(config, ConnectionStatus(ok=True)) + self.records: dict[str, dict[str, Any]] = {} + + def create(self, endpoint: str, record: Mapping[str, Any]) -> Mapping[str, Any]: + assert endpoint == "accounts" + created = {"Id": "001000000000001AAA", **record} + self.records[created["Id"]] = created + return created + + def update( + self, + endpoint: str, + identity: Mapping[str, str], + changes: Mapping[str, Any], + ) -> Mapping[str, Any]: + assert endpoint == "accounts" + current = self.records[identity["Id"]] + current.update(changes) + return current + + def upsert(self, endpoint: str, record: Mapping[str, Any]) -> Mapping[str, Any]: + assert endpoint == "accounts" + current = self.records.setdefault(str(record["Id"]), {}) + current.update(record) + return current + + def delete(self, endpoint: str, identity: Mapping[str, str]) -> DeleteOutcome: + assert endpoint == "accounts" + if self.records.pop(identity["Id"], None) is None: + return DeleteOutcome.NOT_FOUND + return DeleteOutcome.DELETED + + def _config() -> SourceConfig: return SourceConfig( name="example", @@ -252,3 +294,78 @@ def test_get_deleted_fails_clearly_when_capability_is_absent( assert result.exit_code == 1 assert result.exception is not None assert "does not support operation 'get_deleted'" in str(result.exception) + + +def test_write_dispatches_each_operation_against_stateful_source( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + source = _WritableSource(_config()) + monkeypatch.setattr( + cli_module, + "_load_connector_capabilities", + lambda *_args, **_kwargs: (source.config, SourceCapabilities(source)), + ) + inputs = { + "record": {"Name": "Acme"}, + "identity": {"Id": "001000000000001AAA"}, + "changes": {"Name": "Acme Updated"}, + "upsert": {"Id": "001000000000001AAA", "Name": "Acme Final"}, + } + paths: dict[str, Path] = {} + for name, payload in inputs.items(): + path = tmp_path / f"{name}.json" + path.write_text(json.dumps(payload), encoding="utf-8") + paths[name] = path + + commands = [ + ("create", ["--record", str(paths["record"])], "Acme"), + ( + "update", + ["--identity", str(paths["identity"]), "--changes", str(paths["changes"])], + "Acme Updated", + ), + ("upsert", ["--record", str(paths["upsert"])], "Acme Final"), + ("delete", ["--identity", str(paths["identity"])], "deleted"), + ] + for operation, options, expected in commands: + result = CliRunner().invoke( + app, + [ + "connector", + "write", + "example", + "accounts", + operation, + *options, + "--confirm-write", + "--config", + str(tmp_path / "missing.yaml"), + ], + ) + assert result.exit_code == 0, result.output + assert expected in result.output + + assert source.records == {} + + +def test_write_requires_explicit_confirmation_before_loading_source( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + loaded = False + + def load_source(*_args: object, **_kwargs: object) -> None: + nonlocal loaded + loaded = True + + monkeypatch.setattr(cli_module, "_load_connector_capabilities", load_source) + result = CliRunner().invoke( + app, + ["connector", "write", "example", "accounts", "delete"], + ) + + assert result.exit_code == 1 + assert result.exception is not None + assert "--confirm-write" in str(result.exception) + assert loaded is False From 56fa7dfe85cf084c7a9169f8861a1ceaa524563b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 06:04:29 -0400 Subject: [PATCH 004/163] Update capability handoff --- HANDOFF.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 55c1b360..49c7188e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Synchronized Josh's `upstream/main` onto Harrison's current `origin/main` without dropping either tip. -- Preserved Josh's capability contracts, ticket backlog, and reviewed secret-scan exception. -- Preserved Dander `0.6.0rc1`, its Salesforce CRM overlay, and the accepted release work on main. -- Kept connector capability additions additive to the existing source and plugin contracts. +- Synchronized Josh's accepted capability work into Harrison's protected `main` through PR `#90`. +- Merged explicit JSONL invocation for optional `get_deleted` feeds through protected PR `#87`. +- Added confirmed create/update/upsert/delete invocation in stacked PR `#88`. +- Kept provider writes, scheduled execution, BigQuery deletion, and deployment out of core. +- Used one stateful fake to prove write dispatch and repeat-safe delete outcomes. ## Try It -Run `uv run dander validate`, then inspect a configured connector with -`uv run dander connector inspect PIPELINE`. +Run `uv run dander connector get-deleted SOURCE ENDPOINT --since CURSOR` for a supported feed. +Write-back uses `uv run dander connector write SOURCE ENDPOINT OPERATION` with operation-specific +JSON files and the required `--confirm-write` acknowledgement. ## Checks -- Ruff lint/format and strict mypy passed. -- Full test suite passed: `770 passed`. -- Locked production dependency audit found no known vulnerabilities. -- Platform and stage-zero Terraform format/init/validation passed with backends disabled. -- Wheel/sdist inspection and a clean external source-free install/validation passed. -- The Linux container built and passed CLI, unprivileged-user, and bundled-asset checks. +- Synchronized `main` passed all five protected CI checks. +- Pre-synchronization capability suite, Ruff, formatting, and strict mypy passed. +- The synchronized #87 branch passed its full local suite: `772 passed`. +- The synchronized #88 full local suite passed: `774 passed`. ## Decisions -- Use a merge commit so both repositories' existing commit identities remain visible. -- Keep `0.6.0rc1` as the package version from Harrison's newer release tip. -- Treat this PR as synchronization only; no release, deployment, or schedule mutation is included. +- Keep read and mutation entry points explicit rather than adding them to normal pipeline runs. +- Require file-based JSON and confirmation before loading a source for write-back. +- Make provider plugins own retry and idempotency behavior; core does not retry capability calls. ## Remaining -- Merge this synchronization PR after review and protected CI. -- Rebase or retarget Dander PRs #87 and #88 onto synchronized `main`, preserving their stack order. -- Publication, live acceptance, retained-project changes, and schedules remain separately gated. +- Retarget and merge PR `#88` through protected `main`. +- Update the Salesforce provider stack to the published Dander contract version. +- Publish candidates and run live acceptance only after the complete protected stack passes. ## Review First +- `src/dander/cli/main.py` +- `tests/cli/test_connector_cli.py` - `src/dander/ingestion/capabilities.py` -- `docs/decisions.md` -- `.gitleaksignore` From 475446b39a56f38d8f882dc1c1dab18934756e78 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 16:03:01 -0400 Subject: [PATCH 005/163] Prevent stale release metadata --- .github/workflows/ci.yml | 6 +- .github/workflows/publish.yml | 2 + HANDOFF.md | 42 +++--- README.md | 11 +- docs/getting-started.md | 2 +- docs/known-limitations.md | 8 +- docs/release-audit.md | 11 +- docs/session-resume.md | 6 +- docs/upgrading.md | 2 +- scripts/check_distribution.py | 54 ++++++-- scripts/check_release_metadata.py | 164 ++++++++++++++++++++++++ tests/pipeline/test_graph_deployment.py | 3 +- tests/project/test_scaffold.py | 3 +- tests/test_distribution_contract.py | 3 +- tests/test_release_metadata.py | 48 +++++++ 15 files changed, 310 insertions(+), 55 deletions(-) create mode 100644 scripts/check_release_metadata.py create mode 100644 tests/test_release_metadata.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 087a5f15..33dac015 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,11 +80,13 @@ jobs: terraform_version: 1.9.8 - name: Build and inspect artifacts run: | + python scripts/check_release_metadata.py uv build python scripts/check_distribution.py dist/*.whl dist/*.tar.gz - name: Install and run outside the checkout run: | TEMP_DIR="$(mktemp -d)" + VERSION="$(uv version --short)" INDEX=0 for ARTIFACT in dist/*.whl dist/*.tar.gz; do ENV_DIR="$TEMP_DIR/venv-$INDEX" @@ -93,11 +95,11 @@ jobs: uv pip install --python "$ENV_DIR/bin/python" "$ARTIFACT" ( cd "$TEMP_DIR" - "$ENV_DIR/bin/dander" --version | grep -F "dander 0.6.0rc1" + "$ENV_DIR/bin/dander" --version | grep -F "dander $VERSION" "$ENV_DIR/bin/dander" new "$PROJECT_DIR" cd "$PROJECT_DIR" "$ENV_DIR/bin/dander" validate - grep -F "ARG DANDER_VERSION=0.6.0rc1" Dockerfile + grep -F "ARG DANDER_VERSION=$VERSION" Dockerfile "$ENV_DIR/bin/python" -c 'import dander; assert "site-packages" in dander.__file__' ) INDEX=$((INDEX + 1)) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a81dea16..d29685b4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,6 +17,8 @@ jobs: run: | VERSION="$(uv version --short)" test "$GITHUB_REF" = "refs/tags/v$VERSION" + - name: Reject stale public release metadata + run: python scripts/check_release_metadata.py - name: Build wheel and source distribution run: uv build - name: Validate distribution contents diff --git a/HANDOFF.md b/HANDOFF.md index 49c7188e..61283615 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Synchronized Josh's accepted capability work into Harrison's protected `main` through PR `#90`. -- Merged explicit JSONL invocation for optional `get_deleted` feeds through protected PR `#87`. -- Added confirmed create/update/upsert/delete invocation in stacked PR `#88`. -- Kept provider writes, scheduled execution, BigQuery deletion, and deployment out of core. -- Used one stateful fake to prove write dispatch and repeat-safe delete outcomes. +- Added a release-metadata check covering package metadata, public docs, changelog, and templates. +- Made CI and artifact validation derive the version from `pyproject.toml`. +- Made wheel and sdist validation reject stale PyPI descriptions. +- Synchronized public candidate documentation on `0.6.0rc1` while retaining `0.5.1` as stable alpha. +- Added a regression test for the prior `0.1.0` README drift. ## Try It -Run `uv run dander connector get-deleted SOURCE ENDPOINT --since CURSOR` for a supported feed. -Write-back uses `uv run dander connector write SOURCE ENDPOINT OPERATION` with operation-specific -JSON files and the required `--confirm-write` acknowledgement. +Run `python3 scripts/check_release_metadata.py`. Build with `uv build`, then pass the wheel and +sdist to `python3 scripts/check_distribution.py`. ## Checks -- Synchronized `main` passed all five protected CI checks. -- Pre-synchronization capability suite, Ruff, formatting, and strict mypy passed. -- The synchronized #87 branch passed its full local suite: `772 passed`. -- The synchronized #88 full local suite passed: `774 passed`. +- Ruff lint and format checks passed across the repository. +- Strict mypy passed across `src` and `tests`. +- Full test suite passed: `766 passed`. +- Wheel and sdist built and passed identity, contents, hygiene, and description validation. +- `git diff --check` passed. ## Decisions -- Keep read and mutation entry points explicit rather than adding them to normal pipeline runs. -- Require file-based JSON and confirmation before loading a source for write-back. -- Make provider plugins own retry and idempotency behavior; core does not retry capability calls. +- PyPI release descriptions are immutable, so the live correction must arrive in a new release. +- `pyproject.toml` is the single version source; publication-facing copy must match it exactly. +- `0.6.0rc1` is the prepared candidate; `0.5.1` remains the latest stable alpha. ## Remaining -- Retarget and merge PR `#88` through protected `main`. -- Update the Salesforce provider stack to the published Dander contract version. -- Publish candidates and run live acceptance only after the complete protected stack passes. +- Review and merge the release-metadata pull request after protected CI passes. +- Publish `0.6.0rc1` only with explicit approval for the full candidate, not merely its metadata. +- Verify the live PyPI page after publication. ## Review First -- `src/dander/cli/main.py` -- `tests/cli/test_connector_cli.py` -- `src/dander/ingestion/capabilities.py` +- `scripts/check_release_metadata.py` +- `scripts/check_distribution.py` +- `.github/workflows/publish.yml` diff --git a/README.md b/README.md index 13c9fecc..fe6552e3 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,13 @@ Operator-facing documentation: Python 3.12 (app + CLI) · BigQuery SQL (transforms) · Terraform/HCL (infra) · YAML (config). -## Install the latest public release +## Install the current public candidate The Python distribution is named `dander-platform` because the `dander` name on PyPI belongs to a different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.5.1 +uv tool install dander-platform==0.6.0rc1 dander --version dander new my-data-platform cd my-data-platform @@ -623,9 +623,10 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.5.1` is a working alpha: its source-free GCP platform, four retained daily pipelines, -connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface -have completed bounded live proofs. The limits above still make it **unsuitable for production**. +Dander `0.6.0rc1` is the current public candidate: its source-free GCP platform, four retained +daily pipelines, connector plugins, transform/test engine, run ledger, metadata spine, and Druff +operator interface have completed bounded live proofs. The limits above still make it +**unsuitable for production**. The named HR, compensation, and customer systems describe connector categories or bounded test implementations; they do not imply that this repository came from, connects to, or contains data from an existing company. Normal provenance, licensing, and privacy review still applies before diff --git a/docs/getting-started.md b/docs/getting-started.md index 77f6e4c1..4681eca3 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.5.1 +uv tool install dander-platform==0.6.0rc1 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 5c813822..e130622f 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -1,6 +1,6 @@ # Known limitations -Dander `0.5.x` is alpha and proves a focused GCP-native vertical slice. It is not yet suitable for +Dander `0.6.x` is alpha and proves a focused GCP-native vertical slice. It is not yet suitable for an unattended production system containing business-critical data. ## Ingestion and schemas @@ -54,6 +54,6 @@ an unattended production system containing business-critical data. ## Support boundary -`0.5.x` receives fixes only. New connectors, commands, subsystems, manifest capabilities, and -broader writer orchestration enter through `0.6.0` or a later minor. Only the latest patch in the -current public `0.x` minor is supported; see [SECURITY.md](../SECURITY.md). +`0.6.x` receives candidate fixes only until the minor is promoted. Dander `0.5.1` remains the +latest stable alpha during candidate acceptance. Only the newest candidate and the latest patch +in the current stable `0.x` minor are supported; see [SECURITY.md](../SECURITY.md). diff --git a/docs/release-audit.md b/docs/release-audit.md index 59db4470..29e7081b 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -1,6 +1,6 @@ # Dander Platform Release Audit -Audited on 2026-08-05 against the product promise in `steering/00-project-overview.md`. +Audited on 2026-08-06 against the product promise in `steering/00-project-overview.md`. “Live-proven” means the behavior was observed in a disposable provider account or retained GCP project. “Implemented” means automated tests cover the contract while an optional provider or cloud path remains outside the live proof. @@ -23,7 +23,8 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander: `0.5.1`; public Salesforce and ServiceNow plugins: `0.2.0`. +- Public Dander candidate: `0.6.0rc1`; latest stable alpha: `0.5.1`; public Salesforce and + ServiceNow plugins: `0.2.0`. - Retained source-free Dander image: `sha256:3220623…8995`, built with Dander `0.5.0` and both plugin `0.2.0` packages. Dander `0.5.1` changes only catalog recommendations, so the runtime image was intentionally not replaced. @@ -39,8 +40,10 @@ cloud path remains outside the live proof. - Dander provisions inside an existing billing-linked GCP project; it does not create a project or attach billing. Its managed cost guard is simulation-first and is not a spending cap. -- Salesforce supports one read-only Accounts slice. ServiceNow supports one read-only incidents - slice. Provider write-back and broad object/table coverage are not claimed. +- The `0.6.0rc1` Salesforce example supports read-only Accounts, Contacts, Opportunities, and + Users; the retained soak still exercises its existing Accounts pipeline. ServiceNow supports + one read-only incidents slice. Provider write-back and broad object/table coverage are not + claimed. - NetSuite is simulator-validated only; Workday lacks real-tenant acceptance. Live Storage Write and optional Dataplex publication are not implied by the retained proof. - Druff is a public static browser shell. Its privileged operations remain in an operator-started, diff --git a/docs/session-resume.md b/docs/session-resume.md index 11943a53..c77f0035 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -1,12 +1,12 @@ -# Session Resume — 2026-08-05 +# Session Resume — 2026-08-06 Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `docs/release-audit.md` before changing code or cloud resources. ## Public releases -- Dander `0.5.1` is the newest supported package. It corrects the curated connector catalog and - does not change the `0.5.0` runtime. +- Dander `0.6.0rc1` is the newest public candidate. Dander `0.5.1` remains the latest stable alpha + while candidate acceptance is in progress. - Salesforce and ServiceNow connector plugins are public at `0.2.0` and require Dander `0.5.x`. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, execution, catalog, operation-authoring, and deployment-preview work. diff --git a/docs/upgrading.md b/docs/upgrading.md index 0fda853c..ec10dd1a 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.5.1" +export DANDER_TARGET_VERSION="0.6.0rc1" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 622a2f97..64b82711 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -6,11 +6,11 @@ import argparse import email import tarfile +import tomllib import zipfile from pathlib import Path, PurePosixPath -EXPECTED_NAME = "dander-platform" -EXPECTED_VERSION = "0.6.0rc1" +ROOT = Path(__file__).resolve().parents[1] WHEEL_REQUIRED = { "dander/templates/project/.dockerignore", "dander/templates/project/.gitignore", @@ -49,12 +49,13 @@ def main() -> None: parser.add_argument("wheel", type=Path) parser.add_argument("sdist", type=Path) arguments = parser.parse_args() - _check_wheel(arguments.wheel) - _check_sdist(arguments.sdist) - print(f"Validated {EXPECTED_NAME} {EXPECTED_VERSION} wheel and sdist.") + expected_name, expected_version = _project_identity() + _check_wheel(arguments.wheel, expected_name=expected_name, expected_version=expected_version) + _check_sdist(arguments.sdist, expected_name=expected_name, expected_version=expected_version) + print(f"Validated {expected_name} {expected_version} wheel and sdist.") -def _check_wheel(path: Path) -> None: +def _check_wheel(path: Path, *, expected_name: str, expected_version: str) -> None: with zipfile.ZipFile(path) as archive: names = set(archive.namelist()) _reject_unsafe(names) @@ -63,10 +64,14 @@ def _check_wheel(path: Path) -> None: if len(metadata_paths) != 1: raise ValueError("Wheel must contain exactly one METADATA file") metadata = email.message_from_bytes(archive.read(metadata_paths[0])) - _check_metadata(metadata["Name"], metadata["Version"]) + _check_metadata( + metadata, + expected_name=expected_name, + expected_version=expected_version, + ) -def _check_sdist(path: Path) -> None: +def _check_sdist(path: Path, *, expected_name: str, expected_version: str) -> None: with tarfile.open(path, mode="r:gz") as archive: members = archive.getmembers() if any(member.issym() or member.islnk() for member in members): @@ -83,7 +88,11 @@ def _check_sdist(path: Path) -> None: if metadata_file is None: raise ValueError("Source distribution is missing PKG-INFO") metadata = email.message_from_binary_file(metadata_file) - _check_metadata(metadata["Name"], metadata["Version"]) + _check_metadata( + metadata, + expected_name=expected_name, + expected_version=expected_version, + ) def _reject_unsafe(names: set[str]) -> None: @@ -105,9 +114,32 @@ def _require(names: set[str], required: set[str], *, kind: str) -> None: raise ValueError(f"Missing {kind} assets: {', '.join(missing)}") -def _check_metadata(name: str | None, version: str | None) -> None: - if name != EXPECTED_NAME or version != EXPECTED_VERSION: +def _check_metadata( + metadata: email.message.Message, + *, + expected_name: str, + expected_version: str, +) -> None: + name = metadata["Name"] + version = metadata["Version"] + if name != expected_name or version != expected_version: raise ValueError(f"Unexpected distribution identity: {name} {version}") + description = metadata.get_payload() + required_description = { + f"uv tool install dander-platform=={expected_version}", + f"Dander `{expected_version}` is the current public candidate", + } + if not isinstance(description, str): + raise ValueError("Distribution description is not text") + missing = sorted(marker for marker in required_description if marker not in description) + if missing: + raise ValueError(f"Distribution description is stale: {', '.join(missing)}") + + +def _project_identity() -> tuple[str, str]: + config = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + project = config["project"] + return project["name"], project["version"] if __name__ == "__main__": diff --git a/scripts/check_release_metadata.py b/scripts/check_release_metadata.py new file mode 100644 index 00000000..3aebed95 --- /dev/null +++ b/scripts/check_release_metadata.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Reject stale public release versions before Dander is published.""" + +from __future__ import annotations + +import argparse +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class VersionReference: + path: str + label: str + pattern: re.Pattern[str] + + +EXACT_REFERENCES = ( + VersionReference( + "README.md", + "README install command", + re.compile(r"uv tool install dander-platform==(?P[^\s]+)"), + ), + VersionReference( + "README.md", + "README status", + re.compile(r"Dander `(?P[^`]+)` is the current public candidate"), + ), + VersionReference( + "docs/getting-started.md", + "quickstart install command", + re.compile(r"uv tool install dander-platform==(?P[^\s]+)"), + ), + VersionReference( + "docs/upgrading.md", + "upgrade target", + re.compile(r'DANDER_TARGET_VERSION="(?P[^"]+)"'), + ), + VersionReference( + "docs/session-resume.md", + "session public candidate", + re.compile(r"Dander `(?P[^`]+)` is the newest public candidate"), + ), + VersionReference( + "docs/release-audit.md", + "release audit public candidate", + re.compile(r"Public Dander candidate: `(?P[^`]+)`"), + ), +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="Repository root to validate.", + ) + arguments = parser.parse_args() + errors = release_metadata_errors(arguments.root.resolve()) + if errors: + raise SystemExit("Release metadata is inconsistent:\n- " + "\n- ".join(errors)) + project = _project(arguments.root.resolve()) + print(f"Validated release metadata for {project['name']} {project['version']}.") + + +def release_metadata_errors(root: Path) -> list[str]: + try: + project = _project(root) + except (OSError, KeyError, tomllib.TOMLDecodeError) as error: + return [f"could not read project metadata: {error}"] + + name = project.get("name") + version = project.get("version") + if name != "dander-platform": + return [f"pyproject project name is {name!r}; expected 'dander-platform'"] + if not isinstance(version, str) or not re.fullmatch( + r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+|\.post\d+|\.dev\d+)?", version + ): + return [f"pyproject project version is not a supported release version: {version!r}"] + + errors: list[str] = [] + for reference in EXACT_REFERENCES: + errors.extend(_check_reference(root, reference, expected=version)) + + release_line = re.match(r"(?P\d+\.\d+)\.", version) + assert release_line is not None + errors.extend( + _check_single_match( + root / "docs/known-limitations.md", + "known-limitations release line", + re.compile(r"Dander `(?P\d+\.\d+\.x)` is alpha"), + expected=f"{release_line.group('line')}.x", + ) + ) + + changelog = _read(root / "CHANGELOG.md", errors) + if changelog is not None and not re.search( + rf"^## {re.escape(version)}(?:\s|$)", changelog, flags=re.MULTILINE + ): + errors.append(f"CHANGELOG.md has no release heading for {version}") + + template = _read(root / "src/dander/templates/project/Dockerfile", errors) + if template is not None: + placeholder = "ARG DANDER_VERSION=__DANDER_DISTRIBUTION_VERSION__" + if template.count(placeholder) != 1: + errors.append( + "starter Dockerfile must contain exactly one Dander distribution " + "version placeholder" + ) + + return errors + + +def _project(root: Path) -> dict[str, object]: + config = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8")) + return cast("dict[str, object]", config["project"]) + + +def _check_reference(root: Path, reference: VersionReference, *, expected: str) -> list[str]: + return _check_single_match( + root / reference.path, + reference.label, + reference.pattern, + expected=expected, + ) + + +def _check_single_match( + path: Path, + label: str, + pattern: re.Pattern[str], + *, + expected: str, +) -> list[str]: + errors: list[str] = [] + content = _read(path, errors) + if content is None: + return errors + matches = pattern.findall(content) + if len(matches) != 1: + return [f"{label} must appear exactly once in {path.name}; found {len(matches)}"] + actual = matches[0] + if actual != expected: + return [f"{label} uses {actual}; package metadata uses {expected}"] + return [] + + +def _read(path: Path, errors: list[str]) -> str | None: + try: + return path.read_text(encoding="utf-8") + except OSError as error: + errors.append(f"could not read {path}: {error}") + return None + + +if __name__ == "__main__": + main() diff --git a/tests/pipeline/test_graph_deployment.py b/tests/pipeline/test_graph_deployment.py index 206bbc06..70cb7006 100644 --- a/tests/pipeline/test_graph_deployment.py +++ b/tests/pipeline/test_graph_deployment.py @@ -6,6 +6,7 @@ import pytest +from dander import __version__ from dander.pipeline.graph_deployment import ( GraphDeploymentError, GraphDeploymentPreviewer, @@ -161,7 +162,7 @@ def test_preview_builds_source_free_snapshot_and_isolates_exact_plan(tmp_path: P ).preview(store, expected_revision=store.load().revision) assert observations["source_free"] is True - assert "ARG DANDER_VERSION=0.6.0rc1" in observations["dockerfile"] + assert f"ARG DANDER_VERSION={__version__}" in observations["dockerfile"] assert "RUN dander plugins install --config dander.yaml" in observations["dockerfile"] assert observations["graph"] == _GRAPH assert observations["publish"] == ("proof-project", "us-central1", "candidate") diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index 73eacca8..a9940f7f 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -6,6 +6,7 @@ import pytest +from dander import __version__ from dander.project import ProjectScaffoldError, load_project_config, scaffold_project if TYPE_CHECKING: @@ -20,7 +21,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: assert manifest.pipelines["greenhouse_jobs"].paused assert manifest.platform.safety.require_guarded_free_tier is False dockerfile = (project / "Dockerfile").read_text(encoding="utf-8") - assert "ARG DANDER_VERSION=0.6.0rc1" in dockerfile + assert f"ARG DANDER_VERSION={__version__}" in dockerfile assert "RUN dander plugins install --config dander.yaml" in dockerfile assert "COPY --chown=65532:65532 dander.yaml ./dander.yaml" in dockerfile assert "COPY --chown=65532:65532 connectors ./connectors" in dockerfile diff --git a/tests/test_distribution_contract.py b/tests/test_distribution_contract.py index 1e41385a..be41b9eb 100644 --- a/tests/test_distribution_contract.py +++ b/tests/test_distribution_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations import tomllib +from importlib.metadata import version from pathlib import Path ROOT = Path(__file__).parents[1] @@ -12,7 +13,7 @@ def test_distribution_identity_and_version_are_public_release() -> None: project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] assert project["name"] == "dander-platform" - assert project["version"] == "0.6.0rc1" + assert project["version"] == version("dander-platform") assert project["scripts"]["dander"] == "dander.cli.main:app" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py new file mode 100644 index 00000000..f45b0994 --- /dev/null +++ b/tests/test_release_metadata.py @@ -0,0 +1,48 @@ +"""Release-facing versions stay synchronized with package metadata.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from scripts.check_release_metadata import release_metadata_errors + +ROOT_FILES = ( + "CHANGELOG.md", + "README.md", + "docs/getting-started.md", + "docs/known-limitations.md", + "docs/release-audit.md", + "docs/session-resume.md", + "docs/upgrading.md", + "pyproject.toml", + "src/dander/templates/project/Dockerfile", +) + + +def test_release_metadata_matches_package_version() -> None: + root = Path(__file__).parents[1] + + assert release_metadata_errors(root) == [] + + +def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: + root = Path(__file__).parents[1] + for relative in ROOT_FILES: + source = root / relative + destination = tmp_path / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + readme = tmp_path / "README.md" + readme.write_text( + readme.read_text(encoding="utf-8").replace( + "dander-platform==0.6.0rc1", + "dander-platform==0.1.0", + 1, + ), + encoding="utf-8", + ) + + assert release_metadata_errors(tmp_path) == [ + "README install command uses 0.1.0; package metadata uses 0.6.0rc1" + ] From 0d264065e45e6f8c8a7a96a404e5b23ca312d7d9 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 16:07:37 -0400 Subject: [PATCH 006/163] Use Python 3 for release checks --- .github/workflows/ci.yml | 2 +- .github/workflows/publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33dac015..a26dc8fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: terraform_version: 1.9.8 - name: Build and inspect artifacts run: | - python scripts/check_release_metadata.py + python3 scripts/check_release_metadata.py uv build python scripts/check_distribution.py dist/*.whl dist/*.tar.gz - name: Install and run outside the checkout diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d29685b4..9c15b106 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,7 +18,7 @@ jobs: VERSION="$(uv version --short)" test "$GITHUB_REF" = "refs/tags/v$VERSION" - name: Reject stale public release metadata - run: python scripts/check_release_metadata.py + run: python3 scripts/check_release_metadata.py - name: Build wheel and source distribution run: uv build - name: Validate distribution contents From b69fd638bd38ce1fa8a6dad94428f364bc66497c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Thu, 6 Aug 2026 16:08:29 -0400 Subject: [PATCH 007/163] Record release check results --- HANDOFF.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HANDOFF.md b/HANDOFF.md index 61283615..e3fc5f01 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -17,8 +17,9 @@ sdist to `python3 scripts/check_distribution.py`. - Ruff lint and format checks passed across the repository. - Strict mypy passed across `src` and `tests`. -- Full test suite passed: `766 passed`. +- Full test suite passed: `776 passed`. - Wheel and sdist built and passed identity, contents, hygiene, and description validation. +- The built wheel installed and started outside the repository checkout. - `git diff --check` passed. ## Decisions From 941b093b60c7f0b4780aec8d4c58451f38039c7a Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 01:32:41 -0400 Subject: [PATCH 008/163] Preserve managed Dataplex schema aspects --- CHANGELOG.md | 5 +++++ HANDOFF.md | 37 ++++++++++++++++------------------ README.md | 7 ++++--- src/dander/catalog/README.md | 10 +++++---- src/dander/catalog/dataplex.py | 15 +++++++++++--- tests/catalog/test_dataplex.py | 7 ++++++- 6 files changed, 50 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b8f6ada..a7901f5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ capabilities enter through the next minor release. ## Unreleased +### Fixed + +- Preserve BigQuery's Google-managed required schema aspect during Dataplex publication while + continuing to publish Dander's optional overview, contacts, and generic metadata aspects. + ## 0.6.0rc1 — 2026-08-06 (alpha) ### Added diff --git a/HANDOFF.md b/HANDOFF.md index e3fc5f01..2260dcfb 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,37 @@ ## Finished -- Added a release-metadata check covering package metadata, public docs, changelog, and templates. -- Made CI and artifact validation derive the version from `pyproject.toml`. -- Made wheel and sdist validation reject stale PyPI descriptions. -- Synchronized public candidate documentation on `0.6.0rc1` while retaining `0.5.1` as stable alpha. -- Added a regression test for the prior `0.1.0` README drift. +- Kept BigQuery's required `schema` aspect Google-managed during Dataplex publication. +- Continued publishing Dander's optional `overview`, `contacts`, and `generic` aspects. +- Added a regression assertion for the exact Dataplex request keys. +- Corrected the public catalog documentation and changelog. ## Try It -Run `python3 scripts/check_release_metadata.py`. Build with `uv build`, then pass the wheel and -sdist to `python3 scripts/check_distribution.py`. +Run `uv run pytest tests/catalog/test_dataplex.py tests/cli/test_catalog_cli.py -q` and inspect +`DataplexCatalogPublisher.request_for()` to confirm the schema key is absent. ## Checks -- Ruff lint and format checks passed across the repository. +- Ruff lint and format checks passed. - Strict mypy passed across `src` and `tests`. - Full test suite passed: `776 passed`. -- Wheel and sdist built and passed identity, contents, hygiene, and description validation. -- The built wheel installed and started outside the repository checkout. -- `git diff --check` passed. +- Terraform platform and stage-zero validation passed. +- Wheel and sdist built, validated, and the wheel installed and generated a valid project outside the checkout. ## Decisions -- PyPI release descriptions are immutable, so the live correction must arrive in a new release. -- `pyproject.toml` is the single version source; publication-facing copy must match it exactly. -- `0.6.0rc1` is the prepared candidate; `0.5.1` remains the latest stable alpha. +- Dander's local metadata spine still carries declared column schema. +- Dataplex publication enriches optional metadata without rewriting BigQuery's required system schema. ## Remaining -- Review and merge the release-metadata pull request after protected CI passes. -- Publish `0.6.0rc1` only with explicit approval for the full candidate, not merely its metadata. -- Verify the live PyPI page after publication. +- Merge the protected fix PR. +- Publish `0.6.0rc2` from a release-only commit. +- Repeat the failed live catalog scenario and the bounded candidate smoke suite. ## Review First -- `scripts/check_release_metadata.py` -- `scripts/check_distribution.py` -- `.github/workflows/publish.yml` +- `src/dander/catalog/dataplex.py` +- `tests/catalog/test_dataplex.py` +- `README.md` diff --git a/README.md b/README.md index fe6552e3..f3fba4ba 100644 --- a/README.md +++ b/README.md @@ -558,9 +558,10 @@ uv run dander catalog \ --output .dander/catalog.json ``` -Local compilation is the default. `--publish-dataplex` explicitly attaches overview, contacts, -schema, and generic system aspects to the corresponding BigQuery entry; it can be combined with -`--guarded-free-tier`. Publication never deletes unrelated aspects. Google currently makes +Local compilation is the default. `--publish-dataplex` explicitly attaches the optional overview, +contacts, and generic system aspects to the corresponding BigQuery entry; BigQuery's required +schema aspect remains Google-managed. It can be combined with `--guarded-free-tier`, and +publication never deletes unrelated aspects. Google currently makes Knowledge Catalog API calls free but charges for stored aspect metadata, so cloud mutation is not implicit. See [Knowledge Catalog pricing](https://cloud.google.com/products/knowledge-catalog/pricing) and [Dataplex aspect management](https://docs.cloud.google.com/dataplex/docs/enrich-entries-metadata). diff --git a/src/dander/catalog/README.md b/src/dander/catalog/README.md index 2e949ed6..7fd51086 100644 --- a/src/dander/catalog/README.md +++ b/src/dander/catalog/README.md @@ -6,7 +6,8 @@ additional representations: - an atomic BigQuery/SQLite pipeline snapshot containing source, model, lineage, test, and metric definitions; - a deterministic, versioned JSON semantic registry for agents and local tooling; -- Dataplex Knowledge Catalog system aspects for overview, contacts, schema, and generic metadata. +- Dataplex Knowledge Catalog optional system aspects for overview, contacts, and generic metadata; + BigQuery continues to manage its required schema aspect. Local compilation is the default and requires no catalog API: @@ -41,8 +42,9 @@ uv run dander catalog \ --guarded-free-tier ``` -The publisher uses `modifyEntry` for the first-party BigQuery system entry, updates only the four -generated aspect keys, and leaves unrelated aspects untouched. It does not create custom aspect -types. Dataplex API calls are free, but Google charges for stored aspect metadata; see the current +The publisher uses `modifyEntry` for the first-party BigQuery system entry, updates only the three +optional generated aspect keys, leaves BigQuery's required schema and all unrelated aspects +untouched, and does not create custom aspect types. Dataplex API calls are free, but Google charges +for stored aspect metadata; see the current [Knowledge Catalog pricing](https://cloud.google.com/products/knowledge-catalog/pricing). Therefore Dander never publishes merely because `catalog` was run. diff --git a/src/dander/catalog/dataplex.py b/src/dander/catalog/dataplex.py index 39bbb11d..ed4686b7 100644 --- a/src/dander/catalog/dataplex.py +++ b/src/dander/catalog/dataplex.py @@ -19,6 +19,7 @@ _CONTACTS = "dataplex-types.global.contacts" _SCHEMA = "dataplex-types.global.schema" _GENERIC = "dataplex-types.global.generic" +_BIGQUERY_MANAGED_REQUIRED_ASPECTS = frozenset({_SCHEMA}) class CatalogPublishError(RuntimeError): @@ -91,7 +92,7 @@ def generate(self, asset: CatalogAsset) -> tuple[GeneratedAspect, ...]: class DataplexCatalogPublisher: - """Attach generated aspects to first-party BigQuery catalog entries.""" + """Attach optional generated aspects to first-party BigQuery catalog entries.""" def __init__( self, @@ -112,7 +113,7 @@ def request_for(self, asset: CatalogAsset) -> dataplex_v1.ModifyEntryRequest: """Build an aspect-only request that preserves all unrelated aspects.""" if asset.project != self._project: raise CatalogPublishError("Catalog asset belongs to a different project") - aspects = self._generator.generate(asset) + aspects = self._publishable_aspects(asset) entry_name = _bigquery_entry_name(asset, location=self._location) entry = dataplex_v1.Entry( name=entry_name, @@ -157,9 +158,17 @@ def normalized_aspects(self, asset: CatalogAsset) -> dict[str, object]: return { key: _normalize_value(aspect.data) for key, aspect in sorted(entry.aspects.items()) - if key in {generated.key for generated in self._generator.generate(asset)} + if key in {generated.key for generated in self._publishable_aspects(asset)} } + def _publishable_aspects(self, asset: CatalogAsset) -> tuple[GeneratedAspect, ...]: + """Exclude required aspects that Google manages for BigQuery system entries.""" + return tuple( + aspect + for aspect in self._generator.generate(asset) + if aspect.key not in _BIGQUERY_MANAGED_REQUIRED_ASPECTS + ) + def _bigquery_entry_name(asset: CatalogAsset, *, location: str) -> str: entry_id = ( diff --git a/tests/catalog/test_dataplex.py b/tests/catalog/test_dataplex.py index 50a5e86c..aea45345 100644 --- a/tests/catalog/test_dataplex.py +++ b/tests/catalog/test_dataplex.py @@ -118,7 +118,13 @@ def test_publisher_targets_bigquery_system_entry_and_only_generated_aspects() -> assert request.name == "projects/valid-project-123/locations/us" assert request.update_mask.paths == ["aspects"] assert request.delete_missing_aspects is False + assert set(request.aspect_keys) == { + "dataplex-types.global.contacts", + "dataplex-types.global.generic", + "dataplex-types.global.overview", + } assert set(request.aspect_keys) == set(request.entry.aspects) + assert "dataplex-types.global.schema" not in request.entry.aspects generic = request.entry.aspects["dataplex-types.global.generic"].data assert generic["type"] == "dander-view" assert generic["system"] == "greenhouse_job_board" @@ -140,7 +146,6 @@ def test_publisher_reads_back_normalized_aspects() -> None: "dataplex-types.global.contacts", "dataplex-types.global.generic", "dataplex-types.global.overview", - "dataplex-types.global.schema", } From fdf91fc58c2aa492b3cd133d7047c45f873c9d84 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 01:37:57 -0400 Subject: [PATCH 009/163] Prepare Dander 0.6.0rc2 --- CHANGELOG.md | 2 ++ HANDOFF.md | 31 +++++++++++++++---------------- README.md | 4 ++-- docs/getting-started.md | 2 +- docs/release-audit.md | 4 ++-- docs/session-resume.md | 4 ++-- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 ++-- uv.lock | 2 +- 10 files changed, 29 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7901f5b..3f4abf76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ capabilities enter through the next minor release. ## Unreleased +## 0.6.0rc2 — 2026-08-07 (alpha) + ### Fixed - Preserve BigQuery's Google-managed required schema aspect during Dataplex publication while diff --git a/HANDOFF.md b/HANDOFF.md index 2260dcfb..8fc277de 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,36 @@ ## Finished -- Kept BigQuery's required `schema` aspect Google-managed during Dataplex publication. -- Continued publishing Dander's optional `overview`, `contacts`, and `generic` aspects. -- Added a regression assertion for the exact Dataplex request keys. -- Corrected the public catalog documentation and changelog. +- Prepared Dander `0.6.0rc2` after the protected Dataplex fix merged. +- Synchronized package metadata, lockfile, install/upgrade examples, candidate records, and release notes. +- Kept `src/dander` unchanged from the accepted fix on `main`. ## Try It -Run `uv run pytest tests/catalog/test_dataplex.py tests/cli/test_catalog_cli.py -q` and inspect -`DataplexCatalogPublisher.request_for()` to confirm the schema key is absent. +Run `python3 scripts/check_release_metadata.py`, then build with `uv build` and inspect the artifacts +with `python3 scripts/check_distribution.py dist/*.whl dist/*.tar.gz`. ## Checks +- Release metadata check passed for `0.6.0rc2`. - Ruff lint and format checks passed. - Strict mypy passed across `src` and `tests`. - Full test suite passed: `776 passed`. -- Terraform platform and stage-zero validation passed. -- Wheel and sdist built, validated, and the wheel installed and generated a valid project outside the checkout. +- Wheel and sdist built, validated, and installed outside the checkout. ## Decisions -- Dander's local metadata spine still carries declared column schema. -- Dataplex publication enriches optional metadata without rewriting BigQuery's required system schema. +- `0.6.0rc2` replaces `rc1` because live acceptance exposed a packaged Dataplex runtime defect. +- The release branch changes no runtime source after the protected fix merge. ## Remaining -- Merge the protected fix PR. -- Publish `0.6.0rc2` from a release-only commit. -- Repeat the failed live catalog scenario and the bounded candidate smoke suite. +- Merge the protected release PR and publish/tag `0.6.0rc2`. +- Build and deploy a source-free `rc2` image in the isolated project through a reviewed plan. +- Repeat the catalog scenario and bounded smoke suite before stable promotion. ## Review First -- `src/dander/catalog/dataplex.py` -- `tests/catalog/test_dataplex.py` -- `README.md` +- `pyproject.toml` +- `CHANGELOG.md` +- `scripts/check_release_metadata.py` diff --git a/README.md b/README.md index f3fba4ba..9b295fa1 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.6.0rc1 +uv tool install dander-platform==0.6.0rc2 dander --version dander new my-data-platform cd my-data-platform @@ -624,7 +624,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.6.0rc1` is the current public candidate: its source-free GCP platform, four retained +Dander `0.6.0rc2` is the current public candidate: its source-free GCP platform, four retained daily pipelines, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. The limits above still make it **unsuitable for production**. diff --git a/docs/getting-started.md b/docs/getting-started.md index 4681eca3..a4510e09 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.6.0rc1 +uv tool install dander-platform==0.6.0rc2 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 29e7081b..0ddee6ae 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander candidate: `0.6.0rc1`; latest stable alpha: `0.5.1`; public Salesforce and +- Public Dander candidate: `0.6.0rc2`; latest stable alpha: `0.5.1`; public Salesforce and ServiceNow plugins: `0.2.0`. - Retained source-free Dander image: `sha256:3220623…8995`, built with Dander `0.5.0` and both plugin `0.2.0` packages. Dander `0.5.1` changes only catalog recommendations, so the runtime @@ -40,7 +40,7 @@ cloud path remains outside the live proof. - Dander provisions inside an existing billing-linked GCP project; it does not create a project or attach billing. Its managed cost guard is simulation-first and is not a spending cap. -- The `0.6.0rc1` Salesforce example supports read-only Accounts, Contacts, Opportunities, and +- The `0.6.0rc2` Salesforce example supports read-only Accounts, Contacts, Opportunities, and Users; the retained soak still exercises its existing Accounts pipeline. ServiceNow supports one read-only incidents slice. Provider write-back and broad object/table coverage are not claimed. diff --git a/docs/session-resume.md b/docs/session-resume.md index c77f0035..3c78fde7 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -1,11 +1,11 @@ -# Session Resume — 2026-08-06 +# Session Resume — 2026-08-07 Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `docs/release-audit.md` before changing code or cloud resources. ## Public releases -- Dander `0.6.0rc1` is the newest public candidate. Dander `0.5.1` remains the latest stable alpha +- Dander `0.6.0rc2` is the newest public candidate. Dander `0.5.1` remains the latest stable alpha while candidate acceptance is in progress. - Salesforce and ServiceNow connector plugins are public at `0.2.0` and require Dander `0.5.x`. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index ec10dd1a..3393e32c 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.6.0rc1" +export DANDER_TARGET_VERSION="0.6.0rc2" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index e770b298..06996be7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.6.0rc1" +version = "0.6.0rc2" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index f45b0994..f5ba75c8 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -36,7 +36,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.6.0rc1", + "dander-platform==0.6.0rc2", "dander-platform==0.1.0", 1, ), @@ -44,5 +44,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.6.0rc1" + "README install command uses 0.1.0; package metadata uses 0.6.0rc2" ] diff --git a/uv.lock b/uv.lock index 8dfa346b..1ceedc0f 100644 --- a/uv.lock +++ b/uv.lock @@ -520,7 +520,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.6.0rc1" +version = "0.6.0rc2" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 7835e2ab2a9c8590562a7200e01f358f8e32c9b7 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 03:13:24 -0400 Subject: [PATCH 010/163] Prepare Dander 0.6.0 beta release --- CHANGELOG.md | 18 +++++++++++++- HANDOFF.md | 34 +++++++++++++------------- README.md | 14 +++++------ SECURITY.md | 2 +- docs/getting-started.md | 2 +- docs/known-limitations.md | 9 +++---- docs/release-audit.md | 16 +++++++----- docs/releasing.md | 4 +-- docs/salesforce.md | 4 +-- docs/session-resume.md | 11 ++++++--- docs/upgrading.md | 2 +- examples/salesforce/dander.yaml | 2 +- pyproject.toml | 4 +-- scripts/check_distribution.py | 2 +- scripts/check_release_metadata.py | 12 ++++----- src/dander/plugins/catalog.py | 21 +++++++++------- src/dander/templates/project/README.md | 2 +- tests/cli/test_plugins_cli.py | 6 ++--- tests/plugins/test_catalog.py | 18 +++++++------- tests/project/test_scaffold.py | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 22 files changed, 109 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f4abf76..2636ef98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,27 @@ # Changelog Release notes for Dander are kept here and copied into the matching GitHub Release. Dander follows -semantic versioning while it is alpha: released minor lines receive fixes only, and new product +semantic versioning before 1.0: released minor lines receive fixes only, and new product capabilities enter through the next minor release. ## Unreleased +## 0.6.0 — 2026-08-07 (beta) + +### Acceptance + +- Promote the accepted `0.6.0rc2` runtime without functional product-code changes. +- Complete source-free hosted Salesforce ingestion for Accounts, Contacts, Opportunities, and + Users, governed transforms and tests, inclusive replay, visible soft-deletion tombstones, + released leases, staging cleanup, Dataplex publication, and Terraform no-drift. +- Confirm the stable Salesforce `0.3.0` and ServiceNow `0.2.1` plugin packages install together + outside their repositories and preserve plugin API version 1. + +### Classification + +- Move Dander's public package and documentation from Alpha to Beta while retaining the explicit + pre-1.0 limitations and newest-patch support policy. + ## 0.6.0rc2 — 2026-08-07 (alpha) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 8fc277de..1688cd96 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,36 @@ ## Finished -- Prepared Dander `0.6.0rc2` after the protected Dataplex fix merged. -- Synchronized package metadata, lockfile, install/upgrade examples, candidate records, and release notes. -- Kept `src/dander` unchanged from the accepted fix on `main`. +- Prepared the Dander `0.6.0` Beta release from the accepted `0.6.0rc2` runtime. +- Pinned the stable Salesforce `0.3.0` and ServiceNow `0.2.1` connector releases. +- Updated release metadata, public Beta labeling, acceptance records, and generated-project guidance. +- Kept runtime behavior unchanged; only curated catalog data and packaged templates changed under `src/dander`. ## Try It -Run `python3 scripts/check_release_metadata.py`, then build with `uv build` and inspect the artifacts -with `python3 scripts/check_distribution.py dist/*.whl dist/*.tar.gz`. +Run `python3 scripts/check_release_metadata.py`, build with `uv build`, and install the artifacts in +a temporary environment outside this checkout. ## Checks -- Release metadata check passed for `0.6.0rc2`. -- Ruff lint and format checks passed. -- Strict mypy passed across `src` and `tests`. -- Full test suite passed: `776 passed`. -- Wheel and sdist built, validated, and installed outside the checkout. +- Release metadata, Ruff lint/format, strict mypy, and all 776 tests passed. +- Root and stage-zero Terraform format/initialization/validation passed with backends disabled. +- Wheel and sdist built, passed distribution inspection, and installed source-free outside the checkout. +- Public Salesforce `0.3.0` and ServiceNow `0.2.1` installed together outside their repositories. ## Decisions -- `0.6.0rc2` replaces `rc1` because live acceptance exposed a packaged Dataplex runtime defect. -- The release branch changes no runtime source after the protected fix merge. +- Stable `0.6.0` preserves the accepted `rc2` runtime and moves Dander's public status to Beta. +- The curated catalog now advertises only the stable, Dander-`0.6.x` connector pins. ## Remaining -- Merge the protected release PR and publish/tag `0.6.0rc2`. -- Build and deploy a source-free `rc2` image in the isolated project through a reviewed plan. -- Repeat the catalog scenario and bounded smoke suite before stable promotion. +- Merge the protected release PR, tag and publish `0.6.0`, and create the GitHub Release. +- Deploy the public stable source-free image in the isolated project and repeat bounded smoke/no-drift checks. +- Upgrade the retained project only through its separately reviewed plan. ## Review First -- `pyproject.toml` - `CHANGELOG.md` -- `scripts/check_release_metadata.py` +- `pyproject.toml` +- `src/dander/plugins/catalog.py` diff --git a/README.md b/README.md index 9b295fa1..4cc5bf7c 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CI](https://github.com/harrisonoconnorhover/dander/actions/workflows/ci.yml/badge.svg)](https://github.com/harrisonoconnorhover/dander/actions/workflows/ci.yml) > [!WARNING] -> **Alpha software.** Dander is being stabilized through a retained proof project and an +> **Beta software.** Dander has completed source-free candidate acceptance and continues through a > operator trial. Use a disposable GCP project, review every Terraform plan, and read > the [known limitations](https://github.com/harrisonoconnorhover/dander/blob/main/docs/known-limitations.md) > before relying on it. Only the latest patch in the current `0.x` minor is supported. @@ -57,13 +57,13 @@ Operator-facing documentation: Python 3.12 (app + CLI) · BigQuery SQL (transforms) · Terraform/HCL (infra) · YAML (config). -## Install the current public candidate +## Install the current public release The Python distribution is named `dander-platform` because the `dander` name on PyPI belongs to a different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.6.0rc2 +uv tool install dander-platform==0.6.0 dander --version dander new my-data-platform cd my-data-platform @@ -624,10 +624,10 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.6.0rc2` is the current public candidate: its source-free GCP platform, four retained -daily pipelines, connector plugins, transform/test engine, run ledger, metadata spine, and Druff -operator interface have completed bounded live proofs. The limits above still make it -**unsuitable for production**. +Dander `0.6.0` is the current public beta: its source-free GCP platform, connector plugins, +transform/test engine, run ledger, metadata spine, and Druff operator interface have completed +bounded live proofs. It remains pre-1.0 software; review the documented limitations before using +business-critical data. The named HR, compensation, and customer systems describe connector categories or bounded test implementations; they do not imply that this repository came from, connects to, or contains data from an existing company. Normal provenance, licensing, and privacy review still applies before diff --git a/SECURITY.md b/SECURITY.md index d88fb5aa..87547124 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security Policy -Dander is alpha software. Use a dedicated GCP project, least-privilege credentials, immutable +Dander is beta software. Use a dedicated GCP project, least-privilege credentials, immutable container digests, and reviewed Terraform plans. Never put provider tokens, Terraform state, raw source rows, or recovery codes in an issue or pull request. diff --git a/docs/getting-started.md b/docs/getting-started.md index a4510e09..c290f90f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.6.0rc2 +uv tool install dander-platform==0.6.0 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/known-limitations.md b/docs/known-limitations.md index e130622f..98a10900 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -1,7 +1,7 @@ # Known limitations -Dander `0.6.x` is alpha and proves a focused GCP-native vertical slice. It is not yet suitable for -an unattended production system containing business-critical data. +Dander `0.6.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; +evaluate these limits before using it for an unattended system containing business-critical data. ## Ingestion and schemas @@ -54,6 +54,5 @@ an unattended production system containing business-critical data. ## Support boundary -`0.6.x` receives candidate fixes only until the minor is promoted. Dander `0.5.1` remains the -latest stable alpha during candidate acceptance. Only the newest candidate and the latest patch -in the current stable `0.x` minor are supported; see [SECURITY.md](../SECURITY.md). +`0.6.x` is the current supported beta line. Only its newest patch is supported; a newer minor's +newest candidate may additionally receive acceptance fixes. See [SECURITY.md](../SECURITY.md). diff --git a/docs/release-audit.md b/docs/release-audit.md index 0ddee6ae..8ca79b93 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -1,6 +1,6 @@ # Dander Platform Release Audit -Audited on 2026-08-06 against the product promise in `steering/00-project-overview.md`. +Audited on 2026-08-07 against the product promise in `steering/00-project-overview.md`. “Live-proven” means the behavior was observed in a disposable provider account or retained GCP project. “Implemented” means automated tests cover the contract while an optional provider or cloud path remains outside the live proof. @@ -12,7 +12,7 @@ cloud path remains outside the live proof. | One CLI and typed project manifest | Live-proven | `dander.yaml` declares four daily connector pipelines and one executable graph; `dander validate`, provisioning, execution, and metadata inspection use the same CLI. | | Batteries-included GCP infrastructure | Live-proven | Reviewed Terraform owns remote state, Artifact Registry, BigQuery datasets, per-pipeline IAM, Cloud Run jobs, Scheduler jobs, Secret Manager containers, failure alerts, and the optional cost guard. | | Additive multi-pipeline hosting | Live-proven | Greenhouse, HubSpot, Salesforce, and ServiceNow coexist with distinct jobs, schedules, and runtime identities. The graph job is separately deployed and paused. | -| Independently installed connectors | Live-proven | Public Salesforce and ServiceNow `0.2.0` packages are exact-pinned, discovered through Dander plugin API v1, installed into a source-free image, and exercised against disposable provider accounts. | +| Independently installed connectors | Live-proven | Public Salesforce `0.3.0` and ServiceNow `0.2.1` packages are exact-pinned, discovered through Dander plugin API v1, installed into a source-free image, and exercised against disposable provider accounts. | | Shared enterprise authentication | Live-proven for retained plugins | Dander core supplies Salesforce OAuth2 JWT and ServiceNow OAuth2 client credentials while Terraform grants each runtime access only to its declared Secret Manager resources. | | Bounded ingestion and idempotent BigQuery writes | Live-proven for hosted SCD1 | Greenhouse, HubSpot, Salesforce, and ServiceNow have completed hosted ingestion and replay checks without duplicate destination keys; Salesforce applies its cursor server-side. | | Owned transforms and tests | Live-proven | Retained pipelines execute Dander's `ref()` DAG and declared BigQuery assertions after ingestion. | @@ -23,8 +23,12 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander candidate: `0.6.0rc2`; latest stable alpha: `0.5.1`; public Salesforce and - ServiceNow plugins: `0.2.0`. +- Public Dander beta: `0.6.0`; public Salesforce plugin: `0.3.0`; public ServiceNow plugin: + `0.2.1`. +- Isolated candidate acceptance used Dander `0.6.0rc2` and the public plugin candidates in a + source-free image. Salesforce ingested all four endpoints, published five governed models and + Dataplex metadata, captured update and soft-delete state, replayed without duplicate keys or + cursor regression, released its lease, removed staging, and finished with Terraform no-drift. - Retained source-free Dander image: `sha256:3220623…8995`, built with Dander `0.5.0` and both plugin `0.2.0` packages. Dander `0.5.1` changes only catalog recommendations, so the runtime image was intentionally not replaced. @@ -40,7 +44,7 @@ cloud path remains outside the live proof. - Dander provisions inside an existing billing-linked GCP project; it does not create a project or attach billing. Its managed cost guard is simulation-first and is not a spending cap. -- The `0.6.0rc2` Salesforce example supports read-only Accounts, Contacts, Opportunities, and +- The `0.6.0` Salesforce example supports read-only Accounts, Contacts, Opportunities, and Users; the retained soak still exercises its existing Accounts pipeline. ServiceNow supports one read-only incidents slice. Provider write-back and broad object/table coverage are not claimed. @@ -51,7 +55,7 @@ cloud path remains outside the live proof. ## Verdict -Dander demonstrates its intended alpha vertical slice: one manifest and CLI reconcile an owned +Dander demonstrates its intended beta vertical slice: one manifest and CLI reconcile an owned GCP platform, independently installed connectors ingest into BigQuery, Dander transforms and tests the data, one metadata spine records what happened, and Druff authors and operates a bounded graph without becoming a second runtime. Production hardening and broader connector coverage remain diff --git a/docs/releasing.md b/docs/releasing.md index e20ba77e..edc2205c 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -26,7 +26,7 @@ broadly changes packaging, provisioning, orchestration, state, cursors, concurre Tests, evidence tooling, and documentation may change without a new candidate only when packaged runtime behavior is unchanged. -## Alpha release lines +## Pre-1.0 release lines - Published minor lines contain fixes only: installation, upgrades, drift, cursor/lease correctness, schemas, staging cleanup, CLI accuracy, security, and documentation that blocks @@ -35,7 +35,7 @@ runtime behavior is unchanged. next minor release. - Only the latest patch in the current `0.x` minor is supported. A functional patch uses its own candidate, public-artifact proof, protected publication, and clean upgrade verification. -- GitHub Release notes must match `CHANGELOG.md`, identify the release as alpha, link the known +- GitHub Release notes must match `CHANGELOG.md`, identify the release status, link the known limitations, and name the exact tag and commit. ## Initial `0.1.0` gates and post-release soak diff --git a/docs/salesforce.md b/docs/salesforce.md index 5c8cf082..0f624d0c 100644 --- a/docs/salesforce.md +++ b/docs/salesforce.md @@ -53,14 +53,14 @@ add its governed projection to the relevant model. Undeclared fields fail before ## Project configuration -After `dander-connector-salesforce 0.3.0rc1` is published, a new candidate project can use: +With `dander-connector-salesforce 0.3.0`, a project can use: ```yaml version: 1 plugins: salesforce: distribution: dander-connector-salesforce - version: 0.3.0rc1 + version: 0.3.0 pipelines: salesforce_crm: source: salesforce diff --git a/docs/session-resume.md b/docs/session-resume.md index 3c78fde7..306840b2 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,9 +5,9 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.6.0rc2` is the newest public candidate. Dander `0.5.1` remains the latest stable alpha - while candidate acceptance is in progress. -- Salesforce and ServiceNow connector plugins are public at `0.2.0` and require Dander `0.5.x`. +- Dander `0.6.0` is the current public beta. +- Salesforce `0.3.0` and ServiceNow `0.2.1` are the current public connector releases and support + Dander `0.6.x`. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, execution, catalog, operation-authoring, and deployment-preview work. @@ -34,6 +34,11 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Latest operating evidence +- On 2026-08-07, the isolated source-free `0.6.0rc2` image completed four-endpoint Salesforce + ingestion, governed transforms/tests, updates, soft-deletion tombstones, replay, Dataplex + publication, lease and staging cleanup, and a final no-drift plan. ServiceNow also completed a + compatibility smoke run in the same image. + - On 2026-08-05, scheduled executions `dander-greenhouse-public-m2pz2`, `dander-hubspot-companies-cmdbz`, `dander-salesforce-accounts-jljwj`, and `dander-servicenow-incidents-6g72x` all completed successfully. diff --git a/docs/upgrading.md b/docs/upgrading.md index 3393e32c..8233eadf 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.6.0rc2" +export DANDER_TARGET_VERSION="0.6.0" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/examples/salesforce/dander.yaml b/examples/salesforce/dander.yaml index 759ebd9c..0edfd933 100644 --- a/examples/salesforce/dander.yaml +++ b/examples/salesforce/dander.yaml @@ -9,7 +9,7 @@ platform: plugins: salesforce: distribution: dander-connector-salesforce - version: 0.3.0rc1 + version: 0.3.0 pipelines: salesforce_crm: diff --git a/pyproject.toml b/pyproject.toml index 06996be7..f9010c46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.6.0rc2" +version = "0.6.0" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" @@ -8,7 +8,7 @@ license = "Apache-2.0" authors = [{ name = "Harrison O'Connor-Hover" }] keywords = ["bigquery", "data-platform", "etl", "gcp", "terraform"] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: Information Technology", diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 64b82711..3363fb19 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -127,7 +127,7 @@ def _check_metadata( description = metadata.get_payload() required_description = { f"uv tool install dander-platform=={expected_version}", - f"Dander `{expected_version}` is the current public candidate", + f"Dander `{expected_version}` is the current public beta", } if not isinstance(description, str): raise ValueError("Distribution description is not text") diff --git a/scripts/check_release_metadata.py b/scripts/check_release_metadata.py index 3aebed95..4ec9f835 100644 --- a/scripts/check_release_metadata.py +++ b/scripts/check_release_metadata.py @@ -29,7 +29,7 @@ class VersionReference: VersionReference( "README.md", "README status", - re.compile(r"Dander `(?P[^`]+)` is the current public candidate"), + re.compile(r"Dander `(?P[^`]+)` is the current public beta"), ), VersionReference( "docs/getting-started.md", @@ -43,13 +43,13 @@ class VersionReference: ), VersionReference( "docs/session-resume.md", - "session public candidate", - re.compile(r"Dander `(?P[^`]+)` is the newest public candidate"), + "session public release", + re.compile(r"Dander `(?P[^`]+)` is the current public beta"), ), VersionReference( "docs/release-audit.md", - "release audit public candidate", - re.compile(r"Public Dander candidate: `(?P[^`]+)`"), + "release audit public release", + re.compile(r"Public Dander beta: `(?P[^`]+)`"), ), ) @@ -95,7 +95,7 @@ def release_metadata_errors(root: Path) -> list[str]: _check_single_match( root / "docs/known-limitations.md", "known-limitations release line", - re.compile(r"Dander `(?P\d+\.\d+\.x)` is alpha"), + re.compile(r"Dander `(?P\d+\.\d+\.x)` is beta"), expected=f"{release_line.group('line')}.x", ) ) diff --git a/src/dander/plugins/catalog.py b/src/dander/plugins/catalog.py index b568406e..aea3bf76 100644 --- a/src/dander/plugins/catalog.py +++ b/src/dander/plugins/catalog.py @@ -79,17 +79,20 @@ def as_dict( CatalogConnector( connector_id="salesforce", display_name="Salesforce", - description="Bounded Salesforce Bulk API 2.0 Accounts ingestion with replay cursors.", + description=( + "Bounded Salesforce Bulk API 2.0 Accounts, Contacts, Opportunities, and Users " + "ingestion with replay cursors." + ), distribution="dander-connector-salesforce", - version="0.2.0", - dander_specifier=">=0.5.0,<0.6", - support_status="first-party-alpha", + version="0.3.0", + dander_specifier=">=0.6.0,<0.7", + support_status="first-party-beta", validation_status="provider-validated", documentation_url=( "https://github.com/harrisonoconnorhover/" "dander-connector-salesforce#dander-salesforce-connector" ), - pypi_url="https://pypi.org/project/dander-connector-salesforce/0.2.0/", + pypi_url="https://pypi.org/project/dander-connector-salesforce/0.3.0/", repository_url=("https://github.com/harrisonoconnorhover/dander-connector-salesforce"), ), CatalogConnector( @@ -97,15 +100,15 @@ def as_dict( display_name="ServiceNow", description="Read-only ServiceNow Table API incident ingestion with stable paging.", distribution="dander-connector-servicenow", - version="0.2.0", - dander_specifier=">=0.5.0,<0.6", - support_status="first-party-alpha", + version="0.2.1", + dander_specifier=">=0.6.0,<0.7", + support_status="first-party-beta", validation_status="provider-validated", documentation_url=( "https://github.com/harrisonoconnorhover/" "dander-connector-servicenow#dander-servicenow-connector" ), - pypi_url="https://pypi.org/project/dander-connector-servicenow/0.2.0/", + pypi_url="https://pypi.org/project/dander-connector-servicenow/0.2.1/", repository_url=("https://github.com/harrisonoconnorhover/dander-connector-servicenow"), ), ) diff --git a/src/dander/templates/project/README.md b/src/dander/templates/project/README.md index f2e94472..8fe8125a 100644 --- a/src/dander/templates/project/README.md +++ b/src/dander/templates/project/README.md @@ -3,7 +3,7 @@ This project was created by `dander new`. It starts with one paused, credential-free Greenhouse Job Board pipeline so infrastructure changes remain reviewable before any schedule is enabled. -> **Alpha:** review Dander's +> **Beta:** review Dander's > [known limitations](https://github.com/harrisonoconnorhover/dander/blob/main/docs/known-limitations.md) > and use a disposable GCP project. Only the latest patch in the current `0.x` minor is supported. diff --git a/tests/cli/test_plugins_cli.py b/tests/cli/test_plugins_cli.py index 49796725..b538b982 100644 --- a/tests/cli/test_plugins_cli.py +++ b/tests/cli/test_plugins_cli.py @@ -125,8 +125,8 @@ def test_plugins_search_lists_curated_exact_package_pins() -> None: result = CliRunner().invoke(app, ["plugins", "search"]) assert result.exit_code == 0, result.output - assert "dander-connector-salesforce==0.2.0" in result.output - assert "dander-connector-servicenow==0.2.0" in result.output + assert "dander-connector-salesforce==0.3.0" in result.output + assert "dander-connector-servicenow==0.2.1" in result.output def test_plugins_search_filters_and_reports_no_match() -> None: @@ -134,7 +134,7 @@ def test_plugins_search_filters_and_reports_no_match() -> None: missing = CliRunner().invoke(app, ["plugins", "search", "not-a-connector"]) assert filtered.exit_code == 0, filtered.output - assert "dander-connector-servicenow==0.2.0" in filtered.output + assert "dander-connector-servicenow==0.2.1" in filtered.output assert "dander-connector-salesforce" not in filtered.output assert missing.exit_code == 0, missing.output assert "No curated connectors match" in missing.output diff --git a/tests/plugins/test_catalog.py b/tests/plugins/test_catalog.py index b1e9bae5..dae14897 100644 --- a/tests/plugins/test_catalog.py +++ b/tests/plugins/test_catalog.py @@ -27,17 +27,17 @@ def _installed_salesforce(version: str = "0.2.0") -> InstalledConnectorPlugin: ) -def test_curated_catalog_has_exact_public_packages_and_dander_05_compatibility() -> None: - catalog = build_plugin_catalog(dander_version="0.5.1") +def test_curated_catalog_has_exact_public_packages_and_dander_06_compatibility() -> None: + catalog = build_plugin_catalog(dander_version="0.6.0") connectors = cast("list[dict[str, object]]", catalog["connectors"]) assert catalog["schema_version"] == 1 - assert catalog["dander_version"] == "0.5.1" + assert catalog["dander_version"] == "0.6.0" assert {connector["id"] for connector in connectors} == {"salesforce", "servicenow"} - assert all(connector["version"] == "0.2.0" for connector in connectors) - assert all(connector["dander_specifier"] == ">=0.5.0,<0.6" for connector in connectors) + assert {connector["version"] for connector in connectors} == {"0.3.0", "0.2.1"} + assert all(connector["dander_specifier"] == ">=0.6.0,<0.7" for connector in connectors) assert all(connector["compatible"] is True for connector in connectors) - assert all(connector["support_status"] == "first-party-alpha" for connector in connectors) + assert all(connector["support_status"] == "first-party-beta" for connector in connectors) assert all(connector["validation_status"] == "provider-validated" for connector in connectors) assert all( str(connector["pypi_url"]).startswith("https://pypi.org/") for connector in connectors @@ -68,14 +68,14 @@ def test_catalog_installation_status_uses_only_validated_manifest_plugins() -> N def test_catalog_marks_unsupported_dander_version_incompatible() -> None: - catalog = build_plugin_catalog(dander_version="0.6.0") + catalog = build_plugin_catalog(dander_version="0.7.0") connectors = cast("list[dict[str, object]]", catalog["connectors"]) assert all(connector["compatible"] is False for connector in connectors) -def test_catalog_marks_dander_04_incompatible_with_current_connectors() -> None: - catalog = build_plugin_catalog(dander_version="0.4.0") +def test_catalog_marks_dander_05_incompatible_with_current_connectors() -> None: + catalog = build_plugin_catalog(dander_version="0.5.1") connectors = cast("list[dict[str, object]]", catalog["connectors"]) assert all(connector["compatible"] is False for connector in connectors) diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index a9940f7f..6cb4a1a5 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -52,7 +52,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: ): assert (project / relative).is_file() salesforce = load_project_config(project / "examples" / "salesforce" / "dander.yaml") - assert salesforce.plugins["salesforce"].version == "0.3.0rc1" + assert salesforce.plugins["salesforce"].version == "0.3.0" assert set(salesforce.pipelines) == {"salesforce_crm"} assert not list(project.rglob("*.tfplan")) assert not list(project.rglob("*.tfstate")) diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index f5ba75c8..7441d554 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -36,7 +36,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.6.0rc2", + "dander-platform==0.6.0", "dander-platform==0.1.0", 1, ), @@ -44,5 +44,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.6.0rc2" + "README install command uses 0.1.0; package metadata uses 0.6.0" ] diff --git a/uv.lock b/uv.lock index 1ceedc0f..9e5aaea7 100644 --- a/uv.lock +++ b/uv.lock @@ -520,7 +520,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.6.0rc2" +version = "0.6.0" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 46f22d730aa4cc9adfe2128be2b8dc0c89d50e49 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 03:52:30 -0400 Subject: [PATCH 011/163] Establish cloud portability baseline --- HANDOFF.md | 38 +++++++------ docs/cloud-portability-baseline.md | 17 ++++++ docs/cloud-portability-slos.md | 47 +++++++++++++++ docs/decisions.md | 11 ++++ steering/00-project-overview.md | 31 ++++++---- .../test_gcp_compatibility_baseline.py | 57 +++++++++++++++++++ 6 files changed, 173 insertions(+), 28 deletions(-) create mode 100644 docs/cloud-portability-baseline.md create mode 100644 docs/cloud-portability-slos.md create mode 100644 tests/portability/test_gcp_compatibility_baseline.py diff --git a/HANDOFF.md b/HANDOFF.md index 1688cd96..0d288ddc 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,38 @@ ## Finished -- Prepared the Dander `0.6.0` Beta release from the accepted `0.6.0rc2` runtime. -- Pinned the stable Salesforce `0.3.0` and ServiceNow `0.2.1` connector releases. -- Updated release metadata, public Beta labeling, acceptance records, and generated-project guidance. -- Kept runtime behavior unchanged; only curated catalog data and packaged templates changed under `src/dander`. +- Established Dander `0.6.0` as the frozen GCP/Cloud Run/BigQuery compatibility baseline. +- Recorded the cloud-selectable product direction without weakening existing version 1 behavior. +- Added initial correctness, bounded-operation, service, and cost qualification objectives. +- Added one cross-layer characterization suite for BigQuery fencing/state, Cloud Run projection, + CLI identity, and packaged infrastructure. ## Try It -Run `python3 scripts/check_release_metadata.py`, build with `uv build`, and install the artifacts in -a temporary environment outside this checkout. +Run `uv run pytest -q tests/portability/test_gcp_compatibility_baseline.py` to exercise the +cross-layer baseline before refactoring a portability boundary. ## Checks -- Release metadata, Ruff lint/format, strict mypy, and all 776 tests passed. -- Root and stage-zero Terraform format/initialization/validation passed with backends disabled. -- Wheel and sdist built, passed distribution inspection, and installed source-free outside the checkout. -- Public Salesforce `0.3.0` and ServiceNow `0.2.1` installed together outside their repositories. +- Ruff lint/format and strict mypy passed. +- All 780 tests passed. +- Root and stage-zero Terraform format, backend-disabled initialization, and validation passed. +- Release metadata plus wheel/sdist build and distribution inspection passed. ## Decisions -- Stable `0.6.0` preserves the accepted `rc2` runtime and moves Dander's public status to Beta. -- The curated catalog now advertises only the stable, Dander-`0.6.x` connector pins. +- GCP/Cloud Run/BigQuery remains the primary compatibility profile. +- New combinations remain unsupported until their exact conformance, identity, and live gates pass. +- Paid scale tests require a separately approved ceiling and publish measured, digest-bound results. ## Remaining -- Merge the protected release PR, tag and publish `0.6.0`, and create the GitHub Release. -- Deploy the public stable source-free image in the isolated project and repeat bounded smoke/no-drift checks. -- Upgrade the retained project only through its separately reviewed plan. +- Merge this no-runtime-change baseline through protected main. +- Rebase the isolated OCI runtime-contract ticket onto the merged baseline. +- Continue Phase 1 as focused invocation, inspection, artifact, projection, and Cloud Run PRs. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `src/dander/plugins/catalog.py` +- `docs/cloud-portability-slos.md` +- `tests/portability/test_gcp_compatibility_baseline.py` +- `steering/00-project-overview.md` diff --git a/docs/cloud-portability-baseline.md b/docs/cloud-portability-baseline.md new file mode 100644 index 00000000..eb99878e --- /dev/null +++ b/docs/cloud-portability-baseline.md @@ -0,0 +1,17 @@ +# GCP Compatibility Baseline + +Cloud portability begins from the released `0.6.0` GCP profile. This baseline intentionally makes +no runtime change. It identifies the behavior that each refactor must preserve until a reviewed +profile migration says otherwise. + +| Boundary | Compatibility behavior | Direct characterization | +|---|---|---| +| CLI | `dander run PIPELINE` resolves version 1 manifests and executes the named hosted pipeline | `tests/cli/test_run_command.py` | +| BigQuery publication | SCD1 stages with expiration, DML-touches the exact lease token, and merges in the same transaction | `tests/writer/test_bigquery_writer.py` | +| State | run history records one terminal state; leases and watermarks reject stale ownership | `tests/state/test_run_history.py`, `tests/state/test_lease.py`, `tests/state/test_bigquery_watermark.py` | +| Cloud Run | one job and scheduler per pipeline receive the declared resources, retry limit, batch size, and guarded-runtime flag | `tests/infra/test_multi_pipeline_runtime.py` | +| Distribution | `dander-platform` exposes the `dander` CLI and packages every clean Terraform project asset explicitly | `tests/test_distribution_contract.py` | + +`tests/portability/test_gcp_compatibility_baseline.py` binds those five boundaries together as the +Phase 0 release invariant. Provider work adds contracts around them; it does not silently alter +their current semantics. diff --git a/docs/cloud-portability-slos.md b/docs/cloud-portability-slos.md new file mode 100644 index 00000000..43d0d9ab --- /dev/null +++ b/docs/cloud-portability-slos.md @@ -0,0 +1,47 @@ +# Cloud Portability Qualification Objectives + +These are release gates for a named Dander platform profile, not a claim that every cloud/service +combination is supported. GCP/Cloud Run/BigQuery is the initial compatibility profile. A new profile +publishes its measured results, image digest, provider regions and service shapes before its status +can change from unsupported or experimental to supported. + +## Hard correctness objectives + +- Deterministic conformance fixtures produce exactly equal normalized rows and schema across every + warehouse claimed by the profile. No mismatch is tolerated. +- Inclusive cursor replay, duplicate launcher delivery, and a crash after destination commit but + before cursor commit create zero duplicate business keys and never regress a cursor. +- Controlled overlap produces one owner, a truthful skipped outcome for the loser, and zero stale + publications. Every writer mode must reject an old fencing token after a newer claim. +- Every catchable execution produces one sanitized terminal run record. SIGKILL is reconciled by + the next successful lease owner and can never leave a successful terminal record. + +## Bounded operation objectives + +- The bounded-memory benchmark uses input at least ten times larger than the container memory limit + and must complete without exceeding 80% of that limit after warm-up. The configured batch size, + row width, schema depth, and observed peak resident memory are part of the result. +- Run-scoped staging is deleted on handled completion or failure. Staging left by process death has + a provider-enforced expiration of no more than 24 hours. +- A retryable provider failure exhausts a documented finite retry budget, emits a stable failure + code, and leaves the next invocation replay-safe. No operator edits leases, cursors, or staging. +- A scheduled run must finish within its configured hard deadline and before its next scheduled + occurrence. The provider-side launcher, not the container alone, enforces that deadline. + +## Service and cost evidence + +- During qualification, at least 99% of scheduled executions unaffected by an intentionally + revoked credential or a documented source outage finish successfully, and 100% of failures are + visible in run history and provider logs. Fewer than 100 qualifying runs is reported as a sample, + not represented as statistical proof. +- Bulk, incremental, transform, and concurrent benchmarks record rows, bytes, duration, peak + memory, provider work metrics, and estimated cost. There is no universal throughput or cost + promise until measured evidence supports one. +- Paid scale tests require a separately approved ceiling. A provider budget or billing alert is + evidence and notification, never described as a hard spending cap. + +## Promotion rule + +A profile is supported only when all hard correctness objectives pass on the exact release +candidate and the benchmark report documents every bounded-operation objective. Failed or missing +evidence leaves the profile experimental or unsupported; it does not weaken the objective. diff --git a/docs/decisions.md b/docs/decisions.md index 93d1db3b..e97ec58b 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-07 — Cloud-selectable direction preserves a named GCP compatibility profile + +- Portability is introduced through a versioned OCI runtime contract and named, validated platform + profiles; logical pipelines do not gain provider conditionals. +- GCP/Cloud Run/BigQuery/Dataplex/GCP Secret Manager remains the primary compatibility profile. + Existing version 1 manifests and resource addresses remain compatible while the new contracts + are proven incrementally. +- A launcher, warehouse, state, catalog, or secret combination is unsupported until the exact + combination passes its conformance, identity, failure, and live-profile gates. Interfaces and + simulators alone do not create a support claim. + ## 2026-08-04 — Graph deployment preview is artifact-writing but non-applyable - Graph Save remains a local conditional file write. Candidate creation is a separate explicit diff --git a/steering/00-project-overview.md b/steering/00-project-overview.md index ffbf8009..2351cffa 100644 --- a/steering/00-project-overview.md +++ b/steering/00-project-overview.md @@ -6,10 +6,11 @@ ## One-liner -Dander is an open-source, GCP-native EL(T) suite for reading from SaaS systems +Dander is an open-source, cloud-selectable EL(T) suite for reading from SaaS systems (Salesforce, Workday, Greenhouse, NetSuite, Marketo, Xactly, …) and ingesting them -**efficiently and idempotently into BigQuery** — a focused, self-owned replacement for -Informatica, and a customizable stand-in for dbt/SQLMesh transformation. +**efficiently and idempotently into an explicitly supported warehouse profile** — a focused, +self-owned replacement for Informatica, and a customizable stand-in for dbt/SQLMesh +transformation. GCP, Cloud Run, and BigQuery remain the primary compatibility profile. ## Why this exists @@ -17,8 +18,9 @@ Informatica, and a customizable stand-in for dbt/SQLMesh transformation. - **dbt Core is free but not fully ours**, and the transformation OSS landscape consolidated under one vendor (Fivetran acquired Census, Tobiko/SQLMesh, and dbt Labs across 2025–2026). Owning the transform layer removes vendor-consolidation risk. -- We run on **GCP** and land everything in **BigQuery**. GCP-first, with clean provider - abstractions so AWS/Azure can be added later without a rewrite. +- The proven production profile runs on **GCP** and lands in **BigQuery**. Portability is added + through named, separately qualified platform profiles; a new provider is never implied by a + generic interface or local mock alone. ## Modules @@ -28,13 +30,17 @@ Informatica, and a customizable stand-in for dbt/SQLMesh transformation. | **Ingestion (hybrid)** | Each source is a **config object** (base URL, auth ref, endpoints, pagination, incremental cursor, field mappings). Standard REST sources run on **dlt**; enterprise sources (Workday/NetSuite/Xactly) use hand-rolled `EnterpriseSource` extractors. Both behind the `Source` interface. Rate limiting/backoff per source; inferred type casting to BigQuery types with per-field overrides. | | **BigQuery Writer** | Multiple write patterns: SCD1 (MERGE), SCD2 (versioned rows), daily snapshot (partitioned append), incremental (watermark). Storage Write API vs load jobs per workload. | | **Transform** | dbt-replacement: Jinja2 `ref()` templating → parsed dependency DAG → topological execution. Materializations reuse the Writer patterns. Generic tests (not-null/unique/accepted-values/relationships). One YAML per model feeds SQL + Dataplex catalog aspects + semantic registry. | -| **Bootstrap CLI** | pip-installable; wraps **Terraform** to provision Secret Manager, service accounts + least-privilege IAM (Workload Identity Federation), a compute target (Cloud Run jobs), and BigQuery datasets. Provider-abstracted for future AWS/Azure. | +| **Bootstrap CLI** | pip-installable; wraps **Terraform** to provision a selected, named platform profile. The current compatibility profile provisions Secret Manager, service accounts + least-privilege IAM (Workload Identity Federation), Cloud Run jobs, and BigQuery datasets. | | **Orchestration/State** | `PipelineExecutor` owns ingest → transform/tests → metadata and one truthful lifecycle record; Cloud Scheduler invokes isolated Cloud Run jobs; BigQuery/SQLite persist cursors, run history, and atomic catalog snapshots. | | **Metadata spine** | One typed source/model definition projects source endpoints, models, columns, lineage, tests, governed metrics, local JSON, optional Dataplex aspects, and the durable `dander_meta` catalog. | ## Scope discipline (non-goals) -- Not a general-purpose "everything" tool. We read APIs → land in BigQuery. That's the core. +- Not a general-purpose "everything" tool. We read APIs into qualified warehouse profiles; the + GCP/BigQuery profile remains the compatibility baseline until another profile passes its live + release gate. +- Do not claim a Cartesian mix of launchers, warehouses, state backends, catalogs, and secret + providers. Only named combinations in the tested compatibility matrix are supported. - Prove the pattern on **low-friction sources first** (Greenhouse, Marketo) end-to-end before tackling ugly auth/data shapes (Workday, NetSuite). - Borrow vs. build is decided per-module in the Decision Log — don't reinvent pagination/retry @@ -43,9 +49,10 @@ Informatica, and a customizable stand-in for dbt/SQLMesh transformation. ## Tech stack - **Python 3.12+** — primary application language. See `languages/python.md`. -- **BigQuery Standard SQL** — transforms. See `languages/sql.md`. -- **Terraform (HCL)** — infrastructure-as-code. See `languages/terraform.md`. -- **GCP Secret Manager** — the only place real credentials live. +- **BigQuery Standard SQL** — the current transform compatibility dialect. See `languages/sql.md`. +- **Terraform (HCL)** — provider-separated infrastructure-as-code. See `languages/terraform.md`. +- **GCP Secret Manager** — the current compatibility-profile secret store; later profiles must use + an explicitly qualified reference-based secret provider. ## Compliance note (read before open-sourcing) @@ -65,6 +72,10 @@ material, credentials, or non-public data is introduced. Append newest at top. Format: `- YYYY-MM-DD — decision — rationale`. +- 2026-08-07 — **Dander becomes cloud-selectable without weakening the GCP contract** — a + versioned OCI runtime and named deployment profiles will separate logical pipelines from cloud + projection. GCP/Cloud Run/BigQuery remains the primary compatibility profile, and each added + combination stays unsupported until its adapter, identity, launcher, and live proof pass. - 2026-08-02 — **Workday acceptance begins with a three-operation RaaS contract** — token issuance plus workers and organizations custom reports are simulated over real loopback HTTP; tenant auth, prompt aliases, and permissions remain unproven until a narrow live acceptance. diff --git a/tests/portability/test_gcp_compatibility_baseline.py b/tests/portability/test_gcp_compatibility_baseline.py new file mode 100644 index 00000000..7209dbba --- /dev/null +++ b/tests/portability/test_gcp_compatibility_baseline.py @@ -0,0 +1,57 @@ +"""Cross-layer characterization of the released GCP compatibility profile.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +ROOT = Path(__file__).parents[2] + + +def _normalized(path: Path) -> str: + return "\n".join( + " ".join(line.split()) for line in path.read_text(encoding="utf-8").splitlines() + ) + + +def test_bigquery_publication_keeps_transactional_lease_fencing() -> None: + writer = (ROOT / "src/dander/writer/bigquery.py").read_text(encoding="utf-8") + fencing = (ROOT / "src/dander/concurrency.py").read_text(encoding="utf-8") + + assert '"BEGIN TRANSACTION;\\n"' in writer + assert "ASSERT @@row_count = 1 AS 'Dander pipeline lease lost'" in fencing + assert 'f"MERGE `{target_id}` AS target\\n"' in writer + assert "_STAGING_TTL = timedelta(days=1)" in writer + + +def test_bigquery_state_keeps_lease_and_watermark_compare_and_set() -> None: + lease = (ROOT / "src/dander/state/lease.py").read_text(encoding="utf-8") + watermark = (ROOT / "src/dander/state/watermark.py").read_text(encoding="utf-8") + + assert "fencing_token = fencing_token + 1" in lease + assert "fencing_token = @fencing_token" in lease + assert "@cursor AS last_cursor" in watermark + assert "target.last_cursor IS NOT DISTINCT FROM @expected" in watermark + assert "ASSERT @@row_count = 1 AS 'Dander watermark boundary changed'" in watermark + + +def test_cloud_run_projection_keeps_existing_version_one_behavior() -> None: + module = _normalized(ROOT / "infra/modules/scheduled-job/main.tf") + + assert 'resource "google_cloud_run_v2_job" "ingestion" {' in module + assert "task_count = 1" in module + assert "parallelism = 1" in module + assert '["run", each.key, "--config", "/app/dander.yaml"]' in module + assert 'var.require_guarded_free_tier ? ["--guarded-free-tier"] : []' in module + assert '["--batch-rows", tostring(var.runtime_batch_rows)]' in module + + +def test_cli_and_distribution_keep_the_public_gcp_compatibility_surface() -> None: + config = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + project = config["project"] + packaged = config["tool"]["hatch"]["build"]["targets"]["wheel"]["force-include"] + + assert project["name"] == "dander-platform" + assert project["scripts"]["dander"] == "dander.cli.main:app" + assert "infra/main.tf" in packaged + assert "infra/modules/scheduled-job/main.tf" in packaged From c5f7abc4afe519f38eafd170d2b90f52856d7a59 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 03:56:52 -0400 Subject: [PATCH 012/163] Define OCI runtime contract v1 --- HANDOFF.md | 37 +-- docs/oci-runtime-contract.md | 58 +++++ src/dander/cli/main.py | 2 + src/dander/cli/run_command.py | 16 +- src/dander/cli/runtime_command.py | 158 +++++++++++++ src/dander/executor.py | 4 +- src/dander/runtime_contract.py | 363 ++++++++++++++++++++++++++++++ src/dander/state/failure.py | 16 ++ tests/cli/test_runtime_cli.py | 185 +++++++++++++++ tests/state/test_failure.py | 39 ++++ tests/test_executor.py | 12 + tests/test_runtime_contract.py | 149 ++++++++++++ 12 files changed, 1015 insertions(+), 24 deletions(-) create mode 100644 docs/oci-runtime-contract.md create mode 100644 src/dander/cli/runtime_command.py create mode 100644 src/dander/runtime_contract.py create mode 100644 tests/cli/test_runtime_cli.py create mode 100644 tests/test_runtime_contract.py diff --git a/HANDOFF.md b/HANDOFF.md index 0d288ddc..da7a8b27 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,39 @@ ## Finished -- Established Dander `0.6.0` as the frozen GCP/Cloud Run/BigQuery compatibility baseline. -- Recorded the cloud-selectable product direction without weakening existing version 1 behavior. -- Added initial correctness, bounded-operation, service, and cost qualification objectives. -- Added one cross-layer characterization suite for BigQuery fencing/state, Cloud Run projection, - CLI identity, and packaged infrastructure. +- Added the launcher-neutral `io.dander.runtime/v1` invocation and JSON-Line event contract. +- Added stable success, invalid, permanent, retryable, and cancellation exit codes. +- Preserved launcher run IDs through the existing executor and run ledger. +- Added validated Cloud Run/local context, one-based attempt normalization, and bounded signal cleanup. +- Kept ordinary `dander run` behavior and the GCP compatibility profile unchanged. ## Try It -Run `uv run pytest -q tests/portability/test_gcp_compatibility_baseline.py` to exercise the -cross-layer baseline before refactoring a portability boundary. +Run `dander runtime execute --contract io.dander.runtime/v1 --pipeline PIPELINE --platform gcp` +inside a generated project. See `docs/oci-runtime-contract.md` for launcher variables and outputs. ## Checks - Ruff lint/format and strict mypy passed. -- All 780 tests passed. +- All 790 tests passed. - Root and stage-zero Terraform format, backend-disabled initialization, and validation passed. -- Release metadata plus wheel/sdist build and distribution inspection passed. +- Wheel/sdist inspection and source-free wheel installation with `dander runtime --help` passed. ## Decisions -- GCP/Cloud Run/BigQuery remains the primary compatibility profile. -- New combinations remain unsupported until their exact conformance, identity, and live gates pass. -- Paid scale tests require a separately approved ceiling and publish measured, digest-bound results. +- Runtime events contain identifiers and aggregate counts only; unrestricted exception text, + cursor values, rows, credentials, and query bodies remain outside the contract. +- Cloud Run's zero-based task attempt is normalized to Dander's one-based launcher attempt. +- Phase 1 accepts only the existing `gcp` profile until later profiles pass their own gates. ## Remaining -- Merge this no-runtime-change baseline through protected main. -- Rebase the isolated OCI runtime-contract ticket onto the merged baseline. -- Continue Phase 1 as focused invocation, inspection, artifact, projection, and Cloud Run PRs. +- Merge this focused runtime-contract ticket through protected main. +- Add `dander runtime inspect` and the credential-free local OCI conformance command separately. +- Add artifact metadata/SBOM, execution projection, then Cloud Run projection parity. ## Review First -- `docs/cloud-portability-slos.md` -- `tests/portability/test_gcp_compatibility_baseline.py` -- `steering/00-project-overview.md` +- `src/dander/runtime_contract.py` +- `src/dander/cli/runtime_command.py` +- `tests/cli/test_runtime_cli.py` diff --git a/docs/oci-runtime-contract.md b/docs/oci-runtime-contract.md new file mode 100644 index 00000000..d36efa53 --- /dev/null +++ b/docs/oci-runtime-contract.md @@ -0,0 +1,58 @@ +# OCI Runtime Contract v1 + +`io.dander.runtime/v1` is the stable boundary between a launcher and one Dander pipeline process. +It does not provision infrastructure and it does not call launcher APIs. + +## Invocation + +```text +dander runtime execute \ + --contract io.dander.runtime/v1 \ + --pipeline PIPELINE_ID \ + --platform gcp \ + --config /app/dander.yaml +``` + +Phase 1 accepts only the existing `gcp` compatibility profile. Later profiles must enter through +the same versioned command after their platform configuration and adapters are implemented. + +Launchers may set these non-secret correlation values: + +| Variable | Default | Validation | +|---|---|---| +| `DANDER_RUN_ID` | launcher execution ID or a generated UUID | 1–128 identifier characters | +| `DANDER_LAUNCHER` | `cloud_run` when Cloud Run context exists, otherwise `local` | 1–64 identifier characters | +| `DANDER_LAUNCHER_EXECUTION_ID` | Cloud Run execution ID when present | 1–256 characters, no control characters | +| `DANDER_ATTEMPT` | one-based Cloud Run attempt or `1` | integer 1–1000 | +| `DANDER_SHARD_INDEX` | Cloud Run task index or `0` | integer 0–9999 and less than shard count | +| `DANDER_SHARD_COUNT` | Cloud Run task count or `1` | integer 1–10000 | +| `DANDER_DEADLINE_AT` | absent | timezone-aware ISO-8601 timestamp | +| `DANDER_PRINCIPAL` | absent | 1–256 characters, no control characters | + +These values are identifiers and context, never secret material. A launcher must not place tokens, +credentials, source rows, SQL, or request bodies in them. + +## Events + +Standard output contains JSON Lines. A validated invocation emits `runtime.started`, followed by +exactly one `runtime.completed` when the process can terminate normally. The terminal status is +`succeeded`, `skipped`, or `failed` and includes only aggregate counts, stable failure codes, and a +retryability decision. Cursor values, record contents, credential material, and unrestricted +exception text are excluded. + +Invalid contract or identifier input fails before `runtime.started`. `SIGTERM` and `SIGINT` request +bounded normal cleanup and produce `interrupted_run` when time remains. `SIGKILL` cannot produce a +terminal event; lease expiry and the next owner reconcile that run. + +## Exit codes + +| Code | Meaning | +|---:|---| +| 0 | succeeded or intentionally skipped | +| 1 | permanent runtime failure | +| 2 | invalid invocation or configuration | +| 75 | retryable runtime failure | +| 130 | graceful cancellation requested | + +Launchers own their finite whole-process retry budget. An exit code never permits an unbounded +retry loop. diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 9f47a5c6..d064f18d 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -56,6 +56,7 @@ build_source_adapter, execute_run, ) +from dander.cli.runtime_command import runtime_app from dander.core.config import Settings from dander.evidence import EvidenceBundle, EvidenceManifest, ProofEvidence, ProofStatus from dander.ingestion import ( @@ -128,6 +129,7 @@ app.add_typer(graph_app, name="graph") app.add_typer(plugins_app, name="plugins") app.add_typer(connector_app, name="connector") +app.add_typer(runtime_app, name="runtime") console = Console() _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") _DEFAULT_CONNECTORS_DIR = Path("connectors") diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 764d81c1..7a5565cd 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -120,19 +120,25 @@ class _ControlStores: metadata: MetadataStore | None -def execute_run(options: RunOptions, *, console: Console) -> None: +def execute_run( + options: RunOptions, + *, + console: Console, + run_id: str | None = None, + render: bool = True, +) -> PipelineExecutionResult | None: """Resolve, execute, and render one ``dander run`` request.""" resolved = _resolve_run(options) if options.dry_run: _render_dry_run(options, resolved, console=console) - return + return None if not resolved.project: raise ClickException("GCP project is required via --project or GCP_PROJECT_ID") _verify_safety(options, resolved) executor = _build_executor(options, resolved) try: - result = executor.execute() + result = executor.execute() if run_id is None else executor.execute(run_id=run_id) except ( CatalogPublishError, SemanticRegistryError, @@ -141,7 +147,9 @@ def execute_run(options: RunOptions, *, console: Console) -> None: GraphRuntimeError, ) as error: raise ClickException(str(error)) from error - _render_result(result, resolved.graph_plan, console=console) + if render: + _render_result(result, resolved.graph_plan, console=console) + return result def _resolve_run(options: RunOptions) -> _ResolvedRun: diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py new file mode 100644 index 00000000..36188e9a --- /dev/null +++ b/src/dander/cli/runtime_command.py @@ -0,0 +1,158 @@ +"""CLI implementation for the versioned OCI runtime boundary.""" + +from __future__ import annotations + +from pathlib import Path + +import typer +from click import ClickException +from rich.console import Console + +from dander.cli.run_command import RunOptions, execute_run +from dander.runtime_contract import ( + RUNTIME_CONTRACT, + LauncherContext, + RuntimeCancelledError, + RuntimeContractError, + RuntimeEvent, + RuntimeExitCode, + graceful_signal_handlers, + is_retryable_failure, + validate_runtime_contract, + validate_runtime_identifier, +) +from dander.state import RunStage, classify_failure + +runtime_app = typer.Typer( + help="Execute and inspect Dander's launcher-neutral OCI runtime contract.", + no_args_is_help=True, +) +_CONSOLE = Console() + + +@runtime_app.command("execute") +def execute_runtime( + contract: str = typer.Option(..., "--contract", help=f"Must be {RUNTIME_CONTRACT}."), + pipeline: str = typer.Option(..., "--pipeline", help="Pipeline ID from dander.yaml."), + platform: str = typer.Option( + ..., + "--platform", + help="Compatibility profile; Phase 1 supports the existing 'gcp' profile.", + ), + project_config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + connectors_dir: Path = typer.Option(Path("connectors"), "--connectors-dir"), # noqa: B008 + models_dir: Path = typer.Option(Path("models"), "--models-dir"), # noqa: B008 + project: str | None = typer.Option(None, "--project", help="Override GCP_PROJECT_ID."), + dataset: str | None = typer.Option(None, "--dataset", help="Override BQ_DATASET_RAW."), + guarded_free_tier: bool = typer.Option(False, "--guarded-free-tier"), + batch_rows: int = typer.Option(10_000, "--batch-rows", min=1, max=100_000), + budget_name: str = typer.Option("dander-sbx-cap", "--budget-name", hidden=True), +) -> None: + """Execute one pipeline and emit only versioned, non-sensitive JSON Lines.""" + try: + validate_runtime_contract(contract) + validate_runtime_identifier(pipeline, label="pipeline id") + validate_runtime_identifier(platform, label="platform") + if platform != "gcp": + raise RuntimeContractError( + "runtime contract v1 currently supports only the 'gcp' compatibility profile" + ) + context = LauncherContext.from_environment() + except RuntimeContractError as error: + raise typer.BadParameter(str(error)) from error + + typer.echo( + RuntimeEvent.started( + context=context, + pipeline_id=pipeline, + platform=platform, + ).to_json() + ) + options = RunOptions( + pipeline_or_source=pipeline, + project=project, + dataset=dataset, + connectors_dir=connectors_dir, + project_config=project_config, + dry_run=False, + sandbox=False, + guarded_free_tier=guarded_free_tier, + batch_rows=batch_rows, + budget_name=budget_name, + state_path=Path(".dander/state.db"), + build_models=False, + models_dir=models_dir, + selected_models=None, + catalog_output=None, + publish_dataplex=False, + dataplex_location="us", + ) + try: + with graceful_signal_handlers(): + result = execute_run( + options, + console=_CONSOLE, + run_id=context.run_id, + render=False, + ) + except RuntimeCancelledError: + typer.echo( + RuntimeEvent.failed( + context=context, + pipeline_id=pipeline, + platform=platform, + stage="cancelled", + failure_code="interrupted_run", + retryable=True, + ).to_json() + ) + raise typer.Exit(code=RuntimeExitCode.CANCELLED) from None + except ClickException as error: + failure = classify_failure(error, stage=RunStage.INGEST, run_id=context.run_id) + if failure.code != "unexpected_error": + retryable = is_retryable_failure(failure.code) + typer.echo( + RuntimeEvent.failed( + context=context, + pipeline_id=pipeline, + platform=platform, + stage="runtime", + failure_code=failure.code, + retryable=retryable, + ).to_json() + ) + code = ( + RuntimeExitCode.RETRYABLE_FAILURE + if retryable + else RuntimeExitCode.PERMANENT_FAILURE + ) + raise typer.Exit(code=code) from None + typer.echo( + RuntimeEvent.failed( + context=context, + pipeline_id=pipeline, + platform=platform, + stage="configuration", + failure_code="invalid_configuration", + retryable=False, + ).to_json() + ) + raise typer.Exit(code=RuntimeExitCode.INVALID_INVOCATION) from None + except Exception as error: + failure = classify_failure(error, stage=RunStage.INGEST, run_id=context.run_id) + retryable = is_retryable_failure(failure.code) + typer.echo( + RuntimeEvent.failed( + context=context, + pipeline_id=pipeline, + platform=platform, + stage="runtime", + failure_code=failure.code, + retryable=retryable, + ).to_json() + ) + code = RuntimeExitCode.RETRYABLE_FAILURE if retryable else RuntimeExitCode.PERMANENT_FAILURE + raise typer.Exit(code=code) from None + + assert result is not None + typer.echo(RuntimeEvent.completed(result, context=context, platform=platform).to_json()) diff --git a/src/dander/executor.py b/src/dander/executor.py index afe073d4..41d7f307 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -90,9 +90,9 @@ def __init__( self._dataplex_publisher = dataplex_publisher self._leases = leases - def execute(self) -> PipelineExecutionResult: + def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: """Execute every enabled stage and record one truthful terminal outcome.""" - run_id = uuid4().hex + run_id = run_id or uuid4().hex stage = RunStage.INGEST endpoints = extracted = affected = models = assertions = assets = 0 self._history.start( diff --git a/src/dander/runtime_contract.py b/src/dander/runtime_contract.py new file mode 100644 index 00000000..716e27e2 --- /dev/null +++ b/src/dander/runtime_contract.py @@ -0,0 +1,363 @@ +"""Versioned, launcher-neutral contract for the Dander OCI runtime.""" + +from __future__ import annotations + +import json +import os +import re +import signal +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import IntEnum +from typing import TYPE_CHECKING, Any, Literal +from uuid import uuid4 + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping + from types import FrameType + + from dander.executor import PipelineExecutionResult + +RUNTIME_CONTRACT = "io.dander.runtime/v1" +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_SAFE_LAUNCHER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +_MAX_OPAQUE_LENGTH = 256 + + +class RuntimeExitCode(IntEnum): + """Stable process outcomes consumed by launchers.""" + + SUCCESS = 0 + PERMANENT_FAILURE = 1 + INVALID_INVOCATION = 2 + RETRYABLE_FAILURE = 75 + CANCELLED = 130 + + +class RuntimeContractError(ValueError): + """Raised when a launcher supplies an invalid runtime-contract value.""" + + +class RuntimeCancelledError(RuntimeError): + """Raised after a catchable process signal requests graceful cancellation.""" + + def __init__(self, signal_name: str) -> None: + super().__init__("runtime cancellation requested") + self.signal_name = signal_name + + +@dataclass(frozen=True, slots=True) +class LauncherContext: + """Validated, non-secret launcher correlation supplied to one execution.""" + + run_id: str + launcher: str + execution_id: str | None + attempt: int + shard_index: int + shard_count: int + deadline_at: str | None + principal: str | None + + @classmethod + def from_environment(cls, environment: Mapping[str, str] | None = None) -> LauncherContext: + """Resolve Dander variables with Cloud Run's native context as a compatibility source.""" + values = os.environ if environment is None else environment + launcher = values.get("DANDER_LAUNCHER") or ( + "cloud_run" if values.get("CLOUD_RUN_EXECUTION") else "local" + ) + execution_id = values.get("DANDER_LAUNCHER_EXECUTION_ID") or values.get( + "CLOUD_RUN_EXECUTION" + ) + run_id = values.get("DANDER_RUN_ID") or execution_id or uuid4().hex + attempt_value = values.get("DANDER_ATTEMPT") + if attempt_value is None: + cloud_run_attempt = values.get("CLOUD_RUN_TASK_ATTEMPT") + attempt_value = ( + str( + _bounded_integer( + cloud_run_attempt, + label="Cloud Run task attempt", + minimum=0, + maximum=999, + ) + + 1 + ) + if cloud_run_attempt is not None + else "1" + ) + shard_index_value = ( + values.get("DANDER_SHARD_INDEX") or values.get("CLOUD_RUN_TASK_INDEX") or "0" + ) + shard_count_value = ( + values.get("DANDER_SHARD_COUNT") or values.get("CLOUD_RUN_TASK_COUNT") or "1" + ) + deadline_at = values.get("DANDER_DEADLINE_AT") + principal = values.get("DANDER_PRINCIPAL") + + _require_identifier(run_id, label="run id") + if not _SAFE_LAUNCHER.fullmatch(launcher): + raise RuntimeContractError( + "launcher must be 1-64 characters using letters, numbers, '.', '_', or '-'" + ) + _require_opaque(execution_id, label="launcher execution id") + _require_opaque(principal, label="principal") + attempt = _bounded_integer(attempt_value, label="attempt", minimum=1, maximum=1000) + shard_index = _bounded_integer( + shard_index_value, label="shard index", minimum=0, maximum=9999 + ) + shard_count = _bounded_integer( + shard_count_value, label="shard count", minimum=1, maximum=10_000 + ) + if shard_index >= shard_count: + raise RuntimeContractError("shard index must be less than shard count") + if deadline_at is not None: + _parse_deadline(deadline_at) + return cls( + run_id=run_id, + launcher=launcher, + execution_id=execution_id, + attempt=attempt, + shard_index=shard_index, + shard_count=shard_count, + deadline_at=deadline_at, + principal=principal, + ) + + def dimensions(self) -> dict[str, object]: + """Return the presentation-safe correlation dimensions included in runtime events.""" + dimensions: dict[str, object] = { + "launcher": self.launcher, + "launcher_execution_id": self.execution_id, + "attempt": self.attempt, + "shard_index": self.shard_index, + "shard_count": self.shard_count, + } + if self.deadline_at is not None: + dimensions["deadline_at"] = self.deadline_at + if self.principal is not None: + dimensions["principal"] = self.principal + return dimensions + + +@dataclass(frozen=True, slots=True) +class RuntimeEvent: + """One JSON-Line runtime event with an intentionally non-sensitive payload.""" + + event: Literal["runtime.started", "runtime.completed"] + timestamp: str + run_id: str + pipeline_id: str + platform: str + stage: str + dimensions: dict[str, object] + status: Literal["succeeded", "skipped", "failed"] | None = None + outputs: dict[str, object] | None = None + failure_code: str | None = None + retryable: bool | None = None + + @classmethod + def started( + cls, + *, + context: LauncherContext, + pipeline_id: str, + platform: str, + ) -> RuntimeEvent: + """Build the first event for a validated execution request.""" + return cls( + event="runtime.started", + timestamp=_timestamp(), + run_id=context.run_id, + pipeline_id=pipeline_id, + platform=platform, + stage="starting", + dimensions=context.dimensions(), + ) + + @classmethod + def completed( + cls, + result: PipelineExecutionResult, + *, + context: LauncherContext, + platform: str, + ) -> RuntimeEvent: + """Build the terminal success/overlap record from non-sensitive aggregates.""" + endpoints = [ + { + "name": endpoint.endpoint, + "extracted_rows": endpoint.extracted, + "affected_rows": endpoint.affected, + "cursor_committed": endpoint.committed_cursor is not None, + } + for endpoint in result.ingestion.endpoints + ] + return cls( + event="runtime.completed", + timestamp=_timestamp(), + run_id=result.run_id, + pipeline_id=result.pipeline_id, + platform=platform, + stage="complete", + dimensions=context.dimensions(), + status="skipped" if result.skipped else "succeeded", + outputs={ + "source": result.ingestion.source, + "endpoints": endpoints, + "models": list(result.models), + "metrics": { + "endpoints": len(endpoints), + "extracted_rows": sum( + endpoint.extracted for endpoint in result.ingestion.endpoints + ), + "affected_rows": sum( + endpoint.affected for endpoint in result.ingestion.endpoints + ), + "models": len(result.models), + "assertions": result.assertions, + "assets": result.assets, + }, + }, + retryable=False, + ) + + @classmethod + def failed( + cls, + *, + context: LauncherContext, + pipeline_id: str, + platform: str, + stage: str, + failure_code: str, + retryable: bool, + ) -> RuntimeEvent: + """Build a terse terminal failure without copying exception text.""" + return cls( + event="runtime.completed", + timestamp=_timestamp(), + run_id=context.run_id, + pipeline_id=pipeline_id, + platform=platform, + stage=stage, + dimensions=context.dimensions(), + status="failed", + outputs={}, + failure_code=failure_code, + retryable=retryable, + ) + + def to_json(self) -> str: + """Render one compact JSON line suitable for launchers and log processors.""" + payload: dict[str, object] = { + "contract": RUNTIME_CONTRACT, + "event": self.event, + "timestamp": self.timestamp, + "run_id": self.run_id, + "pipeline_id": self.pipeline_id, + "platform": self.platform, + "stage": self.stage, + "dimensions": self.dimensions, + } + if self.status is not None: + payload["status"] = self.status + if self.outputs is not None: + payload["outputs"] = self.outputs + if self.failure_code is not None: + payload["failure_code"] = self.failure_code + if self.retryable is not None: + payload["retryable"] = self.retryable + return json.dumps(payload, separators=(",", ":"), sort_keys=True) + + +def validate_runtime_contract(value: str) -> None: + """Reject unknown invocation contracts before any provider or project access.""" + if value != RUNTIME_CONTRACT: + raise RuntimeContractError( + f"unsupported runtime contract {value!r}; expected {RUNTIME_CONTRACT!r}" + ) + + +def validate_runtime_identifier(value: str, *, label: str) -> None: + """Validate a pipeline or compatibility-profile identifier.""" + _require_identifier(value, label=label) + + +def is_retryable_failure(code: str) -> bool: + """Return the conservative v1 whole-process retry policy for a stable failure code.""" + return code in { + "catalog_failed", + "destination_write_failed", + "extraction_failed", + "lease_failed", + "rate_limited", + } + + +@contextmanager +def graceful_signal_handlers() -> Iterator[None]: + """Translate catchable termination signals into bounded normal exception cleanup.""" + previous: dict[ + signal.Signals, + Callable[[int, FrameType | None], Any] | int | signal.Handlers | None, + ] = {} + + def cancel(signum: int, _frame: object) -> None: + raise RuntimeCancelledError(signal.Signals(signum).name) + + for handled in (signal.SIGTERM, signal.SIGINT): + previous[handled] = signal.getsignal(handled) + signal.signal(handled, cancel) + try: + yield + finally: + for handled, handler in previous.items(): + signal.signal(handled, handler) + + +def _require_identifier(value: str, *, label: str) -> None: + if not _SAFE_IDENTIFIER.fullmatch(value): + raise RuntimeContractError( + f"{label} must be 1-128 characters using letters, numbers, '.', '_', ':', or '-'" + ) + + +def _require_opaque(value: str | None, *, label: str) -> None: + if value is None: + return + if ( + not value + or len(value) > _MAX_OPAQUE_LENGTH + or any(ord(character) < 32 for character in value) + ): + raise RuntimeContractError( + f"{label} must be 1-{_MAX_OPAQUE_LENGTH} characters without control characters" + ) + + +def _bounded_integer(value: str, *, label: str, minimum: int, maximum: int) -> int: + try: + parsed = int(value) + except ValueError as error: + raise RuntimeContractError(f"{label} must be an integer") from error + if not minimum <= parsed <= maximum: + raise RuntimeContractError(f"{label} must be between {minimum} and {maximum}") + return parsed + + +def _parse_deadline(value: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise RuntimeContractError( + "deadline must be an ISO-8601 timestamp with a timezone" + ) from error + if parsed.tzinfo is None: + raise RuntimeContractError("deadline must be an ISO-8601 timestamp with a timezone") + return parsed + + +def _timestamp() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/src/dander/state/failure.py b/src/dander/state/failure.py index 9be350bb..150b3897 100644 --- a/src/dander/state/failure.py +++ b/src/dander/state/failure.py @@ -25,6 +25,12 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu names = {item.__class__.__name__ for item in chain} statuses = {status for item in chain if (status := _status_code(item)) is not None} + if "RuntimeCancelledError" in names: + return _details( + "interrupted_run", + f"The runtime received a cancellation signal. A fresh run can retry safely; " + f"inspect logs for run {run_id}.", + ) if 401 in statuses or names & {"Unauthenticated", "Unauthorized"}: return _details( "authentication_failed", @@ -55,6 +61,16 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu "destination_write_failed", f"BigQuery rejected a destination write. Inspect logs for run {run_id}.", ) + if names & {"TransformProjectError", "TransformRunError", "GraphRuntimeError"}: + return _details( + "transform_failed", + f"A transform failed. Inspect logs for run {run_id}.", + ) + if names & {"CatalogPublishError", "SemanticRegistryError"}: + return _details( + "catalog_failed", + f"Metadata or catalog publication failed. Inspect logs for run {run_id}.", + ) if stage.value == "transform": code = ( "test_failed" if _has_known_phrase(chain, "data tests failed") else "transform_failed" diff --git a/tests/cli/test_runtime_cli.py b/tests/cli/test_runtime_cli.py new file mode 100644 index 00000000..25a820b8 --- /dev/null +++ b/tests/cli/test_runtime_cli.py @@ -0,0 +1,185 @@ +"""OCI runtime invocation, output, and exit-code contract.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, cast + +from click import ClickException +from typer.testing import CliRunner + +import dander.cli.runtime_command as runtime_module +from dander.cli.main import app +from dander.executor import PipelineExecutionResult +from dander.runtime import EndpointRunResult, PipelineRunResult +from dander.runtime_contract import RuntimeCancelledError, RuntimeExitCode + +if TYPE_CHECKING: + from collections.abc import Callable + + from pytest import MonkeyPatch + from rich.console import Console + from typer.testing import Result + + from dander.cli.run_command import RunOptions + + +def _invoke( + monkeypatch: MonkeyPatch, + execute: Callable[..., PipelineExecutionResult | None], +) -> Result: + monkeypatch.setattr(runtime_module, "execute_run", execute) + return CliRunner().invoke( + app, + [ + "runtime", + "execute", + "--contract", + "io.dander.runtime/v1", + "--pipeline", + "greenhouse_jobs", + "--platform", + "gcp", + "--project", + "unit-project", + "--batch-rows", + "2500", + ], + env={ + "DANDER_RUN_ID": "cloud-run:execution-42", + "DANDER_LAUNCHER": "cloud_run", + }, + ) + + +def test_runtime_execute_uses_launcher_run_id_and_emits_json_lines( + monkeypatch: MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + def execute( + options: RunOptions, + *, + console: Console, + run_id: str | None = None, + render: bool = True, + ) -> PipelineExecutionResult: + captured.update(options=options, console=console, run_id=run_id, render=render) + assert run_id is not None + return PipelineExecutionResult( + run_id=run_id, + pipeline_id=options.pipeline_or_source, + ingestion=PipelineRunResult( + run_id=run_id, + source="greenhouse_job_board", + endpoints=( + EndpointRunResult( + endpoint="jobs", + extracted=4, + affected=4, + committed_cursor=None, + ), + ), + ), + models=("stg_greenhouse__jobs",), + assertions=1, + assets=1, + ) + + result = _invoke(monkeypatch, execute) + + assert result.exit_code == RuntimeExitCode.SUCCESS, result.output + events = [json.loads(line) for line in result.output.splitlines()] + assert [event["event"] for event in events] == ["runtime.started", "runtime.completed"] + assert events[-1]["run_id"] == "cloud-run:execution-42" + assert events[-1]["outputs"]["metrics"]["extracted_rows"] == 4 + options = cast("RunOptions", captured["options"]) + assert options.pipeline_or_source == "greenhouse_jobs" + assert options.project == "unit-project" + assert options.batch_rows == 2500 + assert captured["render"] is False + + +def test_runtime_execute_sanitizes_retryable_failure(monkeypatch: MonkeyPatch) -> None: + class _Response: + status_code = 429 + + class HttpFailureError(RuntimeError): + response = _Response() + + def fail(*args: object, **kwargs: object) -> None: + raise HttpFailureError("credential-value") + + result = _invoke(monkeypatch, fail) + + assert result.exit_code == RuntimeExitCode.RETRYABLE_FAILURE + terminal = json.loads(result.output.splitlines()[-1]) + assert terminal["failure_code"] == "rate_limited" + assert terminal["retryable"] is True + assert "credential-value" not in result.output + + +def test_runtime_execute_distinguishes_invalid_configuration( + monkeypatch: MonkeyPatch, +) -> None: + def fail(*args: object, **kwargs: object) -> None: + raise ClickException("private configuration detail") + + result = _invoke(monkeypatch, fail) + + assert result.exit_code == RuntimeExitCode.INVALID_INVOCATION + terminal = json.loads(result.output.splitlines()[-1]) + assert terminal["failure_code"] == "invalid_configuration" + assert "private configuration detail" not in result.output + + +def test_runtime_execute_preserves_wrapped_execution_failure_classification( + monkeypatch: MonkeyPatch, +) -> None: + class TransformRunError(RuntimeError): + pass + + def fail(*args: object, **kwargs: object) -> None: + error = ClickException("private transform detail") + error.__cause__ = TransformRunError("source row must not escape") + raise error + + result = _invoke(monkeypatch, fail) + + assert result.exit_code == RuntimeExitCode.PERMANENT_FAILURE + terminal = json.loads(result.output.splitlines()[-1]) + assert terminal["failure_code"] == "transform_failed" + assert "private transform detail" not in result.output + assert "source row" not in result.output + + +def test_runtime_execute_reports_graceful_cancellation(monkeypatch: MonkeyPatch) -> None: + def cancel(*args: object, **kwargs: object) -> None: + raise RuntimeCancelledError("SIGTERM") + + result = _invoke(monkeypatch, cancel) + + assert result.exit_code == RuntimeExitCode.CANCELLED + terminal = json.loads(result.output.splitlines()[-1]) + assert terminal["failure_code"] == "interrupted_run" + assert terminal["stage"] == "cancelled" + + +def test_runtime_execute_rejects_unknown_contract_before_start() -> None: + result = CliRunner().invoke( + app, + [ + "runtime", + "execute", + "--contract", + "io.dander.runtime/v2", + "--pipeline", + "greenhouse_jobs", + "--platform", + "gcp", + ], + ) + + assert result.exit_code == RuntimeExitCode.INVALID_INVOCATION + assert "unsupported runtime contract" in result.output + assert "runtime.started" not in result.output diff --git a/tests/state/test_failure.py b/tests/state/test_failure.py index fca62304..73879357 100644 --- a/tests/state/test_failure.py +++ b/tests/state/test_failure.py @@ -42,3 +42,42 @@ def test_unknown_failure_is_stage_specific_and_points_to_run_logs() -> None: "Metadata or catalog publication failed. Inspect logs for run run-123." ) assert "customer payload" not in failure.summary + + +def test_runtime_cancellation_is_safe_and_retryable_by_a_fresh_run() -> None: + class RuntimeCancelledError(RuntimeError): + pass + + failure = classify_failure( + RuntimeCancelledError("signal detail must not persist"), + stage=RunStage.INGEST, + run_id="run-123", + ) + + assert failure.code == "interrupted_run" + assert failure.summary == ( + "The runtime received a cancellation signal. A fresh run can retry safely; " + "inspect logs for run run-123." + ) + assert "signal detail" not in failure.summary + + +def test_wrapped_runtime_stage_errors_keep_stable_codes() -> None: + class TransformRunError(RuntimeError): + pass + + class CatalogPublishError(RuntimeError): + pass + + transform_wrapper = RuntimeError("wrapper") + transform_wrapper.__cause__ = TransformRunError("private transform detail") + catalog_wrapper = RuntimeError("wrapper") + catalog_wrapper.__cause__ = CatalogPublishError("private catalog detail") + + transform = classify_failure(transform_wrapper, stage=RunStage.INGEST, run_id="run-123") + catalog = classify_failure(catalog_wrapper, stage=RunStage.INGEST, run_id="run-123") + + assert transform.code == "transform_failed" + assert catalog.code == "catalog_failed" + assert "private" not in transform.summary + assert "private" not in catalog.summary diff --git a/tests/test_executor.py b/tests/test_executor.py index 58ec1974..ff0bcf10 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -239,6 +239,18 @@ def test_executor_records_one_truthful_complete_lifecycle(tmp_path: Path) -> Non assert assets[0]["metrics"][0]["calculation"] == "COUNT(DISTINCT `id`)" +def test_executor_preserves_launcher_supplied_run_id(tmp_path: Path) -> None: + _models(tmp_path) + history = _History() + + result = _executor(tmp_path, history=history, metadata=_Metadata()).execute( + run_id="launcher-run-42" + ) + + assert result.run_id == "launcher-run-42" + assert history.started == ("launcher-run-42", "example", "example_pipeline") + + def test_executor_marks_transform_failure_without_claiming_ingestion_only_success( tmp_path: Path, ) -> None: diff --git a/tests/test_runtime_contract.py b/tests/test_runtime_contract.py new file mode 100644 index 00000000..04c2dfa5 --- /dev/null +++ b/tests/test_runtime_contract.py @@ -0,0 +1,149 @@ +"""Stable JSON and launcher-context boundary for the Dander OCI runtime.""" + +from __future__ import annotations + +import json +import signal + +import pytest + +from dander.executor import PipelineExecutionResult +from dander.runtime import EndpointRunResult, PipelineRunResult +from dander.runtime_contract import ( + RUNTIME_CONTRACT, + LauncherContext, + RuntimeCancelledError, + RuntimeContractError, + RuntimeEvent, + graceful_signal_handlers, + validate_runtime_contract, +) + + +def _context() -> LauncherContext: + return LauncherContext.from_environment( + { + "DANDER_RUN_ID": "cloud-run:execution-42", + "DANDER_LAUNCHER": "cloud_run", + "DANDER_LAUNCHER_EXECUTION_ID": "execution-42", + "DANDER_ATTEMPT": "2", + "DANDER_SHARD_INDEX": "0", + "DANDER_SHARD_COUNT": "1", + "DANDER_DEADLINE_AT": "2026-08-07T12:00:00Z", + "DANDER_PRINCIPAL": "service-account:runtime@example.invalid", + } + ) + + +def _result(*, skipped: bool = False) -> PipelineExecutionResult: + return PipelineExecutionResult( + run_id="cloud-run:execution-42", + pipeline_id="example_pipeline", + ingestion=PipelineRunResult( + run_id="cloud-run:execution-42", + source="example", + endpoints=( + EndpointRunResult( + endpoint="widgets", + extracted=3, + affected=2, + committed_cursor="2026-08-06T12:00:00Z", + ), + ), + ), + models=("stg_widgets",), + assertions=2, + assets=1, + skipped=skipped, + ) + + +def test_runtime_events_are_json_lines_without_cursor_values() -> None: + started = json.loads( + RuntimeEvent.started( + context=_context(), pipeline_id="example_pipeline", platform="gcp" + ).to_json() + ) + completed_json = RuntimeEvent.completed(_result(), context=_context(), platform="gcp").to_json() + completed = json.loads(completed_json) + + assert started["contract"] == RUNTIME_CONTRACT + assert started["event"] == "runtime.started" + assert started["dimensions"] == { + "attempt": 2, + "deadline_at": "2026-08-07T12:00:00Z", + "launcher": "cloud_run", + "launcher_execution_id": "execution-42", + "principal": "service-account:runtime@example.invalid", + "shard_count": 1, + "shard_index": 0, + } + assert completed["event"] == "runtime.completed" + assert completed["status"] == "succeeded" + assert completed["outputs"]["metrics"]["extracted_rows"] == 3 + assert completed["outputs"]["endpoints"][0]["cursor_committed"] is True + assert "2026-08-06" not in completed_json + + +def test_runtime_event_distinguishes_overlap_and_sanitized_failure() -> None: + skipped = json.loads( + RuntimeEvent.completed(_result(skipped=True), context=_context(), platform="gcp").to_json() + ) + failed_json = RuntimeEvent.failed( + context=_context(), + pipeline_id="example_pipeline", + platform="gcp", + stage="runtime", + failure_code="authentication_failed", + retryable=False, + ).to_json() + failed = json.loads(failed_json) + + assert skipped["status"] == "skipped" + assert failed["status"] == "failed" + assert failed["failure_code"] == "authentication_failed" + assert failed["outputs"] == {} + assert "credential-value" not in failed_json + + +def test_launcher_context_validates_values_and_cloud_run_fallbacks() -> None: + context = LauncherContext.from_environment( + { + "DANDER_LAUNCHER": "cloud_run", + "CLOUD_RUN_EXECUTION": "execution-123", + "CLOUD_RUN_TASK_ATTEMPT": "3", + "CLOUD_RUN_TASK_INDEX": "1", + "CLOUD_RUN_TASK_COUNT": "2", + "DANDER_DEADLINE_AT": "2026-08-07T12:00:00Z", + } + ) + + assert context.run_id == "execution-123" + assert context.execution_id == "execution-123" + assert context.attempt == 4 + assert context.shard_index == 1 + + with pytest.raises(RuntimeContractError, match="shard index"): + LauncherContext.from_environment({"DANDER_SHARD_INDEX": "2", "DANDER_SHARD_COUNT": "2"}) + with pytest.raises(RuntimeContractError, match="run id"): + LauncherContext.from_environment({"DANDER_RUN_ID": "unsafe run\nnext-line"}) + with pytest.raises(RuntimeContractError, match="deadline"): + LauncherContext.from_environment({"DANDER_DEADLINE_AT": "2026-08-07T12:00:00"}) + with pytest.raises(RuntimeContractError, match="attempt"): + LauncherContext.from_environment({"DANDER_ATTEMPT": "0"}) + + first_cloud_run_attempt = LauncherContext.from_environment({"CLOUD_RUN_TASK_ATTEMPT": "0"}) + assert first_cloud_run_attempt.attempt == 1 + + +def test_contract_version_fails_closed() -> None: + validate_runtime_contract(RUNTIME_CONTRACT) + with pytest.raises(RuntimeContractError, match="unsupported runtime contract"): + validate_runtime_contract("io.dander.runtime/v2") + + +def test_sigterm_handler_becomes_graceful_cancellation() -> None: + with graceful_signal_handlers(), pytest.raises(RuntimeCancelledError) as raised: + signal.raise_signal(signal.SIGTERM) + + assert raised.value.signal_name == "SIGTERM" From d4fed7e046cc154bf21e43424007271553bc2c33 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 04:15:44 -0400 Subject: [PATCH 013/163] Add runtime inspection and conformance --- HANDOFF.md | 36 ++--- docs/oci-runtime-contract.md | 11 ++ src/dander/cli/runtime_command.py | 34 +++++ src/dander/executor.py | 13 +- src/dander/runtime_inspection.py | 233 ++++++++++++++++++++++++++++++ tests/test_executor.py | 2 +- tests/test_runtime_inspection.py | 159 ++++++++++++++++++++ 7 files changed, 467 insertions(+), 21 deletions(-) create mode 100644 src/dander/runtime_inspection.py create mode 100644 tests/test_runtime_inspection.py diff --git a/HANDOFF.md b/HANDOFF.md index da7a8b27..e145c3ee 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Added the launcher-neutral `io.dander.runtime/v1` invocation and JSON-Line event contract. -- Added stable success, invalid, permanent, retryable, and cancellation exit codes. -- Preserved launcher run IDs through the existing executor and run ledger. -- Added validated Cloud Run/local context, one-based attempt normalization, and bounded signal cleanup. -- Kept ordinary `dander run` behavior and the GCP compatibility profile unchanged. +- Added provider-free `dander runtime inspect` for installed build, adapter, and plugin metadata. +- Added `dander runtime conformance`, a credential-free local executor/run-ledger/event probe. +- Proved the probe writes only its declared SQLite file and refuses to overwrite existing state. +- Proved graceful SIGTERM translation inside the local conformance path. +- Narrowed `PipelineExecutor` to an ingestion protocol without changing runtime behavior. ## Try It -Run `dander runtime execute --contract io.dander.runtime/v1 --pipeline PIPELINE --platform gcp` -inside a generated project. See `docs/oci-runtime-contract.md` for launcher variables and outputs. +Run `dander runtime inspect --config dander.yaml`, then `dander runtime conformance`. Supply an +empty `--work-dir` only when you want to retain the probe's `state.db` for inspection. ## Checks - Ruff lint/format and strict mypy passed. -- All 790 tests passed. +- All 799 tests passed. - Root and stage-zero Terraform format, backend-disabled initialization, and validation passed. -- Wheel/sdist inspection and source-free wheel installation with `dander runtime --help` passed. +- Wheel/sdist inspection and source-free wheel conformance passed outside the checkout. +- A non-root, read-only local OCI image ran conformance and inspection successfully. ## Decisions -- Runtime events contain identifiers and aggregate counts only; unrestricted exception text, - cursor values, rows, credentials, and query bodies remain outside the contract. -- Cloud Run's zero-based task attempt is normalized to Dander's one-based launcher attempt. -- Phase 1 accepts only the existing `gcp` profile until later profiles pass their own gates. +- Inspection loads declared plugin metadata but never constructs a source or resolves a secret. +- The conformance pipeline exercises the real executor and SQLite ledger with a deterministic + in-process ingestion summary; it performs no network or provider access. ## Remaining -- Merge this focused runtime-contract ticket through protected main. -- Add `dander runtime inspect` and the credential-free local OCI conformance command separately. -- Add artifact metadata/SBOM, execution projection, then Cloud Run projection parity. +- Merge this focused ticket through protected main. +- Add OCI annotations, digest recording, capability manifest, SBOM, and artifact checks. +- Add the cloud-neutral execution projection, then make Cloud Run consume it with parity. ## Review First -- `src/dander/runtime_contract.py` +- `src/dander/runtime_inspection.py` - `src/dander/cli/runtime_command.py` -- `tests/cli/test_runtime_cli.py` +- `tests/test_runtime_inspection.py` diff --git a/docs/oci-runtime-contract.md b/docs/oci-runtime-contract.md index d36efa53..beb695ac 100644 --- a/docs/oci-runtime-contract.md +++ b/docs/oci-runtime-contract.md @@ -56,3 +56,14 @@ terminal event; lease expiry and the next owner reconcile that run. Launchers own their finite whole-process retry budget. An exit code never permits an unbounded retry loop. + +## Inspection and local conformance + +`dander runtime inspect --config dander.yaml` reports the installed Dander version, active +compatibility adapters, ingestion engines, and explicitly pinned connector plugins. It validates +package entry points but never constructs a source, resolves a secret, or contacts a provider. + +`dander runtime conformance` runs one deterministic executor lifecycle with local SQLite state, +parses the versioned start/completion events, and verifies graceful `SIGTERM` translation. With an +explicit empty `--work-dir`, its only filesystem write is `state.db`; without one it uses and +removes a temporary directory. The probe uses no credentials, network, or cloud resources. diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index 36188e9a..c492d128 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -2,7 +2,9 @@ from __future__ import annotations +import tempfile from pathlib import Path +from typing import Annotated import typer from click import ClickException @@ -21,6 +23,7 @@ validate_runtime_contract, validate_runtime_identifier, ) +from dander.runtime_inspection import inspect_runtime, run_local_conformance from dander.state import RunStage, classify_failure runtime_app = typer.Typer( @@ -30,6 +33,37 @@ _CONSOLE = Console() +@runtime_app.command("inspect") +def inspect_runtime_command( + project_config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 +) -> None: + """Report installed runtime, adapter, and plugin metadata without provider access.""" + try: + inspection = inspect_runtime(project_config) + except (RuntimeContractError, ValueError) as error: + raise typer.BadParameter(str(error)) from error + typer.echo(inspection.to_json()) + + +@runtime_app.command("conformance") +def conformance_runtime_command( + work_dir: Annotated[ + Path | None, + typer.Option("--work-dir", help="Directory in which the probe may write state.db."), + ] = None, +) -> None: + """Run a credential-free local executor, event, filesystem, and signal probe.""" + try: + if work_dir is not None: + result = run_local_conformance(work_dir) + else: + with tempfile.TemporaryDirectory(prefix="dander-runtime-conformance-") as directory: + result = run_local_conformance(Path(directory)) + except RuntimeContractError as error: + raise typer.BadParameter(str(error)) from error + typer.echo(result.to_json()) + + @runtime_app.command("execute") def execute_runtime( contract: str = typer.Option(..., "--contract", help=f"Must be {RUNTIME_CONTRACT}."), diff --git a/src/dander/executor.py b/src/dander/executor.py index 41d7f307..53ea9d25 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -19,7 +19,6 @@ from dander.catalog import CatalogAsset, MetadataStore from dander.concurrency import OwnershipGuard from dander.ingestion import SourceConfig - from dander.runtime import PipelineRunner from dander.state import LeaseStore, RunHistoryStore _LOGGER = logging.getLogger(__name__) @@ -36,6 +35,16 @@ def build( """Build models and run their assertions.""" +class _IngestionRunner(Protocol): + def run( + self, + *, + run_id: str | None = None, + ownership: OwnershipGuard | None = None, + ) -> PipelineRunResult: + """Run configured endpoints and return a non-sensitive summary.""" + + class _DataplexPublisher(Protocol): def publish(self, asset: CatalogAsset) -> str: """Project one canonical asset into Dataplex.""" @@ -62,7 +71,7 @@ def __init__( *, pipeline_id: str, source_config: SourceConfig, - ingestion: PipelineRunner, + ingestion: _IngestionRunner, history: RunHistoryStore, project: str, models_dir: Path, diff --git a/src/dander/runtime_inspection.py b/src/dander/runtime_inspection.py new file mode 100644 index 00000000..89b30566 --- /dev/null +++ b/src/dander/runtime_inspection.py @@ -0,0 +1,233 @@ +"""Provider-free runtime inspection and local lifecycle conformance.""" + +from __future__ import annotations + +import json +import signal +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import uuid4 + +from dander import __version__ +from dander.executor import PipelineExecutor +from dander.ingestion import SourceConfig +from dander.plugins import ConnectorPluginRegistry, load_connector_plugins +from dander.project import load_project_config +from dander.runtime import EndpointRunResult, PipelineRunResult +from dander.runtime_contract import ( + RUNTIME_CONTRACT, + LauncherContext, + RuntimeCancelledError, + RuntimeContractError, + RuntimeEvent, + graceful_signal_handlers, +) +from dander.state import RunStatus, SqliteRunHistoryStore + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + + from dander.concurrency import OwnershipGuard + +_PLATFORM_PROFILES = ("gcp",) +_LAUNCHERS = ("cloud_run", "local") +_WAREHOUSES = ("bigquery",) +_STATE_BACKENDS = ("bigquery", "sqlite") +_CATALOGS = ("dataplex", "local_json", "none") +_SECRET_PROVIDERS = ("environment", "gcp_secret_manager") + + +@dataclass(frozen=True, slots=True) +class RuntimeInspection: + """Non-secret installed runtime and plugin metadata.""" + + contract: str + build: Mapping[str, object] + adapters: Mapping[str, object] + plugins: tuple[Mapping[str, object], ...] + + def to_json(self) -> str: + """Render deterministic inspection JSON for humans and launchers.""" + return json.dumps( + { + "contract": self.contract, + "build": self.build, + "adapters": self.adapters, + "plugins": self.plugins, + }, + separators=(",", ":"), + sort_keys=True, + ) + + +@dataclass(frozen=True, slots=True) +class RuntimeConformance: + """Normalized result of the credential-free local lifecycle probe.""" + + contract: str + status: str + run_id: str + filesystem_writes: tuple[str, ...] + signal: str + events: tuple[str, ...] + + def to_json(self) -> str: + """Render one deterministic conformance report.""" + return json.dumps( + { + "contract": self.contract, + "status": self.status, + "run_id": self.run_id, + "filesystem_writes": self.filesystem_writes, + "signal": self.signal, + "events": self.events, + }, + separators=(",", ":"), + sort_keys=True, + ) + + +def inspect_runtime(config: Path) -> RuntimeInspection: + """Inspect configured packages without constructing sources or contacting providers.""" + manifest = load_project_config(config) + registry = load_connector_plugins(manifest.plugins) + return RuntimeInspection( + contract=RUNTIME_CONTRACT, + build={ + "distribution": "dander-platform", + "version": __version__, + "python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + }, + adapters={ + "platform_profiles": _PLATFORM_PROFILES, + "launchers": _LAUNCHERS, + "warehouses": _WAREHOUSES, + "state_backends": _STATE_BACKENDS, + "catalogs": _CATALOGS, + "secret_providers": _SECRET_PROVIDERS, + "ingestion_engines": registry.engines, + }, + plugins=_plugin_metadata(registry), + ) + + +def run_local_conformance(work_dir: Path) -> RuntimeConformance: + """Run one deterministic executor lifecycle using only local SQLite state.""" + work_dir.mkdir(parents=True, exist_ok=True) + state_path = work_dir / "state.db" + if state_path.exists(): + raise RuntimeContractError(f"conformance state already exists: {state_path}") + + before = _relative_files(work_dir) + run_id = f"runtime-conformance-{uuid4().hex}" + source_config = SourceConfig( + name="runtime_conformance", + base_url="https://runtime-conformance.invalid", + auth_strategy="none", + endpoints=[], + ) + history = SqliteRunHistoryStore(state_path) + result = PipelineExecutor( + pipeline_id="runtime_conformance", + source_config=source_config, + ingestion=_ConformanceIngestion(), + history=history, + project="runtime-conformance", + models_dir=work_dir, + selected_models=None, + build_models=False, + ).execute(run_id=run_id) + record = history.recent(pipeline_id="runtime_conformance", limit=1)[0] + if record.status is not RunStatus.SUCCEEDED or record.run_id != run_id: + raise RuntimeContractError("local conformance run history did not finish successfully") + + context = LauncherContext.from_environment( + { + "DANDER_RUN_ID": run_id, + "DANDER_LAUNCHER": "local_conformance", + } + ) + started = RuntimeEvent.started( + context=context, + pipeline_id="runtime_conformance", + platform="gcp", + ) + completed = RuntimeEvent.completed(result, context=context, platform="gcp") + for event in (started, completed): + json.loads(event.to_json()) + + observed_signal = _probe_signal_translation() + writes = tuple(sorted(_relative_files(work_dir) - before)) + if writes != ("state.db",): + raise RuntimeContractError( + "local conformance wrote outside its one declared SQLite state file" + ) + return RuntimeConformance( + contract=RUNTIME_CONTRACT, + status="succeeded", + run_id=run_id, + filesystem_writes=writes, + signal=observed_signal, + events=(started.event, completed.event), + ) + + +class _ConformanceIngestion: + def run( + self, + *, + run_id: str | None = None, + ownership: OwnershipGuard | None = None, + ) -> PipelineRunResult: + del ownership + if run_id is None: + raise RuntimeContractError("conformance executor did not supply a run id") + return PipelineRunResult( + run_id=run_id, + source="runtime_conformance", + endpoints=( + EndpointRunResult( + endpoint="fixture", + extracted=2, + affected=2, + committed_cursor=None, + ), + ), + ) + + +def _plugin_metadata(registry: ConnectorPluginRegistry) -> tuple[Mapping[str, object], ...]: + return tuple( + { + "plugin_id": installed.plugin.plugin_id, + "api_version": installed.plugin.api_version, + "engine": installed.plugin.engine, + "display_name": installed.plugin.display_name, + "distribution": installed.distribution, + "version": installed.version, + "connectors": tuple( + { + "connector_id": connector.connector_id, + "display_name": connector.display_name, + "endpoints": tuple(endpoint.endpoint_id for endpoint in connector.endpoints), + } + for connector in installed.plugin.connectors + ), + } + for installed in registry.plugins + ) + + +def _probe_signal_translation() -> str: + try: + with graceful_signal_handlers(): + signal.raise_signal(signal.SIGTERM) + except RuntimeCancelledError as error: + return error.signal_name + raise RuntimeContractError("SIGTERM did not enter the graceful cancellation path") + + +def _relative_files(root: Path) -> set[str]: + return {path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file()} diff --git a/tests/test_executor.py b/tests/test_executor.py index ff0bcf10..85c612f5 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -208,7 +208,7 @@ def _executor( return PipelineExecutor( pipeline_id="example_pipeline", source_config=source, - ingestion=_Ingestion(), # type: ignore[arg-type] + ingestion=_Ingestion(), history=history, project="valid-project-123", models_dir=models_dir, diff --git a/tests/test_runtime_inspection.py b/tests/test_runtime_inspection.py new file mode 100644 index 00000000..e2212c04 --- /dev/null +++ b/tests/test_runtime_inspection.py @@ -0,0 +1,159 @@ +"""Read-only runtime inspection and credential-free local conformance.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +from typer.testing import CliRunner + +import dander.runtime_inspection as inspection_module +from dander.cli.main import app +from dander.plugins import ( + ConnectorDescriptor, + ConnectorEndpointDescriptor, + ConnectorPlugin, + ConnectorPluginRegistry, + InstalledConnectorPlugin, +) +from dander.runtime_contract import RUNTIME_CONTRACT, RuntimeContractError +from dander.runtime_inspection import inspect_runtime, run_local_conformance + +if TYPE_CHECKING: + from pathlib import Path + + from dander.ingestion import Source, SourceConfig + from dander.security import AuthStrategy + + +def _manifest(path: Path) -> Path: + config = path / "dander.yaml" + config.write_text( + """\ +version: 1 +platform: + safety: + require_guarded_free_tier: false +pipelines: + runtime_conformance: + source: example + models: [] + build_models: false +""", + encoding="utf-8", + ) + return config + + +def test_runtime_inspect_reports_active_metadata_without_provider_access(tmp_path: Path) -> None: + inspection = json.loads(inspect_runtime(_manifest(tmp_path)).to_json()) + + assert inspection["contract"] == RUNTIME_CONTRACT + assert inspection["build"]["distribution"] == "dander-platform" + assert inspection["adapters"]["platform_profiles"] == ["gcp"] + assert inspection["adapters"]["launchers"] == ["cloud_run", "local"] + assert "bigquery" in inspection["adapters"]["warehouses"] + assert "dlt" in inspection["adapters"]["ingestion_engines"] + assert inspection["plugins"] == [] + + +def test_runtime_inspect_reports_declared_plugin_without_constructing_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def source_factory(config: SourceConfig, auth: AuthStrategy) -> Source: + del config, auth + raise AssertionError("inspection must not construct a source") + + plugin = ConnectorPlugin( + plugin_id="example", + api_version=1, + engine="example_engine", + display_name="Example", + source_factory=source_factory, + connectors=( + ConnectorDescriptor( + connector_id="example", + display_name="Example", + engine="example_engine", + endpoints=( + ConnectorEndpointDescriptor( + endpoint_id="widgets", + display_name="Widgets", + ), + ), + ), + ), + ) + registry = ConnectorPluginRegistry( + source_factories={"example_engine": source_factory}, + plugins=( + InstalledConnectorPlugin( + plugin=plugin, + distribution="dander-connector-example", + version="1.2.3", + ), + ), + ) + monkeypatch.setattr(inspection_module, "load_connector_plugins", lambda _pins: registry) + + inspection = json.loads(inspect_runtime(_manifest(tmp_path)).to_json()) + + assert inspection["plugins"] == [ + { + "api_version": 1, + "connectors": [ + { + "connector_id": "example", + "display_name": "Example", + "endpoints": ["widgets"], + } + ], + "display_name": "Example", + "distribution": "dander-connector-example", + "engine": "example_engine", + "plugin_id": "example", + "version": "1.2.3", + } + ] + + +def test_local_conformance_runs_one_pipeline_and_limits_filesystem_writes( + tmp_path: Path, +) -> None: + result = run_local_conformance(tmp_path) + + assert result.contract == RUNTIME_CONTRACT + assert result.status == "succeeded" + assert result.events == ("runtime.started", "runtime.completed") + assert result.filesystem_writes == ("state.db",) + assert result.signal == "SIGTERM" + assert {path.name for path in tmp_path.iterdir()} == {"state.db"} + + +def test_local_conformance_never_overwrites_existing_state(tmp_path: Path) -> None: + state = tmp_path / "state.db" + state.write_text("operator-owned", encoding="utf-8") + + with pytest.raises(RuntimeContractError, match="already exists"): + run_local_conformance(tmp_path) + + assert state.read_text(encoding="utf-8") == "operator-owned" + + +def test_runtime_inspect_and_conformance_cli_emit_one_json_document(tmp_path: Path) -> None: + runner = CliRunner() + inspected = runner.invoke(app, ["runtime", "inspect", "--config", str(_manifest(tmp_path))]) + conformance_dir = tmp_path / "probe" + conformed = runner.invoke( + app, + ["runtime", "conformance", "--work-dir", str(conformance_dir)], + ) + + assert inspected.exit_code == 0, inspected.output + assert conformed.exit_code == 0, conformed.output + assert json.loads(inspected.output)["contract"] == RUNTIME_CONTRACT + assert json.loads(conformed.output)["status"] == "succeeded" + assert inspected.stderr == "" + assert conformed.stderr == "" From 9c019e0e3c9a617584ce31b464354dc6a5abc05e Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 05:11:03 -0400 Subject: [PATCH 014/163] Record verifiable OCI runtime artifacts --- .github/workflows/ci.yml | 2 + Dockerfile | 17 +++- HANDOFF.md | 41 ++++---- docs/oci-artifact-contract.md | 20 ++++ pyproject.toml | 1 + src/dander/bootstrap/project.py | 113 ++++++++++++++++++++- src/dander/cli/main.py | 5 +- src/dander/runtime-capabilities.json | 12 +++ src/dander/runtime_inspection.py | 82 +++++++++++---- src/dander/templates/project/Dockerfile | 15 ++- tests/bootstrap/test_project_bootstrap.py | 37 ++++++- tests/infra/test_multi_pipeline_runtime.py | 13 +++ tests/project/test_scaffold.py | 3 + tests/test_runtime_inspection.py | 39 +++++++ 14 files changed, 358 insertions(+), 42 deletions(-) create mode 100644 docs/oci-artifact-contract.md create mode 100644 src/dander/runtime-capabilities.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a26dc8fa..27f8b9ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,8 @@ jobs: - name: Verify runtime contract and bundled proof assets run: | test "$(docker run --rm --entrypoint /usr/bin/id dander-ci:${{ github.sha }} -u)" = "65532" + docker run --rm --read-only --tmpfs /tmp dander-ci:${{ github.sha }} runtime conformance \ + | grep -F '"status":"succeeded"' docker run --rm --entrypoint /usr/bin/test dander-ci:${{ github.sha }} -f /app/connectors/hubspot_test.yaml docker run --rm --entrypoint /usr/bin/test dander-ci:${{ github.sha }} -f /app/models/staging/stg_hubspot__companies.yml - name: Scan image diff --git a/Dockerfile b/Dockerfile index 3e6f19dc..3c6ad622 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,24 @@ FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de +ARG DANDER_VERSION=development +ARG DANDER_BUILD_REVISION=unknown +ARG DANDER_BUILD_CREATED=1970-01-01T00:00:00Z + +LABEL org.opencontainers.image.title="Dander" \ + org.opencontainers.image.description="Self-hosted data platform runtime" \ + org.opencontainers.image.source="https://github.com/harrisonoconnorhover/dander" \ + org.opencontainers.image.documentation="https://github.com/harrisonoconnorhover/dander/blob/main/docs/getting-started.md" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.version="${DANDER_VERSION}" \ + org.opencontainers.image.revision="${DANDER_BUILD_REVISION}" \ + org.opencontainers.image.created="${DANDER_BUILD_CREATED}" + ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ UV_COMPILE_BYTECODE=1 \ - UV_LINK_MODE=copy + UV_LINK_MODE=copy \ + DANDER_BUILD_REVISION=${DANDER_BUILD_REVISION} \ + DANDER_BUILD_CREATED=${DANDER_BUILD_CREATED} WORKDIR /app diff --git a/HANDOFF.md b/HANDOFF.md index e145c3ee..ee398dbf 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,42 @@ ## Finished -- Added provider-free `dander runtime inspect` for installed build, adapter, and plugin metadata. -- Added `dander runtime conformance`, a credential-free local executor/run-ledger/event probe. -- Proved the probe writes only its declared SQLite file and refuses to overwrite existing state. -- Proved graceful SIGTERM translation inside the local conformance path. -- Narrowed `PipelineExecutor` to an ingestion protocol without changing runtime behavior. +- Added OCI source, documentation, license, version, revision, and creation annotations. +- Packaged and validated `io.dander.runtime.capabilities/v1` through `runtime inspect`. +- Made image publication attach SBOM/provenance and record index plus runnable-platform digests. +- Made artifact-record writes atomic and exposed their path through `dander image-publish`. +- Added CI execution of runtime conformance as UID 65532 on a read-only root filesystem. ## Try It -Run `dander runtime inspect --config dander.yaml`, then `dander runtime conformance`. Supply an -empty `--work-dir` only when you want to retain the probe's `state.db` for inspection. +Build the root Dockerfile with `DANDER_BUILD_REVISION` and `DANDER_BUILD_CREATED`, then run +`docker run --rm --read-only --tmpfs /tmp IMAGE runtime conformance`. A real +`dander image-publish` writes `.dander/runtime-artifact.json` after registry verification. ## Checks -- Ruff lint/format and strict mypy passed. -- All 799 tests passed. -- Root and stage-zero Terraform format, backend-disabled initialization, and validation passed. -- Wheel/sdist inspection and source-free wheel conformance passed outside the checkout. -- A non-root, read-only local OCI image ran conformance and inspection successfully. +- Ruff format/lint and strict mypy passed. +- All 804 tests passed. +- Root and stage-zero Terraform formatting, backend-disabled initialization, and validation passed. +- Wheel/sdist inspection confirmed the packaged capability manifest. +- A local OCI image passed inspection and conformance as non-root with a read-only root filesystem. ## Decisions -- Inspection loads declared plugin metadata but never constructs a source or resolves a secret. -- The conformance pipeline exercises the real executor and SQLite ledger with a deterministic - in-process ingestion summary; it performs no network or provider access. +- The source-free build-context digest is the artifact revision; the registry digest remains the + immutable deployment identity. +- Attestation descriptors use `unknown/unknown` and are excluded from runnable-platform records. +- Publication fails closed after push if the registry manifest or local artifact record is invalid. ## Remaining - Merge this focused ticket through protected main. -- Add OCI annotations, digest recording, capability manifest, SBOM, and artifact checks. -- Add the cloud-neutral execution projection, then make Cloud Run consume it with parity. +- Add the cloud-neutral execution projection. +- Make Cloud Run consume the projection and prove GCP parity. +- Add multi-architecture build/copy support in its separately planned phase. ## Review First +- `src/dander/bootstrap/project.py` - `src/dander/runtime_inspection.py` -- `src/dander/cli/runtime_command.py` -- `tests/test_runtime_inspection.py` +- `.github/workflows/ci.yml` diff --git a/docs/oci-artifact-contract.md b/docs/oci-artifact-contract.md new file mode 100644 index 00000000..aa0b551d --- /dev/null +++ b/docs/oci-artifact-contract.md @@ -0,0 +1,20 @@ +# OCI Artifact Contract + +Dander publishes the runtime as a non-root OCI image and records each successful publication in +`.dander/runtime-artifact.json`. The record uses schema `io.dander.runtime.artifact/v1` and contains +the immutable index reference, each runnable platform-manifest digest, the build-context revision, +creation time, and whether SBOM and provenance attestations were requested. The local record is an +operator artifact and is intentionally ignored by Git. + +Published images carry standard OCI source, documentation, license, version, revision, and creation +annotations. The packaged `runtime-capabilities.json` describes the adapters present in that build; +`dander runtime inspect` validates and reports it without accessing a cloud provider or secret. + +`dander image-publish` builds through BuildKit with SBOM and provenance enabled, pushes the image, +resolves the registry digest, inspects the resulting OCI index, and only then writes the artifact +record atomically. If post-push verification fails, the command reports that the image may exist but +does not report a successful artifact publication. + +The runtime executes as UID/GID `65532:65532`. CI runs the local conformance probe with a read-only +root filesystem and a temporary `/tmp`, so runtime code must declare durable state through an +adapter rather than writing into the image filesystem. diff --git a/pyproject.toml b/pyproject.toml index f9010c46..cf582736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,7 @@ include = [ "docs/known-limitations.md", "docs/netsuite-simulator.md", "docs/operator-soak.md", + "docs/oci-artifact-contract.md", "docs/rollback.md", "docs/upgrading.md", "examples", diff --git a/src/dander/bootstrap/project.py b/src/dander/bootstrap/project.py index 457b917b..740e0773 100644 --- a/src/dander/bootstrap/project.py +++ b/src/dander/bootstrap/project.py @@ -3,9 +3,12 @@ from __future__ import annotations import hashlib +import json +import os import re import subprocess import time +from datetime import UTC, datetime from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: @@ -17,6 +20,9 @@ _REGION = re.compile(r"^[a-z][a-z0-9-]{1,62}[a-z0-9]$") _ACCOUNT = re.compile(r"^[^\s@]+@[^\s@]+$") _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_PLATFORM_PART = re.compile(r"^[a-z0-9][a-z0-9_.-]*$") +_ARTIFACT_SCHEMA = "io.dander.runtime.artifact/v1" +_SOURCE_URL = "https://github.com/harrisonoconnorhover/dander" class ProjectBootstrapError(RuntimeError): @@ -151,6 +157,12 @@ class RuntimeImagePublisher: def __init__(self, repository_dir: Path, *, runner: _Runner | None = None) -> None: self._repository_dir = repository_dir.resolve() self._runner = runner or _subprocess_runner + self._artifact_record_path: Path | None = None + + @property + def artifact_record_path(self) -> Path | None: + """Return the local artifact record written by the latest successful publication.""" + return self._artifact_record_path def publish( self, @@ -161,6 +173,7 @@ def publish( impersonate_service_account: str = "", require_source_free: bool = False, ) -> str: + self._artifact_record_path = None if not _PROJECT_ID.fullmatch(project) or not _REGION.fullmatch(region): raise ProjectBootstrapError("Invalid runtime image project or region") if not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", tag_prefix): @@ -173,7 +186,9 @@ def publish( ) host = f"{region}-docker.pkg.dev" repository = f"{host}/{project}/dander/dander" - tag = f"{tag_prefix}-{self._content_digest()[:12]}" + revision = self._content_digest() + created = datetime.now(UTC).isoformat().replace("+00:00", "Z") + tag = f"{tag_prefix}-{revision[:12]}" tagged_image = f"{repository}:{tag}" try: if impersonate_service_account: @@ -222,6 +237,12 @@ def publish( "build", "--platform", "linux/amd64", + "--build-arg", + f"DANDER_BUILD_REVISION={revision}", + "--build-arg", + f"DANDER_BUILD_CREATED={created}", + "--sbom=true", + "--provenance=mode=max", "--push", "-t", tagged_image, @@ -256,7 +277,38 @@ def publish( digest = described.stdout.strip() if not _DIGEST.fullmatch(digest): raise ProjectBootstrapError("Artifact Registry returned an invalid image digest") - return f"{repository}@{digest}" + immutable_image = f"{repository}@{digest}" + try: + inspected = self._runner( + ("docker", "buildx", "imagetools", "inspect", "--raw", tagged_image), + cwd=self._repository_dir, + check=True, + capture_output=True, + text=True, + ) + platforms = _platform_manifests(inspected.stdout, default_digest=digest) + record_path = self._repository_dir / ".dander" / "runtime-artifact.json" + _write_artifact_record( + record_path, + { + "schema": _ARTIFACT_SCHEMA, + "image": immutable_image, + "tagged_image": tagged_image, + "index_digest": digest, + "platform_manifests": platforms, + "source": _SOURCE_URL, + "revision": revision, + "created": created, + "sbom_attached": True, + "provenance_attached": True, + }, + ) + except (OSError, json.JSONDecodeError, subprocess.CalledProcessError) as error: + raise ProjectBootstrapError( + "Runtime image was pushed but its artifact record could not be verified" + ) from error + self._artifact_record_path = record_path + return immutable_image def _content_digest(self) -> str: hasher = hashlib.sha256() @@ -316,6 +368,63 @@ def _is_build_context_file(path: Path) -> bool: ) +def _platform_manifests(raw: str, *, default_digest: str) -> dict[str, str]: + document = json.loads(raw) + if not isinstance(document, dict): + raise ProjectBootstrapError("Runtime image manifest is not an OCI document") + descriptors = document.get("manifests") + if descriptors is None: + if not _DIGEST.fullmatch(default_digest): + raise ProjectBootstrapError("Runtime image manifest has an invalid digest") + return {"linux/amd64": default_digest} + if not isinstance(descriptors, list): + raise ProjectBootstrapError("Runtime image index has invalid platform manifests") + + platforms: dict[str, str] = {} + for descriptor in descriptors: + if not isinstance(descriptor, dict) or not isinstance(descriptor.get("platform"), dict): + raise ProjectBootstrapError("Runtime image index has an invalid descriptor") + platform = descriptor["platform"] + operating_system = platform.get("os") + architecture = platform.get("architecture") + if operating_system == "unknown" and architecture == "unknown": + continue + digest = descriptor.get("digest") + if ( + not isinstance(operating_system, str) + or not _PLATFORM_PART.fullmatch(operating_system) + or not isinstance(architecture, str) + or not _PLATFORM_PART.fullmatch(architecture) + or not isinstance(digest, str) + or not _DIGEST.fullmatch(digest) + ): + raise ProjectBootstrapError("Runtime image index has an invalid platform descriptor") + variant = platform.get("variant") + if variant is not None and ( + not isinstance(variant, str) or not _PLATFORM_PART.fullmatch(variant) + ): + raise ProjectBootstrapError("Runtime image index has an invalid platform variant") + key = f"{operating_system}/{architecture}" + if variant: + key = f"{key}/{variant}" + if key in platforms: + raise ProjectBootstrapError("Runtime image index repeats a platform") + platforms[key] = digest + if not platforms: + raise ProjectBootstrapError("Runtime image index contains no runnable platform") + return dict(sorted(platforms.items())) + + +def _write_artifact_record(path: Path, record: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + try: + temporary.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + def active_admin_member(*, cwd: Path, runner: _Runner | None = None) -> str: """Resolve the active gcloud user as the stage-zero impersonation administrator.""" command_runner = runner or _subprocess_runner diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index d064f18d..57759ca1 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -962,7 +962,8 @@ def image_publish( ): raise typer.Abort() try: - image = RuntimeImagePublisher(config.resolve().parent).publish( + publisher = RuntimeImagePublisher(config.resolve().parent) + image = publisher.publish( project=project, region=resolved_region, tag_prefix=tag_prefix, @@ -972,6 +973,8 @@ def image_publish( except ProjectBootstrapError as error: raise ClickException(str(error)) from error console.print(f"[green]Published immutable runtime image:[/green] {image}") + if publisher.artifact_record_path is not None: + console.print(f"Artifact record: {publisher.artifact_record_path}") next_command = [ "dander", "init-platform-plan", diff --git a/src/dander/runtime-capabilities.json b/src/dander/runtime-capabilities.json new file mode 100644 index 00000000..0c8e7d23 --- /dev/null +++ b/src/dander/runtime-capabilities.json @@ -0,0 +1,12 @@ +{ + "schema": "io.dander.runtime.capabilities/v1", + "contract": "io.dander.runtime/v1", + "adapters": { + "platform_profiles": ["gcp"], + "launchers": ["cloud_run", "local"], + "warehouses": ["bigquery"], + "state_backends": ["bigquery", "sqlite"], + "catalogs": ["dataplex", "local_json", "none"], + "secret_providers": ["environment", "gcp_secret_manager"] + } +} diff --git a/src/dander/runtime_inspection.py b/src/dander/runtime_inspection.py index 89b30566..61b6478c 100644 --- a/src/dander/runtime_inspection.py +++ b/src/dander/runtime_inspection.py @@ -3,9 +3,13 @@ from __future__ import annotations import json +import os +import re import signal import sys from dataclasses import dataclass +from datetime import datetime +from importlib import resources from typing import TYPE_CHECKING from uuid import uuid4 @@ -31,12 +35,8 @@ from dander.concurrency import OwnershipGuard -_PLATFORM_PROFILES = ("gcp",) -_LAUNCHERS = ("cloud_run", "local") -_WAREHOUSES = ("bigquery",) -_STATE_BACKENDS = ("bigquery", "sqlite") -_CATALOGS = ("dataplex", "local_json", "none") -_SECRET_PROVIDERS = ("environment", "gcp_secret_manager") +_REVISION = re.compile(r"^(?:unknown|[0-9a-f]{40}|[0-9a-f]{64})$") +_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") @dataclass(frozen=True, slots=True) @@ -44,6 +44,7 @@ class RuntimeInspection: """Non-secret installed runtime and plugin metadata.""" contract: str + capability_manifest: str build: Mapping[str, object] adapters: Mapping[str, object] plugins: tuple[Mapping[str, object], ...] @@ -53,6 +54,7 @@ def to_json(self) -> str: return json.dumps( { "contract": self.contract, + "capability_manifest": self.capability_manifest, "build": self.build, "adapters": self.adapters, "plugins": self.plugins, @@ -93,20 +95,16 @@ def inspect_runtime(config: Path) -> RuntimeInspection: """Inspect configured packages without constructing sources or contacting providers.""" manifest = load_project_config(config) registry = load_connector_plugins(manifest.plugins) + capabilities = _load_capabilities() + capability_adapters = capabilities["adapters"] + if not isinstance(capability_adapters, dict): + raise RuntimeContractError("runtime capability manifest is incompatible") return RuntimeInspection( contract=RUNTIME_CONTRACT, - build={ - "distribution": "dander-platform", - "version": __version__, - "python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", - }, + capability_manifest=str(capabilities["schema"]), + build=_build_metadata(), adapters={ - "platform_profiles": _PLATFORM_PROFILES, - "launchers": _LAUNCHERS, - "warehouses": _WAREHOUSES, - "state_backends": _STATE_BACKENDS, - "catalogs": _CATALOGS, - "secret_providers": _SECRET_PROVIDERS, + **capability_adapters, "ingestion_engines": registry.engines, }, plugins=_plugin_metadata(registry), @@ -220,6 +218,56 @@ def _plugin_metadata(registry: ConnectorPluginRegistry) -> tuple[Mapping[str, ob ) +def _build_metadata() -> dict[str, object]: + build: dict[str, object] = { + "distribution": "dander-platform", + "version": __version__, + "python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + } + revision = os.environ.get("DANDER_BUILD_REVISION") + if revision is not None: + if not _REVISION.fullmatch(revision): + raise RuntimeContractError("DANDER_BUILD_REVISION is not a valid OCI revision") + build["revision"] = revision + created = os.environ.get("DANDER_BUILD_CREATED") + if created is not None: + try: + parsed = datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError as error: + raise RuntimeContractError( + "DANDER_BUILD_CREATED is not an ISO-8601 timestamp" + ) from error + if parsed.tzinfo is None: + raise RuntimeContractError("DANDER_BUILD_CREATED must include a timezone") + build["created"] = created + digest = os.environ.get("DANDER_IMAGE_DIGEST") + if digest is not None: + if not _DIGEST.fullmatch(digest): + raise RuntimeContractError("DANDER_IMAGE_DIGEST is not a valid sha256 digest") + build["image_digest"] = digest + return build + + +def _load_capabilities() -> Mapping[str, object]: + try: + raw = ( + resources.files("dander") + .joinpath("runtime-capabilities.json") + .read_text(encoding="utf-8") + ) + capabilities = json.loads(raw) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeContractError("runtime capability manifest is unavailable") from error + if ( + not isinstance(capabilities, dict) + or capabilities.get("schema") != "io.dander.runtime.capabilities/v1" + or capabilities.get("contract") != RUNTIME_CONTRACT + or not isinstance(capabilities.get("adapters"), dict) + ): + raise RuntimeContractError("runtime capability manifest is incompatible") + return capabilities + + def _probe_signal_translation() -> str: try: with graceful_signal_handlers(): diff --git a/src/dander/templates/project/Dockerfile b/src/dander/templates/project/Dockerfile index 2b2aec19..2a54c54a 100644 --- a/src/dander/templates/project/Dockerfile +++ b/src/dander/templates/project/Dockerfile @@ -1,9 +1,22 @@ FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de ARG DANDER_VERSION=__DANDER_DISTRIBUTION_VERSION__ +ARG DANDER_BUILD_REVISION=unknown +ARG DANDER_BUILD_CREATED=1970-01-01T00:00:00Z + +LABEL org.opencontainers.image.title="Dander" \ + org.opencontainers.image.description="Self-hosted data platform runtime" \ + org.opencontainers.image.source="https://github.com/harrisonoconnorhover/dander" \ + org.opencontainers.image.documentation="https://github.com/harrisonoconnorhover/dander/blob/main/docs/getting-started.md" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.version="${DANDER_VERSION}" \ + org.opencontainers.image.revision="${DANDER_BUILD_REVISION}" \ + org.opencontainers.image.created="${DANDER_BUILD_CREATED}" ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + DANDER_BUILD_REVISION=${DANDER_BUILD_REVISION} \ + DANDER_BUILD_CREATED=${DANDER_BUILD_CREATED} WORKDIR /app diff --git a/tests/bootstrap/test_project_bootstrap.py b/tests/bootstrap/test_project_bootstrap.py index 47972863..4a7f30cc 100644 --- a/tests/bootstrap/test_project_bootstrap.py +++ b/tests/bootstrap/test_project_bootstrap.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import subprocess from typing import TYPE_CHECKING @@ -42,6 +43,27 @@ def __call__( return subprocess.CompletedProcess(args, 0 if self.bucket_exists else 1, stdout="") if args[:4] == ("gcloud", "artifacts", "docker", "images"): return subprocess.CompletedProcess(args, 0, stdout="sha256:" + "a" * 64 + "\n") + if args[:4] == ("docker", "buildx", "imagetools", "inspect"): + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "digest": "sha256:" + "b" * 64, + "platform": {"os": "linux", "architecture": "amd64"}, + }, + { + "digest": "sha256:" + "c" * 64, + "platform": {"os": "unknown", "architecture": "unknown"}, + }, + ], + } + ), + ) if args[:3] == ("gcloud", "auth", "list"): return subprocess.CompletedProcess(args, 0, stdout="operator@example.invalid\n") if args[:3] == ("gcloud", "auth", "print-access-token"): @@ -77,7 +99,8 @@ def test_runtime_image_publisher_returns_an_immutable_digest(tmp_path: Path) -> (path / "content.txt").write_text(directory, encoding="utf-8") runner = _Runner() - image = RuntimeImagePublisher(tmp_path, runner=runner).publish( + publisher = RuntimeImagePublisher(tmp_path, runner=runner) + image = publisher.publish( project="unit-project", region="us-central1", ) @@ -87,6 +110,18 @@ def test_runtime_image_publisher_returns_an_immutable_digest(tmp_path: Path) -> command for command in runner.commands if command[:3] == ("docker", "buildx", "build") ) assert "--platform" in build and "linux/amd64" in build and "--push" in build + assert "--sbom=true" in build + assert "--provenance=mode=max" in build + assert any(part.startswith("DANDER_BUILD_REVISION=") for part in build) + assert any(part.startswith("DANDER_BUILD_CREATED=") for part in build) + assert publisher.artifact_record_path == tmp_path / ".dander" / "runtime-artifact.json" + record = json.loads(publisher.artifact_record_path.read_text(encoding="utf-8")) + assert record["schema"] == "io.dander.runtime.artifact/v1" + assert record["image"] == image + assert record["index_digest"] == "sha256:" + "a" * 64 + assert record["platform_manifests"] == {"linux/amd64": "sha256:" + "b" * 64} + assert record["sbom_attached"] is True + assert record["provenance_attached"] is True def test_runtime_image_publisher_accepts_installed_project_context(tmp_path: Path) -> None: diff --git a/tests/infra/test_multi_pipeline_runtime.py b/tests/infra/test_multi_pipeline_runtime.py index dff69f60..c64398e1 100644 --- a/tests/infra/test_multi_pipeline_runtime.py +++ b/tests/infra/test_multi_pipeline_runtime.py @@ -99,6 +99,19 @@ def test_container_carries_the_project_manifest() -> None: assert "infra/.terraform/" in dockerignore +def test_container_declares_the_oci_runtime_artifact_contract() -> None: + dockerfile = (ROOT / "Dockerfile").read_text(encoding="utf-8") + + assert "org.opencontainers.image.source=" in dockerfile + assert "org.opencontainers.image.documentation=" in dockerfile + assert "org.opencontainers.image.licenses=" in dockerfile + assert "org.opencontainers.image.revision=" in dockerfile + assert "org.opencontainers.image.created=" in dockerfile + assert "DANDER_BUILD_REVISION" in dockerfile + assert "DANDER_BUILD_CREATED" in dockerfile + assert "USER 65532:65532" in dockerfile + + def test_bootstrap_identity_can_manage_monitoring_alerts() -> None: bootstrap = (ROOT / "infra/bootstrap-admin/main.tf").read_text(encoding="utf-8") diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index 6cb4a1a5..01e25ede 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -27,6 +27,9 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: assert "COPY --chown=65532:65532 connectors ./connectors" in dockerfile assert "COPY --chown=65532:65532 graphs ./graphs" in dockerfile assert "COPY --chown=65532:65532 models ./models" in dockerfile + assert "org.opencontainers.image.revision=" in dockerfile + assert "org.opencontainers.image.created=" in dockerfile + assert "USER 65532:65532" in dockerfile assert dockerfile.index("COPY --chown=65532:65532 dander.yaml") < dockerfile.index( "dander plugins install" ) diff --git a/tests/test_runtime_inspection.py b/tests/test_runtime_inspection.py index e2212c04..083bc199 100644 --- a/tests/test_runtime_inspection.py +++ b/tests/test_runtime_inspection.py @@ -50,6 +50,7 @@ def test_runtime_inspect_reports_active_metadata_without_provider_access(tmp_pat inspection = json.loads(inspect_runtime(_manifest(tmp_path)).to_json()) assert inspection["contract"] == RUNTIME_CONTRACT + assert inspection["capability_manifest"] == "io.dander.runtime.capabilities/v1" assert inspection["build"]["distribution"] == "dander-platform" assert inspection["adapters"]["platform_profiles"] == ["gcp"] assert inspection["adapters"]["launchers"] == ["cloud_run", "local"] @@ -58,6 +59,44 @@ def test_runtime_inspect_reports_active_metadata_without_provider_access(tmp_pat assert inspection["plugins"] == [] +def test_runtime_inspect_reports_validated_build_metadata( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DANDER_BUILD_REVISION", "a" * 64) + monkeypatch.setenv("DANDER_BUILD_CREATED", "2026-08-07T08:00:00Z") + monkeypatch.setenv("DANDER_IMAGE_DIGEST", "sha256:" + "b" * 64) + + build = json.loads(inspect_runtime(_manifest(tmp_path)).to_json())["build"] + + assert build["revision"] == "a" * 64 + assert build["created"] == "2026-08-07T08:00:00Z" + assert build["image_digest"] == "sha256:" + "b" * 64 + + +@pytest.mark.parametrize( + ("name", "value", "message"), + [ + ("DANDER_BUILD_REVISION", "secret revision", "valid OCI revision"), + ("DANDER_BUILD_CREATED", "not-a-date", "ISO-8601 timestamp"), + ("DANDER_IMAGE_DIGEST", "private digest", "valid sha256 digest"), + ], +) +def test_runtime_inspect_rejects_invalid_build_metadata_without_echoing_it( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, + message: str, +) -> None: + monkeypatch.setenv(name, value) + + with pytest.raises(RuntimeContractError, match=message) as raised: + inspect_runtime(_manifest(tmp_path)) + + assert value not in str(raised.value) + + def test_runtime_inspect_reports_declared_plugin_without_constructing_source( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From ab1f814c22cca69e6ddc5a7dadc95043cea7f1f2 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 05:50:57 -0400 Subject: [PATCH 015/163] Define cloud-neutral execution projections --- HANDOFF.md | 39 +- docs/decisions.md | 10 + docs/execution-projection.md | 23 + pyproject.toml | 1 + src/dander/deployment/__init__.py | 33 ++ src/dander/deployment/projection.py | 505 ++++++++++++++++++ tests/deployment/test_execution_projection.py | 152 ++++++ 7 files changed, 742 insertions(+), 21 deletions(-) create mode 100644 docs/execution-projection.md create mode 100644 src/dander/deployment/__init__.py create mode 100644 src/dander/deployment/projection.py create mode 100644 tests/deployment/test_execution_projection.py diff --git a/HANDOFF.md b/HANDOFF.md index ee398dbf..37349222 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,42 +2,39 @@ ## Finished -- Added OCI source, documentation, license, version, revision, and creation annotations. -- Packaged and validated `io.dander.runtime.capabilities/v1` through `runtime inspect`. -- Made image publication attach SBOM/provenance and record index plus runnable-platform digests. -- Made artifact-record writes atomic and exposed their path through `dander image-publish`. -- Added CI execution of runtime conformance as UID 65532 on a read-only root filesystem. +- Added immutable `io.dander.execution/v1` templates for launcher-neutral hosted intent. +- Separated non-secret environment, secret references, workload identity, and run correlation. +- Separated runtime retry intent from launcher retry intent. +- Added explicit Cloud Run capability/limit validation that fails before planning. +- Added deterministic version 1 manifest projection without changing Terraform or live resources. ## Try It -Build the root Dockerfile with `DANDER_BUILD_REVISION` and `DANDER_BUILD_CREATED`, then run -`docker run --rm --read-only --tmpfs /tmp IMAGE runtime conformance`. A real -`dander image-publish` writes `.dander/runtime-artifact.json` after registry verification. +Load `dander.yaml`, then call `build_gcp_v1_execution_templates()` with an immutable image and GCP +project. Inspect `ExecutionTemplate.as_dict()` or bind a run through `ExecutionTemplate.bind()`. ## Checks - Ruff format/lint and strict mypy passed. -- All 804 tests passed. +- All 814 tests passed. - Root and stage-zero Terraform formatting, backend-disabled initialization, and validation passed. -- Wheel/sdist inspection confirmed the packaged capability manifest. -- A local OCI image passed inspection and conformance as non-root with a read-only root filesystem. +- Wheel/sdist build and distribution inspection passed. ## Decisions -- The source-free build-context digest is the artifact revision; the registry digest remains the - immutable deployment identity. -- Attestation descriptors use `unknown/unknown` and are excluded from runnable-platform records. -- Publication fails closed after push if the registry manifest or local artifact record is invalid. +- Deployment-time templates and run-time correlation are separate immutable objects. +- Secret references never enter the non-secret environment mapping. +- Cloud Run advertises only Dander's current one-task/one-worker behavior; unsupported requested + fields fail instead of being silently ignored. ## Remaining - Merge this focused ticket through protected main. -- Add the cloud-neutral execution projection. -- Make Cloud Run consume the projection and prove GCP parity. -- Add multi-architecture build/copy support in its separately planned phase. +- Make Cloud Run Terraform consume this projection with exact behavior parity. +- Run local/Cloud Run normalized outcome, signal, replay, and no-drift proof. ## Review First -- `src/dander/bootstrap/project.py` -- `src/dander/runtime_inspection.py` -- `.github/workflows/ci.yml` +- `src/dander/deployment/projection.py` +- `tests/deployment/test_execution_projection.py` +- `docs/execution-projection.md` diff --git a/docs/decisions.md b/docs/decisions.md index e97ec58b..f0040876 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-07 — Launchers consume a validated execution projection + +- **Decision:** Compile hosted intent into immutable `io.dander.execution/v1` templates containing + runtime, identity, resource, scheduling, networking, and observability fields. Bind run-specific + correlation only when an execution starts. +- **Validation:** Each launcher declares exact capabilities and rejects unsupported fields before + planning; requested values are never silently ignored. +- **Compatibility:** Version 1 manifests compile to the existing GCP profile. Cloud Run initially + advertises only the behavior Dander already provisions, including one task and one parallel worker. + ## 2026-08-07 — Cloud-selectable direction preserves a named GCP compatibility profile - Portability is introduced through a versioned OCI runtime contract and named, validated platform diff --git a/docs/execution-projection.md b/docs/execution-projection.md new file mode 100644 index 00000000..1a7cf0dd --- /dev/null +++ b/docs/execution-projection.md @@ -0,0 +1,23 @@ +# Execution Projection v1 + +`io.dander.execution/v1` is Dander's immutable, cloud-neutral launcher input. An execution template +contains the OCI digest, runtime command, pipeline/profile identifiers, configuration location, +non-secret environment, secret references, workload identity, resources, separate runtime and +launcher retry counts, task/schedule settings, network intent, labels, and observability targets. + +Run IDs, launcher execution IDs, attempts, shards, and deadlines are bound when a launcher starts +an execution. They are validated through the same runtime contract and projected through +`DANDER_*` environment variables. Secret references are kept separately and never appear in that +environment mapping. + +Every launcher declares explicit capabilities and limits. Projection validation fails before +planning if the launcher cannot honor requested CPU, memory, deadline, retries, task count, +parallelism, ephemeral storage, network placement, schedule behavior, or extension. `cloud_run` +initially declares only the behavior already supported by Dander's GCP profile: one task, one +parallel worker, the existing CPU/memory/deadline/retry range, schedules with time zones, and no +separately configurable ephemeral storage or network placement. + +Version 1 `dander.yaml` manifests compile deterministically to this projection while retaining the +existing GCP/BigQuery compatibility profile. A later configuration migration will separate logical +pipeline intent from environment-specific deployment settings; it does not change this runtime +boundary. diff --git a/pyproject.toml b/pyproject.toml index cf582736..2b4fb675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,7 @@ include = [ "dander.yaml", "docs/getting-started.md", "docs/connector-plugins.md", + "docs/execution-projection.md", "docs/known-limitations.md", "docs/netsuite-simulator.md", "docs/operator-soak.md", diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py new file mode 100644 index 00000000..0ea539a7 --- /dev/null +++ b/src/dander/deployment/__init__.py @@ -0,0 +1,33 @@ +"""Cloud-neutral deployment and execution projections.""" + +from dander.deployment.projection import ( + CLOUD_RUN_CAPABILITIES, + EXECUTION_PROJECTION_SCHEMA, + ExecutionProjectionError, + ExecutionRequest, + ExecutionTemplate, + LauncherCapabilities, + NetworkPlacement, + ObservabilityProjection, + ResourceProjection, + ScheduleProjection, + SecretReference, + build_gcp_v1_execution_templates, + validate_launcher_projection, +) + +__all__ = [ + "CLOUD_RUN_CAPABILITIES", + "EXECUTION_PROJECTION_SCHEMA", + "ExecutionProjectionError", + "ExecutionRequest", + "ExecutionTemplate", + "LauncherCapabilities", + "NetworkPlacement", + "ObservabilityProjection", + "ResourceProjection", + "ScheduleProjection", + "SecretReference", + "build_gcp_v1_execution_templates", + "validate_launcher_projection", +] diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py new file mode 100644 index 00000000..e59d1b83 --- /dev/null +++ b/src/dander/deployment/projection.py @@ -0,0 +1,505 @@ +"""Immutable, cloud-neutral execution templates and launcher limit validation.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Literal + +from dander import __version__ +from dander.runtime_contract import RUNTIME_CONTRACT, LauncherContext + +if TYPE_CHECKING: + from dander.project import DanderProject + +EXECUTION_PROJECTION_SCHEMA = "io.dander.execution/v1" +_IMMUTABLE_IMAGE = re.compile(r"^[a-z0-9.-]+/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$") +_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_PROFILE = re.compile(r"^[a-z][a-z0-9_-]{0,62}$") +_ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") +_GCP_PROJECT = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +_SERVICE_ACCOUNT = re.compile( + r"^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z][a-z0-9-]{4,28}[a-z0-9]" + r"\.iam\.gserviceaccount\.com$" +) +_GCP_SECRET_REFERENCE = re.compile( + r"^gcp-sm://projects/[a-z][a-z0-9-]{4,28}[a-z0-9]/secrets/" + r"[A-Za-z][A-Za-z0-9_-]{0,254}/versions/(?:latest|[1-9][0-9]*)$" +) +_REFERENCE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,1023}$") +_SECRET_SCHEMES = { + "environment": "env://", + "gcp_secret_manager": "gcp-sm://", + "aws_secret_manager": "aws-sm://", + "azure_key_vault": "azure-kv://", + "oci_vault": "oci-vault://", +} + + +class ExecutionProjectionError(ValueError): + """A projection is unsafe, incomplete, or unsupported by its launcher.""" + + +@dataclass(frozen=True, slots=True) +class SecretReference: + """One launcher-projected secret reference; never a resolved value.""" + + provider: Literal[ + "environment", + "gcp_secret_manager", + "aws_secret_manager", + "azure_key_vault", + "oci_vault", + ] + reference: str + + def __post_init__(self) -> None: + scheme = _SECRET_SCHEMES.get(self.provider) + if ( + scheme is None + or not self.reference.startswith(scheme) + or not _REFERENCE.fullmatch(self.reference) + ): + raise ExecutionProjectionError("invalid secret reference") + if self.provider == "gcp_secret_manager" and not _GCP_SECRET_REFERENCE.fullmatch( + self.reference + ): + raise ExecutionProjectionError("invalid GCP Secret Manager reference") + + +@dataclass(frozen=True, slots=True) +class ResourceProjection: + """Portable resource and retry intent for one launcher task.""" + + cpu_millis: int + memory_mib: int + ephemeral_storage_mib: int | None + deadline_seconds: int + runtime_retry_count: int + launcher_retry_count: int + + def __post_init__(self) -> None: + values = ( + self.cpu_millis, + self.memory_mib, + self.deadline_seconds, + self.runtime_retry_count, + self.launcher_retry_count, + ) + if any(isinstance(value, bool) or not isinstance(value, int) for value in values): + raise ExecutionProjectionError("resource and retry values must be integers") + if self.ephemeral_storage_mib is not None and ( + isinstance(self.ephemeral_storage_mib, bool) + or not isinstance(self.ephemeral_storage_mib, int) + or self.ephemeral_storage_mib < 1 + ): + raise ExecutionProjectionError("ephemeral storage must be a positive MiB quantity") + if min(self.cpu_millis, self.memory_mib, self.deadline_seconds) < 1: + raise ExecutionProjectionError("CPU, memory, and deadline must be positive") + if min(self.runtime_retry_count, self.launcher_retry_count) < 0: + raise ExecutionProjectionError("retry counts must not be negative") + + +@dataclass(frozen=True, slots=True) +class ScheduleProjection: + """Scheduling and task-parallelism intent independent of a scheduler API.""" + + task_count: int + maximum_parallelism: int + expression: str | None + time_zone: str | None + paused: bool + + def __post_init__(self) -> None: + if ( + isinstance(self.task_count, bool) + or isinstance(self.maximum_parallelism, bool) + or self.task_count < 1 + or self.maximum_parallelism < 1 + or self.maximum_parallelism > self.task_count + ): + raise ExecutionProjectionError("task count and parallelism are inconsistent") + if (self.expression is None) != (self.time_zone is None): + raise ExecutionProjectionError("schedule and time zone must be supplied together") + if self.expression is not None and ( + not self.expression.strip() or not self.time_zone or not self.time_zone.strip() + ): + raise ExecutionProjectionError("schedule and time zone must not be blank") + + +@dataclass(frozen=True, slots=True) +class NetworkPlacement: + """Portable network request plus explicitly named provider extensions.""" + + placement: str | None = None + extensions: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + _validate_pairs(self.extensions, label="network extension") + + +@dataclass(frozen=True, slots=True) +class ObservabilityProjection: + """Launcher-neutral log, metric, alert, and retention destinations.""" + + log_destination: str + metric_namespace: str + alert_target: str | None + retention_days: int | None + + def __post_init__(self) -> None: + if not self.log_destination.strip() or not self.metric_namespace.strip(): + raise ExecutionProjectionError("log and metric destinations must not be blank") + if self.retention_days is not None and self.retention_days < 1: + raise ExecutionProjectionError("log retention must be a positive day count") + + +@dataclass(frozen=True, slots=True) +class ExecutionTemplate: + """One immutable launcher input before run-specific correlation exists.""" + + schema: str + contract: str + pipeline_id: str + profile_id: str + launcher: str + image: str + command: tuple[str, ...] + configuration_reference: str + environment: tuple[tuple[str, str], ...] + secret_bindings: tuple[tuple[str, SecretReference], ...] + workload_identity: str + resources: ResourceProjection + schedule: ScheduleProjection + network: NetworkPlacement + labels: tuple[tuple[str, str], ...] + observability: ObservabilityProjection + extensions: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + if self.schema != EXECUTION_PROJECTION_SCHEMA or self.contract != RUNTIME_CONTRACT: + raise ExecutionProjectionError("unsupported execution projection contract") + if not _IDENTIFIER.fullmatch(self.pipeline_id) or not _PROFILE.fullmatch(self.profile_id): + raise ExecutionProjectionError("invalid pipeline or profile identifier") + if not _PROFILE.fullmatch(self.launcher): + raise ExecutionProjectionError("invalid launcher identifier") + if not _IMMUTABLE_IMAGE.fullmatch(self.image): + raise ExecutionProjectionError("execution image must use an immutable sha256 digest") + if not self.command or any(not part for part in self.command): + raise ExecutionProjectionError("execution command must not be empty") + expected_command = ( + "runtime", + "execute", + "--contract", + self.contract, + "--pipeline", + self.pipeline_id, + "--platform", + self.profile_id, + ) + if self.command[: len(expected_command)] != expected_command: + raise ExecutionProjectionError("execution command does not match projection identity") + if not ( + self.configuration_reference.startswith("/") + or self.configuration_reference.startswith(("gs://", "s3://", "az://", "oci://")) + ): + raise ExecutionProjectionError("configuration reference must be a path or object URI") + if not self.workload_identity.strip() or len(self.workload_identity) > 512: + raise ExecutionProjectionError("workload identity must not be blank") + environment_names = _validate_pairs(self.environment, label="environment") + secret_names = _validate_pairs(self.secret_bindings, label="secret binding") + if any(not _ENVIRONMENT_NAME.fullmatch(name) for name in environment_names | secret_names): + raise ExecutionProjectionError("environment names must use uppercase shell syntax") + if environment_names & secret_names: + raise ExecutionProjectionError("secret and non-secret environment names overlap") + _validate_pairs(self.labels, label="label") + _validate_pairs(self.extensions, label="extension") + + @property + def digest(self) -> str: + """Return the immutable OCI index digest without the repository prefix.""" + return self.image.rsplit("@", maxsplit=1)[1] + + def bind( + self, + *, + run_id: str, + launcher_execution_id: str | None, + attempt: int, + shard_index: int = 0, + shard_count: int = 1, + deadline_at: str | None = None, + ) -> ExecutionRequest: + """Bind one validated run-specific context without changing deployment intent.""" + values = { + "DANDER_RUN_ID": run_id, + "DANDER_LAUNCHER": self.launcher, + "DANDER_ATTEMPT": str(attempt), + "DANDER_SHARD_INDEX": str(shard_index), + "DANDER_SHARD_COUNT": str(shard_count), + "DANDER_PRINCIPAL": self.workload_identity, + } + if launcher_execution_id is not None: + values["DANDER_LAUNCHER_EXECUTION_ID"] = launcher_execution_id + if deadline_at is not None: + values["DANDER_DEADLINE_AT"] = deadline_at + return ExecutionRequest( + template=self, + context=LauncherContext.from_environment(values), + ) + + def as_dict(self) -> dict[str, object]: + """Render a deterministic provider-neutral mapping for launchers and tests.""" + return { + "schema": self.schema, + "contract": self.contract, + "pipeline_id": self.pipeline_id, + "profile_id": self.profile_id, + "launcher": self.launcher, + "image": self.image, + "command": list(self.command), + "configuration_reference": self.configuration_reference, + "environment": dict(self.environment), + "secret_bindings": { + name: asdict(reference) for name, reference in self.secret_bindings + }, + "workload_identity": self.workload_identity, + "resources": asdict(self.resources), + "schedule": asdict(self.schedule), + "network": { + "placement": self.network.placement, + "extensions": dict(self.network.extensions), + }, + "labels": dict(self.labels), + "observability": asdict(self.observability), + "extensions": dict(self.extensions), + } + + +@dataclass(frozen=True, slots=True) +class ExecutionRequest: + """A template bound to one validated launcher execution attempt.""" + + template: ExecutionTemplate + context: LauncherContext + + def environment(self) -> dict[str, str]: + """Project correlation variables without exposing any secret reference or value.""" + values = dict(self.template.environment) + values.update( + { + "DANDER_RUN_ID": self.context.run_id, + "DANDER_LAUNCHER": self.context.launcher, + "DANDER_ATTEMPT": str(self.context.attempt), + "DANDER_SHARD_INDEX": str(self.context.shard_index), + "DANDER_SHARD_COUNT": str(self.context.shard_count), + "DANDER_PRINCIPAL": self.template.workload_identity, + } + ) + if self.context.execution_id is not None: + values["DANDER_LAUNCHER_EXECUTION_ID"] = self.context.execution_id + if self.context.deadline_at is not None: + values["DANDER_DEADLINE_AT"] = self.context.deadline_at + return dict(sorted(values.items())) + + +@dataclass(frozen=True, slots=True) +class LauncherCapabilities: + """Explicit projection limits for one launcher implementation.""" + + launcher: str + cpu_millis: frozenset[int] + minimum_memory_mib: int + maximum_memory_mib: int + maximum_deadline_seconds: int + maximum_launcher_retries: int + maximum_task_count: int + maximum_parallelism: int + supports_ephemeral_storage: bool + supports_schedules: bool + supports_time_zones: bool + supports_network_placement: bool + extension_names: frozenset[str] = frozenset() + + +CLOUD_RUN_CAPABILITIES = LauncherCapabilities( + launcher="cloud_run", + cpu_millis=frozenset({1_000, 2_000, 4_000, 6_000, 8_000}), + minimum_memory_mib=128, + maximum_memory_mib=32_768, + maximum_deadline_seconds=86_400, + maximum_launcher_retries=10, + maximum_task_count=1, + maximum_parallelism=1, + supports_ephemeral_storage=False, + supports_schedules=True, + supports_time_zones=True, + supports_network_placement=False, +) + + +def validate_launcher_projection( + template: ExecutionTemplate, + capabilities: LauncherCapabilities, +) -> None: + """Fail before planning when a launcher cannot honor the exact projection.""" + if template.launcher != capabilities.launcher: + raise ExecutionProjectionError("projection launcher does not match its capabilities") + resources = template.resources + schedule = template.schedule + if resources.cpu_millis not in capabilities.cpu_millis: + raise ExecutionProjectionError("launcher does not support the requested CPU") + if ( + not capabilities.minimum_memory_mib + <= resources.memory_mib + <= (capabilities.maximum_memory_mib) + ): + raise ExecutionProjectionError("launcher does not support the requested memory") + if resources.deadline_seconds > capabilities.maximum_deadline_seconds: + raise ExecutionProjectionError("launcher does not support the requested deadline") + if resources.launcher_retry_count > capabilities.maximum_launcher_retries: + raise ExecutionProjectionError("launcher does not support the requested retry count") + if resources.ephemeral_storage_mib is not None and not ( + capabilities.supports_ephemeral_storage + ): + raise ExecutionProjectionError("launcher cannot configure ephemeral storage") + if schedule.task_count > capabilities.maximum_task_count: + raise ExecutionProjectionError("launcher does not support the requested task count") + if schedule.maximum_parallelism > capabilities.maximum_parallelism: + raise ExecutionProjectionError("launcher does not support the requested parallelism") + if schedule.expression is not None and not capabilities.supports_schedules: + raise ExecutionProjectionError("launcher does not support schedules") + if schedule.time_zone is not None and not capabilities.supports_time_zones: + raise ExecutionProjectionError("launcher does not support schedule time zones") + if template.network.placement is not None and not capabilities.supports_network_placement: + raise ExecutionProjectionError("launcher cannot configure network placement") + extension_names = {name for name, _value in template.extensions} + extension_names.update(name for name, _value in template.network.extensions) + if unknown := extension_names - capabilities.extension_names: + raise ExecutionProjectionError( + f"launcher does not support projection extension {sorted(unknown)[0]!r}" + ) + + +def build_gcp_v1_execution_templates( + manifest: DanderProject, + *, + image: str, + project: str, + dataset: str = "raw", + metadata_dataset: str = "dander_meta", + alert_target: str | None = None, +) -> dict[str, ExecutionTemplate]: + """Project the legacy manifest into exact current GCP/Cloud Run execution intent.""" + if not _GCP_PROJECT.fullmatch(project): + raise ExecutionProjectionError("invalid GCP project identifier") + expanded = manifest.terraform_pipelines() + templates: dict[str, ExecutionTemplate] = {} + for pipeline_id, pipeline in expanded.items(): + account_id = str(pipeline["runtime_service_account_id"]) + identity = f"{account_id}@{project}.iam.gserviceaccount.com" + if not _SERVICE_ACCOUNT.fullmatch(identity): + raise ExecutionProjectionError("invalid Cloud Run workload identity") + command: tuple[str, ...] = ( + "runtime", + "execute", + "--contract", + RUNTIME_CONTRACT, + "--pipeline", + pipeline_id, + "--platform", + "gcp", + "--config", + "/app/dander.yaml", + "--models-dir", + "/app/models", + "--batch-rows", + str(manifest.platform.runtime.batch_rows), + ) + if manifest.platform.safety.require_guarded_free_tier: + command = (*command, "--guarded-free-tier") + secret_env = pipeline["secret_env"] + if not isinstance(secret_env, Mapping): + raise ExecutionProjectionError("pipeline secret bindings are invalid") + template = ExecutionTemplate( + schema=EXECUTION_PROJECTION_SCHEMA, + contract=RUNTIME_CONTRACT, + pipeline_id=pipeline_id, + profile_id="gcp", + launcher="cloud_run", + image=image, + command=command, + configuration_reference="/app/dander.yaml", + environment=tuple( + sorted( + { + "BQ_DATASET_METADATA": metadata_dataset, + "BQ_DATASET_RAW": dataset, + "DANDER_IMAGE_DIGEST": image.rsplit("@", maxsplit=1)[-1], + "DANDER_LAUNCHER": "cloud_run", + "DANDER_PRINCIPAL": identity, + "GCP_PROJECT_ID": project, + }.items() + ) + ), + secret_bindings=tuple( + ( + str(environment_name), + SecretReference( + provider="gcp_secret_manager", + reference=( + f"gcp-sm://projects/{project}/secrets/{secret_id}/versions/latest" + ), + ), + ) + for environment_name, secret_id in sorted(secret_env.items()) + ), + workload_identity=identity, + resources=ResourceProjection( + cpu_millis=manifest.platform.runtime.cpu * 1_000, + memory_mib=_memory_mib(manifest.platform.runtime.memory), + ephemeral_storage_mib=None, + deadline_seconds=manifest.platform.runtime.timeout_seconds, + runtime_retry_count=0, + launcher_retry_count=manifest.platform.runtime.max_retries, + ), + schedule=ScheduleProjection( + task_count=1, + maximum_parallelism=1, + expression=str(pipeline["schedule"]), + time_zone=str(pipeline["time_zone"]), + paused=bool(pipeline["paused"]), + ), + network=NetworkPlacement(), + labels=( + ("dander_version", __version__), + ("image_digest", image.rsplit("@", maxsplit=1)[-1]), + ("pipeline", pipeline_id), + ("profile", "gcp"), + ), + observability=ObservabilityProjection( + log_destination="cloud_logging", + metric_namespace="run.googleapis.com", + alert_target=alert_target, + retention_days=None, + ), + ) + validate_launcher_projection(template, CLOUD_RUN_CAPABILITIES) + templates[pipeline_id] = template + return templates + + +def _validate_pairs(values: tuple[tuple[str, object], ...], *, label: str) -> set[str]: + keys = [key for key, _value in values] + if keys != sorted(set(keys)) or any(not isinstance(key, str) or not key for key in keys): + raise ExecutionProjectionError(f"{label} names must be unique and sorted") + return set(keys) + + +def _memory_mib(value: str) -> int: + match = re.fullmatch(r"([1-9][0-9]*)(Mi|Gi)", value) + if match is None: + raise ExecutionProjectionError("runtime memory must use Mi or Gi") + quantity = int(match.group(1)) + return quantity if match.group(2) == "Mi" else quantity * 1_024 diff --git a/tests/deployment/test_execution_projection.py b/tests/deployment/test_execution_projection.py new file mode 100644 index 00000000..4ac6e209 --- /dev/null +++ b/tests/deployment/test_execution_projection.py @@ -0,0 +1,152 @@ +"""Cloud-neutral execution projection and fail-closed launcher limits.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from dander.deployment import ( + CLOUD_RUN_CAPABILITIES, + ExecutionProjectionError, + ExecutionTemplate, + ResourceProjection, + build_gcp_v1_execution_templates, + validate_launcher_projection, +) +from dander.project import load_project_config +from dander.runtime_contract import RUNTIME_CONTRACT + +_IMAGE = "us-central1-docker.pkg.dev/unit-project/dander/dander@sha256:" + "a" * 64 + + +def _template(root: Path) -> ExecutionTemplate: + manifest = load_project_config(root / "dander.yaml") + return build_gcp_v1_execution_templates( + manifest, + image=_IMAGE, + project="unit-project", + alert_target="operator@example.invalid", + )["greenhouse_jobs"] + + +def test_v1_gcp_projection_preserves_current_hosted_intent() -> None: + template = _template(Path(__file__).parents[2]) + + assert template.schema == "io.dander.execution/v1" + assert template.contract == RUNTIME_CONTRACT + assert template.launcher == "cloud_run" + assert template.image == _IMAGE + assert template.command[:8] == ( + "runtime", + "execute", + "--contract", + RUNTIME_CONTRACT, + "--pipeline", + "greenhouse_jobs", + "--platform", + "gcp", + ) + assert template.resources.cpu_millis == 1_000 + assert template.resources.memory_mib == 512 + assert template.resources.runtime_retry_count == 0 + assert template.resources.launcher_retry_count == 1 + assert template.schedule.task_count == 1 + assert template.schedule.maximum_parallelism == 1 + assert dict(template.environment)["DANDER_IMAGE_DIGEST"] == "sha256:" + "a" * 64 + assert template.secret_bindings == () + observability = template.as_dict()["observability"] + assert isinstance(observability, dict) + assert observability["log_destination"] == "cloud_logging" + + +def test_projection_contains_secret_references_but_no_values(tmp_path: Path) -> None: + config = tmp_path / "dander.yaml" + config.write_text( + """\ +version: 1 +platform: + safety: + require_guarded_free_tier: false +pipelines: + salesforce_crm: + source: salesforce + models: [] + build_models: false + secrets: + SALESFORCE_KEY: salesforce-private-key +""", + encoding="utf-8", + ) + + template = build_gcp_v1_execution_templates( + load_project_config(config), image=_IMAGE, project="unit-project" + )["salesforce_crm"] + + reference = dict(template.secret_bindings)["SALESFORCE_KEY"] + assert reference.reference == ( + "gcp-sm://projects/unit-project/secrets/salesforce-private-key/versions/latest" + ) + assert "SALESFORCE_KEY" not in dict(template.environment) + + +def test_run_binding_projects_validated_correlation_environment() -> None: + request = _template(Path(__file__).parents[2]).bind( + run_id="run-123", + launcher_execution_id="execution-456", + attempt=2, + deadline_at="2026-08-07T09:00:00Z", + ) + + environment = request.environment() + assert environment["DANDER_RUN_ID"] == "run-123" + assert environment["DANDER_LAUNCHER_EXECUTION_ID"] == "execution-456" + assert environment["DANDER_ATTEMPT"] == "2" + assert environment["DANDER_SHARD_INDEX"] == "0" + assert environment["DANDER_SHARD_COUNT"] == "1" + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("cpu_millis", 3_000, "CPU"), + ("memory_mib", 64, "memory"), + ("deadline_seconds", 86_401, "deadline"), + ("launcher_retry_count", 11, "retry"), + ("ephemeral_storage_mib", 1_024, "ephemeral"), + ], +) +def test_cloud_run_rejects_unsupported_projection_fields( + field: str, + value: int, + message: str, +) -> None: + template = _template(Path(__file__).parents[2]) + resources = replace(template.resources, **{field: value}) + + with pytest.raises(ExecutionProjectionError, match=message): + validate_launcher_projection(replace(template, resources=resources), CLOUD_RUN_CAPABILITIES) + + +def test_projection_rejects_mutable_image_reference() -> None: + root = Path(__file__).parents[2] + + with pytest.raises(ExecutionProjectionError, match="immutable"): + build_gcp_v1_execution_templates( + load_project_config(root / "dander.yaml"), + image="us-central1-docker.pkg.dev/unit-project/dander/dander:latest", + project="unit-project", + ) + + +def test_resource_projection_rejects_negative_runtime_retries() -> None: + with pytest.raises(ExecutionProjectionError, match="must not be negative"): + ResourceProjection( + cpu_millis=1_000, + memory_mib=512, + ephemeral_storage_mib=None, + deadline_seconds=300, + runtime_retry_count=-1, + launcher_retry_count=0, + ) From 72b33c09bf8e21c0162d3c778dd04cce5434300b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 05:58:23 -0400 Subject: [PATCH 016/163] Project Cloud Run jobs through runtime contract --- HANDOFF.md | 36 ++++---- docs/decisions.md | 11 +++ docs/execution-projection.md | 5 ++ infra/README.md | 15 ++-- infra/main.tf | 1 + infra/modules/scheduled-job/main.tf | 87 ++++++++++--------- infra/modules/scheduled-job/variables.tf | 70 +++++++++++++++ infra/sandbox.auto.tfvars.example | 2 + infra/variables.tf | 66 ++++++++++++++ src/dander/bootstrap/terraform.py | 24 +++++ src/dander/cli/runtime_command.py | 3 +- src/dander/deployment/__init__.py | 2 + src/dander/deployment/projection.py | 49 +++++++++-- tests/bootstrap/test_terraform.py | 27 ++++++ tests/cli/test_runtime_cli.py | 3 + tests/deployment/test_execution_projection.py | 12 +++ tests/infra/test_multi_pipeline_runtime.py | 15 ++-- .../test_gcp_compatibility_baseline.py | 14 +-- 18 files changed, 353 insertions(+), 89 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 37349222..b05dc201 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Added immutable `io.dander.execution/v1` templates for launcher-neutral hosted intent. -- Separated non-secret environment, secret references, workload identity, and run correlation. -- Separated runtime retry intent from launcher retry intent. -- Added explicit Cloud Run capability/limit validation that fails before planning. -- Added deterministic version 1 manifest projection without changing Terraform or live resources. +- Made GCP planning compile one execution projection per version 1 pipeline. +- Made Cloud Run consume projected image, command, resources, schedule, environment, and secrets. +- Added Terraform identity, observability, and unsupported-capability preconditions. +- Switched hosted commands to `runtime execute` while preserving model catalog output and safety. +- Preserved every existing Terraform resource address and plan-first CLI input. ## Try It -Load `dander.yaml`, then call `build_gcp_v1_execution_templates()` with an immutable image and GCP -project. Inspect `ExecutionTemplate.as_dict()` or bind a run through `ExecutionTemplate.bind()`. +Run the usual `dander init-platform-plan` with an immutable image. The saved plan now receives +validated `io.dander.execution/v1` objects; direct hand-authored execution projections are not a +supported operator path. ## Checks - Ruff format/lint and strict mypy passed. -- All 814 tests passed. -- Root and stage-zero Terraform formatting, backend-disabled initialization, and validation passed. -- Wheel/sdist build and distribution inspection passed. +- All 815 tests passed. +- Root and stage-zero Terraform formatting and validation passed. +- Focused projection/bootstrap/runtime tests passed with exact command and limit assertions. ## Decisions -- Deployment-time templates and run-time correlation are separate immutable objects. -- Secret references never enter the non-secret environment mapping. -- Cloud Run advertises only Dander's current one-task/one-worker behavior; unsupported requested - fields fail instead of being silently ignored. +- Python is the projection compiler; Terraform validates and maps the resulting immutable intent. +- Cloud Run native execution variables remain the source for dynamic run IDs, attempts, and shards. +- Existing direct Terraform input examples are reference-only; the supported plan path is the CLI. ## Remaining - Merge this focused ticket through protected main. -- Make Cloud Run Terraform consume this projection with exact behavior parity. -- Run local/Cloud Run normalized outcome, signal, replay, and no-drift proof. +- Publish a source-free candidate and run local/isolated Cloud Run normalized-outcome parity. +- Prove signal, replay, cleanup, and final no-drift behavior before closing Phase 1. ## Review First - `src/dander/deployment/projection.py` -- `tests/deployment/test_execution_projection.py` -- `docs/execution-projection.md` +- `src/dander/bootstrap/terraform.py` +- `infra/modules/scheduled-job/main.tf` diff --git a/docs/decisions.md b/docs/decisions.md index f0040876..d2fc2607 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-07 — Cloud Run consumes the shared execution projection + +- **Decision:** The GCP plan compiler passes `io.dander.execution/v1` templates into Terraform; + Cloud Run no longer reconstructs runtime commands, limits, schedules, environment, or secret + references independently. +- **Compatibility:** Version 1 manifests and existing Terraform resource addresses remain stable. + The runtime command now emits the versioned JSON event contract while preserving ingestion, + selected transforms/tests, registry output, Dataplex publication, safety, and batch settings. +- **Fail closed:** Terraform validates the provider/profile and rejects unsupported networking, + ephemeral-storage, task, parallelism, retry, identity, secret, or observability projections. + ## 2026-08-07 — Launchers consume a validated execution projection - **Decision:** Compile hosted intent into immutable `io.dander.execution/v1` templates containing diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 1a7cf0dd..ba668203 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -21,3 +21,8 @@ Version 1 `dander.yaml` manifests compile deterministically to this projection w existing GCP/BigQuery compatibility profile. A later configuration migration will separate logical pipeline intent from environment-specific deployment settings; it does not change this runtime boundary. + +The GCP plan path supplies these templates to Terraform. Cloud Run consumes their image, command, +resources, schedule, environment, secret references, identity, labels, and observability contract. +Run-specific IDs and attempts continue to come from Cloud Run's native execution context and are +normalized by the OCI runtime contract. diff --git a/infra/README.md b/infra/README.md index 38fe0d05..875392ad 100644 --- a/infra/README.md +++ b/infra/README.md @@ -89,13 +89,14 @@ services and delete resources, while delayed billing reports can still exceed th amount. Deploying the Gen 2 function uses billable Cloud Build, Cloud Run, Storage, and Artifact Registry components; a plan never asserts that the result will cost exactly zero. -For hosted pipelines, copy `sandbox.auto.tfvars.example` to ignored `sandbox.auto.tfvars`, supply an -immutable Artifact Registry digest, and keep every new pipeline paused for its first apply. Run -each Cloud Run Job manually, verify its guarded write, selected transform tests, and registry -compilation, then enable only the proven schedule in a reviewed plan. Each runtime identity can -create BigQuery jobs, edit the shared Dander datasets, inspect Pub/Sub guard wiring, read billing -budget metadata, and access only its declared secrets. Each scheduler identity can invoke only its -pipeline's named Cloud Run Job. +For hosted pipelines, keep `sandbox.auto.tfvars.example` only as a non-secret Terraform input-shape +reference. Use `dander init-platform-plan` so Dander compiles the manifest into validated execution +projections and supplies them to Terraform; do not copy the example into a direct plan. Keep every +new pipeline paused for its first apply. Run each Cloud Run Job manually, verify its guarded write, +selected transform tests, and registry compilation, then enable only the proven schedule in a +reviewed plan. Each runtime identity can create BigQuery jobs, edit the shared Dander datasets, +inspect Pub/Sub guard wiring, read billing budget metadata, and access only its declared secrets. +Each scheduler identity can invoke only its pipeline's named Cloud Run Job. ## Rules (see `steering/01-security.md` and `steering/languages/terraform.md`) diff --git a/infra/main.tf b/infra/main.tf index d4c4f238..62e4d9a4 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -21,6 +21,7 @@ module "scheduled_job" { runtime_batch_rows = var.runtime_batch_rows require_guarded_free_tier = var.require_guarded_free_tier pipelines = var.pipelines + execution_projections = var.execution_projections failure_alert_email = var.failure_alert_email transform_dataset_ids = setsubtract( toset(var.datasets), diff --git a/infra/modules/scheduled-job/main.tf b/infra/modules/scheduled-job/main.tf index afffd004..3cb69c9f 100644 --- a/infra/modules/scheduled-job/main.tf +++ b/infra/modules/scheduled-job/main.tf @@ -177,70 +177,71 @@ resource "google_cloud_run_v2_job" "ingestion" { location = var.region deletion_protection = false labels = { - module = "scheduled-job" - owner = "dander" - pipeline = replace(each.key, "_", "-") + module = "scheduled-job" + owner = "dander" + pipeline = replace(each.key, "_", "-") + profile = var.execution_projections[each.key].labels.profile + dander_version = replace(var.execution_projections[each.key].labels.dander_version, ".", "-") + image_revision = substr(trimprefix(var.execution_projections[each.key].labels.image_digest, "sha256:"), 0, 12) } template { - task_count = 1 - parallelism = 1 + task_count = var.execution_projections[each.key].schedule.task_count + parallelism = var.execution_projections[each.key].schedule.maximum_parallelism template { service_account = google_service_account.runtime[each.key].email - timeout = "${var.runtime_timeout_seconds}s" - max_retries = var.runtime_max_retries + timeout = "${var.execution_projections[each.key].resources.deadline_seconds}s" + max_retries = var.execution_projections[each.key].resources.launcher_retry_count containers { - image = var.container_image - args = concat( - ["run", each.key, "--config", "/app/dander.yaml"], - var.require_guarded_free_tier ? ["--guarded-free-tier"] : [], - ["--batch-rows", tostring(var.runtime_batch_rows)], - each.value.build_models ? concat( - ["--build-models", "--models-dir", "/app/models"], - flatten([ - for model in each.value.models : ["--select-model", model] - ]), - ["--catalog-output", "/tmp/dander-catalog.json"], - ) : [], - each.value.publish_dataplex ? ["--publish-dataplex"] : [], - ) + image = var.execution_projections[each.key].image + args = var.execution_projections[each.key].command resources { limits = { - cpu = tostring(var.runtime_cpu) - memory = var.runtime_memory + cpu = tostring(var.execution_projections[each.key].resources.cpu_millis / 1000) + memory = "${var.execution_projections[each.key].resources.memory_mib}Mi" } } - env { - name = "GCP_PROJECT_ID" - value = var.project_id - } - env { - name = "BQ_DATASET_RAW" - value = var.dataset_id - } - env { - name = "BQ_DATASET_METADATA" - value = "dander_meta" - } - env { - name = "DANDER_PRINCIPAL" - value = google_service_account.runtime[each.key].email + dynamic "env" { + for_each = var.execution_projections[each.key].environment + content { + name = env.key + value = env.value + } } dynamic "env" { - for_each = each.value.secret_env + for_each = var.execution_projections[each.key].secret_bindings content { name = env.key - value = "projects/${var.project_id}/secrets/${env.value}/versions/latest" + value = trimprefix(env.value.reference, "gcp-sm://") } } } } } + lifecycle { + precondition { + condition = ( + var.execution_projections[each.key].workload_identity == google_service_account.runtime[each.key].email && + var.execution_projections[each.key].observability.log_destination == "cloud_logging" && + var.execution_projections[each.key].observability.metric_namespace == "run.googleapis.com" && + var.execution_projections[each.key].observability.alert_target == (local.failure_alerts_enabled ? var.failure_alert_email : null) && + var.execution_projections[each.key].observability.retention_days == null && + toset(keys(var.execution_projections[each.key].secret_bindings)) == toset(keys(each.value.secret_env)) && + alltrue([ + for env_name, binding in var.execution_projections[each.key].secret_bindings : + binding.provider == "gcp_secret_manager" && + binding.reference == "gcp-sm://projects/${var.project_id}/secrets/${each.value.secret_env[env_name]}/versions/latest" + ]) + ) + error_message = "Cloud Run cannot honor this execution projection exactly." + } + } + depends_on = [ data.google_artifact_registry_repository.images, google_bigquery_dataset_iam_member.runtime_writer, @@ -269,9 +270,9 @@ resource "google_cloud_scheduler_job" "ingestion" { region = var.region name = "${each.value.job_name}-daily" description = "Run Dander pipeline ${each.key}" - schedule = each.value.schedule - time_zone = each.value.time_zone - paused = each.value.paused + schedule = var.execution_projections[each.key].schedule.expression + time_zone = var.execution_projections[each.key].schedule.time_zone + paused = var.execution_projections[each.key].schedule.paused attempt_deadline = "180s" retry_config { diff --git a/infra/modules/scheduled-job/variables.tf b/infra/modules/scheduled-job/variables.tf index 7d08f2d9..0a5c6194 100644 --- a/infra/modules/scheduled-job/variables.tf +++ b/infra/modules/scheduled-job/variables.tf @@ -128,6 +128,76 @@ variable "pipelines" { } } +variable "execution_projections" { + description = "Validated cloud-neutral execution templates keyed by pipeline id." + type = map(object({ + schema = string + contract = string + pipeline_id = string + profile_id = string + launcher = string + image = string + command = list(string) + configuration_reference = string + environment = map(string) + secret_bindings = map(object({ + provider = string + reference = string + })) + workload_identity = string + resources = object({ + cpu_millis = number + memory_mib = number + ephemeral_storage_mib = number + deadline_seconds = number + runtime_retry_count = number + launcher_retry_count = number + }) + schedule = object({ + task_count = number + maximum_parallelism = number + expression = string + time_zone = string + paused = bool + }) + network = object({ + placement = string + extensions = map(string) + }) + labels = map(string) + observability = object({ + log_destination = string + metric_namespace = string + alert_target = string + retention_days = number + }) + extensions = map(string) + })) + + validation { + condition = ( + toset(keys(var.execution_projections)) == toset(keys(var.pipelines)) && + alltrue([ + for id, projection in var.execution_projections : + projection.schema == "io.dander.execution/v1" && + projection.contract == "io.dander.runtime/v1" && + projection.pipeline_id == id && + projection.profile_id == "gcp" && + projection.launcher == "cloud_run" && + can(regex("@sha256:[0-9a-f]{64}$", projection.image)) && + projection.resources.ephemeral_storage_mib == null && + projection.resources.runtime_retry_count == 0 && + projection.schedule.task_count == 1 && + projection.schedule.maximum_parallelism == 1 && + projection.network.placement == null && + length(projection.network.extensions) == 0 && + length(projection.extensions) == 0 + ]) + ) + error_message = "Every pipeline requires one supported GCP/Cloud Run execution projection." + } +} + variable "failure_alert_email" { type = string description = "Operator email receiving Cloud Run Job failure notifications; empty disables alerting." diff --git a/infra/sandbox.auto.tfvars.example b/infra/sandbox.auto.tfvars.example index 7b81031c..7702a776 100644 --- a/infra/sandbox.auto.tfvars.example +++ b/infra/sandbox.auto.tfvars.example @@ -1,3 +1,5 @@ +# Input-shape reference only. Use `dander init-platform-plan`; the CLI compiles and supplies the +# required execution_projections value from dander.yaml. enable_scheduled_job = true billing_account_id = "BILLING_ACCOUNT_ID" runtime_container_image = "REGION-docker.pkg.dev/PROJECT/dander/dander@sha256:IMAGE_DIGEST" diff --git a/infra/variables.tf b/infra/variables.tf index 83666dc9..3b66d2d9 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -139,6 +139,72 @@ variable "pipelines" { default = {} } +variable "execution_projections" { + description = "Validated cloud-neutral execution templates keyed by pipeline id." + type = map(object({ + schema = string + contract = string + pipeline_id = string + profile_id = string + launcher = string + image = string + command = list(string) + configuration_reference = string + environment = map(string) + secret_bindings = map(object({ + provider = string + reference = string + })) + workload_identity = string + resources = object({ + cpu_millis = number + memory_mib = number + ephemeral_storage_mib = number + deadline_seconds = number + runtime_retry_count = number + launcher_retry_count = number + }) + schedule = object({ + task_count = number + maximum_parallelism = number + expression = string + time_zone = string + paused = bool + }) + network = object({ + placement = string + extensions = map(string) + }) + labels = map(string) + observability = object({ + log_destination = string + metric_namespace = string + alert_target = string + retention_days = number + }) + extensions = map(string) + })) + default = {} + + validation { + condition = !var.enable_scheduled_job || ( + toset(keys(var.execution_projections)) == toset(keys(var.pipelines)) && + alltrue([ + for id, projection in var.execution_projections : + projection.schema == "io.dander.execution/v1" && + projection.contract == "io.dander.runtime/v1" && + projection.pipeline_id == id && + projection.profile_id == "gcp" && + projection.launcher == "cloud_run" && + can(regex("@sha256:[0-9a-f]{64}$", projection.image)) && + length(projection.command) > 0 && + projection.configuration_reference == "/app/dander.yaml" + ]) + ) + error_message = "Hosted pipelines require one matching validated GCP/Cloud Run execution projection." + } +} + variable "failure_alert_email" { type = string description = "Operator email receiving hosted-pipeline failure notifications; empty disables alerts." diff --git a/src/dander/bootstrap/terraform.py b/src/dander/bootstrap/terraform.py index 90160a89..f5da7811 100644 --- a/src/dander/bootstrap/terraform.py +++ b/src/dander/bootstrap/terraform.py @@ -8,6 +8,8 @@ from json import dumps from typing import TYPE_CHECKING +from dander.deployment import ExecutionProjectionError, build_gcp_execution_templates + if TYPE_CHECKING: from collections.abc import Mapping from pathlib import Path @@ -166,6 +168,26 @@ def execute( len(failure_alert_email) > 254 or not _EMAIL_ADDRESS.fullmatch(failure_alert_email) ): raise TerraformBootstrapError("Invalid failure-alert email address") + execution_projections: dict[str, dict[str, object]] = {} + if enable_runtime: + try: + execution_projections = { + pipeline_id: template.as_dict() + for pipeline_id, template in build_gcp_execution_templates( + expanded_pipelines, + image=container_image, + project=project, + cpu=runtime_cpu, + memory=runtime_memory, + deadline_seconds=runtime_timeout_seconds, + launcher_retry_count=runtime_max_retries, + batch_rows=runtime_batch_rows, + require_guarded_free_tier=require_guarded_free_tier, + alert_target=failure_alert_email or None, + ).items() + } + except ExecutionProjectionError as error: + raise TerraformBootstrapError(str(error)) from error if billing_account_id and not (enable_runtime or enable_cost_guard): raise TerraformBootstrapError( "--billing-account requires --enable-runtime or --enable-cost-guard" @@ -235,6 +257,8 @@ def execute( f"-var=runtime_container_image={container_image}", f"-var=druff_container_image={druff_container_image}", f"-var=pipelines={dumps(expanded_pipelines, sort_keys=True, separators=(',', ':'))}", + "-var=execution_projections=" + f"{dumps(execution_projections, sort_keys=True, separators=(',', ':'))}", f"-var=failure_alert_email={failure_alert_email}", f"-var=secret_ids={dumps(sorted(set(secret_ids)), separators=(',', ':'))}", f"-var=github_repository={github_repository}", diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index c492d128..b53d2686 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -76,6 +76,7 @@ def execute_runtime( project_config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 connectors_dir: Path = typer.Option(Path("connectors"), "--connectors-dir"), # noqa: B008 models_dir: Path = typer.Option(Path("models"), "--models-dir"), # noqa: B008 + catalog_output: Path | None = typer.Option(None, "--catalog-output"), # noqa: B008 project: str | None = typer.Option(None, "--project", help="Override GCP_PROJECT_ID."), dataset: str | None = typer.Option(None, "--dataset", help="Override BQ_DATASET_RAW."), guarded_free_tier: bool = typer.Option(False, "--guarded-free-tier"), @@ -117,7 +118,7 @@ def execute_runtime( build_models=False, models_dir=models_dir, selected_models=None, - catalog_output=None, + catalog_output=catalog_output, publish_dataplex=False, dataplex_location="us", ) diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index 0ea539a7..11356227 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -12,6 +12,7 @@ ResourceProjection, ScheduleProjection, SecretReference, + build_gcp_execution_templates, build_gcp_v1_execution_templates, validate_launcher_projection, ) @@ -28,6 +29,7 @@ "ResourceProjection", "ScheduleProjection", "SecretReference", + "build_gcp_execution_templates", "build_gcp_v1_execution_templates", "validate_launcher_projection", ] diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index e59d1b83..c99454ec 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -392,11 +392,42 @@ def build_gcp_v1_execution_templates( alert_target: str | None = None, ) -> dict[str, ExecutionTemplate]: """Project the legacy manifest into exact current GCP/Cloud Run execution intent.""" + return build_gcp_execution_templates( + manifest.terraform_pipelines(), + image=image, + project=project, + cpu=manifest.platform.runtime.cpu, + memory=manifest.platform.runtime.memory, + deadline_seconds=manifest.platform.runtime.timeout_seconds, + launcher_retry_count=manifest.platform.runtime.max_retries, + batch_rows=manifest.platform.runtime.batch_rows, + require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, + dataset=dataset, + metadata_dataset=metadata_dataset, + alert_target=alert_target, + ) + + +def build_gcp_execution_templates( + pipelines: Mapping[str, Mapping[str, object]], + *, + image: str, + project: str, + cpu: int, + memory: str, + deadline_seconds: int, + launcher_retry_count: int, + batch_rows: int, + require_guarded_free_tier: bool, + dataset: str = "raw", + metadata_dataset: str = "dander_meta", + alert_target: str | None = None, +) -> dict[str, ExecutionTemplate]: + """Build current GCP templates from already validated deployment inputs.""" if not _GCP_PROJECT.fullmatch(project): raise ExecutionProjectionError("invalid GCP project identifier") - expanded = manifest.terraform_pipelines() templates: dict[str, ExecutionTemplate] = {} - for pipeline_id, pipeline in expanded.items(): + for pipeline_id, pipeline in sorted(pipelines.items()): account_id = str(pipeline["runtime_service_account_id"]) identity = f"{account_id}@{project}.iam.gserviceaccount.com" if not _SERVICE_ACCOUNT.fullmatch(identity): @@ -415,9 +446,11 @@ def build_gcp_v1_execution_templates( "--models-dir", "/app/models", "--batch-rows", - str(manifest.platform.runtime.batch_rows), + str(batch_rows), ) - if manifest.platform.safety.require_guarded_free_tier: + if bool(pipeline["build_models"]): + command = (*command, "--catalog-output", "/tmp/dander-catalog.json") + if require_guarded_free_tier: command = (*command, "--guarded-free-tier") secret_env = pipeline["secret_env"] if not isinstance(secret_env, Mapping): @@ -457,12 +490,12 @@ def build_gcp_v1_execution_templates( ), workload_identity=identity, resources=ResourceProjection( - cpu_millis=manifest.platform.runtime.cpu * 1_000, - memory_mib=_memory_mib(manifest.platform.runtime.memory), + cpu_millis=cpu * 1_000, + memory_mib=_memory_mib(memory), ephemeral_storage_mib=None, - deadline_seconds=manifest.platform.runtime.timeout_seconds, + deadline_seconds=deadline_seconds, runtime_retry_count=0, - launcher_retry_count=manifest.platform.runtime.max_retries, + launcher_retry_count=launcher_retry_count, ), schedule=ScheduleProjection( task_count=1, diff --git a/tests/bootstrap/test_terraform.py b/tests/bootstrap/test_terraform.py index 23dc8579..62e720bb 100644 --- a/tests/bootstrap/test_terraform.py +++ b/tests/bootstrap/test_terraform.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import subprocess from typing import TYPE_CHECKING @@ -135,6 +136,32 @@ def fake_run( assert '"greenhouse_jobs"' in pipeline_argument assert '"paused":false' in pipeline_argument assert '"publish_dataplex":true' in pipeline_argument + projection_argument = next( + argument for argument in plan if argument.startswith("-var=execution_projections=") + ) + projections = json.loads(projection_argument.removeprefix("-var=execution_projections=")) + projection = projections["greenhouse_jobs"] + assert projection["image"].endswith("@sha256:" + digest) + assert projection["command"][:8] == [ + "runtime", + "execute", + "--contract", + "io.dander.runtime/v1", + "--pipeline", + "greenhouse_jobs", + "--platform", + "gcp", + ] + assert projection["resources"] == { + "cpu_millis": 2000, + "deadline_seconds": 900, + "ephemeral_storage_mib": None, + "launcher_retry_count": 3, + "memory_mib": 1024, + "runtime_retry_count": 0, + } + assert projection["schedule"]["paused"] is False + assert projection["environment"]["DANDER_LAUNCHER"] == "cloud_run" assert "-var=failure_alert_email=operator@example.invalid" in plan assert '-var=secret_ids=["greenhouse-client-id","greenhouse-client-secret"]' in plan assert "-var=github_repository=WagnerJ-Dev/dander" in plan diff --git a/tests/cli/test_runtime_cli.py b/tests/cli/test_runtime_cli.py index 25a820b8..51d6d08a 100644 --- a/tests/cli/test_runtime_cli.py +++ b/tests/cli/test_runtime_cli.py @@ -44,6 +44,8 @@ def _invoke( "unit-project", "--batch-rows", "2500", + "--catalog-output", + "/tmp/dander-catalog.json", ], env={ "DANDER_RUN_ID": "cloud-run:execution-42", @@ -97,6 +99,7 @@ def execute( assert options.pipeline_or_source == "greenhouse_jobs" assert options.project == "unit-project" assert options.batch_rows == 2500 + assert str(options.catalog_output) == "/tmp/dander-catalog.json" assert captured["render"] is False diff --git a/tests/deployment/test_execution_projection.py b/tests/deployment/test_execution_projection.py index 4ac6e209..663f2e21 100644 --- a/tests/deployment/test_execution_projection.py +++ b/tests/deployment/test_execution_projection.py @@ -56,6 +56,8 @@ def test_v1_gcp_projection_preserves_current_hosted_intent() -> None: assert template.schedule.maximum_parallelism == 1 assert dict(template.environment)["DANDER_IMAGE_DIGEST"] == "sha256:" + "a" * 64 assert template.secret_bindings == () + assert "--catalog-output" in template.command + assert template.command[-1] == "--guarded-free-tier" observability = template.as_dict()["observability"] assert isinstance(observability, dict) assert observability["log_destination"] == "cloud_logging" @@ -140,6 +142,16 @@ def test_projection_rejects_mutable_image_reference() -> None: ) +def test_graph_projection_does_not_request_model_catalog_output() -> None: + templates = build_gcp_v1_execution_templates( + load_project_config(Path(__file__).parents[2] / "dander.yaml"), + image=_IMAGE, + project="unit-project", + ) + + assert "--catalog-output" not in templates["greenhouse_jobs_graph"].command + + def test_resource_projection_rejects_negative_runtime_retries() -> None: with pytest.raises(ExecutionProjectionError, match="must not be negative"): ResourceProjection( diff --git a/tests/infra/test_multi_pipeline_runtime.py b/tests/infra/test_multi_pipeline_runtime.py index c64398e1..da85480c 100644 --- a/tests/infra/test_multi_pipeline_runtime.py +++ b/tests/infra/test_multi_pipeline_runtime.py @@ -11,6 +11,7 @@ def test_root_passes_pipeline_map_and_scopes_secrets_per_runtime() -> None: normalized = "\n".join(" ".join(line.split()) for line in root.splitlines()) assert "pipelines = var.pipelines" in normalized + assert "execution_projections = var.execution_projections" in normalized assert "failure_alert_email = var.failure_alert_email" in normalized assert "runtime_cpu = var.runtime_cpu" in normalized assert "runtime_memory = var.runtime_memory" in normalized @@ -43,12 +44,12 @@ def test_scheduled_module_preserves_greenhouse_and_creates_each_pipeline() -> No in normalized ) assert module.count("for_each = local.guarded_runtime_pipelines") == 2 - assert 'timeout = "${var.runtime_timeout_seconds}s"' in normalized - assert "max_retries = var.runtime_max_retries" in normalized - assert "cpu = tostring(var.runtime_cpu)" in normalized - assert "memory = var.runtime_memory" in normalized - assert 'var.require_guarded_free_tier ? ["--guarded-free-tier"] : []' in normalized - assert '["--batch-rows", tostring(var.runtime_batch_rows)]' in normalized + assert "resources.deadline_seconds}s" in normalized + assert "resources.launcher_retry_count" in normalized + assert "resources.cpu_millis / 1000" in normalized + assert "resources.memory_mib}Mi" in normalized + assert "args = var.execution_projections[each.key].command" in normalized + assert "secret_bindings" in normalized assert 'timeout = "300s"' not in normalized assert "max_retries = 1" not in normalized assert 'cpu = "1"' not in normalized @@ -81,7 +82,7 @@ def test_unguarded_runtime_omits_guard_resources_but_keeps_hosted_platform() -> 'resource "google_monitoring_alert_policy" "pipeline_failure"', ): assert resource in module - assert 'var.require_guarded_free_tier ? ["--guarded-free-tier"] : []' in normalized_module + assert "args = var.execution_projections[each.key].command" in normalized_module assert 'module "bigquery"' in root assert 'module "secret_manager"' in root diff --git a/tests/portability/test_gcp_compatibility_baseline.py b/tests/portability/test_gcp_compatibility_baseline.py index 7209dbba..290ed1fb 100644 --- a/tests/portability/test_gcp_compatibility_baseline.py +++ b/tests/portability/test_gcp_compatibility_baseline.py @@ -37,13 +37,17 @@ def test_bigquery_state_keeps_lease_and_watermark_compare_and_set() -> None: def test_cloud_run_projection_keeps_existing_version_one_behavior() -> None: module = _normalized(ROOT / "infra/modules/scheduled-job/main.tf") + variables = _normalized(ROOT / "infra/modules/scheduled-job/variables.tf") assert 'resource "google_cloud_run_v2_job" "ingestion" {' in module - assert "task_count = 1" in module - assert "parallelism = 1" in module - assert '["run", each.key, "--config", "/app/dander.yaml"]' in module - assert 'var.require_guarded_free_tier ? ["--guarded-free-tier"] : []' in module - assert '["--batch-rows", tostring(var.runtime_batch_rows)]' in module + assert "task_count = var.execution_projections[each.key].schedule.task_count" in module + assert ( + "parallelism = var.execution_projections[each.key].schedule.maximum_parallelism" in module + ) + assert "image = var.execution_projections[each.key].image" in module + assert "args = var.execution_projections[each.key].command" in module + assert "launcher_retry_count" in module + assert "runtime_retry_count == 0" in variables def test_cli_and_distribution_keep_the_public_gcp_compatibility_surface() -> None: From b2a2b6df9024c9aee6ca0e2765a48415898739d5 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 06:05:35 -0400 Subject: [PATCH 017/163] Prepare Dander 0.7.0rc1 --- CHANGELOG.md | 23 +++++++++++++++++++++ HANDOFF.md | 37 +++++++++++++++++----------------- README.md | 4 ++-- docs/getting-started.md | 2 +- docs/known-limitations.md | 4 ++-- docs/release-audit.md | 2 +- docs/session-resume.md | 4 ++-- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 ++-- uv.lock | 2 +- 11 files changed, 54 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2636ef98..cea479de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,29 @@ capabilities enter through the next minor release. ## Unreleased +## 0.7.0rc1 — 2026-08-07 (beta) + +### Added + +- Define the versioned `io.dander.runtime/v1` invocation, event, terminal-result, exit-code, and + graceful-signal contract without changing the ordinary `dander run` interface. +- Add provider-free runtime inspection and a credential-free local executor/run-ledger/event + conformance probe. +- Add standard OCI annotations, a packaged capability manifest, attached SBOM/provenance, and an + atomic artifact record containing immutable index and runnable-platform digests. +- Define immutable `io.dander.execution/v1` templates with separate runtime/launcher retries, + identity, secret references, resources, schedules, network intent, and observability settings. +- Make Cloud Run consume the shared execution projection and reject unsupported fields before + planning. + +### Compatibility + +- Version 1 manifests, GCP/BigQuery behavior, Terraform resource addresses, connector plugin API 1, + and the ordinary CLI remain compatible. +- This candidate adds no non-GCP provider, warehouse, state backend, catalog, or secret resolver. +- Live source-free local/Cloud Run parity is a promotion gate and is not claimed by this metadata + commit. + ## 0.6.0 — 2026-08-07 (beta) ### Acceptance diff --git a/HANDOFF.md b/HANDOFF.md index b05dc201..83f6539b 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,38 @@ ## Finished -- Made GCP planning compile one execution projection per version 1 pipeline. -- Made Cloud Run consume projected image, command, resources, schedule, environment, and secrets. -- Added Terraform identity, observability, and unsupported-capability preconditions. -- Switched hosted commands to `runtime execute` while preserving model catalog output and safety. -- Preserved every existing Terraform resource address and plan-first CLI input. +- Prepared Dander `0.7.0rc1` release metadata from merged Phase 1 code. +- Added candidate notes for the OCI runtime, artifact, execution-projection, and Cloud Run work. +- Updated the lockfile and exact public install/support references. +- Kept `src/dander` functional runtime code unchanged from the reviewed Phase 1 merge. ## Try It -Run the usual `dander init-platform-plan` with an immutable image. The saved plan now receives -validated `io.dander.execution/v1` objects; direct hand-authored execution projections are not a -supported operator path. +After explicit publication approval, tag this exact merged commit `v0.7.0rc1`, publish through the +protected workflow, and install it outside the checkout before any isolated cloud apply. ## Checks +- Release metadata consistency passed. - Ruff format/lint and strict mypy passed. - All 815 tests passed. -- Root and stage-zero Terraform formatting and validation passed. -- Focused projection/bootstrap/runtime tests passed with exact command and limit assertions. +- Wheel/sdist build and distribution inspection passed. +- `src/dander` is byte-unchanged from the accepted Phase 1 merge. ## Decisions -- Python is the projection compiler; Terraform validates and maps the resulting immutable intent. -- Cloud Run native execution variables remain the source for dynamic run IDs, attempts, and shards. -- Existing direct Terraform input examples are reference-only; the supported plan path is the CLI. +- `0.7.0rc1` is the Phase 1 portability candidate; it makes no support claim beyond GCP. +- Candidate publication and isolated Cloud Run mutation remain separate explicit approval gates. ## Remaining -- Merge this focused ticket through protected main. -- Publish a source-free candidate and run local/isolated Cloud Run normalized-outcome parity. -- Prove signal, replay, cleanup, and final no-drift behavior before closing Phase 1. +- Merge the version-only PR through protected main. +- Obtain explicit tag/PyPI approval and publish the immutable candidate. +- Run source-free local/isolated Cloud Run parity, replay, signals, cleanup, and no-drift proof. +- Do not touch the retained project during candidate acceptance. ## Review First -- `src/dander/deployment/projection.py` -- `src/dander/bootstrap/terraform.py` -- `infra/modules/scheduled-job/main.tf` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/known-limitations.md` diff --git a/README.md b/README.md index 4cc5bf7c..e06654f3 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.6.0 +uv tool install dander-platform==0.7.0rc1 dander --version dander new my-data-platform cd my-data-platform @@ -624,7 +624,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.6.0` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.7.0rc1` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/docs/getting-started.md b/docs/getting-started.md index c290f90f..c06edd3b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.6.0 +uv tool install dander-platform==0.7.0rc1 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 98a10900..1df0eed7 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -1,6 +1,6 @@ # Known limitations -Dander `0.6.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; +Dander `0.7.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. ## Ingestion and schemas @@ -54,5 +54,5 @@ evaluate these limits before using it for an unattended system containing busine ## Support boundary -`0.6.x` is the current supported beta line. Only its newest patch is supported; a newer minor's +`0.7.x` is the current supported beta line. Only its newest patch is supported; a newer minor's newest candidate may additionally receive acceptance fixes. See [SECURITY.md](../SECURITY.md). diff --git a/docs/release-audit.md b/docs/release-audit.md index 8ca79b93..2fa455bb 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.6.0`; public Salesforce plugin: `0.3.0`; public ServiceNow plugin: +- Public Dander beta: `0.7.0rc1`; public Salesforce plugin: `0.3.0`; public ServiceNow plugin: `0.2.1`. - Isolated candidate acceptance used Dander `0.6.0rc2` and the public plugin candidates in a source-free image. Salesforce ingested all four endpoints, published five governed models and diff --git a/docs/session-resume.md b/docs/session-resume.md index 306840b2..b3ffcd83 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,9 +5,9 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.6.0` is the current public beta. +- Dander `0.7.0rc1` is the current public beta. - Salesforce `0.3.0` and ServiceNow `0.2.1` are the current public connector releases and support - Dander `0.6.x`. + Dander `0.7.x`. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, execution, catalog, operation-authoring, and deployment-preview work. diff --git a/docs/upgrading.md b/docs/upgrading.md index 8233eadf..473b11f6 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.6.0" +export DANDER_TARGET_VERSION="0.7.0rc1" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 2b4fb675..7336d4a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.6.0" +version = "0.7.0rc1" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 7441d554..8c0c169d 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -36,7 +36,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.6.0", + "dander-platform==0.7.0rc1", "dander-platform==0.1.0", 1, ), @@ -44,5 +44,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.6.0" + "README install command uses 0.1.0; package metadata uses 0.7.0rc1" ] diff --git a/uv.lock b/uv.lock index 9e5aaea7..7cf9d98e 100644 --- a/uv.lock +++ b/uv.lock @@ -520,7 +520,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.6.0" +version = "0.7.0rc1" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 469cca3f153d7901eaeb356dbdd46e1c2587ee66 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 07:07:54 -0400 Subject: [PATCH 018/163] Preserve Dander version during plugin install --- src/dander/cli/main.py | 4 ++++ tests/cli/test_plugins_cli.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 57759ca1..2a83d191 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -213,6 +213,10 @@ def install_plugins( if not requirements: console.print("No connector plugins are declared in dander.yaml.") return + # Keep the package running this command in the resolver transaction. Plugin + # compatibility constraints must fail clearly instead of silently replacing + # Dander with an older release inside a source-free runtime image. + requirements.append(f"dander-platform=={__version__}") uv_executable = shutil.which("uv") command = ( (uv_executable, "pip", "install", "--python", sys.executable, *requirements) diff --git a/tests/cli/test_plugins_cli.py b/tests/cli/test_plugins_cli.py index b538b982..0909455c 100644 --- a/tests/cli/test_plugins_cli.py +++ b/tests/cli/test_plugins_cli.py @@ -9,6 +9,7 @@ from typer.testing import CliRunner import dander.cli.main as cli_module +from dander import __version__ from dander.cli.main import app if TYPE_CHECKING: @@ -64,6 +65,7 @@ def run(command: tuple[str, ...], *, check: bool) -> subprocess.CompletedProcess "pip", "install", "dander-connector-salesforce==0.1.0", + f"dander-platform=={__version__}", ) ] From 73040cdaa7c50d6e059971f05801170dd083440c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 07:13:46 -0400 Subject: [PATCH 019/163] Recommend Dander 0.7 connector candidates --- src/dander/plugins/catalog.py | 12 ++++++------ tests/cli/test_plugins_cli.py | 6 +++--- tests/plugins/test_catalog.py | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dander/plugins/catalog.py b/src/dander/plugins/catalog.py index aea3bf76..23d1004e 100644 --- a/src/dander/plugins/catalog.py +++ b/src/dander/plugins/catalog.py @@ -84,15 +84,15 @@ def as_dict( "ingestion with replay cursors." ), distribution="dander-connector-salesforce", - version="0.3.0", - dander_specifier=">=0.6.0,<0.7", + version="0.3.1rc1", + dander_specifier=">=0.6.0,<0.8", support_status="first-party-beta", validation_status="provider-validated", documentation_url=( "https://github.com/harrisonoconnorhover/" "dander-connector-salesforce#dander-salesforce-connector" ), - pypi_url="https://pypi.org/project/dander-connector-salesforce/0.3.0/", + pypi_url="https://pypi.org/project/dander-connector-salesforce/0.3.1rc1/", repository_url=("https://github.com/harrisonoconnorhover/dander-connector-salesforce"), ), CatalogConnector( @@ -100,15 +100,15 @@ def as_dict( display_name="ServiceNow", description="Read-only ServiceNow Table API incident ingestion with stable paging.", distribution="dander-connector-servicenow", - version="0.2.1", - dander_specifier=">=0.6.0,<0.7", + version="0.2.2rc1", + dander_specifier=">=0.6.0,<0.8", support_status="first-party-beta", validation_status="provider-validated", documentation_url=( "https://github.com/harrisonoconnorhover/" "dander-connector-servicenow#dander-servicenow-connector" ), - pypi_url="https://pypi.org/project/dander-connector-servicenow/0.2.1/", + pypi_url="https://pypi.org/project/dander-connector-servicenow/0.2.2rc1/", repository_url=("https://github.com/harrisonoconnorhover/dander-connector-servicenow"), ), ) diff --git a/tests/cli/test_plugins_cli.py b/tests/cli/test_plugins_cli.py index 0909455c..2dfd34a9 100644 --- a/tests/cli/test_plugins_cli.py +++ b/tests/cli/test_plugins_cli.py @@ -127,8 +127,8 @@ def test_plugins_search_lists_curated_exact_package_pins() -> None: result = CliRunner().invoke(app, ["plugins", "search"]) assert result.exit_code == 0, result.output - assert "dander-connector-salesforce==0.3.0" in result.output - assert "dander-connector-servicenow==0.2.1" in result.output + assert "dander-connector-salesforce==0.3.1rc1" in result.output + assert "dander-connector-servicenow==0.2.2rc1" in result.output def test_plugins_search_filters_and_reports_no_match() -> None: @@ -136,7 +136,7 @@ def test_plugins_search_filters_and_reports_no_match() -> None: missing = CliRunner().invoke(app, ["plugins", "search", "not-a-connector"]) assert filtered.exit_code == 0, filtered.output - assert "dander-connector-servicenow==0.2.1" in filtered.output + assert "dander-connector-servicenow==0.2.2rc1" in filtered.output assert "dander-connector-salesforce" not in filtered.output assert missing.exit_code == 0, missing.output assert "No curated connectors match" in missing.output diff --git a/tests/plugins/test_catalog.py b/tests/plugins/test_catalog.py index dae14897..fd71d8f9 100644 --- a/tests/plugins/test_catalog.py +++ b/tests/plugins/test_catalog.py @@ -27,15 +27,15 @@ def _installed_salesforce(version: str = "0.2.0") -> InstalledConnectorPlugin: ) -def test_curated_catalog_has_exact_public_packages_and_dander_06_compatibility() -> None: - catalog = build_plugin_catalog(dander_version="0.6.0") +def test_curated_catalog_has_exact_public_candidates_and_dander_07_compatibility() -> None: + catalog = build_plugin_catalog(dander_version="0.7.0rc1") connectors = cast("list[dict[str, object]]", catalog["connectors"]) assert catalog["schema_version"] == 1 - assert catalog["dander_version"] == "0.6.0" + assert catalog["dander_version"] == "0.7.0rc1" assert {connector["id"] for connector in connectors} == {"salesforce", "servicenow"} - assert {connector["version"] for connector in connectors} == {"0.3.0", "0.2.1"} - assert all(connector["dander_specifier"] == ">=0.6.0,<0.7" for connector in connectors) + assert {connector["version"] for connector in connectors} == {"0.3.1rc1", "0.2.2rc1"} + assert all(connector["dander_specifier"] == ">=0.6.0,<0.8" for connector in connectors) assert all(connector["compatible"] is True for connector in connectors) assert all(connector["support_status"] == "first-party-beta" for connector in connectors) assert all(connector["validation_status"] == "provider-validated" for connector in connectors) @@ -68,7 +68,7 @@ def test_catalog_installation_status_uses_only_validated_manifest_plugins() -> N def test_catalog_marks_unsupported_dander_version_incompatible() -> None: - catalog = build_plugin_catalog(dander_version="0.7.0") + catalog = build_plugin_catalog(dander_version="0.8.0") connectors = cast("list[dict[str, object]]", catalog["connectors"]) assert all(connector["compatible"] is False for connector in connectors) From f87ddc1e866c11f242a40a23b99f4aa1efff194f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 07:21:03 -0400 Subject: [PATCH 020/163] Report connector plugin count accurately --- src/dander/cli/main.py | 3 ++- tests/cli/test_plugins_cli.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 2a83d191..9b578337 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -213,6 +213,7 @@ def install_plugins( if not requirements: console.print("No connector plugins are declared in dander.yaml.") return + plugin_count = len(requirements) # Keep the package running this command in the resolver transaction. Plugin # compatibility constraints must fail clearly instead of silently replacing # Dander with an older release inside a source-free runtime image. @@ -236,7 +237,7 @@ def install_plugins( load_connector_plugins(manifest.plugins) except ConnectorPluginError as error: raise ClickException(f"Installed connector plugins are incompatible: {error}") from error - console.print(f"[green]Installed {len(requirements)} connector plugin(s).[/green]") + console.print(f"[green]Installed {plugin_count} connector plugin(s).[/green]") @plugins_app.command("scaffold") diff --git a/tests/cli/test_plugins_cli.py b/tests/cli/test_plugins_cli.py index 2dfd34a9..7063346a 100644 --- a/tests/cli/test_plugins_cli.py +++ b/tests/cli/test_plugins_cli.py @@ -58,6 +58,7 @@ def run(command: tuple[str, ...], *, check: bool) -> subprocess.CompletedProcess result = CliRunner().invoke(app, ["plugins", "install", "--config", str(manifest)]) assert result.exit_code == 0, result.output + assert "Installed 1 connector plugin(s)." in result.output assert captured == [ ( sys.executable, From 46033fb93231b11f113d03fb85445ce3ee097c75 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 07:21:32 -0400 Subject: [PATCH 021/163] Prepare Dander 0.7.0rc2 --- CHANGELOG.md | 14 ++++++++++++++ HANDOFF.md | 28 ++++++++++++---------------- README.md | 4 ++-- docs/getting-started.md | 2 +- docs/release-audit.md | 4 ++-- docs/salesforce.md | 4 ++-- docs/session-resume.md | 6 +++--- docs/upgrading.md | 2 +- examples/salesforce/dander.yaml | 2 +- pyproject.toml | 2 +- tests/project/test_scaffold.py | 2 +- tests/test_release_metadata.py | 4 ++-- uv.lock | 2 +- 13 files changed, 43 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cea479de..732d0a57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ capabilities enter through the next minor release. ## Unreleased +## 0.7.0rc2 — 2026-08-07 (beta) + +### Fixed + +- Keep the running `dander-platform` version in the connector-plugin resolver transaction so an + incompatible plugin fails clearly instead of silently downgrading a source-free runtime image. +- Recommend the published Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` compatibility + candidates, which preserve plugin API v1 and extend support through Dander `0.7.x`. + +### Acceptance + +- Reject `0.7.0rc1` after clean source-free installation exposed the plugin-driven runtime + downgrade; no rc1 image was deployed. + ## 0.7.0rc1 — 2026-08-07 (beta) ### Added diff --git a/HANDOFF.md b/HANDOFF.md index 83f6539b..b1256e90 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,33 +2,29 @@ ## Finished -- Prepared Dander `0.7.0rc1` release metadata from merged Phase 1 code. -- Added candidate notes for the OCI runtime, artifact, execution-projection, and Cloud Run work. -- Updated the lockfile and exact public install/support references. -- Kept `src/dander` functional runtime code unchanged from the reviewed Phase 1 merge. +- Prepared replacement Dander candidate `0.7.0rc2` after rejecting rc1 before deployment. +- Prevented connector installation from silently downgrading the running Dander package. +- Published compatible Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` candidates. +- Updated the curated catalog, Salesforce example, and public version references. ## Try It -After explicit publication approval, tag this exact merged commit `v0.7.0rc1`, publish through the -protected workflow, and install it outside the checkout before any isolated cloud apply. +Tag the merged release commit `v0.7.0rc2`, publish through the protected workflow, and install it +outside the checkout before any isolated cloud apply. ## Checks -- Release metadata consistency passed. -- Ruff format/lint and strict mypy passed. -- All 815 tests passed. -- Wheel/sdist build and distribution inspection passed. -- `src/dander` is byte-unchanged from the accepted Phase 1 merge. +- Release metadata, all 815 tests, Ruff lint/format, strict mypy, and package build passed. +- Connector candidate workflows and clean external resolution with `0.7.0rc1` passed. ## Decisions -- `0.7.0rc1` is the Phase 1 portability candidate; it makes no support claim beyond GCP. -- Candidate publication and isolated Cloud Run mutation remain separate explicit approval gates. +- `0.7.0rc1` is immutable but rejected because its source-free plugin install downgraded Dander. +- `0.7.0rc2` replaces rc1 without expanding the Phase 1 portability scope. ## Remaining -- Merge the version-only PR through protected main. -- Obtain explicit tag/PyPI approval and publish the immutable candidate. +- Merge the release PR through protected main and publish the immutable candidate. - Run source-free local/isolated Cloud Run parity, replay, signals, cleanup, and no-drift proof. - Do not touch the retained project during candidate acceptance. @@ -36,4 +32,4 @@ protected workflow, and install it outside the checkout before any isolated clou - `CHANGELOG.md` - `pyproject.toml` -- `docs/known-limitations.md` +- `src/dander/cli/main.py` diff --git a/README.md b/README.md index e06654f3..a495c4c3 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.7.0rc1 +uv tool install dander-platform==0.7.0rc2 dander --version dander new my-data-platform cd my-data-platform @@ -624,7 +624,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.7.0rc1` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.7.0rc2` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/docs/getting-started.md b/docs/getting-started.md index c06edd3b..13c4a126 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.7.0rc1 +uv tool install dander-platform==0.7.0rc2 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 2fa455bb..850c6b8e 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,8 +23,8 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.7.0rc1`; public Salesforce plugin: `0.3.0`; public ServiceNow plugin: - `0.2.1`. +- Public Dander beta: `0.7.0rc2`; public Salesforce candidate: `0.3.1rc1`; public ServiceNow + candidate: `0.2.2rc1`. - Isolated candidate acceptance used Dander `0.6.0rc2` and the public plugin candidates in a source-free image. Salesforce ingested all four endpoints, published five governed models and Dataplex metadata, captured update and soft-delete state, replayed without duplicate keys or diff --git a/docs/salesforce.md b/docs/salesforce.md index 0f624d0c..15b5c27c 100644 --- a/docs/salesforce.md +++ b/docs/salesforce.md @@ -53,14 +53,14 @@ add its governed projection to the relevant model. Undeclared fields fail before ## Project configuration -With `dander-connector-salesforce 0.3.0`, a project can use: +With `dander-connector-salesforce 0.3.1rc1`, a Dander `0.7` candidate project can use: ```yaml version: 1 plugins: salesforce: distribution: dander-connector-salesforce - version: 0.3.0 + version: 0.3.1rc1 pipelines: salesforce_crm: source: salesforce diff --git a/docs/session-resume.md b/docs/session-resume.md index b3ffcd83..96c88891 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,9 +5,9 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.7.0rc1` is the current public beta. -- Salesforce `0.3.0` and ServiceNow `0.2.1` are the current public connector releases and support - Dander `0.7.x`. +- Dander `0.7.0rc2` is the current public beta. +- Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` are the current public connector candidates for + Dander `0.7.x`; stable `0.3.0` and `0.2.1` remain the accepted Dander `0.6` releases. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, execution, catalog, operation-authoring, and deployment-preview work. diff --git a/docs/upgrading.md b/docs/upgrading.md index 473b11f6..663a4dca 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.7.0rc1" +export DANDER_TARGET_VERSION="0.7.0rc2" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/examples/salesforce/dander.yaml b/examples/salesforce/dander.yaml index 0edfd933..0fbb6fe9 100644 --- a/examples/salesforce/dander.yaml +++ b/examples/salesforce/dander.yaml @@ -9,7 +9,7 @@ platform: plugins: salesforce: distribution: dander-connector-salesforce - version: 0.3.0 + version: 0.3.1rc1 pipelines: salesforce_crm: diff --git a/pyproject.toml b/pyproject.toml index 7336d4a3..2fb5a1d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.7.0rc1" +version = "0.7.0rc2" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index 01e25ede..0e0a3fc7 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -55,7 +55,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: ): assert (project / relative).is_file() salesforce = load_project_config(project / "examples" / "salesforce" / "dander.yaml") - assert salesforce.plugins["salesforce"].version == "0.3.0" + assert salesforce.plugins["salesforce"].version == "0.3.1rc1" assert set(salesforce.pipelines) == {"salesforce_crm"} assert not list(project.rglob("*.tfplan")) assert not list(project.rglob("*.tfstate")) diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 8c0c169d..e3d9147a 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -36,7 +36,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.7.0rc1", + "dander-platform==0.7.0rc2", "dander-platform==0.1.0", 1, ), @@ -44,5 +44,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.7.0rc1" + "README install command uses 0.1.0; package metadata uses 0.7.0rc2" ] diff --git a/uv.lock b/uv.lock index 7cf9d98e..86d484d3 100644 --- a/uv.lock +++ b/uv.lock @@ -520,7 +520,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.7.0rc1" +version = "0.7.0rc2" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 0cc15fd400b9d017081336d3fa0ce81d39ace8ea Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 10:03:51 -0400 Subject: [PATCH 022/163] docs: record cloud portability phase 1 acceptance --- HANDOFF.md | 34 +++++++------ docs/cloud-portability-phase1-acceptance.md | 54 +++++++++++++++++++++ docs/release-audit.md | 10 ++-- docs/session-resume.md | 9 ++-- docs/spec-alignment.md | 2 +- 5 files changed, 84 insertions(+), 25 deletions(-) create mode 100644 docs/cloud-portability-phase1-acceptance.md diff --git a/HANDOFF.md b/HANDOFF.md index b1256e90..1fa6f81e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,36 @@ ## Finished -- Prepared replacement Dander candidate `0.7.0rc2` after rejecting rc1 before deployment. -- Prevented connector installation from silently downgrading the running Dander package. -- Published compatible Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` candidates. -- Updated the curated catalog, Salesforce example, and public version references. +- Accepted the public source-free Dander `0.7.0rc2` OCI artifact locally and on Cloud Run. +- Proved ServiceNow smoke plus four-endpoint Salesforce ingest, transforms/tests, catalog, replay, + overlap skip, interruption recovery, lease release, and staging cleanup. +- Recorded SIGTERM and SIGKILL behavior without a false successful terminal event. +- Confirmed the final isolated Terraform plan reported exactly `No changes.` ## Try It -Tag the merged release commit `v0.7.0rc2`, publish through the protected workflow, and install it -outside the checkout before any isolated cloud apply. +Review `docs/cloud-portability-phase1-acceptance.md`, then promote the accepted connector patches +and Dander `0.7.0` through version-only protected release PRs. ## Checks -- Release metadata, all 815 tests, Ruff lint/format, strict mypy, and package build passed. -- Connector candidate workflows and clean external resolution with `0.7.0rc1` passed. +- Public package install, image inspection, local conformance, Cloud Run conformance: passed. +- Salesforce: 5 models, 35 assertions, 5 assets; ServiceNow: 1 model, 4 assertions, 1 asset. +- Replay: zero duplicate IDs and unchanged cursors; final Terraform plan: `No changes.` ## Decisions -- `0.7.0rc1` is immutable but rejected because its source-free plugin install downgraded Dander. -- `0.7.0rc2` replaces rc1 without expanding the Phase 1 portability scope. +- `0.7.0rc2` is the accepted Phase 1 candidate; `0.7.0rc1` remains rejected and undeployed. +- Phase 1B remains a separate AWS artifact-copy and keyless-identity feasibility gate. ## Remaining -- Merge the release PR through protected main and publish the immutable candidate. -- Run source-free local/isolated Cloud Run parity, replay, signals, cleanup, and no-drift proof. -- Do not touch the retained project during candidate acceptance. +- Merge this bounded evidence update and publish stable connector/Dander versions. +- Implement and prove Phase 1B without touching the retained project. +- Continue later portability phases only through separate provider/backend PRs. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `src/dander/cli/main.py` +- `docs/cloud-portability-phase1-acceptance.md` +- `docs/release-audit.md` +- `docs/session-resume.md` diff --git a/docs/cloud-portability-phase1-acceptance.md b/docs/cloud-portability-phase1-acceptance.md new file mode 100644 index 00000000..51a383af --- /dev/null +++ b/docs/cloud-portability-phase1-acceptance.md @@ -0,0 +1,54 @@ +# Cloud portability Phase 1 acceptance + +Accepted on 2026-08-07 against public Dander `0.7.0rc2`, Salesforce connector +`0.3.1rc1`, and ServiceNow connector `0.2.2rc1`. The proof used a generated source-free +project in the isolated disposable GCP project; both schedules remained paused. The retained +project was not changed. + +## Artifact and installation + +- Public packages installed outside every repository checkout without copying `src/`. +- Exact plugin installation retained the running Dander candidate instead of downgrading it. +- The source-free image was built once with OCI metadata, SBOM/provenance, a non-root user, and a + read-only-root-filesystem contract. +- Accepted OCI index digest: + `sha256:f807318db2ee9b4ac56690779fdb27df59b8732815e8d294e43c4166fae9312a`. +- Local inspection and Cloud Run resolved the same Linux/AMD64 content. Local and Cloud Run + conformance emitted the same normalized `runtime.started` and `runtime.completed` outcome. + +## Runtime behavior + +- ServiceNow execution `dander-servicenow-incidents-xqtgf` ingested 67 incidents, built one model, + passed four assertions, and published one catalog asset. +- Salesforce execution `dander-salesforce-plugin-w9jvt` ingested five inclusive-boundary records + across Accounts, Contacts, Opportunities, and Users, built five models, passed 35 assertions, + and published five catalog assets. +- Salesforce replay `dander-salesforce-plugin-lcjst` repeated that boundary safely. Raw table + counts remained 17, 21, 32, and 8; every table retained zero duplicate IDs; no cursor regressed. +- Concurrent execution `dander-salesforce-plugin-f8mjs` emitted a successful terminal + `status=skipped` while the replay held the lease. +- A 60-second Cloud Run timeout interrupted `dander-salesforce-plugin-svc4l` after lease + acquisition. Run history recorded sanitized `interrupted_run`, the lease cleared, and no staging + table remained. Recovery execution `dander-salesforce-plugin-6rcdx` then completed all five + models, 35 assertions, and five catalog assets without data or cursor change. + +## Signal and reconciliation evidence + +- Cloud Run delivered a catchable termination and received a retryable `interrupted_run` terminal + event before stopping the task. +- The accepted container was also killed locally with SIGKILL after `runtime.started`; it exited + 137 and emitted no false terminal event. +- The next real Cloud Run lease acquisition reconciled a representative orphaned active run to + `failed/interrupted_run` with the bounded summary, “A later execution acquired the expired + pipeline lease.” + +## Final state + +- Salesforce and ServiceNow leases are released; no run-scoped staging residue exists. +- Alerts, datasets, secrets, IAM, Druff, and paused schedules remain present and unchanged. +- The final manifest-aware Terraform plan reported exactly `No changes.` +- No credential, provider response, source row, Terraform state, or alert address is retained in + this record. + +Phase 1 is accepted. Phase 1B remains a separate artifact-copy and keyless AWS-to-BigQuery proof; +this record makes no AWS support claim. diff --git a/docs/release-audit.md b/docs/release-audit.md index 850c6b8e..47caa24f 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -25,10 +25,12 @@ cloud path remains outside the live proof. - Public Dander beta: `0.7.0rc2`; public Salesforce candidate: `0.3.1rc1`; public ServiceNow candidate: `0.2.2rc1`. -- Isolated candidate acceptance used Dander `0.6.0rc2` and the public plugin candidates in a - source-free image. Salesforce ingested all four endpoints, published five governed models and - Dataplex metadata, captured update and soft-delete state, replayed without duplicate keys or - cursor regression, released its lease, removed staging, and finished with Terraform no-drift. +- Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a + source-free image. The same accepted OCI content passed local and Cloud Run conformance. + Salesforce ingested all four endpoints, published five governed models and Dataplex metadata, + replayed without duplicate keys or cursor regression, skipped an overlapping run, recovered + from a bounded interruption, released its lease, removed staging, and finished with Terraform + no-drift. See `docs/cloud-portability-phase1-acceptance.md`. - Retained source-free Dander image: `sha256:3220623…8995`, built with Dander `0.5.0` and both plugin `0.2.0` packages. Dander `0.5.1` changes only catalog recommendations, so the runtime image was intentionally not replaced. diff --git a/docs/session-resume.md b/docs/session-resume.md index 96c88891..3aa9c2cd 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -34,10 +34,11 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Latest operating evidence -- On 2026-08-07, the isolated source-free `0.6.0rc2` image completed four-endpoint Salesforce - ingestion, governed transforms/tests, updates, soft-deletion tombstones, replay, Dataplex - publication, lease and staging cleanup, and a final no-drift plan. ServiceNow also completed a - compatibility smoke run in the same image. +- On 2026-08-07, the isolated source-free `0.7.0rc2` image completed local/Cloud Run OCI parity, + four-endpoint Salesforce ingestion, governed transforms/tests, replay, overlap skip, + SIGTERM/SIGKILL recovery, Dataplex publication, lease and staging cleanup, and a final no-drift + plan. ServiceNow completed its compatibility smoke in the same image. The bounded record is + `docs/cloud-portability-phase1-acceptance.md`. - On 2026-08-05, scheduled executions `dander-greenhouse-public-m2pz2`, `dander-hubspot-companies-cmdbz`, `dander-salesforce-accounts-jljwj`, and diff --git a/docs/spec-alignment.md b/docs/spec-alignment.md index 157e42cd..2086b282 100644 --- a/docs/spec-alignment.md +++ b/docs/spec-alignment.md @@ -6,7 +6,7 @@ This ledger maps `steering/00-project-overview.md` to current code and observed |---|---|---|---| | Security | Live-proven | Per-pipeline Secret Manager bindings, separate bootstrap/runtime/scheduler identities, guarded billing visibility, exact-origin loopback Druff access, and no service-account keys | Other private vendors require separately approved tenant credentials | | Hybrid ingestion | Live-proven | Config-driven REST/dlt and hand-rolled `Source` paths; hosted Greenhouse, HubSpot, Salesforce, ServiceNow, Odoo, and executable-graph proofs completed | Workday and NetSuite are not real-tenant validated | -| Connector plugins | Live-proven | Exact-pinned Salesforce and ServiceNow `0.2.0` packages install source-free, register through API v1, and expose presentation-safe discovery plus optional reads | The built-in Salesforce fallback is deprecated; no marketplace service or provider write-back | +| Connector plugins | Live-proven | Exact-pinned Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` packages install source-free with Dander `0.7.0rc2`, register through API v1, and expose presentation-safe discovery plus optional reads | The built-in Salesforce fallback is deprecated; no marketplace service or provider write-back | | BigQuery writer | Live-proven for hosted SCD1 and graph replace | Bounded batches, declared schemas, replay-safe SCD1, fenced DML publication, and staged executable-graph replacement | Direct cloud replace and live Storage Write/SCD2 are outside the retained proof | | Transform | Live-proven | Restricted `ref()` DAG, topological execution, materialization, generic tests, and the advertised graph-operation subset ran through Dander | Arbitrary browser-executed code and provider mutations are excluded | | Metadata spine | Live-proven | Atomic per-pipeline catalog snapshots and run history expose sources, models, columns, lineage, tests, metrics, counts, and lifecycle state | Dataplex remains an optional projection and was not mutated in the retained proof | From 28dceaaaeff47b293ae180572bc8b83c44ea0919 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 10:20:21 -0400 Subject: [PATCH 023/163] release: prepare Dander 0.7.0 --- CHANGELOG.md | 10 ++++++++++ HANDOFF.md | 34 ++++++++++++++++----------------- README.md | 4 ++-- docs/getting-started.md | 2 +- docs/release-audit.md | 4 ++-- docs/salesforce.md | 4 ++-- docs/session-resume.md | 6 +++--- docs/spec-alignment.md | 2 +- docs/upgrading.md | 2 +- examples/salesforce/dander.yaml | 2 +- pyproject.toml | 2 +- src/dander/plugins/catalog.py | 8 ++++---- tests/cli/test_plugins_cli.py | 6 +++--- tests/plugins/test_catalog.py | 4 ++-- tests/project/test_scaffold.py | 2 +- tests/test_release_metadata.py | 4 ++-- uv.lock | 2 +- 17 files changed, 53 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 732d0a57..2e03aa99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ capabilities enter through the next minor release. ## Unreleased +## 0.7.0 — 2026-08-07 (beta) + +### Acceptance + +- Promote the accepted `0.7.0rc2` runtime without functional product-code changes. +- Complete source-free local and Cloud Run OCI parity, ServiceNow compatibility, four-endpoint + Salesforce ingest, transforms/tests/catalog, replay, overlap skip, SIGTERM/SIGKILL recovery, + cleanup, and final Terraform no-drift. +- Recommend stable Salesforce `0.3.1` and ServiceNow `0.2.2`, which preserve plugin API v1. + ## 0.7.0rc2 — 2026-08-07 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 1fa6f81e..ece17571 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,34 @@ ## Finished -- Accepted the public source-free Dander `0.7.0rc2` OCI artifact locally and on Cloud Run. -- Proved ServiceNow smoke plus four-endpoint Salesforce ingest, transforms/tests, catalog, replay, - overlap skip, interruption recovery, lease release, and staging cleanup. -- Recorded SIGTERM and SIGKILL behavior without a false successful terminal event. -- Confirmed the final isolated Terraform plan reported exactly `No changes.` +- Prepared stable Dander `0.7.0` from the accepted `0.7.0rc2` lineage. +- Published stable Salesforce `0.3.1` and ServiceNow `0.2.2` connector packages. +- Updated stable package, catalog, example, installation, upgrade, release, and handoff references. +- Kept Dander runtime implementation unchanged from the accepted candidate. ## Try It -Review `docs/cloud-portability-phase1-acceptance.md`, then promote the accepted connector patches -and Dander `0.7.0` through version-only protected release PRs. +Install the built package outside the checkout, generate a project, and verify that exact stable +connector pins install without changing Dander `0.7.0`. ## Checks -- Public package install, image inspection, local conformance, Cloud Run conformance: passed. -- Salesforce: 5 models, 35 assertions, 5 assets; ServiceNow: 1 model, 4 assertions, 1 asset. -- Replay: zero duplicate IDs and unchanged cursors; final Terraform plan: `No changes.` +- Phase 1 source-free local/Cloud Run acceptance and final Terraform no-drift passed. +- All 815 tests, Ruff lint/format, strict mypy, package build, and external stable resolution passed. ## Decisions -- `0.7.0rc2` is the accepted Phase 1 candidate; `0.7.0rc1` remains rejected and undeployed. -- Phase 1B remains a separate AWS artifact-copy and keyless-identity feasibility gate. +- Promote the accepted candidate without a functional runtime change. +- Phase 1B remains a separate AWS artifact-copy and keyless-identity gate. ## Remaining -- Merge this bounded evidence update and publish stable connector/Dander versions. -- Implement and prove Phase 1B without touching the retained project. -- Continue later portability phases only through separate provider/backend PRs. +- Merge through protected main, tag `v0.7.0`, and publish through the protected environment. +- Verify public stable installation and create the GitHub Release. +- Begin Phase 1B in a separate branch without touching the retained project. ## Review First -- `docs/cloud-portability-phase1-acceptance.md` -- `docs/release-audit.md` -- `docs/session-resume.md` +- `CHANGELOG.md` +- `pyproject.toml` +- `src/dander/plugins/catalog.py` diff --git a/README.md b/README.md index a495c4c3..32756141 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.7.0rc2 +uv tool install dander-platform==0.7.0 dander --version dander new my-data-platform cd my-data-platform @@ -624,7 +624,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.7.0rc2` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.7.0` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/docs/getting-started.md b/docs/getting-started.md index 13c4a126..6bb4bfd9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.7.0rc2 +uv tool install dander-platform==0.7.0 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 47caa24f..7ce385ef 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,8 +23,8 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.7.0rc2`; public Salesforce candidate: `0.3.1rc1`; public ServiceNow - candidate: `0.2.2rc1`. +- Public Dander beta: `0.7.0`; public Salesforce connector: `0.3.1`; public ServiceNow connector: + `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. Salesforce ingested all four endpoints, published five governed models and Dataplex metadata, diff --git a/docs/salesforce.md b/docs/salesforce.md index 15b5c27c..2b21bb98 100644 --- a/docs/salesforce.md +++ b/docs/salesforce.md @@ -53,14 +53,14 @@ add its governed projection to the relevant model. Undeclared fields fail before ## Project configuration -With `dander-connector-salesforce 0.3.1rc1`, a Dander `0.7` candidate project can use: +With `dander-connector-salesforce 0.3.1`, a Dander `0.7` project can use: ```yaml version: 1 plugins: salesforce: distribution: dander-connector-salesforce - version: 0.3.1rc1 + version: 0.3.1 pipelines: salesforce_crm: source: salesforce diff --git a/docs/session-resume.md b/docs/session-resume.md index 3aa9c2cd..853fe31c 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,9 +5,9 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.7.0rc2` is the current public beta. -- Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` are the current public connector candidates for - Dander `0.7.x`; stable `0.3.0` and `0.2.1` remain the accepted Dander `0.6` releases. +- Dander `0.7.0` is the current public beta. +- Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander + `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, execution, catalog, operation-authoring, and deployment-preview work. diff --git a/docs/spec-alignment.md b/docs/spec-alignment.md index 2086b282..c06f8dd9 100644 --- a/docs/spec-alignment.md +++ b/docs/spec-alignment.md @@ -6,7 +6,7 @@ This ledger maps `steering/00-project-overview.md` to current code and observed |---|---|---|---| | Security | Live-proven | Per-pipeline Secret Manager bindings, separate bootstrap/runtime/scheduler identities, guarded billing visibility, exact-origin loopback Druff access, and no service-account keys | Other private vendors require separately approved tenant credentials | | Hybrid ingestion | Live-proven | Config-driven REST/dlt and hand-rolled `Source` paths; hosted Greenhouse, HubSpot, Salesforce, ServiceNow, Odoo, and executable-graph proofs completed | Workday and NetSuite are not real-tenant validated | -| Connector plugins | Live-proven | Exact-pinned Salesforce `0.3.1rc1` and ServiceNow `0.2.2rc1` packages install source-free with Dander `0.7.0rc2`, register through API v1, and expose presentation-safe discovery plus optional reads | The built-in Salesforce fallback is deprecated; no marketplace service or provider write-back | +| Connector plugins | Live-proven | Stable Salesforce `0.3.1` and ServiceNow `0.2.2` preserve the exact accepted candidate behavior, register through API v1, and expose presentation-safe discovery plus optional reads | The built-in Salesforce fallback is deprecated; no marketplace service or provider write-back | | BigQuery writer | Live-proven for hosted SCD1 and graph replace | Bounded batches, declared schemas, replay-safe SCD1, fenced DML publication, and staged executable-graph replacement | Direct cloud replace and live Storage Write/SCD2 are outside the retained proof | | Transform | Live-proven | Restricted `ref()` DAG, topological execution, materialization, generic tests, and the advertised graph-operation subset ran through Dander | Arbitrary browser-executed code and provider mutations are excluded | | Metadata spine | Live-proven | Atomic per-pipeline catalog snapshots and run history expose sources, models, columns, lineage, tests, metrics, counts, and lifecycle state | Dataplex remains an optional projection and was not mutated in the retained proof | diff --git a/docs/upgrading.md b/docs/upgrading.md index 663a4dca..fc650168 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.7.0rc2" +export DANDER_TARGET_VERSION="0.7.0" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/examples/salesforce/dander.yaml b/examples/salesforce/dander.yaml index 0fbb6fe9..6c2f16c2 100644 --- a/examples/salesforce/dander.yaml +++ b/examples/salesforce/dander.yaml @@ -9,7 +9,7 @@ platform: plugins: salesforce: distribution: dander-connector-salesforce - version: 0.3.1rc1 + version: 0.3.1 pipelines: salesforce_crm: diff --git a/pyproject.toml b/pyproject.toml index 2fb5a1d2..f31c06ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.7.0rc2" +version = "0.7.0" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/src/dander/plugins/catalog.py b/src/dander/plugins/catalog.py index 23d1004e..b7c0db3d 100644 --- a/src/dander/plugins/catalog.py +++ b/src/dander/plugins/catalog.py @@ -84,7 +84,7 @@ def as_dict( "ingestion with replay cursors." ), distribution="dander-connector-salesforce", - version="0.3.1rc1", + version="0.3.1", dander_specifier=">=0.6.0,<0.8", support_status="first-party-beta", validation_status="provider-validated", @@ -92,7 +92,7 @@ def as_dict( "https://github.com/harrisonoconnorhover/" "dander-connector-salesforce#dander-salesforce-connector" ), - pypi_url="https://pypi.org/project/dander-connector-salesforce/0.3.1rc1/", + pypi_url="https://pypi.org/project/dander-connector-salesforce/0.3.1/", repository_url=("https://github.com/harrisonoconnorhover/dander-connector-salesforce"), ), CatalogConnector( @@ -100,7 +100,7 @@ def as_dict( display_name="ServiceNow", description="Read-only ServiceNow Table API incident ingestion with stable paging.", distribution="dander-connector-servicenow", - version="0.2.2rc1", + version="0.2.2", dander_specifier=">=0.6.0,<0.8", support_status="first-party-beta", validation_status="provider-validated", @@ -108,7 +108,7 @@ def as_dict( "https://github.com/harrisonoconnorhover/" "dander-connector-servicenow#dander-servicenow-connector" ), - pypi_url="https://pypi.org/project/dander-connector-servicenow/0.2.2rc1/", + pypi_url="https://pypi.org/project/dander-connector-servicenow/0.2.2/", repository_url=("https://github.com/harrisonoconnorhover/dander-connector-servicenow"), ), ) diff --git a/tests/cli/test_plugins_cli.py b/tests/cli/test_plugins_cli.py index 7063346a..02db30f7 100644 --- a/tests/cli/test_plugins_cli.py +++ b/tests/cli/test_plugins_cli.py @@ -128,8 +128,8 @@ def test_plugins_search_lists_curated_exact_package_pins() -> None: result = CliRunner().invoke(app, ["plugins", "search"]) assert result.exit_code == 0, result.output - assert "dander-connector-salesforce==0.3.1rc1" in result.output - assert "dander-connector-servicenow==0.2.2rc1" in result.output + assert "dander-connector-salesforce==0.3.1" in result.output + assert "dander-connector-servicenow==0.2.2" in result.output def test_plugins_search_filters_and_reports_no_match() -> None: @@ -137,7 +137,7 @@ def test_plugins_search_filters_and_reports_no_match() -> None: missing = CliRunner().invoke(app, ["plugins", "search", "not-a-connector"]) assert filtered.exit_code == 0, filtered.output - assert "dander-connector-servicenow==0.2.2rc1" in filtered.output + assert "dander-connector-servicenow==0.2.2" in filtered.output assert "dander-connector-salesforce" not in filtered.output assert missing.exit_code == 0, missing.output assert "No curated connectors match" in missing.output diff --git a/tests/plugins/test_catalog.py b/tests/plugins/test_catalog.py index fd71d8f9..ddc82b41 100644 --- a/tests/plugins/test_catalog.py +++ b/tests/plugins/test_catalog.py @@ -27,14 +27,14 @@ def _installed_salesforce(version: str = "0.2.0") -> InstalledConnectorPlugin: ) -def test_curated_catalog_has_exact_public_candidates_and_dander_07_compatibility() -> None: +def test_curated_catalog_has_exact_stable_pins_and_dander_07_compatibility() -> None: catalog = build_plugin_catalog(dander_version="0.7.0rc1") connectors = cast("list[dict[str, object]]", catalog["connectors"]) assert catalog["schema_version"] == 1 assert catalog["dander_version"] == "0.7.0rc1" assert {connector["id"] for connector in connectors} == {"salesforce", "servicenow"} - assert {connector["version"] for connector in connectors} == {"0.3.1rc1", "0.2.2rc1"} + assert {connector["version"] for connector in connectors} == {"0.3.1", "0.2.2"} assert all(connector["dander_specifier"] == ">=0.6.0,<0.8" for connector in connectors) assert all(connector["compatible"] is True for connector in connectors) assert all(connector["support_status"] == "first-party-beta" for connector in connectors) diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index 0e0a3fc7..fc767c7c 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -55,7 +55,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: ): assert (project / relative).is_file() salesforce = load_project_config(project / "examples" / "salesforce" / "dander.yaml") - assert salesforce.plugins["salesforce"].version == "0.3.1rc1" + assert salesforce.plugins["salesforce"].version == "0.3.1" assert set(salesforce.pipelines) == {"salesforce_crm"} assert not list(project.rglob("*.tfplan")) assert not list(project.rglob("*.tfstate")) diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index e3d9147a..af02814b 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -36,7 +36,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.7.0rc2", + "dander-platform==0.7.0", "dander-platform==0.1.0", 1, ), @@ -44,5 +44,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.7.0rc2" + "README install command uses 0.1.0; package metadata uses 0.7.0" ] diff --git a/uv.lock b/uv.lock index 86d484d3..36a8d2ce 100644 --- a/uv.lock +++ b/uv.lock @@ -520,7 +520,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.7.0rc2" +version = "0.7.0" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 73166c09983dabd23f9cff026a53973d87010d4f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 11:04:36 -0400 Subject: [PATCH 024/163] Prove cross-cloud artifact and identity feasibility --- .github/workflows/ci.yml | 7 + CHANGELOG.md | 7 + HANDOFF.md | 39 ++- .../cloud-portability/phase1b/README.md | 88 ++++++ .../phase1b/ecr/.terraform.lock.hcl | 26 ++ .../cloud-portability/phase1b/ecr/main.tf | 33 +++ .../cloud-portability/phase1b/ecr/outputs.tf | 9 + .../phase1b/ecr/terraform.tfvars.example | 2 + .../phase1b/ecr/variables.tf | 21 ++ .../cloud-portability/phase1b/ecr/versions.tf | 21 ++ .../phase1b/smoke/.terraform.lock.hcl | 46 +++ .../cloud-portability/phase1b/smoke/main.tf | 198 +++++++++++++ .../phase1b/smoke/outputs.tf | 35 +++ .../phase1b/smoke/terraform.tfvars.example | 6 + .../phase1b/smoke/variables.tf | 66 +++++ .../phase1b/smoke/versions.tf | 29 ++ docs/decisions.md | 12 + docs/known-limitations.md | 4 + docs/oci-artifact-contract.md | 12 +- scripts/portability/__init__.py | 1 + scripts/portability/oci_copy.py | 169 +++++++++++ .../portability/prepare_phase1b_context.py | 124 ++++++++ .../scan_long_lived_credentials.py | 62 ++++ scripts/portability/wif_bigquery_probe.py | 169 +++++++++++ src/dander/bootstrap/project.py | 43 ++- tests/bootstrap/test_project_bootstrap.py | 95 +++++- tests/portability/test_phase1b_tools.py | 276 ++++++++++++++++++ ...oss-cloud-artifact-identity-feasibility.md | 36 +++ 28 files changed, 1611 insertions(+), 25 deletions(-) create mode 100644 acceptance/cloud-portability/phase1b/README.md create mode 100644 acceptance/cloud-portability/phase1b/ecr/.terraform.lock.hcl create mode 100644 acceptance/cloud-portability/phase1b/ecr/main.tf create mode 100644 acceptance/cloud-portability/phase1b/ecr/outputs.tf create mode 100644 acceptance/cloud-portability/phase1b/ecr/terraform.tfvars.example create mode 100644 acceptance/cloud-portability/phase1b/ecr/variables.tf create mode 100644 acceptance/cloud-portability/phase1b/ecr/versions.tf create mode 100644 acceptance/cloud-portability/phase1b/smoke/.terraform.lock.hcl create mode 100644 acceptance/cloud-portability/phase1b/smoke/main.tf create mode 100644 acceptance/cloud-portability/phase1b/smoke/outputs.tf create mode 100644 acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example create mode 100644 acceptance/cloud-portability/phase1b/smoke/variables.tf create mode 100644 acceptance/cloud-portability/phase1b/smoke/versions.tf create mode 100644 scripts/portability/__init__.py create mode 100644 scripts/portability/oci_copy.py create mode 100644 scripts/portability/prepare_phase1b_context.py create mode 100644 scripts/portability/scan_long_lived_credentials.py create mode 100644 scripts/portability/wif_bigquery_probe.py create mode 100644 tests/portability/test_phase1b_tools.py create mode 100644 tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27f8b9ec..90c016fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,13 @@ jobs: run: | terraform -chdir=bootstrap-admin init -backend=false -input=false terraform -chdir=bootstrap-admin validate + - name: Validate cross-cloud feasibility roots + run: | + terraform fmt -check -recursive ../acceptance/cloud-portability/phase1b + terraform -chdir=../acceptance/cloud-portability/phase1b/ecr init -backend=false -input=false + terraform -chdir=../acceptance/cloud-portability/phase1b/ecr validate + terraform -chdir=../acceptance/cloud-portability/phase1b/smoke init -backend=false -input=false + terraform -chdir=../acceptance/cloud-portability/phase1b/smoke validate - name: Static security analysis uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e03aa99..1abbf9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ capabilities enter through the next minor release. ## Unreleased +### Added + +- Publish source-free runtime images as one `linux/amd64,linux/arm64` OCI index and reject an + incomplete registry result before recording publication success. +- Add an isolated, unscheduled Phase 1B proof for digest-preserving GAR-to-ECR copy and keyless + Fargate-to-BigQuery credential refresh. This is feasibility evidence, not Fargate support. + ## 0.7.0 — 2026-08-07 (beta) ### Acceptance diff --git a/HANDOFF.md b/HANDOFF.md index ece17571..664d305d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,41 @@ ## Finished -- Prepared stable Dander `0.7.0` from the accepted `0.7.0rc2` lineage. -- Published stable Salesforce `0.3.1` and ServiceNow `0.2.2` connector packages. -- Updated stable package, catalog, example, installation, upgrade, release, and handoff references. -- Kept Dander runtime implementation unchanged from the accepted candidate. +- Started Phase 1B from stable `v0.7.0` without touching the docs-only roadmap branch. +- Made source-free image publication require one `linux/amd64,linux/arm64` OCI index. +- Added digest-preserving GAR-to-ECR copy verification and a bounded WIF refresh probe. +- Added isolated ECR and manual Fargate proof roots with separate execution/task roles. +- Built one source-free GAR index; both `amd64` and `arm64` passed local runtime conformance. ## Try It -Install the built package outside the checkout, generate a project, and verify that exact stable -connector pins install without changing Dander `0.7.0`. +Follow `acceptance/cloud-portability/phase1b/README.md` only with disposable AWS/GCP accounts and +saved reviewed plans. No live apply is required for local validation. ## Checks -- Phase 1 source-free local/Cloud Run acceptance and final Terraform no-drift passed. -- All 815 tests, Ruff lint/format, strict mypy, package build, and external stable resolution passed. +- All 825 tests, Ruff, and strict mypy passed. +- Both Phase 1B Terraform roots initialized and validated with locked providers. +- Package build, distribution inspection, and clean external wheel install passed. +- Corrected GAR index `sha256:6cd545fa…be81f` passed both platform conformance and key scans; no + fixable high/critical CVE was found (four unfixed base-image findings remain visible). ## Decisions -- Promote the accepted candidate without a functional runtime change. -- Phase 1B remains a separate AWS artifact-copy and keyless-identity gate. +- A registry digest rewrite fails rather than being accepted as an equivalent image. +- Fargate receives no service or schedule; the proof task role has no AWS permission policy. +- Fargate remains feasibility-only until the later portable BigQuery vertical slice. ## Remaining -- Merge through protected main, tag `v0.7.0`, and publish through the protected environment. -- Verify public stable installation and create the GitHub Release. -- Begin Phase 1B in a separate branch without touching the retained project. +- Authenticate the new AWS account with short-lived CLI credentials. +- Build/copy the source-free index and run the reviewed live Fargate/WIF refresh proof. +- Run image/config/state/log scans and Cloud Run artifact parity. +- Destroy AWS proof resources and require isolated GCP `No changes.` +- Record acceptance, complete protected CI/review, and merge the focused PR. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `src/dander/plugins/catalog.py` +- `src/dander/bootstrap/project.py` +- `scripts/portability/oci_copy.py` +- `acceptance/cloud-portability/phase1b/smoke/main.tf` diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md new file mode 100644 index 00000000..d33fd567 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/README.md @@ -0,0 +1,88 @@ +# Phase 1B — Artifact Copy and Cross-Cloud Identity Proof + +This bounded acceptance proves two assumptions before Dander adds provider factories: one +multi-platform OCI index can move from GAR to ECR without a rebuild, and an isolated Fargate task +can query BigQuery before and after a short-lived Google credential refresh without either cloud's +static access keys. + +This directory is proof infrastructure, not a supported Fargate launcher. It creates no ECS +service or schedule. Use only disposable accounts/projects, saved Terraform plans, and an operator +artifact directory outside the repository. Never commit generated credential configuration, +Terraform state, task descriptions, or logs. + +## Prerequisites + +- Dander `0.7.0`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with + short-lived browser/SSO credentials. +- `crane` pinned to `v0.20.6` (`go install + github.com/google/go-containerregistry/cmd/crane@v0.20.6`). +- A disposable billing-linked GCP project containing the bounded proof table. +- A disposable AWS account. Do not create an AWS access key. + +The tested GCP target is project `project-092b24a8-26a3-4438-8cd`, dataset `dander_raw`, table +`greenhouse_job_board_jobs`. The retained Dander project is out of scope. + +## Reviewed sequence + +1. Initialize `ecr/`, save and review its plan, then apply only that saved plan. Record + `repository_url`. +2. Generate a fresh project from the public package into an operator temp directory. Confirm it + has no `src/` directory. +3. Generate an AWS external-account file for the deterministic provider name and probe service + account. This command writes configuration, not a secret: + + ```console + gcloud iam workload-identity-pools create-cred-config \ + projects/1009770943166/locations/global/workloadIdentityPools/dander-phase1b-aws/providers/fargate \ + --service-account=dander-phase1b-aws@project-092b24a8-26a3-4438-8cd.iam.gserviceaccount.com \ + --aws --output-file="$OPERATOR_DIR/external-account.json" + ``` + +4. Prepare only the generated context. The helper validates `external_account`/`aws1`, rejects + private keys and client secrets, and requests a 600-second impersonated token: + + ```console + python scripts/portability/prepare_phase1b_context.py \ + --project-dir "$PROJECT_DIR" \ + --credential-config "$OPERATOR_DIR/external-account.json" + ``` + +5. From the generated project, use this branch's `dander image-publish` to build one + `linux/amd64,linux/arm64` index in staging GAR. Preserve `.dander/runtime-artifact.json` outside + the repository. +6. Authenticate Docker to GAR and ECR with short-lived tokens. Copy and verify without rebuilding: + + ```console + python scripts/portability/oci_copy.py \ + --source "$GAR_IMAGE_AT_DIGEST" \ + --destination "$ECR_REPOSITORY:phase1b-$SHORT_REVISION" \ + --record "$OPERATOR_DIR/oci-copy.json" + ``` + + The command fails unless the index digest and both runnable platform-manifest digests are + identical after the copy. +7. Set the verified ECR digest in `smoke/terraform.tfvars`, initialize `smoke/`, save and review its + plan, then apply only that saved plan. The execution role can pull/log; the separate task role + has no AWS permission policy and is the only role trusted by Google WIF. +8. Run exactly one Fargate task using the Terraform outputs for cluster, task definition, subnet, + and security group. Wait for terminal status and save `describe-tasks` plus the CloudWatch log + stream outside the repository. +9. Require two `query.completed` events followed by `credential.refresh_observed`. The second + expiry must be later than the first. No event may contain a token, URL, query body, or record. +10. Run `scan_long_lived_credentials.py` over the generated config, both Terraform states, task + definition/description, exported logs, and an extracted image filesystem. Run the normal image + vulnerability and secret scans as well. +11. Run the same GAR index in the isolated Cloud Run proof job and confirm its selected `amd64` + manifest matches the recorded index. Restore the isolated project through its reviewed + manifest-aware plan and require `No changes.` +12. Destroy the smoke stack and ECR root from reviewed destroy plans. If teardown is deferred, the + account must still contain no ECS service or schedule. + +## Pass criteria + +- One build; identical GAR/ECR index and `amd64`/`arm64` digests; no registry rewrite. +- The Fargate task uses the task role while ECS pull/logging uses the execution role. +- BigQuery succeeds before and after observed Google credential expiry/refresh. +- No Google service-account key, AWS access key, or secret value appears in inputs, state, task + definition, logs, or image. +- Both proof roots are destroyed and the isolated GCP platform finishes with no Terraform drift. diff --git a/acceptance/cloud-portability/phase1b/ecr/.terraform.lock.hcl b/acceptance/cloud-portability/phase1b/ecr/.terraform.lock.hcl new file mode 100644 index 00000000..4d4a6054 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/ecr/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = ">= 5.0.0, < 7.0.0" + hashes = [ + "h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} diff --git a/acceptance/cloud-portability/phase1b/ecr/main.tf b/acceptance/cloud-portability/phase1b/ecr/main.tf new file mode 100644 index 00000000..cd1ada84 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/ecr/main.tf @@ -0,0 +1,33 @@ +resource "aws_ecr_repository" "runtime" { + name = var.repository_name + image_tag_mutability = "IMMUTABLE" + force_delete = true + + encryption_configuration { + encryption_type = "AES256" + } + + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_lifecycle_policy" "runtime" { + repository = aws_ecr_repository.runtime.name + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Retain only the five newest feasibility images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = 5 + } + action = { + type = "expire" + } + } + ] + }) +} diff --git a/acceptance/cloud-portability/phase1b/ecr/outputs.tf b/acceptance/cloud-portability/phase1b/ecr/outputs.tf new file mode 100644 index 00000000..4da006cd --- /dev/null +++ b/acceptance/cloud-portability/phase1b/ecr/outputs.tf @@ -0,0 +1,9 @@ +output "repository_url" { + description = "ECR repository to receive the digest-preserving OCI copy." + value = aws_ecr_repository.runtime.repository_url +} + +output "registry_id" { + description = "AWS registry identifier used during ECR login." + value = aws_ecr_repository.runtime.registry_id +} diff --git a/acceptance/cloud-portability/phase1b/ecr/terraform.tfvars.example b/acceptance/cloud-portability/phase1b/ecr/terraform.tfvars.example new file mode 100644 index 00000000..869724c2 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/ecr/terraform.tfvars.example @@ -0,0 +1,2 @@ +aws_region = "us-east-1" +repository_name = "dander-phase1b" diff --git a/acceptance/cloud-portability/phase1b/ecr/variables.tf b/acceptance/cloud-portability/phase1b/ecr/variables.tf new file mode 100644 index 00000000..00cd55ac --- /dev/null +++ b/acceptance/cloud-portability/phase1b/ecr/variables.tf @@ -0,0 +1,21 @@ +variable "aws_region" { + description = "AWS region for the isolated Phase 1B proof." + type = string + default = "us-east-1" + + validation { + condition = can(regex("^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$", var.aws_region)) + error_message = "aws_region must be a valid AWS region name." + } +} + +variable "repository_name" { + description = "Private ECR repository that receives the copied OCI index." + type = string + default = "dander-phase1b" + + validation { + condition = can(regex("^[a-z0-9]+(?:[._/-][a-z0-9]+)*$", var.repository_name)) + error_message = "repository_name must be a valid private ECR repository name." + } +} diff --git a/acceptance/cloud-portability/phase1b/ecr/versions.tf b/acceptance/cloud-portability/phase1b/ecr/versions.tf new file mode 100644 index 00000000..239ce1c5 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/ecr/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.9.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0, < 7.0" + } + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + managed-by = "dander" + purpose = "cloud-portability-phase1b" + } + } +} diff --git a/acceptance/cloud-portability/phase1b/smoke/.terraform.lock.hcl b/acceptance/cloud-portability/phase1b/smoke/.terraform.lock.hcl new file mode 100644 index 00000000..f5bd801b --- /dev/null +++ b/acceptance/cloud-portability/phase1b/smoke/.terraform.lock.hcl @@ -0,0 +1,46 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = ">= 5.0.0, < 7.0.0" + hashes = [ + "h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "7.43.0" + constraints = ">= 6.0.0, < 8.0.0" + hashes = [ + "h1:AwFYLjEh05Jog9DYlEQDacVk2Sjt5/QrKAsymAW0k/Q=", + "zh:0050abe670a076cfc110662ba3e9f8c25366a63df9389dfbe090b5dc5e73a4d2", + "zh:31769c27700334d11e652ddb2a0dfc9efd65df43dbc51a06afc04e10db645bbe", + "zh:33f7211886d44a31d9dc89fbe11f0d2e282da22b509ae25dcfb8c6533eeacff4", + "zh:3ee98b297ec00ad10b1080fddfddbe72c38dec00eece6d56e706fea1e871d648", + "zh:620683cba53b964a882a419c38d40ab4595459b44dc54b9ed18c5bb6b97fbceb", + "zh:7c118cc8ba4c245807b3cbcfd4f5bdb81b1afce83fa1528020208117f7c9518c", + "zh:8263f9aeab9c8bc661a8f6a4963e585ca0a86ed6662424d8f34923fd01c6de18", + "zh:d2ba502aa3da2dad62ae81dd32c97bc279c56a97b9ac3bb3f806303dc0af598d", + "zh:db37f9f900b801ffaad1b7194c4b89de9351879e753b752ca37e12f3e989691f", + "zh:eca80c71ef4a559b94a779365a8d8b91a33d7a805a27ca21c176720085c31d30", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fa91060bb31207fc91645e7de881fa61f106b9d0dfed2f444f158b9cd9420c58", + ] +} diff --git a/acceptance/cloud-portability/phase1b/smoke/main.tf b/acceptance/cloud-portability/phase1b/smoke/main.tf new file mode 100644 index 00000000..d7bc31cd --- /dev/null +++ b/acceptance/cloud-portability/phase1b/smoke/main.tf @@ -0,0 +1,198 @@ +data "aws_caller_identity" "current" {} + +locals { + name = "dander-phase1b" +} + +resource "aws_cloudwatch_log_group" "probe" { + name = "/dander/phase1b" + retention_in_days = 7 +} + +resource "aws_ecs_cluster" "probe" { + name = local.name + + setting { + name = "containerInsights" + value = "enabled" + } +} + +data "aws_iam_policy_document" "ecs_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "execution" { + name = "${local.name}-execution" + assume_role_policy = data.aws_iam_policy_document.ecs_assume.json +} + +resource "aws_iam_role_policy_attachment" "execution" { + role = aws_iam_role.execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +resource "aws_iam_role" "task" { + name = "${local.name}-task" + assume_role_policy = data.aws_iam_policy_document.ecs_assume.json + + # Deliberately no AWS permission policy: temporary task-role identity is exchanged by Google WIF. +} + +resource "aws_vpc" "probe" { + cidr_block = "10.77.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true +} + +resource "aws_internet_gateway" "probe" { + vpc_id = aws_vpc.probe.id +} + +resource "aws_subnet" "public" { + vpc_id = aws_vpc.probe.id + cidr_block = "10.77.1.0/24" + map_public_ip_on_launch = true +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.probe.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.probe.id + } +} + +resource "aws_route_table_association" "public" { + route_table_id = aws_route_table.public.id + subnet_id = aws_subnet.public.id +} + +resource "aws_security_group" "task" { + name = local.name + description = "Outbound-only network access for the Dander Phase 1B probe" + vpc_id = aws_vpc.probe.id + + egress { + description = "HTTPS access to AWS, Google STS, and BigQuery APIs" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "google_iam_workload_identity_pool" "aws" { + workload_identity_pool_id = "dander-phase1b-aws" + display_name = "Dander Phase 1B AWS" + description = "Disposable keyless Fargate-to-Google identity proof" +} + +resource "google_iam_workload_identity_pool_provider" "aws" { + workload_identity_pool_id = google_iam_workload_identity_pool.aws.workload_identity_pool_id + workload_identity_pool_provider_id = "fargate" + display_name = "Dander Phase 1B Fargate" + description = "Trusts only the isolated Dander ECS task role" + + attribute_mapping = { + "google.subject" = "assertion.arn" + "attribute.aws_role" = "assertion.arn.extract('assumed-role/{role}/')" + } + attribute_condition = "assertion.arn.startsWith('arn:aws:sts::${data.aws_caller_identity.current.account_id}:assumed-role/${aws_iam_role.task.name}/')" + + aws { + account_id = data.aws_caller_identity.current.account_id + } +} + +resource "google_service_account" "probe" { + account_id = "dander-phase1b-aws" + display_name = "Dander Phase 1B AWS probe" + description = "Disposable BigQuery reader impersonated by the isolated Fargate task" +} + +resource "google_service_account_iam_member" "aws_workload_identity" { + service_account_id = google_service_account.probe.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.aws.name}/attribute.aws_role/${aws_iam_role.task.name}" +} + +resource "google_project_iam_member" "bigquery_job_user" { + project = var.gcp_project_id + role = "roles/bigquery.jobUser" + member = "serviceAccount:${google_service_account.probe.email}" +} + +resource "google_bigquery_dataset_iam_member" "proof_reader" { + project = var.gcp_project_id + dataset_id = var.proof_dataset + role = "roles/bigquery.dataViewer" + member = "serviceAccount:${google_service_account.probe.email}" +} + +resource "aws_ecs_task_definition" "probe" { + family = local.name + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = 256 + memory = 512 + execution_role_arn = aws_iam_role.execution.arn + task_role_arn = aws_iam_role.task.arn + + runtime_platform { + cpu_architecture = var.cpu_architecture + operating_system_family = "LINUX" + } + + container_definitions = jsonencode([ + { + name = "probe" + image = var.ecr_image + essential = true + readonlyRootFilesystem = true + entryPoint = ["python", "/app/phase1b_probe.py"] + command = [ + "--project", + var.gcp_project_id, + "--dataset", + var.proof_dataset, + "--table", + var.proof_table, + ] + environment = [ + { + name = "GOOGLE_APPLICATION_CREDENTIALS" + value = "/app/gcp-wif.json" + }, + { + name = "GOOGLE_CLOUD_PROJECT" + value = var.gcp_project_id + }, + ] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.probe.name + awslogs-region = var.aws_region + awslogs-stream-prefix = "fargate" + } + } + } + ]) + + depends_on = [ + aws_iam_role_policy_attachment.execution, + google_service_account_iam_member.aws_workload_identity, + google_project_iam_member.bigquery_job_user, + google_bigquery_dataset_iam_member.proof_reader, + ] +} diff --git a/acceptance/cloud-portability/phase1b/smoke/outputs.tf b/acceptance/cloud-portability/phase1b/smoke/outputs.tf new file mode 100644 index 00000000..171225af --- /dev/null +++ b/acceptance/cloud-portability/phase1b/smoke/outputs.tf @@ -0,0 +1,35 @@ +output "cluster_name" { + value = aws_ecs_cluster.probe.name +} + +output "task_definition_arn" { + value = aws_ecs_task_definition.probe.arn +} + +output "task_role_arn" { + value = aws_iam_role.task.arn +} + +output "execution_role_arn" { + value = aws_iam_role.execution.arn +} + +output "subnet_id" { + value = aws_subnet.public.id +} + +output "security_group_id" { + value = aws_security_group.task.id +} + +output "workload_identity_provider_name" { + value = google_iam_workload_identity_pool_provider.aws.name +} + +output "google_service_account_email" { + value = google_service_account.probe.email +} + +output "log_group_name" { + value = aws_cloudwatch_log_group.probe.name +} diff --git a/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example b/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example new file mode 100644 index 00000000..736316ca --- /dev/null +++ b/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example @@ -0,0 +1,6 @@ +aws_region = "us-east-1" +gcp_project_id = "replace-with-disposable-project" +ecr_image = "123456789012.dkr.ecr.us-east-1.amazonaws.com/dander-phase1b@sha256:replace" +proof_dataset = "dander_raw" +proof_table = "greenhouse_job_board_jobs" +cpu_architecture = "ARM64" diff --git a/acceptance/cloud-portability/phase1b/smoke/variables.tf b/acceptance/cloud-portability/phase1b/smoke/variables.tf new file mode 100644 index 00000000..c21c4659 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/smoke/variables.tf @@ -0,0 +1,66 @@ +variable "aws_region" { + description = "AWS region for the isolated Fargate task." + type = string + default = "us-east-1" + + validation { + condition = can(regex("^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$", var.aws_region)) + error_message = "aws_region must be a valid AWS region name." + } +} + +variable "gcp_project_id" { + description = "Disposable GCP project containing the bounded proof table." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{4,28}[a-z0-9]$", var.gcp_project_id)) + error_message = "gcp_project_id must be a valid GCP project ID." + } +} + +variable "ecr_image" { + description = "Immutable ECR OCI-index reference copied from staging GAR." + type = string + + validation { + condition = can(regex( + "^[0-9]{12}\\.dkr\\.ecr\\.[a-z0-9-]+\\.amazonaws\\.com/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$", + var.ecr_image, + )) + error_message = "ecr_image must be an immutable private-ECR sha256 reference." + } +} + +variable "proof_dataset" { + description = "Existing BigQuery dataset the probe may read." + type = string + default = "dander_raw" + + validation { + condition = length(var.proof_dataset) <= 1024 && can(regex("^[A-Za-z_][A-Za-z0-9_]*$", var.proof_dataset)) + error_message = "proof_dataset must be a valid BigQuery dataset ID." + } +} + +variable "proof_table" { + description = "Existing bounded BigQuery table the probe counts twice." + type = string + default = "greenhouse_job_board_jobs" + + validation { + condition = length(var.proof_table) <= 1024 && can(regex("^[A-Za-z_][A-Za-z0-9_]*$", var.proof_table)) + error_message = "proof_table must be a valid BigQuery table ID." + } +} + +variable "cpu_architecture" { + description = "Fargate architecture selected from the copied OCI index." + type = string + default = "ARM64" + + validation { + condition = contains(["ARM64", "X86_64"], var.cpu_architecture) + error_message = "cpu_architecture must be ARM64 or X86_64." + } +} diff --git a/acceptance/cloud-portability/phase1b/smoke/versions.tf b/acceptance/cloud-portability/phase1b/smoke/versions.tf new file mode 100644 index 00000000..b550cf52 --- /dev/null +++ b/acceptance/cloud-portability/phase1b/smoke/versions.tf @@ -0,0 +1,29 @@ +terraform { + required_version = ">= 1.9.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0, < 7.0" + } + google = { + source = "hashicorp/google" + version = ">= 6.0, < 8.0" + } + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + managed-by = "dander" + purpose = "cloud-portability-phase1b" + } + } +} + +provider "google" { + project = var.gcp_project_id +} diff --git a/docs/decisions.md b/docs/decisions.md index d2fc2607..7be5f810 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -579,3 +579,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - The governed fact excludes deleted Opportunities and Accounts while retaining inactive owner dimensions. Existing installations may keep the `salesforce_accounts` pipeline resource ID; new examples use `salesforce_crm`. + +## 2026-08-07 — Cross-cloud feasibility uses one copied multi-platform artifact + +- Runtime image publication now requires both `linux/amd64` and `linux/arm64` manifests in one OCI + index. Phase 1B copies that exact index GAR-to-ECR with `crane`; any index or per-platform digest + rewrite fails the gate rather than being treated as equivalent packaging. +- The proof Fargate task has a pull/log execution role and a distinct policy-free task role. Google + trusts only the exact assumed task-role ARN through Workload Identity Federation and permits it + to impersonate a disposable service account with BigQuery job access and read access to one + proof dataset. +- A proof-only, source-free probe observes a 600-second impersonated Google credential expire and + refresh in one process. This does not make Fargate a supported launcher; that remains Phase 3. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 1df0eed7..66aba3b3 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -3,6 +3,10 @@ Dander `0.7.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. +- AWS ECS/Fargate is not yet a supported Dander launcher. Phase 1B contains an isolated artifact + and keyless-identity feasibility stack only; launcher lifecycle, scheduling, alerts, rollback, + and parity remain gated on the later portable BigQuery vertical slice. + ## Ingestion and schemas - Hosted SCD1 and sandbox replacement consume bounded batches. Direct SCD2, snapshot, diff --git a/docs/oci-artifact-contract.md b/docs/oci-artifact-contract.md index aa0b551d..8a515afb 100644 --- a/docs/oci-artifact-contract.md +++ b/docs/oci-artifact-contract.md @@ -10,10 +10,14 @@ Published images carry standard OCI source, documentation, license, version, rev annotations. The packaged `runtime-capabilities.json` describes the adapters present in that build; `dander runtime inspect` validates and reports it without accessing a cloud provider or secret. -`dander image-publish` builds through BuildKit with SBOM and provenance enabled, pushes the image, -resolves the registry digest, inspects the resulting OCI index, and only then writes the artifact -record atomically. If post-push verification fails, the command reports that the image may exist but -does not report a successful artifact publication. +`dander image-publish` builds one `linux/amd64,linux/arm64` index through BuildKit with SBOM and +provenance enabled, pushes the image, resolves the registry digest, inspects both required runnable +manifests, and only then writes the artifact record atomically. If post-push verification fails, the +command reports that the image may exist but does not report a successful artifact publication. + +Cross-registry promotion copies the digest-qualified index without rebuilding it. A promotion is +valid only when the destination resolves to the same index digest and reports the same runnable +platform-manifest map; registries that rewrite the index fail verification. The runtime executes as UID/GID `65532:65532`. CI runs the local conformance probe with a read-only root filesystem and a temporary `/tmp`, so runtime code must declare durable state through an diff --git a/scripts/portability/__init__.py b/scripts/portability/__init__.py new file mode 100644 index 00000000..1e8dc8ca --- /dev/null +++ b/scripts/portability/__init__.py @@ -0,0 +1 @@ +"""Repeatable tooling for cloud-portability acceptance proofs.""" diff --git a/scripts/portability/oci_copy.py b/scripts/portability/oci_copy.py new file mode 100644 index 00000000..97aac1cf --- /dev/null +++ b/scripts/portability/oci_copy.py @@ -0,0 +1,169 @@ +"""Copy one OCI index between registries and prove that no rebuild occurred.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from collections.abc import Sequence + +_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_DIGEST_REFERENCE = re.compile(r"^(?P[^\s@]+)@(?Psha256:[0-9a-f]{64})$") +_TAG_REFERENCE = re.compile(r"^[^\s@:]+(?:[.:][^\s@/:]+)*(?:/[^\s@/:]+)+:[^\s@/:]+$") +_REQUIRED_PLATFORMS = ("linux/amd64", "linux/arm64") +_RECORD_SCHEMA = "io.dander.portability.oci-copy/v1" + + +class CopyVerificationError(RuntimeError): + """Raised when an OCI copy cannot be proven content-identical.""" + + +class Runner(Protocol): + def __call__(self, command: Sequence[str]) -> bytes: + """Run one command and return stdout bytes.""" + + +def _run(command: Sequence[str]) -> bytes: + try: + result = subprocess.run( + tuple(command), + check=True, + capture_output=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError) as error: + executable = command[0] if command else "registry tool" + raise CopyVerificationError(f"{executable} could not complete the OCI copy") from error + return result.stdout + + +def _read_digest(reference: str, runner: Runner) -> str: + digest = runner(("crane", "digest", reference)).decode("utf-8").strip() + if not _DIGEST.fullmatch(digest): + raise CopyVerificationError(f"Registry returned an invalid digest for {reference!r}") + return digest + + +def _platform_manifests(raw: bytes) -> dict[str, str]: + try: + document = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise CopyVerificationError("Registry returned an invalid OCI manifest") from error + if not isinstance(document, dict) or not isinstance(document.get("manifests"), list): + raise CopyVerificationError("Expected a multi-platform OCI index") + + platforms: dict[str, str] = {} + for descriptor in document["manifests"]: + if not isinstance(descriptor, dict) or not isinstance(descriptor.get("platform"), dict): + raise CopyVerificationError("OCI index contains an invalid descriptor") + platform = descriptor["platform"] + operating_system = platform.get("os") + architecture = platform.get("architecture") + if operating_system == "unknown" and architecture == "unknown": + continue + digest = descriptor.get("digest") + if ( + not isinstance(operating_system, str) + or not isinstance(architecture, str) + or not isinstance(digest, str) + or not _DIGEST.fullmatch(digest) + ): + raise CopyVerificationError("OCI index contains an invalid platform manifest") + variant = platform.get("variant") + key = f"{operating_system}/{architecture}" + if isinstance(variant, str) and variant: + key = f"{key}/{variant}" + if key in platforms: + raise CopyVerificationError(f"OCI index repeats platform {key!r}") + platforms[key] = digest + + missing = tuple( + required + for required in _REQUIRED_PLATFORMS + if not any(actual == required or actual.startswith(f"{required}/") for actual in platforms) + ) + if missing: + raise CopyVerificationError( + "OCI index is missing required platform manifests: " + ", ".join(missing) + ) + return dict(sorted(platforms.items())) + + +def copy_and_verify( + *, + source: str, + destination: str, + runner: Runner = _run, +) -> dict[str, object]: + """Copy ``source`` to ``destination`` and return a sanitized verification record.""" + source_match = _DIGEST_REFERENCE.fullmatch(source) + if source_match is None: + raise CopyVerificationError("Source must be an immutable digest-qualified image reference") + if _TAG_REFERENCE.fullmatch(destination) is None: + raise CopyVerificationError("Destination must be a tagged registry image reference") + + expected_digest = source_match.group("digest") + source_digest = _read_digest(source, runner) + if source_digest != expected_digest: + raise CopyVerificationError("Source registry digest does not match the requested image") + source_manifest = runner(("crane", "manifest", source)) + source_platforms = _platform_manifests(source_manifest) + + runner(("crane", "copy", source, destination)) + + destination_digest = _read_digest(destination, runner) + destination_manifest = runner(("crane", "manifest", destination)) + destination_platforms = _platform_manifests(destination_manifest) + if destination_digest != source_digest: + raise CopyVerificationError("Destination registry rewrote the OCI index digest") + if hashlib.sha256(destination_manifest).digest() != hashlib.sha256(source_manifest).digest(): + raise CopyVerificationError("Destination registry changed the OCI index document") + if destination_platforms != source_platforms: + raise CopyVerificationError("Destination platform manifests differ from the source") + + return { + "schema": _RECORD_SCHEMA, + "source": source, + "destination": destination.rsplit(":", 1)[0] + f"@{destination_digest}", + "index_digest": source_digest, + "platform_manifests": source_platforms, + "source_manifest_sha256": hashlib.sha256(source_manifest).hexdigest(), + "destination_manifest_sha256": hashlib.sha256(destination_manifest).hexdigest(), + "copied_without_rebuild": True, + } + + +def _write_record(path: Path, record: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + try: + temporary.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", required=True) + parser.add_argument("--destination", required=True) + parser.add_argument("--record", required=True, type=Path) + args = parser.parse_args() + try: + record = copy_and_verify(source=args.source, destination=args.destination) + _write_record(args.record, record) + except CopyVerificationError as error: + print(f"OCI copy verification failed: {error}") + return 1 + print(json.dumps(record, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/portability/prepare_phase1b_context.py b/scripts/portability/prepare_phase1b_context.py new file mode 100644 index 00000000..369427ab --- /dev/null +++ b/scripts/portability/prepare_phase1b_context.py @@ -0,0 +1,124 @@ +"""Add only the Phase 1B probe and non-secret external-account config to a generated project.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +from pathlib import Path + + +class ContextPreparationError(RuntimeError): + """Raised when the source-free proof context is invalid.""" + + +def _atomic_text(path: Path, content: str) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + try: + temporary.write_text(content, encoding="utf-8") + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _credential_config(path: Path, *, token_lifetime_seconds: int) -> dict[str, object]: + try: + config = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ContextPreparationError("Credential configuration is not valid JSON") from error + if not isinstance(config, dict) or config.get("type") != "external_account": + raise ContextPreparationError("Phase 1B requires an external-account credential config") + source = config.get("credential_source") + if not isinstance(source, dict) or source.get("environment_id") != "aws1": + raise ContextPreparationError("Credential configuration is not for an AWS workload") + if "private_key" in config or "client_secret" in config: + raise ContextPreparationError("Credential configuration contains a prohibited secret") + impersonation_url = config.get("service_account_impersonation_url") + if not isinstance(impersonation_url, str) or not impersonation_url.startswith( + "https://iamcredentials.googleapis.com/" + ): + raise ContextPreparationError("Credential configuration must impersonate a service account") + config["service_account_impersonation_options"] = { + "token_lifetime_seconds": token_lifetime_seconds + } + return config + + +def prepare_context( + *, + project_dir: Path, + credential_config: Path, + probe_script: Path, + token_lifetime_seconds: int = 600, +) -> None: + """Prepare an ephemeral generated project for the Phase 1B image build.""" + if (project_dir / "src").exists(): + raise ContextPreparationError("Phase 1B requires a generated source-free project") + dockerfile = project_dir / "Dockerfile" + dockerignore = project_dir / ".dockerignore" + required = (dockerfile, dockerignore, project_dir / "dander.yaml", probe_script) + if any(not path.is_file() for path in required): + raise ContextPreparationError("Generated project or proof tooling is incomplete") + if not 300 <= token_lifetime_seconds <= 900: + raise ContextPreparationError("Proof token lifetime must be between 300 and 900 seconds") + + config = _credential_config( + credential_config, + token_lifetime_seconds=token_lifetime_seconds, + ) + shutil.copyfile(probe_script, project_dir / "phase1b_probe.py") + _atomic_text( + project_dir / "gcp-wif.json", + json.dumps(config, indent=2, sort_keys=True) + "\n", + ) + + dockerfile_content = dockerfile.read_text(encoding="utf-8") + marker = "\nUSER 65532:65532\n" + if marker not in dockerfile_content: + raise ContextPreparationError("Generated Dockerfile has no non-root user boundary") + additions = ( + "\nCOPY --chown=65532:65532 phase1b_probe.py ./phase1b_probe.py\n" + "COPY --chown=65532:65532 gcp-wif.json ./gcp-wif.json\n" + ) + if "COPY --chown=65532:65532 phase1b_probe.py ./phase1b_probe.py" not in dockerfile_content: + dockerfile_content = dockerfile_content.replace(marker, additions + marker, 1) + _atomic_text(dockerfile, dockerfile_content) + + ignore_lines = dockerignore.read_text(encoding="utf-8").rstrip().splitlines() + for line in ("!phase1b_probe.py", "!gcp-wif.json"): + if line not in ignore_lines: + ignore_lines.append(line) + _atomic_text( + dockerignore, + "\n".join(ignore_lines) + "\n", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-dir", required=True, type=Path) + parser.add_argument("--credential-config", required=True, type=Path) + parser.add_argument( + "--probe-script", + type=Path, + default=Path(__file__).with_name("wif_bigquery_probe.py"), + ) + parser.add_argument("--token-lifetime-seconds", type=int, default=600) + args = parser.parse_args() + try: + prepare_context( + project_dir=args.project_dir.resolve(), + credential_config=args.credential_config.resolve(), + probe_script=args.probe_script.resolve(), + token_lifetime_seconds=args.token_lifetime_seconds, + ) + except ContextPreparationError as error: + print(f"Phase 1B context preparation failed: {error}") + return 1 + print(f"Prepared source-free Phase 1B context: {args.project_dir.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/portability/scan_long_lived_credentials.py b/scripts/portability/scan_long_lived_credentials.py new file mode 100644 index 00000000..2aa760ec --- /dev/null +++ b/scripts/portability/scan_long_lived_credentials.py @@ -0,0 +1,62 @@ +"""Fail when Phase 1B proof artifacts contain recognizable long-lived cloud credentials.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +_PATTERNS = { + "aws_static_access_key": re.compile(rb"\bAKIA[0-9A-Z]{16}\b"), + "aws_temporary_access_key": re.compile(rb"\bASIA[0-9A-Z]{16}\b"), + "aws_secret_key_value": re.compile( + rb"(?i)(?:aws_)?secret_access_key\s*[\"']?\s*[:=]\s*[\"'][A-Za-z0-9/+=]{40}[\"']" + ), + "gcp_private_key": re.compile( + rb"-----BEGIN (?:RSA |EC )?PRIVATE KEY-----" + rb"(?:\\n|\r?\n)" + rb"(?:[A-Za-z0-9+/=]{20,}(?:\\n|\r?\n)){2,}" + rb"-----END (?:RSA |EC )?PRIVATE KEY-----" + ), +} + + +def scan(paths: list[Path]) -> list[dict[str, str]]: + """Return sanitized path/pattern findings without returning matched material.""" + findings: list[dict[str, str]] = [] + files: list[Path] = [] + for path in paths: + if path.is_dir(): + files.extend(candidate for candidate in path.rglob("*") if candidate.is_file()) + elif path.is_file(): + files.append(path) + for path in sorted(set(files)): + try: + content = path.read_bytes() + except OSError: + findings.append({"path": str(path), "pattern": "unreadable"}) + continue + for name, pattern in _PATTERNS.items(): + if pattern.search(content): + findings.append({"path": str(path), "pattern": name}) + return findings + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="+", type=Path) + args = parser.parse_args() + findings = scan([path.resolve() for path in args.paths]) + print( + json.dumps( + {"schema": "io.dander.portability.credential-scan/v1", "findings": findings}, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 1 if findings else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/portability/wif_bigquery_probe.py b/scripts/portability/wif_bigquery_probe.py new file mode 100644 index 00000000..c3ed3b5e --- /dev/null +++ b/scripts/portability/wif_bigquery_probe.py @@ -0,0 +1,169 @@ +"""Prove that an AWS task can refresh keyless Google credentials during one process.""" + +from __future__ import annotations + +import argparse +import json +import re +import time +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Protocol + +import google.auth +from google.cloud import bigquery + +if TYPE_CHECKING: + from collections.abc import Callable + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,1023}$") +_PROJECT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) +_SCHEMA = "io.dander.portability.identity-probe/v1" + + +class ProbeError(RuntimeError): + """Raised when the live identity-refresh proof does not satisfy its contract.""" + + +class ExpiringCredentials(Protocol): + expiry: datetime | None + + +class QueryClient(Protocol): + def query_count(self, *, project: str, dataset: str, table: str) -> int: + """Return a bounded count from the proof relation.""" + + +class BigQueryProbeClient: + """Small adapter that keeps the exact proof query out of logs and task configuration.""" + + def __init__(self, client: bigquery.Client) -> None: + self._client = client + + def query_count(self, *, project: str, dataset: str, table: str) -> int: + if not _PROJECT_ID.fullmatch(project) or not all( + _IDENTIFIER.fullmatch(value) for value in (dataset, table) + ): + raise ProbeError("BigQuery proof target is invalid") + query = f"SELECT COUNT(*) AS row_count FROM `{project}.{dataset}.{table}`" + row = next(iter(self._client.query(query).result(max_results=1)), None) + if row is None: + raise ProbeError("BigQuery proof query returned no result") + value = row[0] + if not isinstance(value, int) or value < 0: + raise ProbeError("BigQuery proof query returned an invalid count") + return value + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _expiry(credentials: ExpiringCredentials) -> datetime: + if credentials.expiry is None: + raise ProbeError("Google credentials did not expose an expiry") + return _utc(credentials.expiry) + + +def run_probe( + *, + credentials: ExpiringCredentials, + client: QueryClient, + project: str, + dataset: str, + table: str, + max_wait_seconds: int, + refresh_margin_seconds: int, + now: Callable[[], datetime] = lambda: datetime.now(UTC), + sleep: Callable[[float], None] = time.sleep, + emit: Callable[[dict[str, object]], None], +) -> None: + """Query before and after expiry, requiring the same credentials object to refresh.""" + first_count = client.query_count(project=project, dataset=dataset, table=table) + first_expiry = _expiry(credentials) + first_time = _utc(now()) + wait_seconds = max(0.0, (first_expiry - first_time).total_seconds()) + refresh_margin_seconds + if wait_seconds > max_wait_seconds: + raise ProbeError("Issued credential lifetime exceeds the bounded proof window") + emit( + { + "schema": _SCHEMA, + "event": "query.completed", + "sequence": 1, + "row_count": first_count, + "credential_expiry": first_expiry.isoformat(), + } + ) + + sleep(wait_seconds) + + second_count = client.query_count(project=project, dataset=dataset, table=table) + second_expiry = _expiry(credentials) + if second_expiry <= first_expiry: + raise ProbeError("Google credentials did not refresh after expiry") + emit( + { + "schema": _SCHEMA, + "event": "query.completed", + "sequence": 2, + "row_count": second_count, + "credential_expiry": second_expiry.isoformat(), + } + ) + emit( + { + "schema": _SCHEMA, + "event": "credential.refresh_observed", + "previous_expiry": first_expiry.isoformat(), + "refreshed_expiry": second_expiry.isoformat(), + } + ) + + +def _emit(record: dict[str, object]) -> None: + print(json.dumps(record, sort_keys=True, separators=(",", ":")), flush=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", required=True) + parser.add_argument("--dataset", required=True) + parser.add_argument("--table", required=True) + parser.add_argument("--max-wait-seconds", type=int, default=900) + parser.add_argument("--refresh-margin-seconds", type=int, default=15) + args = parser.parse_args() + if not _PROJECT_ID.fullmatch(args.project) or not all( + _IDENTIFIER.fullmatch(value) for value in (args.dataset, args.table) + ): + _emit({"schema": _SCHEMA, "event": "probe.failed", "failure_code": "invalid_target"}) + return 2 + try: + credentials, _ = google.auth.default(scopes=_SCOPES) + client = BigQueryProbeClient(bigquery.Client(project=args.project, credentials=credentials)) + run_probe( + credentials=credentials, + client=client, + project=args.project, + dataset=args.dataset, + table=args.table, + max_wait_seconds=args.max_wait_seconds, + refresh_margin_seconds=args.refresh_margin_seconds, + emit=_emit, + ) + except Exception as error: # noqa: BLE001 - the probe must sanitize every library failure + _emit( + { + "schema": _SCHEMA, + "event": "probe.failed", + "failure_code": "identity_or_query_failed", + "failure_type": type(error).__name__, + } + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/dander/bootstrap/project.py b/src/dander/bootstrap/project.py index 740e0773..76bce959 100644 --- a/src/dander/bootstrap/project.py +++ b/src/dander/bootstrap/project.py @@ -9,11 +9,11 @@ import subprocess import time from datetime import UTC, datetime +from pathlib import Path from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: from collections.abc import Callable - from pathlib import Path _PROJECT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") _BUCKET_NAME = re.compile(r"^[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]$") @@ -23,6 +23,7 @@ _PLATFORM_PART = re.compile(r"^[a-z0-9][a-z0-9_.-]*$") _ARTIFACT_SCHEMA = "io.dander.runtime.artifact/v1" _SOURCE_URL = "https://github.com/harrisonoconnorhover/dander" +_RUNTIME_PLATFORMS = ("linux/amd64", "linux/arm64") class ProjectBootstrapError(RuntimeError): @@ -236,7 +237,7 @@ def publish( "buildx", "build", "--platform", - "linux/amd64", + ",".join(_RUNTIME_PLATFORMS), "--build-arg", f"DANDER_BUILD_REVISION={revision}", "--build-arg", @@ -287,6 +288,18 @@ def publish( text=True, ) platforms = _platform_manifests(inspected.stdout, default_digest=digest) + missing_platforms = tuple( + required + for required in _RUNTIME_PLATFORMS + if not any( + actual == required or actual.startswith(f"{required}/") for actual in platforms + ) + ) + if missing_platforms: + raise ProjectBootstrapError( + "Runtime image index is missing required platform manifests: " + + ", ".join(missing_platforms) + ) record_path = self._repository_dir / ".dander" / "runtime-artifact.json" _write_artifact_record( record_path, @@ -343,12 +356,13 @@ def _content_digest(self) -> str: ) if (self._repository_dir / name).is_file() ] + files.extend(_explicit_docker_context_files(self._repository_dir)) for root in roots: files.extend( path for path in root.rglob("*") if path.is_file() and _is_build_context_file(path) ) try: - for path in sorted(files): + for path in sorted(set(files)): hasher.update(str(path.relative_to(self._repository_dir)).encode()) hasher.update(path.read_bytes()) except OSError as error: @@ -368,6 +382,29 @@ def _is_build_context_file(path: Path) -> bool: ) +def _explicit_docker_context_files(repository_dir: Path) -> tuple[Path, ...]: + dockerignore = repository_dir / ".dockerignore" + if not dockerignore.is_file(): + return () + try: + lines = dockerignore.read_text(encoding="utf-8").splitlines() + except OSError as error: + raise ProjectBootstrapError("Could not read the Docker build-context rules") from error + files: list[Path] = [] + for raw in lines: + rule = raw.strip() + if not rule.startswith("!") or any(character in rule for character in "*?["): + continue + relative = rule[1:].lstrip("/") + path = Path(relative) + if not relative or relative.endswith("/") or ".." in path.parts: + continue + candidate = repository_dir / path + if candidate.is_file() and _is_build_context_file(candidate): + files.append(candidate) + return tuple(files) + + def _platform_manifests(raw: str, *, default_digest: str) -> dict[str, str]: document = json.loads(raw) if not isinstance(document, dict): diff --git a/tests/bootstrap/test_project_bootstrap.py b/tests/bootstrap/test_project_bootstrap.py index 4a7f30cc..e7c69c24 100644 --- a/tests/bootstrap/test_project_bootstrap.py +++ b/tests/bootstrap/test_project_bootstrap.py @@ -56,6 +56,14 @@ def __call__( "digest": "sha256:" + "b" * 64, "platform": {"os": "linux", "architecture": "amd64"}, }, + { + "digest": "sha256:" + "d" * 64, + "platform": { + "os": "linux", + "architecture": "arm64", + "variant": "v8", + }, + }, { "digest": "sha256:" + "c" * 64, "platform": {"os": "unknown", "architecture": "unknown"}, @@ -109,7 +117,7 @@ def test_runtime_image_publisher_returns_an_immutable_digest(tmp_path: Path) -> build = next( command for command in runner.commands if command[:3] == ("docker", "buildx", "build") ) - assert "--platform" in build and "linux/amd64" in build and "--push" in build + assert "--platform" in build and "linux/amd64,linux/arm64" in build and "--push" in build assert "--sbom=true" in build assert "--provenance=mode=max" in build assert any(part.startswith("DANDER_BUILD_REVISION=") for part in build) @@ -119,7 +127,10 @@ def test_runtime_image_publisher_returns_an_immutable_digest(tmp_path: Path) -> assert record["schema"] == "io.dander.runtime.artifact/v1" assert record["image"] == image assert record["index_digest"] == "sha256:" + "a" * 64 - assert record["platform_manifests"] == {"linux/amd64": "sha256:" + "b" * 64} + assert record["platform_manifests"] == { + "linux/amd64": "sha256:" + "b" * 64, + "linux/arm64/v8": "sha256:" + "d" * 64, + } assert record["sbom_attached"] is True assert record["provenance_attached"] is True @@ -140,6 +151,58 @@ def test_runtime_image_publisher_accepts_installed_project_context(tmp_path: Pat assert image.endswith("@sha256:" + "a" * 64) +def test_runtime_image_publisher_rejects_an_incomplete_platform_index(tmp_path: Path) -> None: + for name in ("Dockerfile", "README.md", "dander.yaml"): + (tmp_path / name).write_text(name, encoding="utf-8") + for directory in ("connectors", "models"): + path = tmp_path / directory + path.mkdir() + (path / "content.txt").write_text(directory, encoding="utf-8") + + class _IncompleteIndexRunner(_Runner): + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: + result = super().__call__( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + input=input, + ) + if args[:4] != ("docker", "buildx", "imagetools", "inspect"): + return result + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "schemaVersion": 2, + "manifests": [ + { + "digest": "sha256:" + "b" * 64, + "platform": {"os": "linux", "architecture": "amd64"}, + } + ], + } + ), + ) + + with pytest.raises(ProjectBootstrapError, match="missing required.*linux/arm64"): + RuntimeImagePublisher(tmp_path, runner=_IncompleteIndexRunner()).publish( + project="unit-project", + region="us-central1", + ) + + def test_runtime_image_publisher_uses_bootstrap_impersonation_without_token_in_args( tmp_path: Path, ) -> None: @@ -231,6 +294,34 @@ def test_runtime_image_tag_tracks_graph_content(tmp_path: Path) -> None: assert all(":candidate-" in tag for tag in tags) +def test_runtime_image_tag_tracks_explicit_extra_build_context_files(tmp_path: Path) -> None: + for name in ("Dockerfile", "dander.yaml"): + (tmp_path / name).write_text(name, encoding="utf-8") + for directory in ("connectors", "models"): + path = tmp_path / directory + path.mkdir() + (path / "content.txt").write_text(directory, encoding="utf-8") + (tmp_path / ".dockerignore").write_text( + "*\n!Dockerfile\n!dander.yaml\n!connectors/\n!models/\n!phase1b_probe.py\n", + encoding="utf-8", + ) + probe = tmp_path / "phase1b_probe.py" + probe.write_text("print('before')\n", encoding="utf-8") + runner = _Runner() + publisher = RuntimeImagePublisher(tmp_path, runner=runner) + + publisher.publish(project="unit-project", region="us-central1", tag_prefix="candidate") + probe.write_text("print('after')\n", encoding="utf-8") + publisher.publish(project="unit-project", region="us-central1", tag_prefix="candidate") + + tags = [ + command[command.index("-t") + 1] + for command in runner.commands + if command[:3] == ("docker", "buildx", "build") + ] + assert tags[0] != tags[1] + + def test_active_admin_member_uses_the_authenticated_gcloud_user(tmp_path: Path) -> None: assert active_admin_member(cwd=tmp_path, runner=_Runner()) == ("user:operator@example.invalid") diff --git a/tests/portability/test_phase1b_tools.py b/tests/portability/test_phase1b_tools.py new file mode 100644 index 00000000..f48b5bbc --- /dev/null +++ b/tests/portability/test_phase1b_tools.py @@ -0,0 +1,276 @@ +"""Focused tests for the live Phase 1B artifact and identity proof tooling.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +import pytest +from scripts.portability.oci_copy import CopyVerificationError, copy_and_verify +from scripts.portability.prepare_phase1b_context import ContextPreparationError, prepare_context +from scripts.portability.scan_long_lived_credentials import scan +from scripts.portability.wif_bigquery_probe import ProbeError, run_probe + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + +_INDEX_DIGEST = "sha256:" + "a" * 64 +_AMD64_DIGEST = "sha256:" + "b" * 64 +_ARM64_DIGEST = "sha256:" + "c" * 64 + + +def _manifest() -> bytes: + return json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "digest": _AMD64_DIGEST, + "platform": {"os": "linux", "architecture": "amd64"}, + }, + { + "digest": _ARM64_DIGEST, + "platform": { + "os": "linux", + "architecture": "arm64", + "variant": "v8", + }, + }, + { + "digest": "sha256:" + "d" * 64, + "platform": {"os": "unknown", "architecture": "unknown"}, + }, + ], + }, + separators=(",", ":"), + ).encode() + + +class _RegistryRunner: + def __init__( + self, + *, + destination_digest: str = _INDEX_DIGEST, + destination_manifest: bytes | None = None, + ) -> None: + self.destination_digest = destination_digest + self.destination_manifest = destination_manifest + self.commands: list[tuple[str, ...]] = [] + + def __call__(self, command: Sequence[str]) -> bytes: + args = tuple(command) + self.commands.append(args) + if args[:2] == ("crane", "copy"): + return b"" + if args[:2] == ("crane", "manifest"): + if "ecr" in args[2] and self.destination_manifest is not None: + return self.destination_manifest + return _manifest() + if args[:2] == ("crane", "digest"): + return (self.destination_digest if "ecr" in args[2] else _INDEX_DIGEST).encode() + raise AssertionError(args) + + +def test_oci_copy_preserves_index_and_platform_digests() -> None: + source = f"us-central1-docker.pkg.dev/unit/dander/dander@{_INDEX_DIGEST}" + destination = "123456789012.dkr.ecr.us-east-1.amazonaws.com/dander:phase1b" + runner = _RegistryRunner() + + record = copy_and_verify(source=source, destination=destination, runner=runner) + + assert record["index_digest"] == _INDEX_DIGEST + assert record["destination"] == destination.rsplit(":", 1)[0] + f"@{_INDEX_DIGEST}" + assert record["platform_manifests"] == { + "linux/amd64": _AMD64_DIGEST, + "linux/arm64/v8": _ARM64_DIGEST, + } + assert record["copied_without_rebuild"] is True + assert runner.commands[2] == ("crane", "copy", source, destination) + assert all("build" not in command for command in runner.commands) + + +def test_oci_copy_fails_when_destination_rewrites_the_index() -> None: + with pytest.raises(CopyVerificationError, match="rewrote"): + copy_and_verify( + source=f"gar.example.invalid/dander@{_INDEX_DIGEST}", + destination="ecr.example.invalid/dander:phase1b", + runner=_RegistryRunner(destination_digest="sha256:" + "e" * 64), + ) + + +def test_oci_copy_fails_when_destination_changes_the_index_document() -> None: + changed = _manifest().replace(b'"schemaVersion":2', b'"schemaVersion": 2') + with pytest.raises(CopyVerificationError, match="changed the OCI index"): + copy_and_verify( + source=f"gar.example.invalid/dander@{_INDEX_DIGEST}", + destination="ecr.example.invalid/dander:phase1b", + runner=_RegistryRunner(destination_manifest=changed), + ) + + +class _Credentials: + def __init__(self, expiry: datetime) -> None: + self.expiry: datetime | None = expiry + + +class _QueryClient: + def __init__(self, credentials: _Credentials) -> None: + self.credentials = credentials + self.calls = 0 + + def query_count(self, *, project: str, dataset: str, table: str) -> int: + assert (project, dataset, table) == ("unit-project", "raw", "records") + self.calls += 1 + if self.calls == 2: + assert self.credentials.expiry is not None + self.credentials.expiry += timedelta(minutes=10) + return 7 + + +def test_wif_probe_observes_expiry_refresh_without_logging_tokens() -> None: + started = datetime(2026, 8, 7, 12, 0, tzinfo=UTC) + credentials = _Credentials(started + timedelta(minutes=10)) + client = _QueryClient(credentials) + records: list[dict[str, object]] = [] + waits: list[float] = [] + + run_probe( + credentials=credentials, + client=client, + project="unit-project", + dataset="raw", + table="records", + max_wait_seconds=700, + refresh_margin_seconds=5, + now=lambda: started, + sleep=waits.append, + emit=records.append, + ) + + assert waits == [605.0] + assert [record["event"] for record in records] == [ + "query.completed", + "query.completed", + "credential.refresh_observed", + ] + assert "token" not in json.dumps(records).lower() + assert client.calls == 2 + + +def test_wif_probe_rejects_an_unbounded_credential_lifetime() -> None: + started = datetime(2026, 8, 7, 12, 0, tzinfo=UTC) + credentials = _Credentials(started + timedelta(hours=1)) + with pytest.raises(ProbeError, match="bounded proof window"): + run_probe( + credentials=credentials, + client=_QueryClient(credentials), + project="unit-project", + dataset="raw", + table="records", + max_wait_seconds=900, + refresh_margin_seconds=5, + now=lambda: started, + sleep=lambda _seconds: None, + emit=lambda _record: None, + ) + + +def _write_generated_project(path: Path) -> None: + path.mkdir() + (path / "Dockerfile").write_text( + "FROM python:3.12-slim\n\nUSER 65532:65532\n", + encoding="utf-8", + ) + (path / ".dockerignore").write_text("*\n!Dockerfile\n", encoding="utf-8") + (path / "dander.yaml").write_text("version: 1\n", encoding="utf-8") + + +def _write_aws_config(path: Path) -> None: + path.write_text( + json.dumps( + { + "type": "external_account", + "audience": ( + "//iam.googleapis.com/projects/123/locations/global/pools/p/providers/a" + ), + "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", + "token_url": "https://sts.googleapis.com/v1/token", + "service_account_impersonation_url": ( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" + "probe@unit.iam.gserviceaccount.com:generateAccessToken" + ), + "credential_source": {"environment_id": "aws1"}, + } + ), + encoding="utf-8", + ) + + +def test_prepare_context_keeps_the_project_source_free_and_bounds_token_lifetime( + tmp_path: Path, +) -> None: + project = tmp_path / "project" + _write_generated_project(project) + config = tmp_path / "external.json" + _write_aws_config(config) + probe = tmp_path / "probe.py" + probe.write_text("print('probe')\n", encoding="utf-8") + + prepare_context( + project_dir=project, + credential_config=config, + probe_script=probe, + ) + prepare_context( + project_dir=project, + credential_config=config, + probe_script=probe, + ) + + assert not (project / "src").exists() + rendered = json.loads((project / "gcp-wif.json").read_text(encoding="utf-8")) + assert rendered["type"] == "external_account" + assert rendered["service_account_impersonation_options"] == {"token_lifetime_seconds": 600} + assert (project / "Dockerfile").read_text(encoding="utf-8").count( + "COPY --chown=65532:65532 phase1b_probe.py" + ) == 1 + assert (project / ".dockerignore").read_text(encoding="utf-8").count("!gcp-wif.json") == 1 + + +def test_prepare_context_rejects_a_service_account_key(tmp_path: Path) -> None: + project = tmp_path / "project" + _write_generated_project(project) + config = tmp_path / "service-account.json" + config.write_text('{"type":"service_account","private_key":"nope"}', encoding="utf-8") + probe = tmp_path / "probe.py" + probe.write_text("pass\n", encoding="utf-8") + + with pytest.raises(ContextPreparationError, match="external-account"): + prepare_context( + project_dir=project, + credential_config=config, + probe_script=probe, + ) + + +def test_long_lived_credential_scan_reports_only_path_and_pattern(tmp_path: Path) -> None: + safe = tmp_path / "external.json" + safe.write_text('{"type":"external_account"}', encoding="utf-8") + unsafe = tmp_path / "bad.json" + fake_key = ( + "-----BEGIN PRIVATE KEY-----\n" + "A" * 64 + "\n" + "B" * 64 + "\n-----END PRIVATE KEY-----" + ) + unsafe.write_text( + json.dumps({"type": "service_account", "private_key": fake_key}), + encoding="utf-8", + ) + + findings = scan([tmp_path]) + + assert findings == [ + {"path": str(unsafe), "pattern": "gcp_private_key"}, + ] + assert "redacted" not in json.dumps(findings) diff --git a/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md b/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md new file mode 100644 index 00000000..87599d23 --- /dev/null +++ b/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md @@ -0,0 +1,36 @@ +# DANDER-77 — Cross-cloud artifact and identity feasibility + +Status: in progress + +## Outcome + +Prove the roadmap's Phase 1B gate before broad provider factories are introduced: publish one +source-free multi-platform OCI index, copy it from GAR to ECR without rebuilding, and run a manual +Fargate BigQuery probe across a real Google credential expiry/refresh using only workload identity. + +## Acceptance criteria + +- [x] `image-publish` builds `linux/amd64` and `linux/arm64` and rejects an incomplete index. +- [x] Digest-copy tooling fails on source mismatch, destination index rewrite, or platform drift. +- [x] The source-free proof context accepts only AWS external-account configuration and bounded + impersonated-token lifetime. +- [x] Isolated Terraform separates ECS execution and task roles, creates no service or schedule, + grants the Google identity only BigQuery job and one-dataset read access, and validates locally. +- [x] Probe and scanner tests cover credential refresh, bounded waits, sanitized output, and + recognizable long-lived cloud credentials. +- [x] One source-free OCI index was built in staging GAR; its recorded `amd64` and `arm64` + manifests both passed the read-only local runtime conformance command. +- [ ] The index and both platform manifests are identical in staging GAR and ECR. +- [ ] Cloud Run selects the recorded `amd64` content and the Fargate task selects a recorded + platform from the same index. +- [ ] A live Fargate task reads BigQuery before and after Google credential refresh. +- [ ] State, task definition, logs, configuration, and image scans find no static cloud key. +- [ ] Both AWS stacks are destroyed, the isolated GCP configuration is restored, and its final + Terraform plan reports `No changes.` + +## Boundaries + +- This is a feasibility proof, not an ECS/Fargate support claim. +- No retained-project mutation, background service, schedule, static key, or provider factory. +- Live evidence is recorded only after the new AWS account is authenticated with short-lived + credentials and every paid apply has a saved, reviewed plan. From c1398b6218a7aabe73070dd69e57a7ffcc44ce39 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 11:20:47 -0400 Subject: [PATCH 025/163] Record Phase 1B Cloud Run parity --- HANDOFF.md | 7 ++++--- ...ER-77-cross-cloud-artifact-identity-feasibility.md | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 664d305d..c269db76 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -6,7 +6,7 @@ - Made source-free image publication require one `linux/amd64,linux/arm64` OCI index. - Added digest-preserving GAR-to-ECR copy verification and a bounded WIF refresh probe. - Added isolated ECR and manual Fargate proof roots with separate execution/task roles. -- Built one source-free GAR index; both `amd64` and `arm64` passed local runtime conformance. +- Proved the same index on Cloud Run as `x86_64`, deleted the temporary job, and restored no drift. ## Try It @@ -20,6 +20,7 @@ saved reviewed plans. No live apply is required for local validation. - Package build, distribution inspection, and clean external wheel install passed. - Corrected GAR index `sha256:6cd545fa…be81f` passed both platform conformance and key scans; no fixable high/critical CVE was found (four unfixed base-image findings remain visible). +- Protected PR checks passed; the isolated GCP final plan reported exactly `No changes.` ## Decisions @@ -30,8 +31,8 @@ saved reviewed plans. No live apply is required for local validation. ## Remaining - Authenticate the new AWS account with short-lived CLI credentials. -- Build/copy the source-free index and run the reviewed live Fargate/WIF refresh proof. -- Run image/config/state/log scans and Cloud Run artifact parity. +- Copy the accepted source-free index and run the reviewed live Fargate/WIF refresh proof. +- Run the AWS state, task-definition, log, configuration, and image scans. - Destroy AWS proof resources and require isolated GCP `No changes.` - Record acceptance, complete protected CI/review, and merge the focused PR. diff --git a/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md b/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md index 87599d23..e0aba7a0 100644 --- a/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md +++ b/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md @@ -28,6 +28,17 @@ Fargate BigQuery probe across a real Google credential expiry/refresh using only - [ ] Both AWS stacks are destroyed, the isolated GCP configuration is restored, and its final Terraform plan reports `No changes.` +## Partial live evidence + +On 2026-08-07, the accepted GAR index +`sha256:6cd545fa8975ed6c49cf38786a1fcfeb96eaa8fa104f156974609d84031be81f` ran in +the isolated project as Cloud Run execution `dander-phase1b-conformance-nwngt`. The container +reported `x86_64`, emitted a successful `io.dander.runtime/v1` conformance result, and exited zero. +The temporary unscheduled job was deleted, both managed schedules remained paused, the evidence +and source-free context contained no recognizable long-lived cloud key, and the final +manifest-aware Terraform plan reported exactly `No changes.` The combined Cloud Run/Fargate and +teardown criteria remain open until the AWS half is complete. + ## Boundaries - This is a feasibility proof, not an ECS/Fargate support claim. From 0a36e187ac296053f39383472fb6f373af48384c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 15:44:52 -0400 Subject: [PATCH 026/163] Complete Phase 1B cross-cloud proof --- CHANGELOG.md | 8 ++ HANDOFF.md | 41 +++++----- .../cloud-portability/phase1b/README.md | 19 +++-- .../phase1b/smoke/terraform.tfvars.example | 4 +- .../phase1b/smoke/variables.tf | 4 +- docs/decisions.md | 6 +- .../portability/prepare_phase1b_context.py | 5 +- scripts/portability/wif_bigquery_probe.py | 75 ++++++++++++++++++- tests/portability/test_phase1b_tools.py | 44 ++++++++++- ...oss-cloud-artifact-identity-feasibility.md | 48 ++++++++---- uv.lock | 6 +- 11 files changed, 203 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1abbf9f9..7d3b4768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ capabilities enter through the next minor release. - Add an isolated, unscheduled Phase 1B proof for digest-preserving GAR-to-ECR copy and keyless Fargate-to-BigQuery credential refresh. This is feasibility evidence, not Fargate support. +### Fixed + +- Adapt Fargate's short-lived ECS task-role credential endpoint to Google Auth's AWS environment + chain in-process, and use Google Auth's supported `service_account_impersonation` field so the + proof token is actually bounded to 600 seconds. +- Point the isolated BigQuery probe defaults at the disposable Salesforce Accounts table used by + the acceptance project; the probe returns only `COUNT(*)` and never record content. + ## 0.7.0 — 2026-08-07 (beta) ### Acceptance diff --git a/HANDOFF.md b/HANDOFF.md index c269db76..360215b2 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,11 +2,11 @@ ## Finished -- Started Phase 1B from stable `v0.7.0` without touching the docs-only roadmap branch. -- Made source-free image publication require one `linux/amd64,linux/arm64` OCI index. -- Added digest-preserving GAR-to-ECR copy verification and a bounded WIF refresh probe. -- Added isolated ECR and manual Fargate proof roots with separate execution/task roles. -- Proved the same index on Cloud Run as `x86_64`, deleted the temporary job, and restored no drift. +- Completed Phase 1B from stable `v0.7.0` without touching the docs-only roadmap branch. +- Proved one source-free AMD64/ARM64 OCI index locally, on Cloud Run, and after byte-identical ECR copy. +- Proved ARM64 Fargate can refresh keyless Google credentials and query bounded BigQuery data twice. +- Corrected Fargate credential sourcing, Google Auth lifetime configuration, and the proof-table default. +- Destroyed every AWS/GCP proof resource and recorded final isolated Terraform `No changes.` ## Try It @@ -15,29 +15,26 @@ saved reviewed plans. No live apply is required for local validation. ## Checks -- All 825 tests, Ruff, and strict mypy passed. -- Both Phase 1B Terraform roots initialized and validated with locked providers. -- Package build, distribution inspection, and clean external wheel install passed. -- Corrected GAR index `sha256:6cd545fa…be81f` passed both platform conformance and key scans; no - fixable high/critical CVE was found (four unfixed base-image findings remain visible). -- Protected PR checks passed; the isolated GCP final plan reported exactly `No changes.` +- All 828 tests, Ruff, strict mypy, and four Terraform validations passed. +- Wheel/sdist build, inspection, outside-checkout installs, project generation, and Terraform validation passed. +- Both image platforms passed runtime conformance and had zero fixable High/Critical findings. +- Dependency and long-lived-credential scans passed; the GitPython lock is updated to fixed `3.1.58`. +- Cloud Run and Fargate exited zero; final AWS inventory was empty and isolated GCP reported no drift. ## Decisions -- A registry digest rewrite fails rather than being accepted as an equivalent image. -- Fargate receives no service or schedule; the proof task role has no AWS permission policy. -- Fargate remains feasibility-only until the later portable BigQuery vertical slice. +- Registry or platform-digest rewrites fail instead of being treated as equivalent packaging. +- ECS task-role credentials remain short-lived, process-only, and separate from pull/log execution access. +- Fargate remains feasibility-only until the portable BigQuery vertical slice. ## Remaining -- Authenticate the new AWS account with short-lived CLI credentials. -- Copy the accepted source-free index and run the reviewed live Fargate/WIF refresh proof. -- Run the AWS state, task-definition, log, configuration, and image scans. -- Destroy AWS proof resources and require isolated GCP `No changes.` -- Record acceptance, complete protected CI/review, and merge the focused PR. +- Review the final Phase 1B diff and protected PR checks. +- Merge focused PR #108 after CI passes. +- Begin Phase 2 only from merged `main` on a separate branch. ## Review First -- `src/dander/bootstrap/project.py` -- `scripts/portability/oci_copy.py` -- `acceptance/cloud-portability/phase1b/smoke/main.tf` +- `scripts/portability/wif_bigquery_probe.py` +- `scripts/portability/prepare_phase1b_context.py` +- `tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md` diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index d33fd567..8b796e7b 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -19,8 +19,9 @@ Terraform state, task descriptions, or logs. - A disposable billing-linked GCP project containing the bounded proof table. - A disposable AWS account. Do not create an AWS access key. -The tested GCP target is project `project-092b24a8-26a3-4438-8cd`, dataset `dander_raw`, table -`greenhouse_job_board_jobs`. The retained Dander project is out of scope. +The tested GCP target is project `project-092b24a8-26a3-4438-8cd`, dataset `raw`, table +`salesforce_accounts`. The probe returns only `COUNT(*)` over disposable test rows; it never logs +record content. The retained Dander project is out of scope. ## Reviewed sequence @@ -47,6 +48,12 @@ The tested GCP target is project `project-092b24a8-26a3-4438-8cd`, dataset `dand --credential-config "$OPERATOR_DIR/external-account.json" ``` + Current Google Auth reads the token lifetime from `service_account_impersonation`; the helper + removes the ignored legacy `service_account_impersonation_options` spelling. On Fargate, the + probe reads only the short-lived ECS task-role document from the fixed link-local credential + origin and exposes it to Google Auth in the current process. It never persists or emits those + values. + 5. From the generated project, use this branch's `dander image-publish` to build one `linux/amd64,linux/arm64` index in staging GAR. Preserve `.dander/runtime-artifact.json` outside the repository. @@ -64,9 +71,11 @@ The tested GCP target is project `project-092b24a8-26a3-4438-8cd`, dataset `dand 7. Set the verified ECR digest in `smoke/terraform.tfvars`, initialize `smoke/`, save and review its plan, then apply only that saved plan. The execution role can pull/log; the separate task role has no AWS permission policy and is the only role trusted by Google WIF. -8. Run exactly one Fargate task using the Terraform outputs for cluster, task definition, subnet, - and security group. Wait for terminal status and save `describe-tasks` plus the CloudWatch log - stream outside the repository. +8. For an accepted candidate, run exactly one Fargate task using the Terraform outputs for + cluster, task definition, subnet, and security group. Wait for terminal status and save + `describe-tasks` plus the CloudWatch log stream outside the repository. Failed candidates are + invalidated, corrected, rebuilt once, and rerun from the beginning; they are not acceptance + evidence. 9. Require two `query.completed` events followed by `credential.refresh_observed`. The second expiry must be later than the first. No event may contain a token, URL, query body, or record. 10. Run `scan_long_lived_credentials.py` over the generated config, both Terraform states, task diff --git a/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example b/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example index 736316ca..8c440f58 100644 --- a/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example +++ b/acceptance/cloud-portability/phase1b/smoke/terraform.tfvars.example @@ -1,6 +1,6 @@ aws_region = "us-east-1" gcp_project_id = "replace-with-disposable-project" ecr_image = "123456789012.dkr.ecr.us-east-1.amazonaws.com/dander-phase1b@sha256:replace" -proof_dataset = "dander_raw" -proof_table = "greenhouse_job_board_jobs" +proof_dataset = "raw" +proof_table = "salesforce_accounts" cpu_architecture = "ARM64" diff --git a/acceptance/cloud-portability/phase1b/smoke/variables.tf b/acceptance/cloud-portability/phase1b/smoke/variables.tf index c21c4659..16a24f29 100644 --- a/acceptance/cloud-portability/phase1b/smoke/variables.tf +++ b/acceptance/cloud-portability/phase1b/smoke/variables.tf @@ -35,7 +35,7 @@ variable "ecr_image" { variable "proof_dataset" { description = "Existing BigQuery dataset the probe may read." type = string - default = "dander_raw" + default = "raw" validation { condition = length(var.proof_dataset) <= 1024 && can(regex("^[A-Za-z_][A-Za-z0-9_]*$", var.proof_dataset)) @@ -46,7 +46,7 @@ variable "proof_dataset" { variable "proof_table" { description = "Existing bounded BigQuery table the probe counts twice." type = string - default = "greenhouse_job_board_jobs" + default = "salesforce_accounts" validation { condition = length(var.proof_table) <= 1024 && can(regex("^[A-Za-z_][A-Za-z0-9_]*$", var.proof_table)) diff --git a/docs/decisions.md b/docs/decisions.md index 7be5f810..d6040494 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -590,4 +590,8 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc to impersonate a disposable service account with BigQuery job access and read access to one proof dataset. - A proof-only, source-free probe observes a 600-second impersonated Google credential expire and - refresh in one process. This does not make Fargate a supported launcher; that remains Phase 3. + refresh in one process. Because Google Auth's AWS supplier does not consume Fargate's ECS + credential endpoint directly, the probe validates that fixed link-local endpoint and exposes + its short-lived task-role values only in the current process. The generated external-account + config uses the library-supported `service_account_impersonation` lifetime field. This does not + make Fargate a supported launcher; that remains Phase 3. diff --git a/scripts/portability/prepare_phase1b_context.py b/scripts/portability/prepare_phase1b_context.py index 369427ab..c40a410d 100644 --- a/scripts/portability/prepare_phase1b_context.py +++ b/scripts/portability/prepare_phase1b_context.py @@ -39,9 +39,8 @@ def _credential_config(path: Path, *, token_lifetime_seconds: int) -> dict[str, "https://iamcredentials.googleapis.com/" ): raise ContextPreparationError("Credential configuration must impersonate a service account") - config["service_account_impersonation_options"] = { - "token_lifetime_seconds": token_lifetime_seconds - } + config.pop("service_account_impersonation_options", None) + config["service_account_impersonation"] = {"token_lifetime_seconds": token_lifetime_seconds} return config diff --git a/scripts/portability/wif_bigquery_probe.py b/scripts/portability/wif_bigquery_probe.py index c3ed3b5e..6ebb5fcc 100644 --- a/scripts/portability/wif_bigquery_probe.py +++ b/scripts/portability/wif_bigquery_probe.py @@ -4,8 +4,11 @@ import argparse import json +import os +import platform import re import time +import urllib.request from datetime import UTC, datetime from typing import TYPE_CHECKING, Protocol @@ -13,12 +16,15 @@ from google.cloud import bigquery if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, MutableMapping _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,1023}$") _PROJECT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") _SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) _SCHEMA = "io.dander.portability.identity-probe/v1" +_ECS_CREDENTIALS_PATH = re.compile(r"^/v2/credentials/[A-Za-z0-9-]{16,128}$") +_ECS_CREDENTIALS_ORIGIN = "http://169.254.170.2" +_TEMPORARY_ACCESS_KEY = re.compile(r"^ASIA[A-Z0-9]{16}$") class ProbeError(RuntimeError): @@ -34,6 +40,62 @@ def query_count(self, *, project: str, dataset: str, table: str) -> int: """Return a bounded count from the proof relation.""" +class CredentialFetcher(Protocol): + def __call__(self, url: str) -> object: + """Return one parsed ECS task-credential response.""" + + +def _fetch_ecs_credentials(url: str) -> object: + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=5) as response: # noqa: S310 + if response.status != 200: + raise ProbeError("ECS task credentials were unavailable") + return json.load(response) + + +def prepare_fargate_task_credentials( + *, + environ: MutableMapping[str, str] = os.environ, + fetch: CredentialFetcher = _fetch_ecs_credentials, +) -> bool: + """Expose short-lived ECS task credentials to Google Auth without persisting them.""" + existing = tuple( + environ.get(name) + for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN") + ) + if all(existing): + return False + if any(existing): + raise ProbeError("AWS credential environment is incomplete") + + relative_uri = environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if relative_uri is None: + return False + if _ECS_CREDENTIALS_PATH.fullmatch(relative_uri) is None: + raise ProbeError("ECS task credential endpoint is invalid") + + document = fetch(f"{_ECS_CREDENTIALS_ORIGIN}{relative_uri}") + if not isinstance(document, dict): + raise ProbeError("ECS task credentials were invalid") + access_key = document.get("AccessKeyId") + secret_key = document.get("SecretAccessKey") + session_token = document.get("Token") + if ( + not isinstance(access_key, str) + or _TEMPORARY_ACCESS_KEY.fullmatch(access_key) is None + or not isinstance(secret_key, str) + or not secret_key + or not isinstance(session_token, str) + or not session_token + ): + raise ProbeError("ECS task credentials were invalid") + + environ["AWS_ACCESS_KEY_ID"] = access_key + environ["AWS_SECRET_ACCESS_KEY"] = secret_key + environ["AWS_SESSION_TOKEN"] = session_token + return True + + class BigQueryProbeClient: """Small adapter that keeps the exact proof query out of logs and task configuration.""" @@ -140,6 +202,17 @@ def main() -> int: _emit({"schema": _SCHEMA, "event": "probe.failed", "failure_code": "invalid_target"}) return 2 try: + task_credentials_loaded = prepare_fargate_task_credentials() + _emit( + { + "schema": _SCHEMA, + "event": "runtime.observed", + "architecture": platform.machine().lower(), + "aws_credential_source": ( + "ecs_task_role" if task_credentials_loaded else "aws_default_chain" + ), + } + ) credentials, _ = google.auth.default(scopes=_SCOPES) client = BigQueryProbeClient(bigquery.Client(project=args.project, credentials=credentials)) run_probe( diff --git a/tests/portability/test_phase1b_tools.py b/tests/portability/test_phase1b_tools.py index f48b5bbc..8f1b1727 100644 --- a/tests/portability/test_phase1b_tools.py +++ b/tests/portability/test_phase1b_tools.py @@ -10,7 +10,11 @@ from scripts.portability.oci_copy import CopyVerificationError, copy_and_verify from scripts.portability.prepare_phase1b_context import ContextPreparationError, prepare_context from scripts.portability.scan_long_lived_credentials import scan -from scripts.portability.wif_bigquery_probe import ProbeError, run_probe +from scripts.portability.wif_bigquery_probe import ( + ProbeError, + prepare_fargate_task_credentials, + run_probe, +) if TYPE_CHECKING: from collections.abc import Sequence @@ -178,6 +182,41 @@ def test_wif_probe_rejects_an_unbounded_credential_lifetime() -> None: ) +def test_fargate_task_credentials_are_short_lived_and_never_emitted() -> None: + relative_uri = "/v2/credentials/12345678-1234-1234-1234-123456789012" + access_key = "ASIA" + "A" * 16 + secret_key = "short-lived-secret" + session_token = "short-lived-session-token" + environment = {"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": relative_uri} + requested_urls: list[str] = [] + + def fetch(url: str) -> object: + requested_urls.append(url) + return { + "AccessKeyId": access_key, + "SecretAccessKey": secret_key, + "Token": session_token, + } + + assert prepare_fargate_task_credentials(environ=environment, fetch=fetch) is True + assert requested_urls == [f"http://169.254.170.2{relative_uri}"] + assert environment["AWS_ACCESS_KEY_ID"] == access_key + assert environment["AWS_SECRET_ACCESS_KEY"] == secret_key + assert environment["AWS_SESSION_TOKEN"] == session_token + + +@pytest.mark.parametrize( + "relative_uri", + ["https://example.invalid/credentials", "/latest/meta-data/iam/security-credentials"], +) +def test_fargate_task_credentials_reject_an_untrusted_endpoint(relative_uri: str) -> None: + with pytest.raises(ProbeError, match="endpoint is invalid"): + prepare_fargate_task_credentials( + environ={"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": relative_uri}, + fetch=lambda _url: pytest.fail("untrusted endpoint must not be fetched"), + ) + + def _write_generated_project(path: Path) -> None: path.mkdir() (path / "Dockerfile").write_text( @@ -233,7 +272,8 @@ def test_prepare_context_keeps_the_project_source_free_and_bounds_token_lifetime assert not (project / "src").exists() rendered = json.loads((project / "gcp-wif.json").read_text(encoding="utf-8")) assert rendered["type"] == "external_account" - assert rendered["service_account_impersonation_options"] == {"token_lifetime_seconds": 600} + assert rendered["service_account_impersonation"] == {"token_lifetime_seconds": 600} + assert "service_account_impersonation_options" not in rendered assert (project / "Dockerfile").read_text(encoding="utf-8").count( "COPY --chown=65532:65532 phase1b_probe.py" ) == 1 diff --git a/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md b/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md index e0aba7a0..2cde733e 100644 --- a/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md +++ b/tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md @@ -1,6 +1,6 @@ # DANDER-77 — Cross-cloud artifact and identity feasibility -Status: in progress +Status: complete ## Outcome @@ -20,24 +20,40 @@ Fargate BigQuery probe across a real Google credential expiry/refresh using only recognizable long-lived cloud credentials. - [x] One source-free OCI index was built in staging GAR; its recorded `amd64` and `arm64` manifests both passed the read-only local runtime conformance command. -- [ ] The index and both platform manifests are identical in staging GAR and ECR. -- [ ] Cloud Run selects the recorded `amd64` content and the Fargate task selects a recorded +- [x] The index and both platform manifests are identical in staging GAR and ECR. +- [x] Cloud Run selects the recorded `amd64` content and the Fargate task selects a recorded platform from the same index. -- [ ] A live Fargate task reads BigQuery before and after Google credential refresh. -- [ ] State, task definition, logs, configuration, and image scans find no static cloud key. -- [ ] Both AWS stacks are destroyed, the isolated GCP configuration is restored, and its final +- [x] A live Fargate task reads BigQuery before and after Google credential refresh. +- [x] State, task definition, logs, configuration, and image scans find no static cloud key. +- [x] Both AWS stacks are destroyed, the isolated GCP configuration is restored, and its final Terraform plan reports `No changes.` -## Partial live evidence - -On 2026-08-07, the accepted GAR index -`sha256:6cd545fa8975ed6c49cf38786a1fcfeb96eaa8fa104f156974609d84031be81f` ran in -the isolated project as Cloud Run execution `dander-phase1b-conformance-nwngt`. The container -reported `x86_64`, emitted a successful `io.dander.runtime/v1` conformance result, and exited zero. -The temporary unscheduled job was deleted, both managed schedules remained paused, the evidence -and source-free context contained no recognizable long-lived cloud key, and the final -manifest-aware Terraform plan reported exactly `No changes.` The combined Cloud Run/Fargate and -teardown criteria remain open until the AWS half is complete. +## Live evidence + +On 2026-08-07, the accepted source-free GAR index was +`sha256:5e026166092ec79d22562781596bc40d7f2e64f89030ac9126000d4614880c82`, with runnable manifests +`sha256:5eb1e1618378c8a9f1671e6746531976cf3225bdf1d8955fa0a4c63e09b192b7` (`linux/amd64`) and +`sha256:4488eecc4fd888379f741553302ef1688a8a3c6537f8b1f59f1c063b3e799fd0` (`linux/arm64`). Both +passed local conformance. Cloud Run execution `dander-phase1b-conformance-b4v57` selected the +recorded AMD64 manifest, reported `x86_64`, completed `io.dander.runtime/v1`, and exited zero. The +temporary unscheduled job was deleted and both managed schedules remained paused. + +The exact index and both runnable manifests copied to private ECR without a registry rewrite. +Fargate task `e3840b6301dd4cce8af5ebce6f7cee40` selected ARM64, reported `aarch64` and +`ecs_task_role`, counted 17 disposable Salesforce Account rows, refreshed its Google credential +from `2026-08-07T19:30:36Z` to `2026-08-07T19:40:51Z`, counted the same 17 rows again, and exited +zero. Its task role had no attached or inline AWS policies; the execution role was limited to the +standard ECS pull/log policy. + +The first live attempts were rejected rather than blessed: the original proof target did not +exist, Google Auth initially tried EC2 metadata instead of Fargate's ECS credential endpoint, and +an ignored impersonation-options spelling produced a 3,600-second token outside the bounded proof +window. Each defect was corrected before building the final candidate. The scanner found no +recognizable long-lived cloud key across generated configuration, Terraform state, task evidence, +logs, or either image filesystem. Reviewed teardown destroyed all 18 smoke resources and both ECR +resources; AWS has no proof cluster, active task definition, log group, role, or repository, and +the disposable Google WIF pool is `DELETED` with its IAM grants removed. The final isolated +manifest-aware Terraform plan reported exactly `No changes.` The retained project was untouched. ## Boundaries diff --git a/uv.lock b/uv.lock index 36a8d2ce..f43d8aa9 100644 --- a/uv.lock +++ b/uv.lock @@ -799,14 +799,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.57" +version = "3.1.58" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/0d/132ed135c871b6bf91adf16a0e43797cd535b81d4973b5d09291c54fc5ee/gitpython-3.1.57.tar.gz", hash = "sha256:c493ec57c0ef6b19743798b6a5af859c71814b524e7e6f97baa2f8e658961488", size = 225898, upload-time = "2026-07-26T07:33:26.351Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/d6/5f358ff283325580c2003a6d953aea18cfe10ae87b46f5ebc80fa3a386dc/gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22", size = 228498, upload-time = "2026-08-04T15:05:49.47Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/6e/2139de986d9c7c3ac86f1f8be43858ce90bdfe2f7175e6c80c650ba15242/gitpython-3.1.57-py3-none-any.whl", hash = "sha256:4ccf7d73c10f5c9e76043fbb2675ac5a1b3ff5b41e648f56bcbed5f63792ecaf", size = 217151, upload-time = "2026-07-26T07:33:24.838Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0c/9d8752098bc442f0726e64aa6135940b3a96809915d1aa4206c1bb97881d/gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f", size = 220183, upload-time = "2026-08-04T15:05:48.025Z" }, ] [[package]] From 6b3154dd241ee6b59b4aa0df9d47319ef84ebff4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 16:22:46 -0400 Subject: [PATCH 027/163] Add named platform profiles and v1 migration --- CHANGELOG.md | 3 + HANDOFF.md | 42 +- README.md | 37 +- docs/cloud-portability-slos.md | 3 +- docs/decisions.md | 14 + docs/getting-started.md | 8 +- docs/platform-profiles.md | 44 ++ docs/upgrading.md | 3 +- src/dander/cli/config_command.py | 101 ++++ src/dander/cli/main.py | 10 +- src/dander/pipeline/graph_deployment.py | 20 +- src/dander/project/__init__.py | 18 + src/dander/project/config.py | 83 ++- src/dander/project/portable_config.py | 494 ++++++++++++++++++ src/dander/templates/project/.dockerignore | 1 + src/dander/templates/project/Dockerfile | 1 + src/dander/templates/project/README.md | 8 +- .../templates/project/dander.platforms.yaml | 37 ++ src/dander/templates/project/dander.yaml | 21 +- tests/project/test_portable_config.py | 157 ++++++ tests/project/test_scaffold.py | 5 + tickets/DANDER-78-platform-profiles.md | 44 ++ 22 files changed, 1071 insertions(+), 83 deletions(-) create mode 100644 docs/platform-profiles.md create mode 100644 src/dander/cli/config_command.py create mode 100644 src/dander/project/portable_config.py create mode 100644 src/dander/templates/project/dander.platforms.yaml create mode 100644 tests/project/test_portable_config.py create mode 100644 tickets/DANDER-78-platform-profiles.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d3b4768..e4aba7ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Add version 2 logical projects plus named platform/deployment configuration, deterministic v1 + migration with a read-only compatibility check, and v2 starter projects. The supported hosted + composition remains GCP/BigQuery/Cloud Run in this change. - Publish source-free runtime images as one `linux/amd64,linux/arm64` OCI index and reject an incomplete registry result before recording publication success. - Add an isolated, unscheduled Phase 1B proof for digest-preserving GAR-to-ECR copy and keyless diff --git a/HANDOFF.md b/HANDOFF.md index 360215b2..17bfc9de 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Completed Phase 1B from stable `v0.7.0` without touching the docs-only roadmap branch. -- Proved one source-free AMD64/ARM64 OCI index locally, on Cloud Run, and after byte-identical ECR copy. -- Proved ARM64 Fargate can refresh keyless Google credentials and query bounded BigQuery data twice. -- Corrected Fargate credential sourcing, Google Auth lifetime configuration, and the proof-table default. -- Destroyed every AWS/GCP proof resource and recorded final isolated Terraform `No changes.` +- Added version 2 logical projects plus named platform/deployment configuration. +- Kept every version 1 manifest compatible through one resolved `DanderProject` view. +- Added deterministic `dander config migrate --check` and guarded atomic migration writes. +- Changed new source-free starter projects to the two-file v2 layout. +- Proved the isolated GCP deployment plans `No changes` before and after migration. ## Try It -Follow `acceptance/cloud-portability/phase1b/README.md` only with disposable AWS/GCP accounts and -saved reviewed plans. No live apply is required for local validation. +Run `dander config migrate --config dander.yaml --check`. On a version 1 project, review the result +and then run the command without `--check`; `dander.platforms.yaml` is never overwritten. ## Checks -- All 828 tests, Ruff, strict mypy, and four Terraform validations passed. -- Wheel/sdist build, inspection, outside-checkout installs, project generation, and Terraform validation passed. -- Both image platforms passed runtime conformance and had zero fixable High/Critical findings. -- Dependency and long-lived-credential scans passed; the GitPython lock is updated to fixed `3.1.58`. -- Cloud Run and Fargate exited zero; final AWS inventory was empty and isolated GCP reported no drift. +- All 832 tests, Ruff, strict mypy across 190 files, and both Terraform roots passed. +- Wheel/sdist build and inspection plus outside-checkout installs and v2 scaffolds passed. +- Dependency audit and local read-only container conformance passed. +- Version 1 and migrated version 2 isolated plans each reported exactly `No changes.` +- Both isolated schedulers remained paused; no apply or retained-project mutation occurred. ## Decisions -- Registry or platform-digest rewrites fail instead of being treated as equivalent packaging. -- ECS task-role credentials remain short-lived, process-only, and separate from pull/log execution access. -- Fargate remains feasibility-only until the portable BigQuery vertical slice. +- Multiple deployments require explicit selection; one deployment resolves deterministically. +- GCP/BigQuery/Cloud Run remains the only supported hosted composition in this slice. +- New provider factories and canonical schemas remain separate Phase 2 PRs. ## Remaining -- Review the final Phase 1B diff and protected PR checks. -- Merge focused PR #108 after CI passes. -- Begin Phase 2 only from merged `main` on a separate branch. +- Review and merge the focused platform-profile PR after protected CI passes. +- Add provider registries/factories from merged main in the next separate PR. +- Keep the repository's retained version 1 manifest unchanged during the compatibility window. ## Review First -- `scripts/portability/wif_bigquery_probe.py` -- `scripts/portability/prepare_phase1b_context.py` -- `tickets/DANDER-77-cross-cloud-artifact-identity-feasibility.md` +- `src/dander/project/portable_config.py` +- `src/dander/cli/config_command.py` +- `tests/project/test_portable_config.py` diff --git a/README.md b/README.md index 32756141..05ce6e89 100644 --- a/README.md +++ b/README.md @@ -326,24 +326,23 @@ method; manually upgrading makes overages beyond remaining credit and free allow path separates each mutation from its saved Terraform plan: create the state bucket once, run `init-admin-plan`, review and run `init-admin-apply`, publish an immutable source-free image with `image-publish`, then run `init-platform-plan`, review, and run `init-platform-apply`. See the -[hosted quickstart](docs/getting-started.md) for copyable commands. Newly generated projects use the -ordinary hosted path without the optional managed cost guard: +[hosted quickstart](docs/getting-started.md) for copyable commands. Newly generated projects keep +portable pipeline intent in `dander.yaml` and GCP deployment settings in +`dander.platforms.yaml`. The standard deployment uses the ordinary hosted path without the +optional managed cost guard: ```yaml -platform: - region: us-central1 - bigquery_location: US - runtime: - cpu: 1 - memory: 512Mi - timeout_seconds: 300 - max_retries: 1 - batch_rows: 10000 - safety: - require_guarded_free_tier: false +deployments: + gcp_cloud_run: + platform: gcp + launcher: + provider: cloud_run + region: us-central1 + safety: + require_guarded_free_tier: false ``` -These repository-owned values configure every hosted job. `batch_rows` bounds both hosted SCD1 +The deployment's repository-owned `runtime` values configure every hosted job. `batch_rows` bounds both hosted SCD1 extraction batches and BigQuery writer requests. Sandbox replacement also consumes the endpoint as bounded batches through a run-scoped staging table. When guarded free tier is required, initialization rejects a disabled cost guard and hosted jobs receive @@ -369,7 +368,8 @@ from runtime identities; only it can provision project resources. Guarded instal additionally use it to delegate each runtime's read-only billing visibility. The default unguarded path does not request billing-account IAM or grant runtime billing/Pub/Sub guard permissions. Dander does not manage, limit, or prevent cloud spending in that configuration. -To opt into the managed guard, set `require_guarded_free_tier: true` and pass +To opt into the managed guard, set +`deployments.gcp_cloud_run.safety.require_guarded_free_tier: true` and pass `--billing-account ABCDEF-123456-ABCDEF`; the caller then needs the additional billing-account permissions required for the reviewed IAM and budget plan. @@ -385,8 +385,11 @@ uv run dander init-platform-plan \ --failure-alert-email operator@example.com ``` -The image must use an immutable SHA-256 digest. `dander.yaml` declares every additive pipeline, -including connector, transform roots, schedule, and secret references. Secret Manager containers +The image must use an immutable SHA-256 digest. In new projects, `dander.yaml` declares logical +pipelines and `dander.platforms.yaml` declares schedules, secret references, resources, and the +selected providers. Version 1 combined manifests remain supported during the compatibility window; +`dander config migrate --check` proves the deterministic split before `dander config migrate` +writes it. See [platform profiles](docs/platform-profiles.md). Secret Manager containers and per-pipeline runtime access are managed by Terraform, but secret values never enter the manifest or Terraform state. GitHub Actions authenticates through repository/ref-constrained OIDC rather than a downloaded key. diff --git a/docs/cloud-portability-slos.md b/docs/cloud-portability-slos.md index 43d0d9ab..c353da9c 100644 --- a/docs/cloud-portability-slos.md +++ b/docs/cloud-portability-slos.md @@ -38,7 +38,8 @@ can change from unsupported or experimental to supported. memory, provider work metrics, and estimated cost. There is no universal throughput or cost promise until measured evidence supports one. - Paid scale tests require a separately approved ceiling. A provider budget or billing alert is - evidence and notification, never described as a hard spending cap. + evidence and notification, never described as a hard spending cap. The default unapproved paid + test budget is $0; every live proof records its separately approved ceiling before mutation. ## Promotion rule diff --git a/docs/decisions.md b/docs/decisions.md index d6040494..b8463cf9 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,19 @@ # Engineering Decisions +## 2026-08-07 — Version 2 separates logical projects from named deployments + +- **Configuration:** One logical project may be resolved through multiple named platform profiles. + `dander.yaml` version 2 owns provider-neutral pipeline intent; `dander.platforms.yaml` owns + provider selections and launcher projection. Multiple deployments require explicit selection. +- **Compatibility:** Version 1 remains loadable. `dander config migrate --check` must prove the + deterministic v2 split resolves to identical current GCP settings, pipeline behavior, stable + resource names, and Terraform projection before either file is written. +- **Provider direction:** The release image will contain all first-party adapters while local Python + installs may use extras. PostgreSQL 15+ is the minimum. Dataplex and Glue may represent + non-native warehouses. OCI uses Container Instances plus a narrow launch Function; OKE remains + the Kubernetes path. Qualification uses `docs/cloud-portability-slos.md`, and paid testing has a + zero-dollar default until a separate per-proof ceiling is approved. + ## 2026-08-07 — Cloud Run consumes the shared execution projection - **Decision:** The GCP plan compiler passes `io.dander.execution/v1` templates into Terraform; diff --git a/docs/getting-started.md b/docs/getting-started.md index 6bb4bfd9..7e6bd0e7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -39,7 +39,7 @@ dander validate dander run greenhouse_jobs --dry-run --project "$DANDER_PROJECT" ``` -The generated manifest keeps the schedule paused. The dry-run performs no credential or network +The generated `dander.platforms.yaml` keeps the schedule paused. The dry-run performs no credential or network access and should identify one Greenhouse endpoint and one selected model. ## Create the Terraform backend once @@ -113,7 +113,8 @@ limiting, or preventing cloud spending; disabling the guard does not prevent or The managed cost guard is optional. To use its simulation-first budget, Pub/Sub notification path, billing-shutoff function, and guarded runtime preflight, set -`platform.safety.require_guarded_free_tier` to `true` and pass `--billing-account`. That opt-in path +`deployments.gcp_cloud_run.safety.require_guarded_free_tier` to `true` in +`dander.platforms.yaml` and pass `--billing-account`. That opt-in path requires permission to grant the bootstrap identity Billing Account Administrator on the linked billing account. Review those additional billing-account and project-level IAM changes before any apply. Selecting GitHub WIF similarly requires Workload Identity Pool Admin during stage zero. @@ -144,7 +145,8 @@ non-empty raw and staging relation proves the live public extraction and transfo ## Enable the schedule only after proof -Change only `pipelines.greenhouse_jobs.paused` to `false` in `dander.yaml`, rerun `dander validate`, +Change only `deployments.gcp_cloud_run.pipelines.greenhouse_jobs.paused` to `false` in +`dander.platforms.yaml`, rerun `dander validate`, then repeat `image-publish`, `init-platform-plan`, plan review, and `init-platform-apply`. The expected platform changes are the scheduler state and immutable image. Repeat `init-platform-plan` afterward and require Terraform to report `No changes.` diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md new file mode 100644 index 00000000..dc0990d8 --- /dev/null +++ b/docs/platform-profiles.md @@ -0,0 +1,44 @@ +# Platform profiles and version 2 projects + +Dander version 2 separates reusable pipeline intent from environment-specific deployment choices: + +- `dander.yaml` contains connector plugins, sources, graphs/models, tests, and catalog intent. +- `dander.platforms.yaml` contains named warehouse, state, catalog, secret, and launcher selections, + plus runtime limits, schedules, secret references, and provider resource names. + +The first supported hosted profile remains BigQuery, BigQuery state, Dataplex (or no cloud +catalog), GCP Secret Manager, and Cloud Run. A provider name outside the installed and +supported contract fails validation; configuration does not imply provider support. + +One logical project may have multiple named platforms and deployments. `dander validate` accepts +`--deployment`; the Python configuration loader accepts the same explicit selection. Other current +GCP commands require a single deployment until the shared provider factories are introduced. +When a project has exactly one deployment, Dander selects it deterministically; multiple +deployments never fall back to an arbitrary default. + +## Migrate a version 1 project + +Version 1 combined manifests remain supported during the compatibility window. First run the +read-only check: + +```bash +dander config migrate --config dander.yaml --check +``` + +The check renders both files in memory, resolves the generated GCP/Cloud Run deployment, and +requires its platform settings, plugin pins, pipeline behavior, Terraform pipeline projection, and +stable resource names to equal the version 1 result. It changes no file. + +Then write the deterministic split: + +```bash +dander config migrate --config dander.yaml +dander validate --config dander.yaml --platforms-config dander.platforms.yaml +``` + +Migration refuses to overwrite an existing `dander.platforms.yaml`. Commit and review both files +together. Existing Terraform addresses do not change merely because the equivalent configuration +is represented in two files. + +The current runtime continues to support only the GCP compatibility composition. Provider +registries and additional adapters arrive in later, separate portability changes. diff --git a/docs/upgrading.md b/docs/upgrading.md index fc650168..103b0da2 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -51,7 +51,8 @@ dander init-platform-apply \ --bootstrap-service-account "dander-bootstrap@$DANDER_PROJECT.iam.gserviceaccount.com" ``` -If `platform.safety.require_guarded_free_tier` remains `true`, also pass the installation's existing +If the v1 `platform.safety.require_guarded_free_tier` setting or the v2 +`deployments..safety.require_guarded_free_tier` setting remains `true`, also pass the installation's existing `--billing-account "$DANDER_BILLING_ACCOUNT"`. Do not change that safety setting merely to simplify an upgrade. diff --git a/src/dander/cli/config_command.py b/src/dander/cli/config_command.py new file mode 100644 index 00000000..c295588d --- /dev/null +++ b/src/dander/cli/config_command.py @@ -0,0 +1,101 @@ +"""Project configuration migration commands.""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from pathlib import Path + +import typer +from click import ClickException + +from dander.project import ProjectConfigError +from dander.project.portable_config import ProjectMigration, prepare_version_one_migration + +config_app = typer.Typer( + help="Validate and migrate versioned project configuration.", + no_args_is_help=True, +) + + +@config_app.command("migrate") +def migrate_config( + project_config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option( # noqa: B008 + None, + "--platforms-config", + help="Output path (defaults beside dander.yaml).", + ), + check: bool = typer.Option( + False, + "--check", + help="Prove deterministic compatibility without writing files.", + ), +) -> None: + """Split a version 1 project into v2 logical and deployment files.""" + resolved_platforms = platforms_config or project_config.with_name("dander.platforms.yaml") + try: + migration = prepare_version_one_migration(project_config) + if resolved_platforms.exists() or resolved_platforms.is_symlink(): + raise ProjectConfigError( + f"Refusing to overwrite platform configuration: {resolved_platforms}" + ) + if check: + typer.echo( + "Migration check passed: version 2 resolves to the same GCP/Cloud Run behavior; " + "no files changed." + ) + return + _write_migration(project_config, resolved_platforms, migration) + except ProjectConfigError as error: + raise ClickException(str(error)) from error + except OSError as error: + raise ClickException("Could not write migrated project configuration") from error + typer.echo(f"Migrated logical project: {project_config}") + typer.echo(f"Created platform configuration: {resolved_platforms}") + typer.echo( + f"Next: dander validate --config {project_config} --platforms-config {resolved_platforms}" + ) + + +def _write_migration( + project_config: Path, + platforms_config: Path, + migration: ProjectMigration, +) -> None: + if hashlib.sha256(project_config.read_bytes()).hexdigest() != migration.source_sha256: + raise ProjectConfigError("dander.yaml changed during migration; no files were written") + project_config.parent.mkdir(parents=True, exist_ok=True) + platforms_config.parent.mkdir(parents=True, exist_ok=True) + logical_temp = _write_temporary(project_config, migration.logical_yaml) + platforms_temp = _write_temporary(platforms_config, migration.platforms_yaml) + try: + if platforms_config.exists() or platforms_config.is_symlink(): + raise ProjectConfigError( + f"Refusing to overwrite platform configuration: {platforms_config}" + ) + platforms_temp.replace(platforms_config) + logical_temp.replace(project_config) + finally: + logical_temp.unlink(missing_ok=True) + platforms_temp.unlink(missing_ok=True) + + +def _write_temporary(target: Path, content: str) -> Path: + descriptor, temporary = tempfile.mkstemp( + prefix=f".{target.name}.", + suffix=".tmp", + dir=target.parent, + text=True, + ) + path = Path(temporary) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + except Exception: + path.unlink(missing_ok=True) + raise + return path diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 9b578337..ed0a67e3 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -44,6 +44,7 @@ SemanticRegistryPublisher, SqliteMetadataStore, ) +from dander.cli.config_command import config_app from dander.cli.init_command import ( InitOptions, execute_init, @@ -124,6 +125,7 @@ graph_app = typer.Typer(help="Open validated pipeline graphs to local visual editors.") plugins_app = typer.Typer(help="Install and inspect explicitly pinned connector plugins.") connector_app = typer.Typer(help="Inspect and check configured connector capabilities.") +app.add_typer(config_app, name="config") app.add_typer(verify_app, name="verify") app.add_typer(metadata_app, name="metadata") app.add_typer(graph_app, name="graph") @@ -179,6 +181,8 @@ def new_project(directory: Path = typer.Argument(...)) -> None: # noqa: B008 @app.command("validate") def validate_project( project_config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + deployment: str | None = typer.Option(None, "--deployment"), connectors_dir: Path = typer.Option( # noqa: B008 _DEFAULT_CONNECTORS_DIR, "--connectors-dir" ), @@ -186,7 +190,11 @@ def validate_project( ) -> None: """Validate the project manifest and all configured connector/model references.""" try: - manifest = load_project_config(project_config) + manifest = load_project_config( + project_config, + platforms_path=platforms_config, + deployment=deployment, + ) manifest.validate_references( project_config.resolve().parent, connectors_dir=connectors_dir, diff --git a/src/dander/pipeline/graph_deployment.py b/src/dander/pipeline/graph_deployment.py index a52e0f1c..f7633acf 100644 --- a/src/dander/pipeline/graph_deployment.py +++ b/src/dander/pipeline/graph_deployment.py @@ -26,7 +26,11 @@ TerraformBootstrap, TerraformBootstrapError, ) -from dander.project import ProjectConfigError, load_project_config +from dander.project import ( + ProjectConfigError, + load_project_config, + prepare_version_one_migration, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -286,6 +290,15 @@ def _copy_source_free_project( for directory in ("connectors", "graphs", "models"): _copy_tree(project_root / directory, destination / directory) shutil.copy2(project_config, destination / "dander.yaml") + platforms_config = project_config.with_name("dander.platforms.yaml") + if platforms_config.is_file() and not platforms_config.is_symlink(): + shutil.copy2(platforms_config, destination / "dander.platforms.yaml") + else: + migration = prepare_version_one_migration(project_config) + (destination / "dander.platforms.yaml").write_text( + migration.platforms_yaml, + encoding="utf-8", + ) dockerfile = ( files("dander").joinpath("templates", "project", "Dockerfile").read_text(encoding="utf-8") ) @@ -349,6 +362,11 @@ def _validate_snapshot( def _deployment_fingerprint(project_root: Path, project_config: Path) -> str: hasher = hashlib.sha256() candidates = [project_config] + platforms_config = project_config.with_name("dander.platforms.yaml") + if platforms_config.is_file(): + if platforms_config.is_symlink(): + raise GraphDeploymentError("Deployment preview inputs must not contain symlinks.") + candidates.append(platforms_config) for directory in ("connectors", "graphs", "models", "infra"): root = project_root / directory if not root.is_dir() or root.is_symlink(): diff --git a/src/dander/project/__init__.py b/src/dander/project/__init__.py index 0d8397e5..e4b6aff6 100644 --- a/src/dander/project/__init__.py +++ b/src/dander/project/__init__.py @@ -11,18 +11,36 @@ ProjectConfigError, load_project_config, ) +from dander.project.portable_config import ( + DanderLogicalProjectV2, + DanderPlatforms, + DeploymentPipelineSpec, + DeploymentSpec, + LogicalPipelineSpec, + PlatformProfileSpec, + ProjectMigration, + prepare_version_one_migration, +) from dander.project.scaffold import ProjectScaffoldError, scaffold_project __all__ = [ "DanderProject", + "DanderLogicalProjectV2", + "DanderPlatforms", + "DeploymentPipelineSpec", + "DeploymentSpec", + "LogicalPipelineSpec", "PipelineResourceNames", "PipelineSpec", "PluginSpec", "PlatformRuntimeSpec", "PlatformSafetySpec", "PlatformSpec", + "PlatformProfileSpec", + "ProjectMigration", "ProjectConfigError", "load_project_config", + "prepare_version_one_migration", "ProjectScaffoldError", "scaffold_project", ] diff --git a/src/dander/project/config.py b/src/dander/project/config.py index d9e72fad..ae5e0163 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -188,14 +188,22 @@ class PlatformSpec(BaseModel): class DanderProject(BaseModel): - """The versioned, repository-owned Dander project manifest.""" + """A validated project resolved for one deployment. + + Version 1 files populate this model directly. Version 2 logical projects are + combined with one named deployment from ``dander.platforms.yaml`` while the + compatibility window remains open. + """ model_config = ConfigDict(extra="forbid") - version: Literal[1] + version: Literal[1, 2] platform: PlatformSpec = Field(default_factory=PlatformSpec) plugins: dict[str, PluginSpec] = Field(default_factory=dict) pipelines: dict[str, PipelineSpec] = Field(min_length=1) + platform_name: str = Field(default="gcp", exclude=True) + deployment_name: str = Field(default="gcp_cloud_run", exclude=True) + deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") @classmethod @@ -214,11 +222,28 @@ def validate_pipeline_ids(cls, values: dict[str, PipelineSpec]) -> dict[str, Pip raise ValueError("pipeline ids must use lowercase letters, numbers, and underscores") return values + @model_validator(mode="after") + def validate_deployed_pipeline_ids(self) -> DanderProject: + if self.deployed_pipeline_ids is None: + return self + if len(self.deployed_pipeline_ids) != len(set(self.deployed_pipeline_ids)): + raise ValueError("deployed pipeline ids must not contain duplicates") + unknown = sorted(set(self.deployed_pipeline_ids) - set(self.pipelines)) + if unknown: + raise ValueError(f"deployed pipeline ids contain unknown pipeline {unknown[0]!r}") + return self + def terraform_pipelines(self) -> dict[str, dict[str, object]]: """Expand human pipeline definitions into the literal Terraform module contract.""" + selected = ( + set(self.pipelines) + if self.deployed_pipeline_ids is None + else set(self.deployed_pipeline_ids) + ) return { pipeline_id: _terraform_pipeline(pipeline_id, pipeline) for pipeline_id, pipeline in sorted(self.pipelines.items()) + if pipeline_id in selected } def validate_references( @@ -296,23 +321,51 @@ def validate_references( ) -def load_project_config(path: Path) -> DanderProject: - """Load and validate one project manifest without reflecting authored values in errors.""" +def load_project_config( + path: Path, + *, + platforms_path: Path | None = None, + deployment: str | None = None, +) -> DanderProject: + """Load one v1 project or resolve one v2 project deployment. + + A v2 project finds ``dander.platforms.yaml`` beside the logical manifest by + default. Multiple deployments require an explicit selection so a command can + never operate against an arbitrary environment. + """ + raw = _load_yaml_mapping(path, label="Dander project configuration") + if raw.get("version") == 2: + from dander.project.portable_config import resolve_version_two_project + + return resolve_version_two_project( + path, + raw, + platforms_path=platforms_path, + deployment=deployment, + ) + try: + return DanderProject.model_validate(raw) + except ValidationError as error: + raise _project_validation_error(path, error) from error + + +def _load_yaml_mapping(path: Path, *, label: str) -> dict[str, object]: try: raw = yaml.safe_load(path.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError) as error: - raise ProjectConfigError(f"Cannot read Dander project configuration: {path}") from error + raise ProjectConfigError(f"Cannot read {label}: {path}") from error if not isinstance(raw, dict): - raise ProjectConfigError(f"Dander project configuration must be a mapping: {path}") - try: - return DanderProject.model_validate(raw) - except ValidationError as error: - locations = sorted( - {".".join(str(part) for part in issue["loc"]) or "" for issue in error.errors()} - ) - raise ProjectConfigError( - f"Invalid Dander project configuration at {path}; check: {', '.join(locations)}" - ) from error + raise ProjectConfigError(f"{label} must be a mapping: {path}") + return raw + + +def _project_validation_error(path: Path, error: ValidationError) -> ProjectConfigError: + locations = sorted( + {".".join(str(part) for part in issue["loc"]) or "" for issue in error.errors()} + ) + return ProjectConfigError( + f"Invalid Dander project configuration at {path}; check: {', '.join(locations)}" + ) def _terraform_pipeline(pipeline_id: str, pipeline: PipelineSpec) -> dict[str, object]: diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py new file mode 100644 index 00000000..334b3a94 --- /dev/null +++ b/src/dander/project/portable_config.py @@ -0,0 +1,494 @@ +"""Version 2 logical projects and named deployment configuration.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated, Literal + +import yaml +from packaging.utils import canonicalize_name +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator + +from dander.project.config import ( + DanderProject, + PipelineResourceNames, + PipelineSpec, + PlatformRuntimeSpec, + PlatformSafetySpec, + PlatformSpec, + PluginSpec, + ProjectConfigError, + _load_yaml_mapping, +) + +_NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") +_PIPELINE_NAME = re.compile(r"^[a-z][a-z0-9_]{1,62}$") +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") +_ENV_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") +_SECRET_ID = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,254}$") + + +class LogicalPipelineSpec(BaseModel): + """Provider-neutral pipeline intent authored in a version 2 project.""" + + model_config = ConfigDict(extra="forbid") + + source: str = Field(pattern=_IDENTIFIER.pattern) + graph: str | None = None + models: list[str] = Field(default_factory=list) + build_models: bool = True + publish_catalog: bool = False + + @field_validator("models") + @classmethod + def validate_models(cls, values: list[str]) -> list[str]: + if any(not _IDENTIFIER.fullmatch(value) for value in values): + raise ValueError("models must contain valid Dander identifiers") + if len(values) != len(set(values)): + raise ValueError("models must not contain duplicates") + return values + + @field_validator("graph") + @classmethod + def validate_graph_path(cls, value: str | None) -> str | None: + if value is None: + return None + path = Path(value) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in value + or path.suffix.lower() not in {".yaml", ".yml", ".json"} + ): + raise ValueError("graph must be a safe relative YAML or JSON path") + return path.as_posix() + + @model_validator(mode="after") + def validate_execution_shape(self) -> LogicalPipelineSpec: + if self.graph is None: + if self.build_models and not self.models: + raise ValueError("models must not be empty when build_models is enabled") + return self + if self.models or self.build_models or self.publish_catalog: + raise ValueError( + "graph pipelines require models=[], build_models=false, and publish_catalog=false" + ) + return self + + +class DanderLogicalProjectV2(BaseModel): + """The portable, provider-neutral ``dander.yaml`` version 2 contract.""" + + model_config = ConfigDict(extra="forbid") + + version: Literal[2] + plugins: dict[str, PluginSpec] = Field(default_factory=dict) + pipelines: dict[str, LogicalPipelineSpec] = Field(min_length=1) + + @field_validator("plugins") + @classmethod + def validate_plugin_ids(cls, values: dict[str, PluginSpec]) -> dict[str, PluginSpec]: + if any(not _PLUGIN_ID.fullmatch(plugin_id) for plugin_id in values): + raise ValueError("plugin ids must use lowercase letters, numbers, and underscores") + distributions = [canonicalize_name(plugin.distribution) for plugin in values.values()] + if len(distributions) != len(set(distributions)): + raise ValueError("plugin distributions must not be declared more than once") + return values + + @field_validator("pipelines") + @classmethod + def validate_pipeline_ids( + cls, values: dict[str, LogicalPipelineSpec] + ) -> dict[str, LogicalPipelineSpec]: + if any(not _PIPELINE_NAME.fullmatch(pipeline_id) for pipeline_id in values): + raise ValueError("pipeline ids must use lowercase letters, numbers, and underscores") + return values + + +class BigQueryWarehouseSpec(BaseModel): + """BigQuery connection identity owned by one named platform profile.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["bigquery"] + location: str = Field(default="US", min_length=1) + + +class BigQueryStateSpec(BaseModel): + """BigQuery durable-state selection for the compatibility profile.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["bigquery"] + + +class DataplexCatalogSpec(BaseModel): + """Dataplex catalog publication selection.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["dataplex"] + + +class NoCatalogSpec(BaseModel): + """Explicit cloud-catalog no-op selection.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["none"] + + +CatalogSpec = Annotated[DataplexCatalogSpec | NoCatalogSpec, Field(discriminator="provider")] + + +class GcpSecretManagerSpec(BaseModel): + """GCP Secret Manager runtime resolution selection.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["gcp_secret_manager"] + + +class EnvironmentSecretSpec(BaseModel): + """Environment-only runtime secret resolution selection.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["environment"] + + +SecretProviderSpec = Annotated[ + GcpSecretManagerSpec | EnvironmentSecretSpec, + Field(discriminator="provider"), +] + + +class PlatformProfileSpec(BaseModel): + """One named combination of data-plane provider selections.""" + + model_config = ConfigDict(extra="forbid") + + warehouse: BigQueryWarehouseSpec + state: BigQueryStateSpec + catalog: CatalogSpec + secrets: SecretProviderSpec + + +class CloudRunLauncherSpec(BaseModel): + """Cloud Run deployment projection for the existing GCP path.""" + + model_config = ConfigDict(extra="forbid") + + provider: Literal["cloud_run"] + region: str = Field(default="us-central1", min_length=1) + + +class DeploymentPipelineSpec(BaseModel): + """Environment-specific projection for one logical pipeline.""" + + model_config = ConfigDict(extra="forbid") + + schedule: str = Field(default="0 9 * * *", min_length=1) + time_zone: str = Field(default="America/New_York", min_length=1) + paused: bool = True + secret_bindings: dict[str, str] = Field(default_factory=dict) + resources: PipelineResourceNames = Field(default_factory=PipelineResourceNames) + + @field_validator("schedule", "time_zone") + @classmethod + def reject_blank_strings(cls, value: str) -> str: + if not value.strip(): + raise ValueError("schedule and time_zone must not be blank") + return value + + @field_validator("secret_bindings") + @classmethod + def validate_secrets(cls, values: dict[str, str]) -> dict[str, str]: + if any( + not _ENV_NAME.fullmatch(env_name) or not _SECRET_ID.fullmatch(secret_id) + for env_name, secret_id in values.items() + ): + raise ValueError( + "secret_bindings must map uppercase environment names to safe secret ids" + ) + return values + + +class DeploymentSpec(BaseModel): + """One launcher projection over a named platform profile.""" + + model_config = ConfigDict(extra="forbid") + + platform: str = Field(pattern=_NAME.pattern) + launcher: CloudRunLauncherSpec + runtime: PlatformRuntimeSpec = Field(default_factory=PlatformRuntimeSpec) + safety: PlatformSafetySpec = Field(default_factory=PlatformSafetySpec) + pipelines: dict[str, DeploymentPipelineSpec] = Field(min_length=1) + + @field_validator("pipelines") + @classmethod + def validate_pipeline_ids( + cls, values: dict[str, DeploymentPipelineSpec] + ) -> dict[str, DeploymentPipelineSpec]: + if any(not _PIPELINE_NAME.fullmatch(pipeline_id) for pipeline_id in values): + raise ValueError("pipeline ids must use lowercase letters, numbers, and underscores") + return values + + +class DanderPlatforms(BaseModel): + """Versioned operator-owned platform and deployment configuration.""" + + model_config = ConfigDict(extra="forbid") + + version: Literal[1] + platforms: dict[str, PlatformProfileSpec] = Field(min_length=1) + deployments: dict[str, DeploymentSpec] = Field(min_length=1) + + @field_validator("platforms", "deployments") + @classmethod + def validate_names(cls, values: dict[str, object]) -> dict[str, object]: + if any(not _NAME.fullmatch(name) for name in values): + raise ValueError("names must use lowercase letters, numbers, and underscores") + return values + + @model_validator(mode="after") + def validate_platform_references(self) -> DanderPlatforms: + unknown = sorted( + { + deployment.platform + for deployment in self.deployments.values() + if deployment.platform not in self.platforms + } + ) + if unknown: + raise ValueError(f"deployments reference unknown platform {unknown[0]!r}") + return self + + +@dataclass(frozen=True, slots=True) +class ProjectMigration: + """Deterministic version 1 to version 2 migration artifacts.""" + + logical_yaml: str + platforms_yaml: str + source_sha256: str + + +def resolve_version_two_project( + project_path: Path, + raw_project: dict[str, object], + *, + platforms_path: Path | None, + deployment: str | None, +) -> DanderProject: + """Resolve one v2 logical project against one explicit deployment.""" + logical = _validate_logical_project(project_path, raw_project) + resolved_platforms_path = platforms_path or project_path.with_name("dander.platforms.yaml") + raw_platforms = _load_yaml_mapping( + resolved_platforms_path, + label="Dander platform configuration", + ) + platforms = _validate_platforms(resolved_platforms_path, raw_platforms) + return _resolve(logical, platforms, deployment=deployment) + + +def prepare_version_one_migration(path: Path) -> ProjectMigration: + """Render deterministic v2 files and prove resolved behavior is unchanged.""" + try: + source = path.read_bytes() + raw = yaml.safe_load(source.decode("utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as error: + raise ProjectConfigError(f"Cannot read Dander project configuration: {path}") from error + if not isinstance(raw, dict): + raise ProjectConfigError(f"Dander project configuration must be a mapping: {path}") + if raw.get("version") != 1: + raise ProjectConfigError("Only a version 1 dander.yaml can be migrated") + try: + legacy = DanderProject.model_validate(raw) + except ValidationError as error: + raise _validation_error(path, error, label="Dander project configuration") from error + + logical_document = _logical_document(legacy) + platforms_document = _platforms_document(legacy) + logical_yaml = yaml.safe_dump(logical_document, sort_keys=False, width=100) + platforms_yaml = yaml.safe_dump(platforms_document, sort_keys=False, width=100) + + logical = _validate_logical_project(path, logical_document) + platforms = _validate_platforms(path.with_name("dander.platforms.yaml"), platforms_document) + migrated = _resolve(logical, platforms, deployment="gcp_cloud_run") + if not _equivalent(legacy, migrated): + raise ProjectConfigError("Generated version 2 configuration changed resolved GCP behavior") + return ProjectMigration( + logical_yaml=logical_yaml, + platforms_yaml=platforms_yaml, + source_sha256=hashlib.sha256(source).hexdigest(), + ) + + +def _validate_logical_project(path: Path, raw: dict[str, object]) -> DanderLogicalProjectV2: + try: + return DanderLogicalProjectV2.model_validate(raw) + except ValidationError as error: + raise _validation_error(path, error, label="Dander project configuration") from error + + +def _validate_platforms(path: Path, raw: dict[str, object]) -> DanderPlatforms: + try: + return DanderPlatforms.model_validate(raw) + except ValidationError as error: + raise _validation_error(path, error, label="Dander platform configuration") from error + + +def _resolve( + logical: DanderLogicalProjectV2, + platforms: DanderPlatforms, + *, + deployment: str | None, +) -> DanderProject: + selected_name = _select_deployment(platforms, deployment) + selected = platforms.deployments[selected_name] + profile = platforms.platforms[selected.platform] + if profile.secrets.provider != "gcp_secret_manager": + raise ProjectConfigError( + "The GCP compatibility runtime currently requires secrets.provider='gcp_secret_manager'" + ) + unknown_pipelines = sorted(set(selected.pipelines) - set(logical.pipelines)) + if unknown_pipelines: + raise ProjectConfigError( + f"Deployment {selected_name!r} references unknown pipeline {unknown_pipelines[0]!r}" + ) + + resolved_pipelines: dict[str, PipelineSpec] = {} + for pipeline_id, pipeline in logical.pipelines.items(): + deployment_pipeline = selected.pipelines.get(pipeline_id) or DeploymentPipelineSpec() + resolved_pipelines[pipeline_id] = PipelineSpec( + source=pipeline.source, + graph=pipeline.graph, + models=pipeline.models, + build_models=pipeline.build_models, + publish_dataplex=(pipeline.publish_catalog and profile.catalog.provider == "dataplex"), + schedule=deployment_pipeline.schedule, + time_zone=deployment_pipeline.time_zone, + paused=deployment_pipeline.paused, + secrets=deployment_pipeline.secret_bindings, + resources=deployment_pipeline.resources, + ) + + return DanderProject( + version=2, + platform=PlatformSpec( + region=selected.launcher.region, + bigquery_location=profile.warehouse.location, + runtime=selected.runtime, + safety=selected.safety, + ), + plugins=logical.plugins, + pipelines=resolved_pipelines, + platform_name=selected.platform, + deployment_name=selected_name, + deployed_pipeline_ids=tuple(sorted(selected.pipelines)), + ) + + +def _select_deployment(platforms: DanderPlatforms, requested: str | None) -> str: + if requested is not None: + if requested not in platforms.deployments: + raise ProjectConfigError(f"Unknown deployment {requested!r}") + return requested + if len(platforms.deployments) != 1: + choices = ", ".join(sorted(platforms.deployments)) + raise ProjectConfigError( + f"Multiple deployments are configured ({choices}); select one explicitly" + ) + return next(iter(platforms.deployments)) + + +def _logical_document(project: DanderProject) -> dict[str, object]: + document: dict[str, object] = {"version": 2} + if project.plugins: + document["plugins"] = { + plugin_id: plugin.model_dump(mode="json") + for plugin_id, plugin in sorted(project.plugins.items()) + } + pipelines: dict[str, object] = {} + for pipeline_id, pipeline in sorted(project.pipelines.items()): + item: dict[str, object] = { + "source": pipeline.source, + "models": list(pipeline.models), + } + if pipeline.graph is not None: + item["graph"] = pipeline.graph + if not pipeline.build_models: + item["build_models"] = False + if pipeline.publish_dataplex: + item["publish_catalog"] = True + pipelines[pipeline_id] = item + document["pipelines"] = pipelines + return document + + +def _platforms_document(project: DanderProject) -> dict[str, object]: + pipelines: dict[str, object] = {} + for pipeline_id, pipeline in sorted(project.pipelines.items()): + item: dict[str, object] = { + "schedule": pipeline.schedule, + "time_zone": pipeline.time_zone, + "paused": pipeline.paused, + } + if pipeline.secrets: + item["secret_bindings"] = dict(sorted(pipeline.secrets.items())) + resources = pipeline.resources.model_dump(mode="json", exclude_none=True) + if resources: + item["resources"] = resources + pipelines[pipeline_id] = item + return { + "version": 1, + "platforms": { + "gcp": { + "warehouse": { + "provider": "bigquery", + "location": project.platform.bigquery_location, + }, + "state": {"provider": "bigquery"}, + "catalog": {"provider": "dataplex"}, + "secrets": {"provider": "gcp_secret_manager"}, + } + }, + "deployments": { + "gcp_cloud_run": { + "platform": "gcp", + "launcher": { + "provider": "cloud_run", + "region": project.platform.region, + }, + "runtime": project.platform.runtime.model_dump(mode="json"), + "safety": project.platform.safety.model_dump(mode="json"), + "pipelines": pipelines, + } + }, + } + + +def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: + return ( + legacy.platform == migrated.platform + and legacy.plugins == migrated.plugins + and legacy.pipelines == migrated.pipelines + and legacy.terraform_pipelines() == migrated.terraform_pipelines() + ) + + +def _validation_error( + path: Path, + error: ValidationError, + *, + label: str, +) -> ProjectConfigError: + locations = sorted( + {".".join(str(part) for part in issue["loc"]) or "" for issue in error.errors()} + ) + return ProjectConfigError(f"Invalid {label} at {path}; check: {', '.join(locations)}") diff --git a/src/dander/templates/project/.dockerignore b/src/dander/templates/project/.dockerignore index 2f3f4ae7..5b14f2cc 100644 --- a/src/dander/templates/project/.dockerignore +++ b/src/dander/templates/project/.dockerignore @@ -1,6 +1,7 @@ * !Dockerfile !dander.yaml +!dander.platforms.yaml !connectors/ !connectors/** !graphs/ diff --git a/src/dander/templates/project/Dockerfile b/src/dander/templates/project/Dockerfile index 2a54c54a..69f39d1d 100644 --- a/src/dander/templates/project/Dockerfile +++ b/src/dander/templates/project/Dockerfile @@ -23,6 +23,7 @@ WORKDIR /app RUN pip install --no-cache-dir "dander-platform==${DANDER_VERSION}" COPY --chown=65532:65532 dander.yaml ./dander.yaml +COPY --chown=65532:65532 dander.platforms.yaml ./dander.platforms.yaml RUN dander plugins install --config dander.yaml diff --git a/src/dander/templates/project/README.md b/src/dander/templates/project/README.md index 8fe8125a..c1ed6245 100644 --- a/src/dander/templates/project/README.md +++ b/src/dander/templates/project/README.md @@ -14,8 +14,10 @@ dander image-publish --project YOUR_GCP_PROJECT --failure-alert-email OPERATOR_E ``` `image-publish` prints the complete `init-platform-plan` command. Review that saved Terraform plan, -then apply it with `init-platform-apply`. The starter -manifest keeps its scheduler paused and does not enable Dander's optional managed cost guard. +then apply it with `init-platform-apply`. `dander.yaml` owns portable pipeline intent; +`dander.platforms.yaml` owns the GCP profile, Cloud Run deployment, schedule, resources, and secret +references. The starter deployment keeps its scheduler paused and does not enable Dander's +optional managed cost guard. Disabling the guard does not prevent or cap cloud charges. Use the public [hosted Greenhouse quickstart](https://github.com/harrisonoconnorhover/dander/blob/main/docs/getting-started.md) @@ -36,7 +38,7 @@ The public UI stores no graph or credentials; start `dander graph serve --origin from this project when you want the browser to open, save, or run one operator-bound graph. `graphs/greenhouse_jobs.yaml` is an inactive Druff-compatible example. To make a pipeline execute -that graph instead of SQL models, set its manifest entry to: +that graph instead of SQL models, set its logical `dander.yaml` entry to: ```yaml source: greenhouse_job_board diff --git a/src/dander/templates/project/dander.platforms.yaml b/src/dander/templates/project/dander.platforms.yaml new file mode 100644 index 00000000..b0cf2e92 --- /dev/null +++ b/src/dander/templates/project/dander.platforms.yaml @@ -0,0 +1,37 @@ +version: 1 + +platforms: + gcp: + warehouse: + provider: bigquery + location: US + state: + provider: bigquery + catalog: + provider: dataplex + secrets: + provider: gcp_secret_manager + +deployments: + gcp_cloud_run: + platform: gcp + launcher: + provider: cloud_run + region: us-central1 + runtime: + cpu: 1 + memory: 512Mi + timeout_seconds: 300 + max_retries: 1 + batch_rows: 10000 + safety: + require_guarded_free_tier: false + pipelines: + greenhouse_jobs: + schedule: "0 9 * * *" + time_zone: America/New_York + paused: true + resources: + job: dander-greenhouse-public + runtime_service_account: dander-runtime + scheduler_service_account: dander-scheduler diff --git a/src/dander/templates/project/dander.yaml b/src/dander/templates/project/dander.yaml index 6d252084..e5880439 100644 --- a/src/dander/templates/project/dander.yaml +++ b/src/dander/templates/project/dander.yaml @@ -1,25 +1,6 @@ -version: 1 - -platform: - region: us-central1 - bigquery_location: US - runtime: - cpu: 1 - memory: 512Mi - timeout_seconds: 300 - max_retries: 1 - batch_rows: 10000 - safety: - require_guarded_free_tier: false +version: 2 pipelines: greenhouse_jobs: source: greenhouse_job_board models: [stg_greenhouse__jobs] - schedule: "0 9 * * *" - time_zone: America/New_York - paused: true - resources: - job: dander-greenhouse-public - runtime_service_account: dander-runtime - scheduler_service_account: dander-scheduler diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py new file mode 100644 index 00000000..af41b011 --- /dev/null +++ b/tests/project/test_portable_config.py @@ -0,0 +1,157 @@ +"""Version 2 platform profile and deterministic migration tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +import yaml +from typer.testing import CliRunner + +from dander.cli.main import app +from dander.project import ProjectConfigError, load_project_config, prepare_version_one_migration + +if TYPE_CHECKING: + from pathlib import Path + +_V1_PROJECT = """ +version: 1 +platform: + region: us-east1 + bigquery_location: EU + runtime: + cpu: 2 + memory: 1Gi + timeout_seconds: 600 + max_retries: 2 + batch_rows: 2048 + safety: + require_guarded_free_tier: false +plugins: + example: + distribution: dander-connector-example + version: 1.2.3 +pipelines: + example_records: + source: example + models: [stg_example__records] + publish_dataplex: true + schedule: "15 4 * * *" + time_zone: UTC + paused: false + secrets: + EXAMPLE_TOKEN: example-token + resources: + job: dander-example-records + runtime_service_account: dander-example-run + scheduler_service_account: dander-example-sched +""".strip() + + +def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( + tmp_path: Path, +) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + original = load_project_config(project_path) + + first = prepare_version_one_migration(project_path) + second = prepare_version_one_migration(project_path) + project_path.write_text(first.logical_yaml, encoding="utf-8") + platforms_path.write_text(first.platforms_yaml, encoding="utf-8") + migrated = load_project_config(project_path) + + assert first == second + assert migrated.version == 2 + assert migrated.platform_name == "gcp" + assert migrated.deployment_name == "gcp_cloud_run" + assert migrated.platform == original.platform + assert migrated.plugins == original.plugins + assert migrated.pipelines == original.pipelines + assert migrated.terraform_pipelines() == original.terraform_pipelines() + + +def test_version_two_requires_explicit_selection_with_multiple_deployments( + tmp_path: Path, +) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["deployments"]["shadow_cloud_run"] = platforms["deployments"]["gcp_cloud_run"].copy() + logical = yaml.safe_load(project_path.read_text(encoding="utf-8")) + logical["pipelines"]["undeployed_records"] = { + "source": "example", + "models": ["stg_example__records"], + } + project_path.write_text(yaml.safe_dump(logical), encoding="utf-8") + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + with pytest.raises(ProjectConfigError, match="Multiple deployments"): + load_project_config(project_path) + + resolved = load_project_config(project_path, deployment="shadow_cloud_run") + assert resolved.deployment_name == "shadow_cloud_run" + assert set(resolved.pipelines) == {"example_records", "undeployed_records"} + assert set(resolved.terraform_pipelines()) == {"example_records"} + + +def test_version_two_rejects_unknown_provider_and_unknown_pipeline(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["platforms"]["gcp"]["warehouse"]["provider"] = "snowflake" + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + with pytest.raises( + ProjectConfigError, + match=r"platforms\.gcp\.warehouse\.provider", + ): + load_project_config(project_path) + + platforms = yaml.safe_load(migration.platforms_yaml) + pipelines = platforms["deployments"]["gcp_cloud_run"]["pipelines"] + pipelines["missing_pipeline"] = pipelines["example_records"] + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + with pytest.raises(ProjectConfigError, match="unknown pipeline 'missing_pipeline'"): + load_project_config(project_path) + + +def test_config_migrate_check_is_read_only_then_write_is_atomic(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + + checked = CliRunner().invoke( + app, + ["config", "migrate", "--config", str(project_path), "--check"], + ) + assert checked.exit_code == 0, checked.output + assert "no files changed" in checked.output + assert project_path.read_text(encoding="utf-8") == _V1_PROJECT + assert not platforms_path.exists() + + migrated = CliRunner().invoke( + app, + ["config", "migrate", "--config", str(project_path)], + ) + assert migrated.exit_code == 0, migrated.output + assert platforms_path.is_file() + resolved = load_project_config(project_path) + assert resolved.version == 2 + assert resolved.terraform_pipelines()["example_records"]["paused"] is False + + refused = CliRunner().invoke( + app, + ["config", "migrate", "--config", str(project_path)], + ) + assert refused.exit_code != 0 + assert refused.exception is not None + assert "Only a version 1" in str(refused.exception) diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index fc767c7c..3fd4b11e 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -20,10 +20,14 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: manifest.validate_references(project) assert manifest.pipelines["greenhouse_jobs"].paused assert manifest.platform.safety.require_guarded_free_tier is False + assert manifest.version == 2 + assert manifest.platform_name == "gcp" + assert manifest.deployment_name == "gcp_cloud_run" dockerfile = (project / "Dockerfile").read_text(encoding="utf-8") assert f"ARG DANDER_VERSION={__version__}" in dockerfile assert "RUN dander plugins install --config dander.yaml" in dockerfile assert "COPY --chown=65532:65532 dander.yaml ./dander.yaml" in dockerfile + assert "COPY --chown=65532:65532 dander.platforms.yaml ./dander.platforms.yaml" in dockerfile assert "COPY --chown=65532:65532 connectors ./connectors" in dockerfile assert "COPY --chown=65532:65532 graphs ./graphs" in dockerfile assert "COPY --chown=65532:65532 models ./models" in dockerfile @@ -39,6 +43,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: ".dockerignore", ".gitignore", "Dockerfile", + "dander.platforms.yaml", "connectors/greenhouse_job_board.yaml", "graphs/greenhouse_jobs.yaml", "infra/main.tf", diff --git a/tickets/DANDER-78-platform-profiles.md b/tickets/DANDER-78-platform-profiles.md new file mode 100644 index 00000000..c912563f --- /dev/null +++ b/tickets/DANDER-78-platform-profiles.md @@ -0,0 +1,44 @@ +--- +id: DANDER-78 +title: Separate logical projects from named platform deployments +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-77] +created: 2026-08-07 +--- + +## Context + +Phase 2 begins by separating portable pipeline intent from environment and launcher projection +without changing the proven version 1 GCP deployment or claiming another provider is supported. + +## Acceptance Criteria + +- [x] Version 2 `dander.yaml` contains only plugins and logical pipeline intent. +- [x] Version 1 `dander.platforms.yaml` validates named platform profiles and deployments. +- [x] Multiple deployments require explicit selection; one deployment resolves deterministically. +- [x] Version 1 combined manifests remain compatible. +- [x] `dander config migrate --check` proves equivalent GCP behavior without writing. +- [x] Migration writes both files deterministically and refuses an existing platform file. +- [x] Newly generated source-free projects include and copy both files. +- [x] Full local tests, lint, typing, packaging, and Terraform validation pass. +- [x] Existing isolated GCP configuration produces a no-change Terraform plan before and after migration. +- [ ] Protected CI passes. + +## Design + +Keep the existing `DanderProject` as a resolved compatibility view consumed by current execution +and Terraform code. Parse v2 logical and platform files through closed Pydantic models, select one +deployment, and flatten only that deployment into the compatibility view. Add provider factories +and broader provider schemas in later reviewable tickets rather than implying support here. + +## Implementation Notes + +The migration materializes existing defaults, maps `publish_dataplex` to portable +`publish_catalog`, and verifies platform, plugin, pipeline, stable-name, and Terraform-projection +equality. The standard scaffold now authors v2; the repository's retained v1 manifest is unchanged. + +## Review Log + +Pending protected review. From 3ee3e9092b42380a7253470a30e6cd98d89703bc Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 16:35:48 -0400 Subject: [PATCH 028/163] Add lazy provider factory registry --- CHANGELOG.md | 2 + HANDOFF.md | 38 ++-- docs/decisions.md | 10 + docs/platform-profiles.md | 8 +- src/dander/providers/__init__.py | 19 ++ src/dander/providers/registry.py | 202 +++++++++++++++++ tests/providers/test_registry.py | 207 ++++++++++++++++++ .../DANDER-79-provider-factory-registry.md | 41 ++++ 8 files changed, 504 insertions(+), 23 deletions(-) create mode 100644 src/dander/providers/__init__.py create mode 100644 src/dander/providers/registry.py create mode 100644 tests/providers/test_registry.py create mode 100644 tickets/DANDER-79-provider-factory-registry.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e4aba7ec..b4706c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ capabilities enter through the next minor release. ### Added +- Add an API-v1 provider-factory registry for warehouse, state, catalog, secret, and launcher + categories with strict configuration validation and lazy implementation loading. - Add version 2 logical projects plus named platform/deployment configuration, deterministic v1 migration with a read-only compatibility check, and v2 starter projects. The supported hosted composition remains GCP/BigQuery/Cloud Run in this change. diff --git a/HANDOFF.md b/HANDOFF.md index 17bfc9de..84073adb 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,35 @@ ## Finished -- Added version 2 logical projects plus named platform/deployment configuration. -- Kept every version 1 manifest compatible through one resolved `DanderProject` view. -- Added deterministic `dander config migrate --check` and guarded atomic migration writes. -- Changed new source-free starter projects to the two-file v2 layout. -- Proved the isolated GCP deployment plans `No changes` before and after migration. +- Added one API-v1 provider-factory registry for all five platform capability categories. +- Added strict duplicate, unknown-provider, config-type, identity, and API-version checks. +- Kept provider implementation imports lazy until the exact provider is built. +- Added focused fake-provider and import-path tests without moving BigQuery runtime behavior. ## Try It -Run `dander config migrate --config dander.yaml --check`. On a version 1 project, review the result -and then run the command without `--check`; `dander.platforms.yaml` is never overwritten. +Create a `ProviderRegistry`, register a lightweight Pydantic configuration model and lazy factory, +then call `parse` before `build`. Parsing does not import the implementation module. ## Checks -- All 832 tests, Ruff, strict mypy across 190 files, and both Terraform roots passed. -- Wheel/sdist build and inspection plus outside-checkout installs and v2 scaffolds passed. -- Dependency audit and local read-only container conformance passed. -- Version 1 and migrated version 2 isolated plans each reported exactly `No changes.` -- Both isolated schedulers remained paused; no apply or retained-project mutation occurred. +- All 843 tests, Ruff, formatting, and strict mypy across 193 source files passed. +- Wheel/sdist build and inspection plus both Terraform-root validations passed. +- The isolated GCP plan reported `No changes`; both schedules remained paused and no apply ran. ## Decisions -- Multiple deployments require explicit selection; one deployment resolves deterministically. -- GCP/BigQuery/Cloud Run remains the only supported hosted composition in this slice. -- New provider factories and canonical schemas remain separate Phase 2 PRs. +- One registry contract covers warehouse, state, catalog, secrets, and launcher categories. +- Factory API version 1 validates construction compatibility independently of config versions. +- Concrete adapters and support claims remain separate provider PRs. ## Remaining -- Review and merge the focused platform-profile PR after protected CI passes. -- Add provider registries/factories from merged main in the next separate PR. -- Keep the repository's retained version 1 manifest unchanged during the compatibility window. +- Open and merge the focused provider-registry PR after protected CI passes. +- Add canonical relation and schema contracts from merged main in the next PR. ## Review First -- `src/dander/project/portable_config.py` -- `src/dander/cli/config_command.py` -- `tests/project/test_portable_config.py` +- `src/dander/providers/registry.py` +- `tests/providers/test_registry.py` +- `tickets/DANDER-79-provider-factory-registry.md` diff --git a/docs/decisions.md b/docs/decisions.md index b8463cf9..68f28c98 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-07 — Provider factories are explicit and lazy + +- **Selection:** Warehouse, state, catalog, secret, and launcher factories use one API-v1 registry + keyed by category and validated provider ID. Duplicate and unknown registrations fail clearly. +- **Loading:** Configuration models remain lightweight. A provider implementation and its SDKs are + imported only when that exact provider is built; merely validating another profile does not load + unselected providers. +- **Boundary:** This registry establishes construction and compatibility rules only. A provider is + not supported until its concrete adapter, conformance suite, and live profile pass separately. + ## 2026-08-07 — Version 2 separates logical projects from named deployments - **Configuration:** One logical project may be resolved through multiple named platform profiles. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index dc0990d8..a1813f86 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -40,5 +40,9 @@ Migration refuses to overwrite an existing `dander.platforms.yaml`. Commit and r together. Existing Terraform addresses do not change merely because the equivalent configuration is represented in two files. -The current runtime continues to support only the GCP compatibility composition. Provider -registries and additional adapters arrive in later, separate portability changes. +The current runtime continues to support only the GCP compatibility composition. The internal +factory contract provides one API-v1 registry across warehouse, state, catalog, secret, and +launcher categories. Registration loads only a small configuration model; selecting and building +a provider loads its implementation and SDK dependencies. The current GCP resolver is not routed +through that boundary yet. The registry is a construction contract, not a support claim; concrete +adapter routing arrives in later, separate portability changes. diff --git a/src/dander/providers/__init__.py b/src/dander/providers/__init__.py new file mode 100644 index 00000000..95deae62 --- /dev/null +++ b/src/dander/providers/__init__.py @@ -0,0 +1,19 @@ +"""Cloud-provider registration and construction contracts.""" + +from dander.providers.registry import ( + PROVIDER_API_VERSION, + ProviderFactory, + ProviderFactoryError, + ProviderKind, + ProviderRegistry, + lazy_provider_factory, +) + +__all__ = [ + "PROVIDER_API_VERSION", + "ProviderFactory", + "ProviderFactoryError", + "ProviderKind", + "ProviderRegistry", + "lazy_provider_factory", +] diff --git a/src/dander/providers/registry.py b/src/dander/providers/registry.py new file mode 100644 index 00000000..20b49431 --- /dev/null +++ b/src/dander/providers/registry.py @@ -0,0 +1,202 @@ +"""Typed, lazy provider-factory registration for platform adapters.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from enum import StrEnum +from importlib import import_module +from typing import TypeVar + +from pydantic import BaseModel, ValidationError + +PROVIDER_API_VERSION = 1 +_PROVIDER_ID = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_IMPORT_PATH = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*:[A-Za-z_][A-Za-z0-9_]*$") + + +class ProviderKind(StrEnum): + """Independent platform capabilities selected by one deployment profile.""" + + WAREHOUSE = "warehouse" + STATE = "state" + CATALOG = "catalog" + SECRETS = "secrets" + LAUNCHER = "launcher" + + +class ProviderFactoryError(ValueError): + """A provider registration, configuration, or factory is invalid.""" + + +ProviderProductT = TypeVar("ProviderProductT") +ProviderBuilder = Callable[[BaseModel, Mapping[str, object]], ProviderProductT] + + +@dataclass(frozen=True, slots=True) +class ProviderFactory[ProviderProductT]: + """One validated API-v1 provider builder loaded only after selection.""" + + kind: ProviderKind + provider_id: str + api_version: int + build: ProviderBuilder[ProviderProductT] + + +ProviderFactoryLoader = Callable[[], ProviderFactory[object]] + + +@dataclass(frozen=True, slots=True) +class _Registration: + config_model: type[BaseModel] + load_factory: ProviderFactoryLoader + + +class ProviderRegistry: + """Explicit provider configurations plus lazy category-specific factories. + + Registration imports only the small configuration model. The provider module and + its SDK dependencies are loaded when ``build`` selects that exact provider. + """ + + def __init__(self) -> None: + self._registrations: dict[tuple[ProviderKind, str], _Registration] = {} + self._factories: dict[tuple[ProviderKind, str], ProviderFactory[object]] = {} + + def register( + self, + *, + kind: ProviderKind, + provider_id: str, + config_model: type[BaseModel], + load_factory: ProviderFactoryLoader, + ) -> None: + """Register one provider exactly once without loading its implementation.""" + _validate_provider_id(provider_id) + if not isinstance(config_model, type) or not issubclass(config_model, BaseModel): + raise ProviderFactoryError("provider config_model must be a Pydantic model") + if not callable(load_factory): + raise ProviderFactoryError("provider factory loader must be callable") + key = (kind, provider_id) + if key in self._registrations: + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} is registered more than once" + ) + self._registrations[key] = _Registration( + config_model=config_model, + load_factory=load_factory, + ) + + def providers(self, kind: ProviderKind) -> tuple[str, ...]: + """List registered providers for one capability in deterministic order.""" + return tuple( + sorted( + provider_id + for registered_kind, provider_id in self._registrations + if registered_kind is kind + ) + ) + + def parse(self, kind: ProviderKind, raw: Mapping[str, object]) -> BaseModel: + """Validate one explicitly selected provider block without importing its SDK.""" + provider_id = raw.get("provider") + if not isinstance(provider_id, str) or not _PROVIDER_ID.fullmatch(provider_id): + raise ProviderFactoryError(f"{kind.value}.provider must be a valid provider id") + registration = self._registration(kind, provider_id) + try: + return registration.config_model.model_validate(dict(raw)) + except ValidationError as error: + locations = sorted( + { + ".".join(str(part) for part in issue["loc"]) or "" + for issue in error.errors() + } + ) + raise ProviderFactoryError( + f"Invalid {kind.value} provider {provider_id!r}; check: {', '.join(locations)}" + ) from error + + def build( + self, + kind: ProviderKind, + config: BaseModel, + *, + context: Mapping[str, object] | None = None, + ) -> object: + """Build only the selected provider with explicitly injected dependencies.""" + provider_id = getattr(config, "provider", None) + if not isinstance(provider_id, str): + raise ProviderFactoryError("provider config is missing its provider id") + registration = self._registration(kind, provider_id) + if not isinstance(config, registration.config_model): + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} received the wrong configuration type" + ) + key = (kind, provider_id) + factory = self._factories.get(key) + if factory is None: + try: + loaded = registration.load_factory() + except Exception as error: + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} factory could not be loaded" + ) from error + factory = _validate_factory(loaded, kind=kind, provider_id=provider_id) + self._factories[key] = factory + return factory.build(config, context or {}) + + def _registration(self, kind: ProviderKind, provider_id: str) -> _Registration: + try: + return self._registrations[(kind, provider_id)] + except KeyError as error: + available = ", ".join(self.providers(kind)) or "none" + raise ProviderFactoryError( + f"Unknown {kind.value} provider {provider_id!r}; available: {available}" + ) from error + + +def lazy_provider_factory(import_path: str) -> ProviderFactoryLoader: + """Create a loader for ``module:attribute`` without importing the module now.""" + if not _IMPORT_PATH.fullmatch(import_path): + raise ProviderFactoryError("provider factory import path must use module:attribute") + module_name, attribute = import_path.split(":", maxsplit=1) + + def load() -> ProviderFactory[object]: + value = getattr(import_module(module_name), attribute) + if not isinstance(value, ProviderFactory): + raise TypeError("provider factory export has the wrong type") + return value + + return load + + +def _validate_factory( + value: object, + *, + kind: ProviderKind, + provider_id: str, +) -> ProviderFactory[object]: + if not isinstance(value, ProviderFactory): + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} loader returned an invalid factory" + ) + if value.api_version != PROVIDER_API_VERSION: + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} uses API version {value.api_version}; " + f"Dander requires {PROVIDER_API_VERSION}" + ) + if value.kind is not kind or value.provider_id != provider_id: + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} factory identity does not match registration" + ) + if not callable(value.build): + raise ProviderFactoryError( + f"Provider {kind.value}.{provider_id} factory build is not callable" + ) + return value + + +def _validate_provider_id(provider_id: str) -> None: + if not _PROVIDER_ID.fullmatch(provider_id): + raise ProviderFactoryError(f"Invalid provider id {provider_id!r}") diff --git a/tests/providers/test_registry.py b/tests/providers/test_registry.py new file mode 100644 index 00000000..88378d6d --- /dev/null +++ b/tests/providers/test_registry.py @@ -0,0 +1,207 @@ +"""Provider registry selection, validation, and lazy-loading tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal + +import pytest +from pydantic import BaseModel, ConfigDict + +from dander.providers import ( + PROVIDER_API_VERSION, + ProviderFactory, + ProviderFactoryError, + ProviderKind, + ProviderRegistry, + lazy_provider_factory, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + + from pytest import MonkeyPatch + + +class ExampleWarehouseConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + provider: Literal["example"] + namespace: str + + +def _factory(*, api_version: int = PROVIDER_API_VERSION) -> ProviderFactory[object]: + def build(config: BaseModel, context: Mapping[str, object]) -> object: + assert isinstance(config, ExampleWarehouseConfig) + return { + "namespace": config.namespace, + "client": context.get("client"), + } + + return ProviderFactory( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + api_version=api_version, + build=build, + ) + + +def test_registry_parses_then_lazily_builds_only_selected_provider() -> None: + loads = 0 + + def load() -> ProviderFactory[object]: + nonlocal loads + loads += 1 + return _factory() + + registry = ProviderRegistry() + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + config_model=ExampleWarehouseConfig, + load_factory=load, + ) + + config = registry.parse( + ProviderKind.WAREHOUSE, + {"provider": "example", "namespace": "analytics"}, + ) + assert loads == 0 + assert registry.providers(ProviderKind.WAREHOUSE) == ("example",) + assert registry.build( + ProviderKind.WAREHOUSE, + config, + context={"client": "injected"}, + ) == {"namespace": "analytics", "client": "injected"} + second = registry.build(ProviderKind.WAREHOUSE, config) + assert isinstance(second, dict) + assert second["namespace"] == "analytics" + assert loads == 1 + + +@pytest.mark.parametrize("kind", list(ProviderKind)) +def test_every_provider_category_uses_the_same_registry_contract(kind: ProviderKind) -> None: + class CategoryConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + provider: Literal["example"] + + registry = ProviderRegistry() + registry.register( + kind=kind, + provider_id="example", + config_model=CategoryConfig, + load_factory=lambda: ProviderFactory( + kind=kind, + provider_id="example", + api_version=PROVIDER_API_VERSION, + build=lambda _config, _context: kind.value, + ), + ) + + config = registry.parse(kind, {"provider": "example"}) + assert registry.build(kind, config) == kind.value + + +def test_registry_rejects_duplicates_unknowns_and_invalid_config_without_values() -> None: + registry = ProviderRegistry() + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + config_model=ExampleWarehouseConfig, + load_factory=_factory, + ) + + with pytest.raises(ProviderFactoryError, match="registered more than once"): + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + config_model=ExampleWarehouseConfig, + load_factory=_factory, + ) + with pytest.raises(ProviderFactoryError, match="Unknown state provider"): + registry.parse(ProviderKind.STATE, {"provider": "missing"}) + with pytest.raises(ProviderFactoryError, match="check: namespace") as captured: + registry.parse(ProviderKind.WAREHOUSE, {"provider": "example"}) + assert "analytics" not in str(captured.value) + + +@pytest.mark.parametrize( + ("factory", "message"), + [ + (_factory(api_version=2), "uses API version 2"), + ( + ProviderFactory( + kind=ProviderKind.STATE, + provider_id="example", + api_version=PROVIDER_API_VERSION, + build=lambda _config, _context: object(), + ), + "identity does not match", + ), + ], +) +def test_registry_rejects_incompatible_factory( + factory: ProviderFactory[object], + message: str, +) -> None: + registry = ProviderRegistry() + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + config_model=ExampleWarehouseConfig, + load_factory=lambda: factory, + ) + config = registry.parse( + ProviderKind.WAREHOUSE, + {"provider": "example", "namespace": "analytics"}, + ) + + with pytest.raises(ProviderFactoryError, match=message): + registry.build(ProviderKind.WAREHOUSE, config) + + +def test_lazy_import_path_loads_only_when_selected( + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + module_path = tmp_path / "example_provider.py" + module_path.write_text( + "from dander.providers import PROVIDER_API_VERSION, ProviderFactory, ProviderKind\n" + "FACTORY = ProviderFactory(\n" + " kind=ProviderKind.WAREHOUSE, provider_id='example',\n" + " api_version=PROVIDER_API_VERSION,\n" + " build=lambda config, context: config.namespace,\n" + ")\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + registry = ProviderRegistry() + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + config_model=ExampleWarehouseConfig, + load_factory=lazy_provider_factory("example_provider:FACTORY"), + ) + config = registry.parse( + ProviderKind.WAREHOUSE, + {"provider": "example", "namespace": "analytics"}, + ) + + assert registry.build(ProviderKind.WAREHOUSE, config) == "analytics" + + +def test_registry_rejects_config_model_from_another_provider() -> None: + class OtherConfig(BaseModel): + provider: Literal["example"] + + registry = ProviderRegistry() + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="example", + config_model=ExampleWarehouseConfig, + load_factory=_factory, + ) + + with pytest.raises(ProviderFactoryError, match="wrong configuration type"): + registry.build(ProviderKind.WAREHOUSE, OtherConfig(provider="example")) diff --git a/tickets/DANDER-79-provider-factory-registry.md b/tickets/DANDER-79-provider-factory-registry.md new file mode 100644 index 00000000..56917422 --- /dev/null +++ b/tickets/DANDER-79-provider-factory-registry.md @@ -0,0 +1,41 @@ +--- +id: DANDER-79 +title: Add lazy provider factory registries +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-78] +created: 2026-08-07 +--- + +## Context + +Platform profiles need one explicit construction boundary before concrete warehouse, state, +catalog, secret, and launcher adapters can move independently. Provider configuration validation +must not import every provider SDK. + +## Acceptance Criteria + +- [x] All five provider categories use the same API-v1 registry contract. +- [x] Provider IDs and configuration models validate before construction. +- [x] Duplicate and unknown providers fail with stable, non-sensitive errors. +- [x] Provider implementations load only after explicit selection. +- [x] Factories validate category, provider ID, API version, and callable construction. +- [x] Deterministic discovery and lazy-load behavior have focused tests. +- [x] Full local validation passes. +- [ ] Protected CI passes. + +## Design + +Keep the registry generic over provider products and inject construction dependencies explicitly. +Each concrete adapter owns its typed product and errors. Register lightweight configuration models +now; provider modules and SDK imports stay behind lazy factory loaders. + +## Implementation Notes + +This PR does not move BigQuery runtime behavior or claim a new provider. Canonical relations and +schemas, adapter bundles, and provider-specific implementations remain separate follow-up changes. + +## Review Log + +Pending protected review. From d7f058e9f54c5b65a03b4f28eaca9ec06c2bac87 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 16:49:18 -0400 Subject: [PATCH 029/163] Add canonical relation and schema contracts --- CHANGELOG.md | 2 + HANDOFF.md | 33 +-- docs/canonical-schema.md | 32 +++ docs/decisions.md | 11 + docs/platform-profiles.md | 4 + src/dander/ingestion/source.py | 14 ++ src/dander/warehouse/__init__.py | 31 +++ src/dander/warehouse/bigquery_compat.py | 153 ++++++++++++ src/dander/warehouse/contracts.py | 235 ++++++++++++++++++ src/dander/writer/base.py | 21 ++ tests/warehouse/test_contracts.py | 190 ++++++++++++++ .../DANDER-80-canonical-relation-schema.md | 42 ++++ 12 files changed, 752 insertions(+), 16 deletions(-) create mode 100644 docs/canonical-schema.md create mode 100644 src/dander/warehouse/__init__.py create mode 100644 src/dander/warehouse/bigquery_compat.py create mode 100644 src/dander/warehouse/contracts.py create mode 100644 tests/warehouse/test_contracts.py create mode 100644 tickets/DANDER-80-canonical-relation-schema.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b4706c1e..dfc4d696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ capabilities enter through the next minor release. ### Added +- Add canonical relation/schema v1 contracts with explicit decimal, timestamp, array, record, and + provider-extension semantics plus a fail-closed BigQuery compatibility mapper. - Add an API-v1 provider-factory registry for warehouse, state, catalog, secret, and launcher categories with strict configuration validation and lazy implementation loading. - Add version 2 logical projects plus named platform/deployment configuration, deterministic v1 diff --git a/HANDOFF.md b/HANDOFF.md index 84073adb..edf78da8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,36 @@ ## Finished -- Added one API-v1 provider-factory registry for all five platform capability categories. -- Added strict duplicate, unknown-provider, config-type, identity, and API-version checks. -- Kept provider implementation imports lazy until the exact provider is built. -- Added focused fake-provider and import-path tests without moving BigQuery runtime behavior. +- Added unrendered `RelationRef` coordinates and the provider `RelationCodec` boundary. +- Added canonical schema v1 for exact scalars, decimals, timestamps, arrays, and records. +- Added ordered provider extensions and duplicate/shape validation. +- Added a fail-closed, one-way mapper for legacy BigQuery raw and writer fields. +- Exposed canonical views without changing authored schemas or BigQuery runtime behavior. ## Try It -Create a `ProviderRegistry`, register a lightweight Pydantic configuration model and lazy factory, -then call `parse` before `build`. Parsing does not import the implementation module. +Call `endpoint.canonical_raw_schema()` or `target.canonical_schema`; use +`target.relation_ref` to pass coordinates to a future provider codec without rendering SQL. ## Checks -- All 843 tests, Ruff, formatting, and strict mypy across 193 source files passed. +- All 861 tests, Ruff, formatting, and strict mypy across 197 source files passed. - Wheel/sdist build and inspection plus both Terraform-root validations passed. -- The isolated GCP plan reported `No changes`; both schedules remained paused and no apply ran. +- The isolated GCP plan reported `No changes`; both schedules stayed paused and no apply ran. ## Decisions -- One registry contract covers warehouse, state, catalog, secrets, and launcher categories. -- Factory API version 1 validates construction compatibility independently of config versions. -- Concrete adapters and support claims remain separate provider PRs. +- Decimal precision/scale and timestamp timezone semantics are mandatory. +- BigQuery `REPEATED` maps to a required canonical array while retaining its original mode. +- Provider-only types require an explicit fallback; silent lossy mapping is forbidden. ## Remaining -- Open and merge the focused provider-registry PR after protected CI passes. -- Add canonical relation and schema contracts from merged main in the next PR. +- Open and merge the focused canonical-schema PR after protected CI passes. +- Add portable SQL/dialect boundaries from merged main in the next PR. ## Review First -- `src/dander/providers/registry.py` -- `tests/providers/test_registry.py` -- `tickets/DANDER-79-provider-factory-registry.md` +- `src/dander/warehouse/contracts.py` +- `src/dander/warehouse/bigquery_compat.py` +- `tests/warehouse/test_contracts.py` diff --git a/docs/canonical-schema.md b/docs/canonical-schema.md new file mode 100644 index 00000000..ade870be --- /dev/null +++ b/docs/canonical-schema.md @@ -0,0 +1,32 @@ +# Canonical relation and schema contract + +Dander schema contract v1 keeps portable meaning separate from provider SQL and SDK objects. +`RelationRef` carries `catalog`, `namespace`, and `name` as unrendered coordinates. A warehouse +adapter's `RelationCodec` validates provider-specific limits and performs all quoting and rendering. + +Canonical fields declare: + +- boolean, signed integer, decimal, floating-point, string, binary, date, time, timestamp, JSON, + array, and record types; +- bit width for integers and floating-point values; +- precision and scale for every decimal; +- timezone semantics and fractional-second precision for time/timestamp values; +- nullable or required cardinality; and +- ordered provider extensions for information that has no portable equivalent. + +Arrays contain one canonical element type. Records contain recursively validated, uniquely named +fields. Lossy mappings are not automatic. + +## BigQuery compatibility + +Existing connector `RawField` and writer `WriteField` declarations remain the authored contract. +Their canonical view maps `NUMERIC` to exact decimal precision/scale, distinguishes `TIMESTAMP` +from timezone-free `DATETIME`, records BigQuery's microsecond precision, and maps `REPEATED` to a +required canonical array. Original BigQuery type and mode remain ordered extensions for +traceability. + +`BIGNUMERIC` (whose documented precision includes a partial 77th digit), `GEOGRAPHY`, `INTERVAL`, +or a future BigQuery-only type fails mapping unless a caller explicitly provides a canonical +fallback. This is intentional: validation must not silently relabel a lossy conversion as +portable. This change does not alter BigQuery writes, transform SQL, or deployed schemas; provider +codecs and runtime bundles consume these contracts in later portability PRs. diff --git a/docs/decisions.md b/docs/decisions.md index 68f28c98..f8fc5f6e 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-07 — Canonical schemas preserve semantics before rendering + +- **Relations:** Internal coordinates use `RelationRef(catalog, namespace, name)`. Only a selected + provider codec may validate provider limits, quote identifiers, or render SQL. +- **Types:** Schema contract v1 distinguishes numeric bit widths, decimal precision/scale, + timestamp timezone/precision semantics, arrays, records, and nullable/required cardinality. + Provider annotations are ordered extensions and never credentials. +- **Compatibility:** Existing BigQuery `RawField` and `WriteField` declarations map one way into the + canonical contract. Unsupported types fail unless the caller supplies an explicit fallback; + authored BigQuery files and runtime behavior remain unchanged in this slice. + ## 2026-08-07 — Provider factories are explicit and lazy - **Selection:** Warehouse, state, catalog, secret, and launcher factories use one API-v1 registry diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index a1813f86..09fbc2a1 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -46,3 +46,7 @@ launcher categories. Registration loads only a small configuration model; select a provider loads its implementation and SDK dependencies. The current GCP resolver is not routed through that boundary yet. The registry is a construction contract, not a support claim; concrete adapter routing arrives in later, separate portability changes. + +Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). +Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; +the selected provider remains responsible for physical identifier rendering and type mapping. diff --git a/src/dander/ingestion/source.py b/src/dander/ingestion/source.py index cb4304bf..8ef6e734 100644 --- a/src/dander/ingestion/source.py +++ b/src/dander/ingestion/source.py @@ -24,6 +24,8 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping + from dander.warehouse import CanonicalField, RelationSchema + class RawField(BaseModel): """One complete field declaration for a raw BigQuery relation.""" @@ -69,6 +71,12 @@ def _validate_nested_contract(self) -> RawField: raise ValueError("nested raw schema field names must be unique") return self + def to_canonical(self) -> CanonicalField: + """Map this legacy BigQuery declaration to canonical schema v1.""" + from dander.warehouse import canonical_field_from_bigquery + + return canonical_field_from_bigquery(self) + class Endpoint(BaseModel): """One retrievable entity within a source (loaded from the connector YAML). @@ -183,6 +191,12 @@ def _validate_raw_schema(self) -> Endpoint: ) return self + def canonical_raw_schema(self) -> RelationSchema: + """Return the declared raw schema through the one-way compatibility mapper.""" + from dander.warehouse import canonical_schema_from_bigquery + + return canonical_schema_from_bigquery(self.raw_schema) + @field_validator("pagination", mode="before") @classmethod def _coerce_bare_pagination_kind(cls, value: object) -> object: diff --git a/src/dander/warehouse/__init__.py b/src/dander/warehouse/__init__.py new file mode 100644 index 00000000..7716060c --- /dev/null +++ b/src/dander/warehouse/__init__.py @@ -0,0 +1,31 @@ +"""Provider-neutral warehouse contracts and compatibility mappings.""" + +from dander.warehouse.bigquery_compat import ( + BigQuerySchemaCompatibilityError, + canonical_field_from_bigquery, + canonical_schema_from_bigquery, +) +from dander.warehouse.contracts import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + ProviderExtension, + RelationCodec, + RelationRef, + RelationSchema, +) + +__all__ = [ + "BigQuerySchemaCompatibilityError", + "CanonicalField", + "CanonicalType", + "FieldCardinality", + "LogicalTypeKind", + "ProviderExtension", + "RelationCodec", + "RelationRef", + "RelationSchema", + "canonical_field_from_bigquery", + "canonical_schema_from_bigquery", +] diff --git a/src/dander/warehouse/bigquery_compat.py b/src/dander/warehouse/bigquery_compat.py new file mode 100644 index 00000000..a28ecd39 --- /dev/null +++ b/src/dander/warehouse/bigquery_compat.py @@ -0,0 +1,153 @@ +"""One-way mapping from existing BigQuery field declarations to canonical schema v1.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol, cast + +from dander.schema import normalize_bigquery_type +from dander.warehouse.contracts import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + ProviderExtension, + RelationSchema, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + +class BigQuerySchemaCompatibilityError(ValueError): + """A BigQuery schema cannot be represented canonically without an explicit fallback.""" + + +class BigQueryFieldLike(Protocol): + """Structural view shared by connector RawField and writer WriteField.""" + + @property + def name(self) -> str: ... + + @property + def data_type(self) -> str: ... + + @property + def mode(self) -> str: ... + + @property + def fields(self) -> Sequence[object]: ... + + +_SCALARS = { + "BOOL": CanonicalType(kind=LogicalTypeKind.BOOLEAN), + "BYTES": CanonicalType(kind=LogicalTypeKind.BINARY), + "DATE": CanonicalType(kind=LogicalTypeKind.DATE), + "FLOAT64": CanonicalType(kind=LogicalTypeKind.FLOAT, bit_width=64), + "INT64": CanonicalType(kind=LogicalTypeKind.INTEGER, bit_width=64), + "JSON": CanonicalType(kind=LogicalTypeKind.JSON), + "STRING": CanonicalType(kind=LogicalTypeKind.STRING), + "TIME": CanonicalType(kind=LogicalTypeKind.TIME, fractional_second_precision=6), + "TIMESTAMP": CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=True, + fractional_second_precision=6, + ), + "DATETIME": CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=False, + fractional_second_precision=6, + ), + "NUMERIC": CanonicalType(kind=LogicalTypeKind.DECIMAL, precision=38, scale=9), +} + + +def canonical_schema_from_bigquery( + fields: Sequence[BigQueryFieldLike], + *, + unsupported_fallbacks: Mapping[str, CanonicalType] | None = None, +) -> RelationSchema: + """Map a complete BigQuery schema without mutating or weakening the source declaration.""" + if not fields: + raise BigQuerySchemaCompatibilityError( + "Canonical schema compatibility requires at least one declared BigQuery field" + ) + return RelationSchema( + fields=tuple( + canonical_field_from_bigquery( + field, + unsupported_fallbacks=unsupported_fallbacks, + ) + for field in fields + ) + ) + + +def canonical_field_from_bigquery( + field: BigQueryFieldLike, + *, + unsupported_fallbacks: Mapping[str, CanonicalType] | None = None, +) -> CanonicalField: + """Map one recursive BigQuery field, rejecting undeclared lossy conversions.""" + data_type = normalize_bigquery_type(field.data_type) + mode = field.mode.upper() + base_type = _canonical_base_type( + data_type, + field.fields, + unsupported_fallbacks=unsupported_fallbacks, + ) + canonical_type = ( + CanonicalType(kind=LogicalTypeKind.ARRAY, element=base_type) + if mode == "REPEATED" + else base_type + ) + if mode not in {"NULLABLE", "REQUIRED", "REPEATED"}: + raise BigQuerySchemaCompatibilityError(f"Unsupported BigQuery field mode: {mode}") + cardinality = FieldCardinality.NULLABLE if mode == "NULLABLE" else FieldCardinality.REQUIRED + return CanonicalField( + name=field.name, + data_type=canonical_type, + cardinality=cardinality, + extensions=( + ProviderExtension(provider="bigquery", name="mode", value=mode), + ProviderExtension(provider="bigquery", name="type", value=data_type), + ), + ) + + +def _canonical_base_type( + data_type: str, + nested_fields: Sequence[object], + *, + unsupported_fallbacks: Mapping[str, CanonicalType] | None, +) -> CanonicalType: + if data_type == "RECORD": + if not nested_fields: + raise BigQuerySchemaCompatibilityError("BigQuery RECORD must declare nested fields") + return CanonicalType( + kind=LogicalTypeKind.RECORD, + fields=tuple( + canonical_field_from_bigquery( + _field_like(field), + unsupported_fallbacks=unsupported_fallbacks, + ) + for field in nested_fields + ), + ) + if nested_fields: + raise BigQuerySchemaCompatibilityError( + f"BigQuery {data_type} field cannot declare nested fields" + ) + if mapped := _SCALARS.get(data_type): + return mapped + fallback = (unsupported_fallbacks or {}).get(data_type) + if fallback is None: + raise BigQuerySchemaCompatibilityError( + f"BigQuery type {data_type!r} requires an explicit canonical fallback" + ) + return fallback + + +def _field_like(value: object) -> BigQueryFieldLike: + if not all(hasattr(value, attribute) for attribute in ("name", "data_type", "mode", "fields")): + raise BigQuerySchemaCompatibilityError("BigQuery schema contains an invalid field") + return cast("BigQueryFieldLike", value) diff --git a/src/dander/warehouse/contracts.py b/src/dander/warehouse/contracts.py new file mode 100644 index 00000000..09b4c9c8 --- /dev/null +++ b/src/dander/warehouse/contracts.py @@ -0,0 +1,235 @@ +"""Canonical relation and schema contracts shared by warehouse adapters.""" + +from __future__ import annotations + +import re +from enum import StrEnum +from typing import Literal, Protocol, Self, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +_CATALOG = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,254}$") +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,254}$") +_PROVIDER = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_EXTENSION_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,126}$") + +ExtensionScalar = str | int | float | bool + + +class LogicalTypeKind(StrEnum): + """Closed set of provider-neutral data types in schema contract v1.""" + + BOOLEAN = "boolean" + INTEGER = "integer" + DECIMAL = "decimal" + FLOAT = "float" + STRING = "string" + BINARY = "binary" + DATE = "date" + TIME = "time" + TIMESTAMP = "timestamp" + JSON = "json" + ARRAY = "array" + RECORD = "record" + + +class FieldCardinality(StrEnum): + """Whether one relation row must contain a value for a field.""" + + NULLABLE = "nullable" + REQUIRED = "required" + + +class ProviderExtension(BaseModel): + """One deterministic provider-specific schema annotation, never a credential.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: str + name: str + value: ExtensionScalar + + @field_validator("provider") + @classmethod + def validate_provider(cls, value: str) -> str: + if not _PROVIDER.fullmatch(value): + raise ValueError("extension provider must be a valid provider id") + return value + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + if not _EXTENSION_NAME.fullmatch(value): + raise ValueError("extension name must use lowercase portable syntax") + return value + + +class CanonicalType(BaseModel): + """Recursive logical type with explicit precision and timestamp semantics.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: LogicalTypeKind + bit_width: Literal[8, 16, 32, 64] | None = None + precision: int | None = None + scale: int | None = None + with_timezone: bool | None = None + fractional_second_precision: int | None = None + element: CanonicalType | None = None + fields: tuple[CanonicalField, ...] = Field(default_factory=tuple) + + @model_validator(mode="after") + def validate_shape(self) -> Self: + if self.kind is LogicalTypeKind.INTEGER: + if self.bit_width is None: + raise ValueError("integer must declare bit_width") + elif self.kind is LogicalTypeKind.FLOAT: + if self.bit_width not in {32, 64}: + raise ValueError("float bit_width must be 32 or 64") + elif self.bit_width is not None: + raise ValueError("bit_width applies only to integer and float") + + if self.kind is LogicalTypeKind.DECIMAL: + if ( + self.precision is None + or self.scale is None + or isinstance(self.precision, bool) + or isinstance(self.scale, bool) + or not 1 <= self.precision <= 1_000 + or not 0 <= self.scale <= self.precision + ): + raise ValueError( + "decimal requires precision and scale with 0 <= scale <= precision" + ) + elif self.precision is not None or self.scale is not None: + raise ValueError("precision and scale apply only to decimal") + + if self.kind is LogicalTypeKind.TIMESTAMP: + if self.with_timezone is None: + raise ValueError("timestamp must declare with_timezone") + elif self.with_timezone is not None: + raise ValueError("with_timezone applies only to timestamp") + + if self.kind in {LogicalTypeKind.TIME, LogicalTypeKind.TIMESTAMP}: + if ( + self.fractional_second_precision is None + or isinstance(self.fractional_second_precision, bool) + or not 0 <= self.fractional_second_precision <= 9 + ): + raise ValueError( + "time and timestamp must declare fractional_second_precision from 0 to 9" + ) + elif self.fractional_second_precision is not None: + raise ValueError("fractional_second_precision applies only to time and timestamp") + + if self.kind is LogicalTypeKind.ARRAY: + if self.element is None: + raise ValueError("array must declare an element type") + elif self.element is not None: + raise ValueError("element applies only to array") + + if self.kind is LogicalTypeKind.RECORD: + if not self.fields: + raise ValueError("record must declare fields") + names = [field.name for field in self.fields] + if len(names) != len(set(names)): + raise ValueError("record field names must be unique") + elif self.fields: + raise ValueError("fields apply only to record") + return self + + +class CanonicalField(BaseModel): + """One named relation field with provider-neutral type and cardinality.""" + + model_config = ConfigDict( + extra="forbid", + frozen=True, + populate_by_name=True, + ) + + name: str + data_type: CanonicalType = Field(alias="type") + cardinality: FieldCardinality = FieldCardinality.NULLABLE + description: str = "" + extensions: tuple[ProviderExtension, ...] = Field(default_factory=tuple) + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + if not _IDENTIFIER.fullmatch(value): + raise ValueError("canonical field name must be a portable identifier") + return value + + @field_validator("extensions") + @classmethod + def validate_extensions( + cls, + values: tuple[ProviderExtension, ...], + ) -> tuple[ProviderExtension, ...]: + keys = [(extension.provider, extension.name) for extension in values] + if len(keys) != len(set(keys)): + raise ValueError("provider extensions must not contain duplicate keys") + if tuple(sorted(keys)) != tuple(keys): + raise ValueError("provider extensions must use deterministic provider/name order") + return values + + +class RelationSchema(BaseModel): + """Versioned canonical schema for one physical relation.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + version: Literal[1] = 1 + fields: tuple[CanonicalField, ...] = Field(min_length=1) + + @field_validator("fields") + @classmethod + def validate_fields(cls, values: tuple[CanonicalField, ...]) -> tuple[CanonicalField, ...]: + names = [field.name for field in values] + if len(names) != len(set(names)): + raise ValueError("relation field names must be unique") + return values + + +class RelationRef(BaseModel): + """Unrendered catalog/namespace/relation coordinates.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + catalog: str + namespace: str + name: str + + @field_validator("catalog") + @classmethod + def validate_catalog(cls, value: str) -> str: + if not _CATALOG.fullmatch(value): + raise ValueError("relation catalog must be a portable catalog identifier") + return value + + @field_validator("namespace", "name") + @classmethod + def validate_identifier(cls, value: str) -> str: + if not _IDENTIFIER.fullmatch(value): + raise ValueError("relation namespace and name must be portable identifiers") + return value + + @property + def coordinates(self) -> tuple[str, str, str]: + """Return stable coordinates without choosing provider quoting or separators.""" + return (self.catalog, self.namespace, self.name) + + +@runtime_checkable +class RelationCodec(Protocol): + """Validate and render canonical coordinates for one warehouse provider.""" + + @property + def provider_id(self) -> str: + """Return the registered warehouse provider identifier.""" + ... + + def render(self, relation: RelationRef) -> str: + """Return one safely quoted provider relation identifier.""" + ... diff --git a/src/dander/writer/base.py b/src/dander/writer/base.py index 15c8844f..685198c0 100644 --- a/src/dander/writer/base.py +++ b/src/dander/writer/base.py @@ -16,6 +16,7 @@ from collections.abc import Iterable, Mapping from dander.concurrency import FencingToken + from dander.warehouse import CanonicalField, RelationRef, RelationSchema class WriteMode(StrEnum): @@ -51,6 +52,12 @@ class WriteField: mode: str = "NULLABLE" fields: tuple[WriteField, ...] = field(default_factory=tuple) + def to_canonical(self) -> CanonicalField: + """Map this legacy BigQuery writer field to canonical schema v1.""" + from dander.warehouse import canonical_field_from_bigquery + + return canonical_field_from_bigquery(self) + @dataclass(frozen=True) class WriteTarget: @@ -63,6 +70,20 @@ class WriteTarget: schema: tuple[WriteField, ...] = field(default_factory=tuple) fence: FencingToken | None = None + @property + def relation_ref(self) -> RelationRef: + """Return the target coordinates without rendering BigQuery SQL.""" + from dander.warehouse import RelationRef + + return RelationRef(catalog=self.project, namespace=self.dataset, name=self.table) + + @property + def canonical_schema(self) -> RelationSchema: + """Return the target schema through the one-way compatibility mapper.""" + from dander.warehouse import canonical_schema_from_bigquery + + return canonical_schema_from_bigquery(self.schema) + class WritePattern(ABC): """Loads a batch of records into a BigQuery target using one ``WriteMode``.""" diff --git a/tests/warehouse/test_contracts.py b/tests/warehouse/test_contracts.py new file mode 100644 index 00000000..9a5cb759 --- /dev/null +++ b/tests/warehouse/test_contracts.py @@ -0,0 +1,190 @@ +"""Canonical warehouse relation and schema contract tests.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from dander.ingestion import Endpoint, RawField +from dander.warehouse import ( + BigQuerySchemaCompatibilityError, + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + ProviderExtension, + RelationRef, + RelationSchema, + canonical_schema_from_bigquery, +) +from dander.writer import WriteField, WriteTarget + + +def test_relation_ref_keeps_coordinates_unrendered() -> None: + relation = RelationRef( + catalog="gcp-project", + namespace="analytics", + name="customer_orders", + ) + + assert relation.coordinates == ("gcp-project", "analytics", "customer_orders") + assert "`" not in "".join(relation.coordinates) + + +@pytest.mark.parametrize( + "payload", + [ + {"catalog": "", "namespace": "analytics", "name": "orders"}, + {"catalog": "project", "namespace": "bad.name", "name": "orders"}, + {"catalog": "project", "namespace": "analytics", "name": "bad-name"}, + ], +) +def test_relation_ref_rejects_nonportable_coordinates(payload: dict[str, str]) -> None: + with pytest.raises(ValidationError): + RelationRef.model_validate(payload) + + +def test_canonical_type_requires_explicit_decimal_timestamp_array_and_record_shape() -> None: + decimal = CanonicalType(kind=LogicalTypeKind.DECIMAL, precision=38, scale=9) + timestamp = CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=True, + fractional_second_precision=6, + ) + array = CanonicalType(kind=LogicalTypeKind.ARRAY, element=decimal) + record = CanonicalType( + kind=LogicalTypeKind.RECORD, + fields=(CanonicalField(name="observed_at", data_type=timestamp),), + ) + + assert array.element == decimal + assert record.fields[0].data_type.with_timezone is True + for payload in ( + {"kind": "decimal", "precision": 10}, + {"kind": "timestamp"}, + {"kind": "integer"}, + {"kind": "float", "bit_width": 16}, + {"kind": "array"}, + {"kind": "record", "fields": []}, + {"kind": "string", "precision": 5}, + ): + with pytest.raises(ValidationError): + CanonicalType.model_validate(payload) + + +def test_schema_rejects_duplicate_fields_and_extensions() -> None: + field = CanonicalField(name="id", data_type=CanonicalType(kind=LogicalTypeKind.STRING)) + with pytest.raises(ValidationError, match="relation field names must be unique"): + RelationSchema(fields=(field, field)) + + extension = ProviderExtension(provider="bigquery", name="type", value="STRING") + with pytest.raises(ValidationError, match="duplicate keys"): + CanonicalField( + name="id", + data_type=CanonicalType(kind=LogicalTypeKind.STRING), + extensions=(extension, extension), + ) + + +def test_bigquery_compatibility_maps_nested_repeated_and_precise_types() -> None: + fields = ( + WriteField(name="id", data_type="INTEGER", mode="REQUIRED"), + WriteField(name="amount", data_type="NUMERIC"), + WriteField(name="observed_at", data_type="TIMESTAMP"), + WriteField(name="local_at", data_type="DATETIME"), + WriteField( + name="events", + data_type="RECORD", + mode="REPEATED", + fields=(WriteField(name="active", data_type="BOOLEAN"),), + ), + ) + + schema = canonical_schema_from_bigquery(fields) + + assert schema.fields[0].data_type.kind is LogicalTypeKind.INTEGER + assert schema.fields[0].cardinality is FieldCardinality.REQUIRED + assert schema.fields[1].data_type == CanonicalType( + kind=LogicalTypeKind.DECIMAL, + precision=38, + scale=9, + ) + assert schema.fields[2].data_type.with_timezone is True + assert schema.fields[2].data_type.fractional_second_precision == 6 + assert schema.fields[3].data_type.with_timezone is False + repeated = schema.fields[4] + assert repeated.cardinality is FieldCardinality.REQUIRED + assert repeated.data_type.kind is LogicalTypeKind.ARRAY + assert repeated.data_type.element is not None + assert repeated.data_type.element.kind is LogicalTypeKind.RECORD + assert repeated.data_type.element.fields[0].data_type.kind is LogicalTypeKind.BOOLEAN + assert [(item.name, item.value) for item in repeated.extensions] == [ + ("mode", "REPEATED"), + ("type", "RECORD"), + ] + + +@pytest.mark.parametrize( + ("bigquery_type", "kind"), + [ + ("BOOL", LogicalTypeKind.BOOLEAN), + ("BYTES", LogicalTypeKind.BINARY), + ("DATE", LogicalTypeKind.DATE), + ("FLOAT64", LogicalTypeKind.FLOAT), + ("INT64", LogicalTypeKind.INTEGER), + ("JSON", LogicalTypeKind.JSON), + ("STRING", LogicalTypeKind.STRING), + ("TIME", LogicalTypeKind.TIME), + ], +) +def test_bigquery_scalar_compatibility_is_explicit( + bigquery_type: str, + kind: LogicalTypeKind, +) -> None: + field = WriteField(name="value", data_type=bigquery_type) + + assert canonical_schema_from_bigquery((field,)).fields[0].data_type.kind is kind + + +def test_bigquery_compatibility_requires_a_declared_schema() -> None: + with pytest.raises(BigQuerySchemaCompatibilityError, match="at least one declared"): + canonical_schema_from_bigquery(()) + + +def test_bigquery_compatibility_rejects_lossy_type_without_explicit_fallback() -> None: + geography = WriteField(name="location", data_type="GEOGRAPHY") + + with pytest.raises(BigQuerySchemaCompatibilityError, match="explicit canonical fallback"): + canonical_schema_from_bigquery((geography,)) + + schema = canonical_schema_from_bigquery( + (geography,), + unsupported_fallbacks={ + "GEOGRAPHY": CanonicalType(kind=LogicalTypeKind.STRING), + }, + ) + assert schema.fields[0].data_type.kind is LogicalTypeKind.STRING + assert schema.fields[0].extensions[1].value == "GEOGRAPHY" + + big_numeric = WriteField(name="amount", data_type="BIGNUMERIC") + with pytest.raises(BigQuerySchemaCompatibilityError, match="explicit canonical fallback"): + canonical_schema_from_bigquery((big_numeric,)) + + +def test_existing_connector_and_writer_models_expose_canonical_views() -> None: + endpoint = Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[RawField(name="id", data_type="STRING", mode="REQUIRED")], + ) + target = WriteTarget( + project="gcp-project", + dataset="raw", + table="records", + schema=(WriteField(name="id", data_type="STRING", mode="REQUIRED"),), + ) + + assert endpoint.raw_schema[0].to_canonical() == endpoint.canonical_raw_schema().fields[0] + assert target.relation_ref.coordinates == ("gcp-project", "raw", "records") + assert target.canonical_schema.fields[0].cardinality is FieldCardinality.REQUIRED diff --git a/tickets/DANDER-80-canonical-relation-schema.md b/tickets/DANDER-80-canonical-relation-schema.md new file mode 100644 index 00000000..79be497d --- /dev/null +++ b/tickets/DANDER-80-canonical-relation-schema.md @@ -0,0 +1,42 @@ +--- +id: DANDER-80 +title: Add canonical relation and schema contracts +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-79] +created: 2026-08-07 +--- + +## Context + +BigQuery project/dataset/table coordinates and type/mode strings cannot remain shared warehouse +boundaries. The portable contract must retain exact semantics before provider-specific rendering. + +## Acceptance Criteria + +- [x] `RelationRef` keeps catalog, namespace, and name unrendered. +- [x] `RelationCodec` makes provider rendering an explicit adapter responsibility. +- [x] Canonical types cover required scalar, decimal, timestamp, array, and record semantics. +- [x] Relation and nested-record fields reject duplicate names. +- [x] Provider extensions are validated, ordered, and duplicate-free. +- [x] Existing BigQuery raw and writer fields expose a one-way canonical view. +- [x] Unsupported BigQuery types fail without an explicit fallback. +- [x] Existing authored schemas and BigQuery execution remain unchanged. +- [x] Full local validation passes. +- [ ] Protected CI passes. + +## Design + +Use frozen Pydantic models so canonical contracts serialize deterministically but still parse normal +YAML/JSON scalar forms. Keep BigQuery compatibility in a one-way mapper accepting the structural +shape shared by `RawField` and `WriteField`. + +## Implementation Notes + +This PR does not add a provider codec implementation, rewrite schema YAML, or route a runtime +through canonical writers. Those moves remain separately characterized adapter work. + +## Review Log + +Pending protected review. From 13394c479a873716d0b1ecc85d2d845935df0788 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 17:12:26 -0400 Subject: [PATCH 030/163] Add explicit portable SQL dialect contract --- CHANGELOG.md | 3 + HANDOFF.md | 37 +- docs/decisions.md | 11 + .../marts/fct_salesforce__opportunities.yml | 1 + models/staging/stg_greenhouse__candidates.yml | 1 + models/staging/stg_greenhouse__jobs.yml | 1 + models/staging/stg_hubspot__companies.yml | 1 + models/staging/stg_netsuite__customers.yml | 1 + models/staging/stg_odoo__partners.yml | 1 + models/staging/stg_salesforce__accounts.yml | 1 + models/staging/stg_salesforce__contacts.yml | 1 + .../staging/stg_salesforce__opportunities.yml | 1 + models/staging/stg_salesforce__users.yml | 1 + models/staging/stg_servicenow__incidents.yml | 1 + src/dander/transform/README.md | 23 +- src/dander/transform/__init__.py | 6 + src/dander/transform/config.py | 3 +- src/dander/transform/dialects.py | 346 ++++++++++++++++++ src/dander/transform/model.py | 14 + src/dander/transform/project.py | 106 +++++- steering/languages/sql.md | 12 +- tests/transform/test_dialects.py | 124 +++++++ tests/transform/test_project.py | 45 +++ tickets/DANDER-81-portable-sql-dialects.md | 44 +++ 24 files changed, 754 insertions(+), 31 deletions(-) create mode 100644 src/dander/transform/dialects.py create mode 100644 tests/transform/test_dialects.py create mode 100644 tickets/DANDER-81-portable-sql-dialects.md diff --git a/CHANGELOG.md b/CHANGELOG.md index dfc4d696..a880e134 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Add explicit `portable`, BigQuery, Snowflake, Redshift, and PostgreSQL model dialect metadata; + preserve BigQuery as the compatibility default and validate portable models through a closed, + deterministic SQL AST before provider rendering. - Add canonical relation/schema v1 contracts with explicit decimal, timestamp, array, record, and provider-extension semantics plus a fail-closed BigQuery compatibility mapper. - Add an API-v1 provider-factory registry for warehouse, state, catalog, secret, and launcher diff --git a/HANDOFF.md b/HANDOFF.md index edf78da8..7ab19164 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,39 @@ ## Finished -- Added unrendered `RelationRef` coordinates and the provider `RelationCodec` boundary. -- Added canonical schema v1 for exact scalars, decimals, timestamps, arrays, and records. -- Added ordered provider extensions and duplicate/shape validation. -- Added a fail-closed, one-way mapper for legacy BigQuery raw and writer fields. -- Exposed canonical views without changing authored schemas or BigQuery runtime behavior. +- Added explicit exact-provider and portable model dialect metadata. +- Preserved BigQuery as the compatibility default and marked repository models explicitly. +- Added a closed portable SQL AST with deterministic ordering, casts, identifiers, and refs. +- Added BigQuery, Snowflake, Redshift, and PostgreSQL rendering for validated portable queries. +- Added positive and rejection fixtures plus exact-dialect mismatch coverage. ## Try It -Call `endpoint.canonical_raw_schema()` or `target.canonical_schema`; use -`target.relation_ref` to pass coordinates to a future provider codec without rendering SQL. +Set `dialect: portable` in a model sidecar, use only `{{ ref('...') }}` relations, then call +`project.compile(model, target_dialect="postgres")`. Existing models need no changes. ## Checks -- All 861 tests, Ruff, formatting, and strict mypy across 197 source files passed. -- Wheel/sdist build and inspection plus both Terraform-root validations passed. +- All 887 tests, Ruff, formatting, and strict mypy across 199 source files passed. +- Wheel/sdist build, inspection, source-free installs, generated-project validation, and all four + Terraform-root validations passed. +- Local container build and read-only runtime conformance passed as UID 65532. - The isolated GCP plan reported `No changes`; both schedules stayed paused and no apply ran. ## Decisions -- Decimal precision/scale and timestamp timezone semantics are mandatory. -- BigQuery `REPEATED` maps to a required canonical array while retaining its original mode. -- Provider-only types require an explicit fallback; silent lossy mapping is forbidden. +- Existing and undeclared models remain exact BigQuery SQL. +- Portable AST nodes are closed; new sqlglot nodes fail until explicitly reviewed. +- Render targets do not imply provider runtime support. ## Remaining -- Open and merge the focused canonical-schema PR after protected CI passes. -- Add portable SQL/dialect boundaries from merged main in the next PR. +- Complete local and isolated GCP validation. +- Open the focused dialect PR and merge only after protected CI passes. +- Compile graph operations through the same provider-neutral boundary in the next PR. ## Review First -- `src/dander/warehouse/contracts.py` -- `src/dander/warehouse/bigquery_compat.py` -- `tests/warehouse/test_contracts.py` +- `src/dander/transform/dialects.py` +- `src/dander/transform/project.py` +- `tests/transform/test_dialects.py` diff --git a/docs/decisions.md b/docs/decisions.md index f8fc5f6e..a8448235 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-07 — Portable SQL is an explicit, closed contract + +- **Compatibility:** Model metadata defaults to exact `bigquery`; repository-owned legacy models + state it explicitly. Exact SQL never translates to another provider. +- **Portability:** `portable` models use only declared `ref()` relations and a closed sqlglot AST. + Dander rejects unknown nodes, implicit null ordering, unstable windows, non-canonical identifiers + and literals, lossy casts, and provider-specific constructs before rendering. +- **Scope:** BigQuery, Snowflake, Redshift, and PostgreSQL are render targets, not support claims. + Provider variants, graph AST reuse, assertions, materializations, and live equivalence remain + independent tickets. + ## 2026-08-07 — Canonical schemas preserve semantics before rendering - **Relations:** Internal coordinates use `RelationRef(catalog, namespace, name)`. Only a selected diff --git a/models/marts/fct_salesforce__opportunities.yml b/models/marts/fct_salesforce__opportunities.yml index 5c941d03..d08a1aa3 100644 --- a/models/marts/fct_salesforce__opportunities.yml +++ b/models/marts/fct_salesforce__opportunities.yml @@ -1,6 +1,7 @@ model: fct_salesforce__opportunities description: Non-deleted Salesforce Opportunity facts with active Account and historical owner data. owner: data-eng +dialect: bigquery materialization: table dataset: marts source_system: salesforce diff --git a/models/staging/stg_greenhouse__candidates.yml b/models/staging/stg_greenhouse__candidates.yml index 30d3c8b1..64a05f90 100644 --- a/models/staging/stg_greenhouse__candidates.yml +++ b/models/staging/stg_greenhouse__candidates.yml @@ -3,6 +3,7 @@ model: stg_greenhouse__candidates description: Cleaned Greenhouse candidates, one row per candidate. owner: data-eng +dialect: bigquery materialization: view source_system: greenhouse sensitivity: pii # candidate PII — treat accordingly (steering/01-security.md) diff --git a/models/staging/stg_greenhouse__jobs.yml b/models/staging/stg_greenhouse__jobs.yml index dbd0610a..c09548e7 100644 --- a/models/staging/stg_greenhouse__jobs.yml +++ b/models/staging/stg_greenhouse__jobs.yml @@ -2,6 +2,7 @@ model: stg_greenhouse__jobs description: Published Greenhouse jobs cleaned to one row per job. owner: data-eng +dialect: bigquery materialization: view dataset: staging source_system: greenhouse_job_board diff --git a/models/staging/stg_hubspot__companies.yml b/models/staging/stg_hubspot__companies.yml index 9968b8dd..6ac0d508 100644 --- a/models/staging/stg_hubspot__companies.yml +++ b/models/staging/stg_hubspot__companies.yml @@ -2,6 +2,7 @@ model: stg_hubspot__companies description: Invented HubSpot proof companies normalized for the Dander authenticated ingestion path. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: hubspot_test diff --git a/models/staging/stg_netsuite__customers.yml b/models/staging/stg_netsuite__customers.yml index e118a087..3a05545b 100644 --- a/models/staging/stg_netsuite__customers.yml +++ b/models/staging/stg_netsuite__customers.yml @@ -1,6 +1,7 @@ model: stg_netsuite__customers description: NetSuite customers normalized from the simulator-validated SuiteQL contract. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: netsuite diff --git a/models/staging/stg_odoo__partners.yml b/models/staging/stg_odoo__partners.yml index 78a05cb1..f9767436 100644 --- a/models/staging/stg_odoo__partners.yml +++ b/models/staging/stg_odoo__partners.yml @@ -1,6 +1,7 @@ model: stg_odoo__partners description: Odoo contacts and companies normalized from the read-only JSON-2 contract. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: odoo diff --git a/models/staging/stg_salesforce__accounts.yml b/models/staging/stg_salesforce__accounts.yml index 627167a7..3c91fdb2 100644 --- a/models/staging/stg_salesforce__accounts.yml +++ b/models/staging/stg_salesforce__accounts.yml @@ -1,6 +1,7 @@ model: stg_salesforce__accounts description: Salesforce Accounts normalized from the read-only QueryAll extraction contract. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: salesforce diff --git a/models/staging/stg_salesforce__contacts.yml b/models/staging/stg_salesforce__contacts.yml index fefb3ec9..79a7cbc2 100644 --- a/models/staging/stg_salesforce__contacts.yml +++ b/models/staging/stg_salesforce__contacts.yml @@ -1,6 +1,7 @@ model: stg_salesforce__contacts description: Salesforce Contacts normalized with Account and owner relationships and tombstones. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: salesforce diff --git a/models/staging/stg_salesforce__opportunities.yml b/models/staging/stg_salesforce__opportunities.yml index 3c0e2ee3..f06995a0 100644 --- a/models/staging/stg_salesforce__opportunities.yml +++ b/models/staging/stg_salesforce__opportunities.yml @@ -1,6 +1,7 @@ model: stg_salesforce__opportunities description: Salesforce Opportunities normalized with revenue, stage, ownership, and tombstones. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: salesforce diff --git a/models/staging/stg_salesforce__users.yml b/models/staging/stg_salesforce__users.yml index 29240293..a69ac2cd 100644 --- a/models/staging/stg_salesforce__users.yml +++ b/models/staging/stg_salesforce__users.yml @@ -1,6 +1,7 @@ model: stg_salesforce__users description: Salesforce Users normalized for owner dimensions while retaining inactive users. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: salesforce diff --git a/models/staging/stg_servicenow__incidents.yml b/models/staging/stg_servicenow__incidents.yml index d5948883..fe45e60f 100644 --- a/models/staging/stg_servicenow__incidents.yml +++ b/models/staging/stg_servicenow__incidents.yml @@ -1,6 +1,7 @@ model: stg_servicenow__incidents description: ServiceNow incidents normalized from the read-only Table API contract. owner: data-eng +dialect: bigquery materialization: incremental dataset: staging source_system: servicenow diff --git a/src/dander/transform/README.md b/src/dander/transform/README.md index 2b8a8f2b..f7a1c577 100644 --- a/src/dander/transform/README.md +++ b/src/dander/transform/README.md @@ -1,15 +1,32 @@ # Transform engine Dander owns the SQL transform layer described in the project decision log. A project is a directory -of BigQuery SQL files with same-named YAML sidecars. The sidecar is the metadata spine shared by -execution, generic tests, Dataplex system-aspect generation, and the semantic registry. +of SQL files with same-named YAML sidecars. The sidecar is the metadata spine shared by execution, +generic tests, catalog publication, and the semantic registry. + +Every sidecar has an authored SQL contract: + +```yaml +dialect: bigquery # portable | bigquery | snowflake | redshift | postgres +``` + +Existing files default to exact `bigquery` SQL and are never silently translated. A model that +opts into `portable` is parsed into one restricted sqlglot AST and may render to the four declared +warehouse dialects. Exact provider SQL compiles only for its matching target. Portable models must +use `ref()` for physical relations; provider functions and ambiguous constructs fail validation. +The current subset includes projections, filters, explicit joins, `UNION ALL`, aggregations, +deterministic windows, closed scalar functions, and strict casts. It intentionally requires +explicit null ordering, Unicode NFC literals, canonical identifiers, decimal `(38, 9)`, and +microsecond time/timestamp precision. Provider variants, generic assertion rendering, and runtime +selection arrive in separate portability slices; this contract alone is not a support claim. ## Build contract 1. Discover every `*.sql` file and validate its `*.yml` or `*.yaml` sidecar. 2. Resolve model `ref()` calls and conventional `raw_` source references. 3. Reject unknown references and cycles before submitting a query. -4. Render refs through a restricted Jinja environment and require one read-only BigQuery query. +4. Render refs through a restricted Jinja environment and require one read-only query matching the + sidecar's exact or portable dialect contract. 5. Materialize selected models and their dependencies as views, tables, or incremental merges in topological order. 6. Run declared not-null, unique, accepted-values, and relationship assertions. diff --git a/src/dander/transform/__init__.py b/src/dander/transform/__init__.py index 095a108a..76ebfff7 100644 --- a/src/dander/transform/__init__.py +++ b/src/dander/transform/__init__.py @@ -10,6 +10,8 @@ TransformConfigError, load_model_metadata, ) +from dander.transform.dialects import PortableSqlError, parse_portable_query, render_portable_query +from dander.transform.model import SqlDialect from dander.transform.project import ( TransformModel, TransformProject, @@ -24,6 +26,7 @@ "MetricAggregation", "MetricMetadata", "ModelMetadata", + "PortableSqlError", "RelationshipMetadata", "TransformConfigError", "TransformModel", @@ -31,5 +34,8 @@ "TransformProjectError", "TransformRunError", "TransformRunResult", + "SqlDialect", "load_model_metadata", + "parse_portable_query", + "render_portable_query", ] diff --git a/src/dander/transform/config.py b/src/dander/transform/config.py index 246bb3a0..dd347439 100644 --- a/src/dander/transform/config.py +++ b/src/dander/transform/config.py @@ -9,7 +9,7 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator -from dander.transform.model import Materialization +from dander.transform.model import Materialization, SqlDialect _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") type Scalar = str | int | float | bool @@ -103,6 +103,7 @@ class ModelMetadata(BaseModel): model: str = Field(pattern=_IDENTIFIER.pattern) description: str = Field(min_length=1) owner: str = Field(min_length=1) + dialect: SqlDialect = SqlDialect.BIGQUERY materialization: Materialization = Materialization.VIEW dataset: str = Field(default="staging", pattern=_IDENTIFIER.pattern) source_system: str = Field(min_length=1) diff --git a/src/dander/transform/dialects.py b/src/dander/transform/dialects.py new file mode 100644 index 00000000..7124ea4e --- /dev/null +++ b/src/dander/transform/dialects.py @@ -0,0 +1,346 @@ +"""Conservative SQL-dialect contract for portable transform models.""" + +from __future__ import annotations + +import re +import unicodedata +from typing import TYPE_CHECKING + +import sqlglot +from sqlglot import exp +from sqlglot.tokenizer_core import Token, TokenType +from sqlglot.tokens import Tokenizer + +from dander.transform.model import SqlDialect + +if TYPE_CHECKING: + from collections.abc import Collection + + +class PortableSqlError(ValueError): + """SQL falls outside Dander's provider-neutral subset.""" + + +_PORTABLE_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") +_TARGET_DIALECTS = frozenset(SqlDialect) - {SqlDialect.PORTABLE} + +# Exact node types are intentional: a new sqlglot construct is rejected until Dander assigns it +# portable semantics and adds positive and negative conformance fixtures. +_PORTABLE_NODES: frozenset[type[exp.Expression]] = frozenset( + { + exp.Abs, + exp.Add, + exp.Alias, + exp.And, + exp.Avg, + exp.Between, + exp.Boolean, + exp.Case, + exp.Cast, + exp.Ceil, + exp.Coalesce, + exp.Column, + exp.Concat, + exp.Count, + exp.CTE, + exp.DataType, + exp.DataTypeParam, + exp.DenseRank, + exp.Distinct, + exp.Div, + exp.EQ, + exp.Floor, + exp.From, + exp.Greatest, + exp.Group, + exp.GT, + exp.GTE, + exp.Having, + exp.Identifier, + exp.If, + exp.In, + exp.Is, + exp.Join, + exp.Lag, + exp.LastValue, + exp.Lead, + exp.Least, + exp.Length, + exp.Like, + exp.Limit, + exp.Literal, + exp.Lower, + exp.LT, + exp.LTE, + exp.Max, + exp.Min, + exp.Mod, + exp.Mul, + exp.Neg, + exp.NEQ, + exp.Not, + exp.Null, + exp.Nullif, + exp.Offset, + exp.Or, + exp.Order, + exp.Ordered, + exp.Paren, + exp.Rank, + exp.Round, + exp.RowNumber, + exp.Select, + exp.Star, + exp.Sub, + exp.Substring, + exp.Subquery, + exp.Sum, + exp.Table, + exp.TableAlias, + exp.Trim, + exp.Tuple, + exp.Union, + exp.Upper, + exp.When, + exp.Where, + exp.Window, + exp.WindowSpec, + exp.With, + } +) + +_PORTABLE_CASTS = frozenset( + { + exp.DataType.Type.BIGINT, + exp.DataType.Type.BOOLEAN, + exp.DataType.Type.DATE, + exp.DataType.Type.DECIMAL, + exp.DataType.Type.DOUBLE, + exp.DataType.Type.TEXT, + exp.DataType.Type.TIME, + exp.DataType.Type.TIMESTAMP, + exp.DataType.Type.TIMESTAMPTZ, + } +) + +_ORDER_TERMINATORS = frozenset( + { + TokenType.FETCH, + TokenType.HAVING, + TokenType.LIMIT, + TokenType.OFFSET, + TokenType.QUALIFY, + TokenType.UNION, + TokenType.WHERE, + } +) + + +def parse_portable_query( + sql: str, + *, + unique_columns: Collection[str] = (), + allowed_relations: Collection[tuple[str, ...]] | None = None, +) -> exp.Query: + """Parse and validate one query against Dander's closed portable SQL subset.""" + try: + expression = sqlglot.parse_one(sql, read="bigquery") + except sqlglot.errors.ParseError as error: + raise PortableSqlError("invalid portable SQL") from error + if not isinstance(expression, exp.Query): + raise PortableSqlError("portable model must contain one read-only query") + + unsupported = sorted( + {type(node).__name__ for node in expression.walk() if type(node) not in _PORTABLE_NODES} + ) + if unsupported: + raise PortableSqlError("unsupported portable SQL nodes: " + ", ".join(unsupported)) + + _validate_identifiers(expression) + _validate_literals(expression) + _validate_casts(expression) + _validate_sets_and_joins(expression) + _validate_stars_and_distinct(expression) + _require_explicit_null_ordering(sql) + _validate_deterministic_windows(expression, unique_columns=frozenset(unique_columns)) + _validate_tables( + expression, + allowed_relations=None if allowed_relations is None else frozenset(allowed_relations), + ) + return expression + + +def render_portable_query(expression: exp.Query, *, target: SqlDialect | str) -> str: + """Render a previously validated portable AST for one supported warehouse dialect.""" + try: + dialect = SqlDialect(target) + except ValueError as error: + raise PortableSqlError(f"unsupported SQL target dialect: {target}") from error + if dialect not in _TARGET_DIALECTS: + raise PortableSqlError("portable is an authored contract, not a render target") + return expression.sql(dialect=dialect.value) + + +def _validate_identifiers(expression: exp.Query) -> None: + for identifier in expression.find_all(exp.Identifier): + value = identifier.this + if identifier.args.get("quoted"): + if isinstance(identifier.parent, exp.Table): + continue + raise PortableSqlError("portable logical identifiers must not be quoted") + if not _PORTABLE_IDENTIFIER.fullmatch(value): + raise PortableSqlError( + "portable logical identifiers must use lowercase snake_case names" + ) + + +def _validate_literals(expression: exp.Query) -> None: + for literal in expression.find_all(exp.Literal): + if literal.is_string and unicodedata.normalize("NFC", literal.this) != literal.this: + raise PortableSqlError("portable string literals must use Unicode NFC normalization") + + +def _validate_casts(expression: exp.Query) -> None: + for cast in expression.find_all(exp.Cast): + if cast.args.get("safe"): + raise PortableSqlError("portable SQL does not permit safe or try casts") + target = cast.args.get("to") + if not isinstance(target, exp.DataType) or target.this not in _PORTABLE_CASTS: + raise PortableSqlError("portable SQL cast target is unsupported") + parameters = tuple(target.expressions) + if target.this is exp.DataType.Type.DECIMAL: + if _integer_parameters(parameters) != (38, 9): + raise PortableSqlError("portable DECIMAL casts must declare DECIMAL(38, 9)") + elif target.this in { + exp.DataType.Type.TIME, + exp.DataType.Type.TIMESTAMP, + exp.DataType.Type.TIMESTAMPTZ, + }: + if _integer_parameters(parameters) != (6,): + raise PortableSqlError("portable time and timestamp casts must declare precision 6") + elif parameters: + raise PortableSqlError("portable scalar casts do not accept type parameters") + + +def _integer_parameters(parameters: tuple[exp.Expression, ...]) -> tuple[int, ...]: + values: list[int] = [] + for parameter in parameters: + value = parameter.this if isinstance(parameter, exp.DataTypeParam) else None + if not isinstance(value, exp.Literal) or value.is_string or not value.this.isdigit(): + return () + values.append(int(value.this)) + return tuple(values) + + +def _validate_sets_and_joins(expression: exp.Query) -> None: + for union in expression.find_all(exp.Union): + if union.args.get("distinct") is not False or union.args.get("by_name"): + raise PortableSqlError("portable SQL supports UNION ALL only") + for join in expression.find_all(exp.Join): + kind = (join.args.get("kind") or "").upper() + side = (join.args.get("side") or "").upper() + method = (join.args.get("method") or "").upper() + if method == "NATURAL": + raise PortableSqlError("portable SQL does not permit NATURAL JOIN") + if kind == "CROSS": + if join.args.get("on") is not None or join.args.get("using"): + raise PortableSqlError("portable CROSS JOIN cannot declare ON or USING") + continue + if kind not in {"", "INNER", "OUTER"}: + raise PortableSqlError("portable SQL join type is unsupported") + if side not in {"", "INNER", "LEFT", "RIGHT", "FULL"}: + raise PortableSqlError("portable SQL join type is unsupported") + if kind == "OUTER" and side not in {"LEFT", "RIGHT", "FULL"}: + raise PortableSqlError("portable OUTER JOIN must declare LEFT, RIGHT, or FULL") + if join.args.get("on") is None and not join.args.get("using"): + raise PortableSqlError("portable JOIN must declare ON or USING") + + +def _validate_stars_and_distinct(expression: exp.Query) -> None: + for star in expression.find_all(exp.Star): + if any(star.args.get(key) for key in ("except_", "replace", "rename")): + raise PortableSqlError("portable wildcard modifiers are unsupported") + for distinct in expression.find_all(exp.Distinct): + if distinct.expressions or distinct.args.get("on"): + raise PortableSqlError("portable SQL does not support DISTINCT ON") + + +def _validate_deterministic_windows( + expression: exp.Query, + *, + unique_columns: frozenset[str], +) -> None: + for window in expression.find_all(exp.Window): + order = window.args.get("order") + if not isinstance(order, exp.Order) or not order.expressions: + raise PortableSqlError("portable windows require deterministic ORDER BY") + final = order.expressions[-1] + tie_breaker = final.this if isinstance(final, exp.Ordered) else None + if not isinstance(tie_breaker, exp.Column) or tie_breaker.name not in unique_columns: + raise PortableSqlError( + "portable window ORDER BY must end with a column declared unique in model tests" + ) + + +def _validate_tables( + expression: exp.Query, + *, + allowed_relations: frozenset[tuple[str, ...]] | None, +) -> None: + cte_names = {cte.alias_or_name for cte in expression.find_all(exp.CTE)} + for table in expression.find_all(exp.Table): + coordinates = tuple(part for part in (table.catalog, table.db, table.name) if part) + if len(coordinates) == 1 and table.name in cte_names: + continue + if len(coordinates) < 2: + raise PortableSqlError("portable models must use Dander ref() relations") + if allowed_relations is not None and coordinates not in allowed_relations: + raise PortableSqlError("portable model contains a relation outside its declared refs") + + +def _require_explicit_null_ordering(sql: str) -> None: + tokens = Tokenizer(dialect="bigquery").tokenize(sql) + for index, token in enumerate(tokens): + if token.token_type is not TokenType.ORDER_BY: + continue + depth = _depth_before(tokens, index) + item: list[Token] = [] + cursor = index + 1 + while cursor < len(tokens): + current = tokens[cursor] + current_depth = _depth_before(tokens, cursor) + if current_depth < depth: + _require_null_clause(item) + break + if current_depth == depth and current.token_type in _ORDER_TERMINATORS: + _require_null_clause(item) + break + if current_depth == depth and current.token_type is TokenType.COMMA: + _require_null_clause(item) + item = [] + else: + item.append(current) + cursor += 1 + else: + _require_null_clause(item) + + +def _depth_before(tokens: list[Token], stop: int) -> int: + depth = 0 + for token in tokens[:stop]: + token_type = token.token_type + if token_type is TokenType.L_PAREN: + depth += 1 + elif token_type is TokenType.R_PAREN: + depth -= 1 + return depth + + +def _require_null_clause(item: list[Token]) -> None: + if not item: + raise PortableSqlError("portable ORDER BY must contain an expression") + words = [token.text.upper() for token in item] + for index, word in enumerate(words[:-1]): + if word == "NULLS" and words[index + 1] in {"FIRST", "LAST"}: + return + raise PortableSqlError("portable ORDER BY expressions must declare NULLS FIRST or NULLS LAST") diff --git a/src/dander/transform/model.py b/src/dander/transform/model.py index be027a71..b9ab551f 100644 --- a/src/dander/transform/model.py +++ b/src/dander/transform/model.py @@ -36,6 +36,20 @@ class Materialization(StrEnum): INCREMENTAL = "incremental" +class SqlDialect(StrEnum): + """Authored SQL contract for one transform model. + + ``portable`` is Dander's deliberately small, validated SQL subset. The other values declare + provider-exact SQL and are never translated to a different warehouse dialect. + """ + + PORTABLE = "portable" + BIGQUERY = "bigquery" + SNOWFLAKE = "snowflake" + REDSHIFT = "redshift" + POSTGRES = "postgres" + + @dataclass class Model: """A single transform model: SQL file + its declared metadata.""" diff --git a/src/dander/transform/project.py b/src/dander/transform/project.py index 2517a33d..97152eb0 100644 --- a/src/dander/transform/project.py +++ b/src/dander/transform/project.py @@ -11,7 +11,8 @@ from sqlglot import exp from dander.transform.config import ModelMetadata, TransformConfigError, load_model_metadata -from dander.transform.model import parse_refs +from dander.transform.dialects import PortableSqlError, parse_portable_query, render_portable_query +from dander.transform.model import SqlDialect, parse_refs if TYPE_CHECKING: from collections.abc import Iterable @@ -155,21 +156,67 @@ def relation_for_ref(self, reference: str) -> str: return self._relation("raw", table) raise TransformProjectError(f"Unknown model reference: {reference}") - def compile(self, model: TransformModel) -> str: - """Render refs and validate that the model is exactly one BigQuery query.""" + def compile( + self, + model: TransformModel, + *, + target_dialect: SqlDialect | str = SqlDialect.BIGQUERY, + ) -> str: + """Render refs and validate one exact or portable read-only model query.""" + try: + target = SqlDialect(target_dialect) + except ValueError as error: + raise TransformProjectError(f"Unknown target SQL dialect: {target_dialect}") from error + if target is SqlDialect.PORTABLE: + raise TransformProjectError("portable is an authored SQL contract, not a target") + authored = model.metadata.dialect + if authored is not SqlDialect.PORTABLE and authored is not target: + raise TransformProjectError( + f"Model {model.name} is {authored.value} SQL and cannot target {target.value}" + ) + + reference_dialect = target if authored is SqlDialect.PORTABLE else authored environment = Environment(undefined=StrictUndefined, autoescape=False) environment.globals.clear() try: - compiled = environment.from_string(model.sql).render(ref=self.relation_for_ref).strip() + compiled = ( + environment.from_string(model.sql) + .render( + ref=lambda reference: self._relation_for_ref( + reference, + dialect=reference_dialect, + ) + ) + .strip() + ) except (TemplateError, TransformProjectError) as error: raise TransformProjectError(f"Cannot compile model {model.name}") from error if compiled.endswith(";"): compiled = compiled[:-1].rstrip() + if authored is SqlDialect.PORTABLE: + unique_columns = {test.column for test in model.metadata.tests if test.unique} + try: + allowed_relations = { + self._relation_coordinates_for_ref(reference, dialect=target) + for reference in model.refs + } + expression = parse_portable_query( + compiled, + unique_columns=unique_columns, + allowed_relations=allowed_relations, + ) + return render_portable_query(expression, target=target) + except PortableSqlError as error: + raise TransformProjectError( + f"Invalid portable SQL in model {model.name}: {error}" + ) from error try: - expression = sqlglot.parse_one(compiled, read="bigquery") + exact_expression = sqlglot.parse_one(compiled, read=authored.value) except sqlglot.errors.ParseError as error: - raise TransformProjectError(f"Invalid BigQuery SQL in model {model.name}") from error - if not isinstance(expression, exp.Query): + raise TransformProjectError( + f"Invalid {authored.value} SQL in model {model.name}" + ) from error + if not isinstance(exact_expression, exp.Query): raise TransformProjectError(f"Model {model.name} must contain one read-only query") return compiled @@ -178,7 +225,52 @@ def _validate_references(self) -> None: for reference in model.refs: self.relation_for_ref(reference) + def _relation_for_ref(self, reference: str, *, dialect: SqlDialect) -> str: + if dialect is SqlDialect.BIGQUERY: + return self.relation_for_ref(reference) + if reference in self.models: + model = self.models[reference] + return self._relation_for_dialect(model.metadata.dataset, model.name, dialect=dialect) + if reference.startswith(_RAW_PREFIX): + table = reference.removeprefix(_RAW_PREFIX) + if _IDENTIFIER.fullmatch(table): + return self._relation_for_dialect("raw", table, dialect=dialect) + raise TransformProjectError(f"Unknown model reference: {reference}") + def _relation(self, dataset: str, table: str) -> str: if not _IDENTIFIER.fullmatch(dataset) or not _IDENTIFIER.fullmatch(table): raise TransformProjectError("Unsafe BigQuery relation identifier") return f"`{self.project_id}.{dataset}.{table}`" + + def _relation_for_dialect(self, dataset: str, table: str, *, dialect: SqlDialect) -> str: + if not _IDENTIFIER.fullmatch(dataset) or not _IDENTIFIER.fullmatch(table): + raise TransformProjectError("Unsafe relation identifier") + relation = exp.Table( + this=exp.to_identifier(table, quoted=True), + db=exp.to_identifier(dataset, quoted=True), + catalog=( + None + if dialect is SqlDialect.POSTGRES + else exp.to_identifier(self.project_id, quoted=True) + ), + ) + return relation.sql(dialect=dialect.value) + + def _relation_coordinates_for_ref( + self, + reference: str, + *, + dialect: SqlDialect, + ) -> tuple[str, ...]: + if reference in self.models: + namespace = self.models[reference].metadata.dataset + name = reference + elif reference.startswith(_RAW_PREFIX) and _IDENTIFIER.fullmatch( + name := reference.removeprefix(_RAW_PREFIX) + ): + namespace = "raw" + else: + raise TransformProjectError(f"Unknown model reference: {reference}") + if dialect is SqlDialect.POSTGRES: + return (namespace, name) + return (self.project_id, namespace, name) diff --git a/steering/languages/sql.md b/steering/languages/sql.md index 3a087249..ea504b64 100644 --- a/steering/languages/sql.md +++ b/steering/languages/sql.md @@ -1,10 +1,11 @@ -# SQL Conventions (BigQuery Standard SQL) +# SQL Conventions -For the Transform module and any hand-written BigQuery SQL. +For the Transform module and hand-written model SQL. ## Dialect & style -- **BigQuery Standard SQL** only (never legacy). +- Declare `dialect` in every repository-owned model sidecar. Existing models use **BigQuery + Standard SQL** (never legacy); new cross-warehouse models may opt into `portable`. - Uppercase keywords (`SELECT`, `FROM`, `LEFT JOIN`, `PARTITION BY`); lowercase identifiers. - **snake_case** for tables and columns. Datasets: `raw` → `staging` → `marts`. - Lead with **CTEs**, not nested subqueries. One transformation per CTE, named for what it produces. @@ -13,6 +14,11 @@ For the Transform module and any hand-written BigQuery SQL. - Explicit column lists — no `SELECT *` in models or anything persisted. - Qualify columns when more than one table is in scope. +Portable models additionally use only Dander `ref()` relations, lowercase unquoted logical +identifiers, explicit `NULLS FIRST`/`NULLS LAST`, and deterministic window tie-breakers declared +unique in model tests. Provider-specific functions or semantics belong in an exact provider model, +not in the portable subset. + ```sql WITH source AS ( SELECT id, updated_at, amount diff --git a/tests/transform/test_dialects.py b/tests/transform/test_dialects.py new file mode 100644 index 00000000..8f291789 --- /dev/null +++ b/tests/transform/test_dialects.py @@ -0,0 +1,124 @@ +"""Portable SQL subset and warehouse-dialect rendering tests.""" + +from __future__ import annotations + +import pytest +from sqlglot import exp + +from dander.transform import PortableSqlError, SqlDialect +from dander.transform.dialects import parse_portable_query, render_portable_query + + +@pytest.mark.parametrize("target", ["bigquery", "snowflake", "redshift", "postgres"]) +def test_portable_projection_filter_join_aggregation_and_render(target: str) -> None: + query = parse_portable_query( + """ + SELECT + account.account_id, + COUNT(*) AS contact_count, + SUM(CAST(contact.score AS DECIMAL(38, 9))) AS total_score, + COALESCE(MAX(contact.title), 'unknown') AS latest_title + FROM `portable.raw.accounts` AS account + LEFT JOIN `portable.raw.contacts` AS contact + ON account.account_id = contact.account_id + WHERE contact.is_deleted = FALSE OR contact.is_deleted IS NULL + GROUP BY account.account_id + HAVING COUNT(*) > 0 + """ + ) + + rendered = render_portable_query(query, target=target) + + assert "contact_count" in rendered + assert "LEFT JOIN" in rendered + assert "GROUP BY" in rendered + assert SqlDialect(target) is not SqlDialect.PORTABLE + + +def test_portable_union_all_and_deterministic_window() -> None: + query = parse_portable_query( + """ + SELECT + id, + ROW_NUMBER() OVER ( + PARTITION BY account_id + ORDER BY updated_at DESC NULLS LAST, id ASC NULLS LAST + ) AS row_num + FROM `portable.raw.contacts` + UNION ALL + SELECT id, 1 AS row_num + FROM `portable.raw.archived_contacts` + """, + unique_columns={"id"}, + ) + + assert isinstance(query, exp.Union) + assert "ROW_NUMBER" in render_portable_query(query, target="snowflake") + + +@pytest.mark.parametrize( + ("sql", "message"), + [ + ("SELECT id FROM `portable.raw.t` ORDER BY id", "NULLS FIRST or NULLS LAST"), + ( + "SELECT ROW_NUMBER() OVER (ORDER BY updated_at NULLS LAST) FROM `portable.raw.t`", + "declared unique", + ), + ( + "SELECT id FROM `portable.raw.a` UNION DISTINCT SELECT id FROM `portable.raw.b`", + "UNION ALL only", + ), + ("SELECT * FROM `portable.raw.a` NATURAL JOIN `portable.raw.b`", "NATURAL JOIN"), + ( + "SELECT * FROM `portable.raw.a` LEFT SEMI JOIN `portable.raw.b` ON a.id = b.id", + "join type is unsupported", + ), + ("SELECT SAFE_CAST(id AS INT64) FROM `portable.raw.t`", "TryCast"), + ( + "SELECT CAST(amount AS NUMERIC(10, 2)) FROM `portable.raw.t`", + r"DECIMAL\(38, 9\)", + ), + ("SELECT CAST(event_at AS TIMESTAMP) FROM `portable.raw.t`", "precision 6"), + ("SELECT `MixedCase` FROM `portable.raw.t`", "must not be quoted"), + ("SELECT MixedCase FROM `portable.raw.t`", "lowercase snake_case"), + ("SELECT 'e\u0301' FROM `portable.raw.t`", "Unicode NFC"), + ("SELECT CURRENT_TIMESTAMP() FROM `portable.raw.t`", "CurrentTimestamp"), + ], +) +def test_portable_subset_rejects_ambiguous_or_provider_specific_sql( + sql: str, + message: str, +) -> None: + with pytest.raises(PortableSqlError, match=message): + parse_portable_query(sql) + + +def test_portable_render_rejects_portable_as_target() -> None: + query = parse_portable_query("SELECT id FROM `portable.raw.t`") + + with pytest.raises(PortableSqlError, match="not a render target"): + render_portable_query(query, target="portable") + + +def test_portable_relations_are_limited_to_declared_refs() -> None: + sql = "SELECT id FROM `portable.raw.contacts`" + + with pytest.raises(PortableSqlError, match="outside its declared refs"): + parse_portable_query( + sql, + allowed_relations={("portable", "raw", "accounts")}, + ) + + +def test_portable_cte_may_reference_a_declared_relation() -> None: + query = parse_portable_query( + """ + WITH source AS ( + SELECT id FROM `portable.raw.contacts` + ) + SELECT id FROM source + """, + allowed_relations={("portable", "raw", "contacts")}, + ) + + assert isinstance(query, exp.Select) diff --git a/tests/transform/test_project.py b/tests/transform/test_project.py index 5e0f66c6..33418212 100644 --- a/tests/transform/test_project.py +++ b/tests/transform/test_project.py @@ -16,6 +16,7 @@ def _write_model( sql: str, *, materialization: str = "view", + dialect: str | None = None, tests: str = "[]", ) -> None: root.mkdir(parents=True, exist_ok=True) @@ -26,6 +27,7 @@ def _write_model( model: {name} description: Safe test model. owner: data-eng + {f"dialect: {dialect}" if dialect is not None else ""} materialization: {materialization} dataset: staging source_system: fixture @@ -62,6 +64,49 @@ def test_load_orders_selected_model_dependencies_and_compiles_refs(tmp_path: Pat assert project.compile(ordered[1]) == ("SELECT id FROM `valid-project-123.staging.base`") +def test_existing_models_default_to_exact_bigquery_sql(tmp_path: Path) -> None: + _write_model(tmp_path, "existing", "SELECT CAST(id AS STRING) AS id") + project = TransformProject.load(tmp_path, project_id="valid-project-123") + model = project.models["existing"] + + assert model.metadata.dialect.value == "bigquery" + with pytest.raises(TransformProjectError, match="cannot target postgres"): + project.compile(model, target_dialect="postgres") + + +@pytest.mark.parametrize("target", ["bigquery", "snowflake", "redshift", "postgres"]) +def test_portable_model_compiles_refs_for_selected_target(tmp_path: Path, target: str) -> None: + _write_model( + tmp_path, + "portable_model", + "SELECT CAST(id AS STRING) AS id FROM {{ ref('raw_fixture') }}", + dialect="portable", + ) + project = TransformProject.load(tmp_path, project_id="valid-project-123") + + compiled = project.compile(project.models["portable_model"], target_dialect=target) + + assert "fixture" in compiled + assert "SELECT" in compiled + + +def test_exact_provider_model_compiles_only_for_declared_target(tmp_path: Path) -> None: + _write_model( + tmp_path, + "provider_model", + "SELECT CAST(id AS TEXT) AS id FROM {{ ref('raw_fixture') }}", + dialect="postgres", + ) + project = TransformProject.load(tmp_path, project_id="valid-project-123") + + assert '"raw"."fixture"' in project.compile( + project.models["provider_model"], + target_dialect="postgres", + ) + with pytest.raises(TransformProjectError, match="cannot target bigquery"): + project.compile(project.models["provider_model"]) + + def test_unknown_reference_fails_during_project_load(tmp_path: Path) -> None: _write_model(tmp_path, "broken", "SELECT id FROM {{ ref('missing') }}") diff --git a/tickets/DANDER-81-portable-sql-dialects.md b/tickets/DANDER-81-portable-sql-dialects.md new file mode 100644 index 00000000..065b593f --- /dev/null +++ b/tickets/DANDER-81-portable-sql-dialects.md @@ -0,0 +1,44 @@ +--- +id: DANDER-81 +title: Add explicit model dialects and portable SQL validation +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-80] +created: 2026-08-07 +--- + +## Context + +Existing transform SQL is intentionally BigQuery-specific. Cross-warehouse execution needs an +explicit portable contract rather than silently translating arbitrary provider SQL. + +## Acceptance Criteria + +- [x] Model metadata declares `portable`, `bigquery`, `snowflake`, `redshift`, or `postgres`. +- [x] Existing models remain exact BigQuery SQL and repository models state that contract. +- [x] Portable SQL parses to one read-only AST and rejects nodes outside a closed allowlist. +- [x] The subset covers projections, filters, joins, `UNION ALL`, aggregations, windows, casts, and + selected scalar functions. +- [x] Physical relations must resolve from declared Dander `ref()` calls. +- [x] Explicit null ordering and unique window tie-breakers are required. +- [x] Identifier, Unicode NFC, decimal, and timestamp precision rules fail closed. +- [x] Valid portable ASTs render to BigQuery, Snowflake, Redshift, and PostgreSQL syntax. +- [x] Exact provider SQL refuses a mismatched target. +- [x] Full local validation passes. +- [ ] Protected CI passes. + +## Design + +Keep the authored SQL and target warehouse separate. Exact SQL is parsed only with its declared +dialect. Portable SQL uses a closed sqlglot expression allowlist; a dependency update cannot admit +a new construct without an intentional code and fixture change. + +## Implementation Notes + +This ticket does not route runtime materialization to a new warehouse, add provider variants, or +compile graph operations. Those remain separate, reviewable slices. + +## Review Log + +Pending protected review. From a75ab26a3b63c7cd04ac79924e3b981bb603a813 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 20:00:38 -0400 Subject: [PATCH 031/163] Compile pipeline graphs through relational AST --- CHANGELOG.md | 3 + HANDOFF.md | 37 ++- docs/decisions.md | 10 + src/dander/pipeline/README.md | 23 +- src/dander/pipeline/__init__.py | 8 +- src/dander/pipeline/compiler.py | 309 ++++++++++++++++------ tests/pipeline/test_compiler.py | 75 +++++- tests/pipeline/test_runtime_bridge.py | 6 +- tickets/DANDER-82-graph-relational-ast.md | 41 +++ 9 files changed, 387 insertions(+), 125 deletions(-) create mode 100644 tickets/DANDER-82-graph-relational-ast.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a880e134..a21c0c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Compile executable graph mappings, joins, transformations, and ordered operations into one + provider-neutral relational AST before dialect rendering; preserve BigQuery runtime output and + fail when another target cannot retain graph safe-cast semantics. - Add explicit `portable`, BigQuery, Snowflake, Redshift, and PostgreSQL model dialect metadata; preserve BigQuery as the compatibility default and validate portable models through a closed, deterministic SQL AST before provider rendering. diff --git a/HANDOFF.md b/HANDOFF.md index 7ab19164..5e5bba66 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,38 @@ ## Finished -- Added explicit exact-provider and portable model dialect metadata. -- Preserved BigQuery as the compatibility default and marked repository models explicitly. -- Added a closed portable SQL AST with deterministic ordering, casts, identifiers, and refs. -- Added BigQuery, Snowflake, Redshift, and PostgreSQL rendering for validated portable queries. -- Added positive and rejection fixtures plus exact-dialect mismatch coverage. +- Replaced graph SQL-fragment construction with one provider-neutral sqlglot relational AST. +- Compiled relations, CTEs, joins, mappings, conditions, casts, and operations as expressions. +- Preserved `CompiledTarget.query` as the compatible BigQuery runtime rendering. +- Added isolated AST access and four-dialect rendering for semantically compatible graphs. +- Added fail-closed safe-cast target checks and graph/runtime-bridge regression coverage. ## Try It -Set `dialect: portable` in a model sidecar, use only `{{ ref('...') }}` relations, then call -`project.compile(model, target_dialect="postgres")`. Existing models need no changes. +Call `compile_target(...)` as before. Inspect `compiled.query_ast` or call +`compiled.render("postgres")`; safe-cast graphs reject targets that cannot preserve semantics. ## Checks -- All 887 tests, Ruff, formatting, and strict mypy across 199 source files passed. -- Wheel/sdist build, inspection, source-free installs, generated-project validation, and all four - Terraform-root validations passed. -- Local container build and read-only runtime conformance passed as UID 65532. +- All 892 tests, Ruff, formatting, and strict mypy across 199 source files passed. +- Wheel/sdist build, inspection, source-free installs, generated project, all four Terraform roots, + and local non-root container conformance passed. - The isolated GCP plan reported `No changes`; both schedules stayed paused and no apply ran. ## Decisions -- Existing and undeclared models remain exact BigQuery SQL. -- Portable AST nodes are closed; new sqlglot nodes fail until explicitly reviewed. -- Render targets do not imply provider runtime support. +- Reuse sqlglot rather than creating another relational AST. +- Preserve BigQuery runtime behavior while provider execution remains separate. +- Reject lossy target rendering; syntax generation is not a support claim. ## Remaining - Complete local and isolated GCP validation. -- Open the focused dialect PR and merge only after protected CI passes. -- Compile graph operations through the same provider-neutral boundary in the next PR. +- Open and merge the focused graph-AST PR after protected CI passes. +- Add normalized telemetry, dependency extras, and runtime adapter assembly next. ## Review First -- `src/dander/transform/dialects.py` -- `src/dander/transform/project.py` -- `tests/transform/test_dialects.py` +- `src/dander/pipeline/compiler.py` +- `tests/pipeline/test_compiler.py` +- `tests/pipeline/test_runtime_bridge.py` diff --git a/docs/decisions.md b/docs/decisions.md index a8448235..c43597a6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-07 — Graph nodes compile to relational expressions before SQL + +- **Boundary:** Source relations, CTEs, joins, projections, scalar transforms, and ordered graph + operations are sqlglot expressions. Graph nodes no longer concatenate BigQuery SQL fragments. +- **Compatibility:** `CompiledTarget.query` remains the BigQuery rendering and the existing + BigQuery writer path is unchanged. Its AST is exposed only as an isolated copy. +- **Fail closed:** Other dialects render only when their semantics are exact. Current safe-cast + nodes reject Snowflake and PostgreSQL instead of silently becoming strict casts; rendering alone + does not claim executable provider support. + ## 2026-08-07 — Portable SQL is an explicit, closed contract - **Compatibility:** Model metadata defaults to exact `bigquery`; repository-owned legacy models diff --git a/src/dander/pipeline/README.md b/src/dander/pipeline/README.md index e73cf506..8862f614 100644 --- a/src/dander/pipeline/README.md +++ b/src/dander/pipeline/README.md @@ -7,7 +7,8 @@ future drag-drop UI and fully code-authored pipelines. A graph is a list of `nod tasks) and `edges` (connections between them, i.e. how data flows). The package defines the on-disk shape (YAML/JSON), validates that a graph is structurally and semantically sound, derives read-only structure (adjacency, topological order), safely compiles the supported graph subset to -BigQuery SQL, and prepares configured target writers. Ingestion, query submission, and scheduled +a provider-neutral relational AST, renders compatible SQL, and prepares configured target writers. +Ingestion, query submission, and scheduled execution remain in the Ingestion/Transform/Writer/Orchestration layers described in `steering/00-project-overview.md`. @@ -23,7 +24,7 @@ code is right and this doc has drifted — please fix it. | `dander.pipeline.graph_service` | Loopback-only, single-file bridge for visual editors. Dander validates conditional saves and can bind that exact graph to one already-deployed Cloud Run job selected at startup. | | `dander.pipeline.node_config` | The discriminated, per-node-type `Node.config` models — `NodeType`, `NodeConfig`, `SourceNodeConfig`, `TransformNodeConfig`, `TargetNodeConfig` — and the routing function `Node` delegates to. See *Typed per-node-type config* below. | | `dander.pipeline.operations` | Typed, ordered, schema-preserving operations for executable transform nodes. | -| `dander.pipeline.compiler` | Safe compilation of executable linear/two-input graphs to explicit BigQuery SQL, plus side-effect-free target-writer preparation. | +| `dander.pipeline.compiler` | Safe compilation of executable linear/two-input graphs to one sqlglot relational AST, compatible dialect rendering, and side-effect-free BigQuery target-writer preparation. | | `dander.pipeline.request_spec` | `SourceNodeConfig`'s declarative request/payload spec — `HttpMethod`, `RequestSpec` — and the secret/field-reference grammar its header/param/body values must follow. See *Source request/payload spec* below. | | `dander.pipeline.errors` | The typed `GraphValidationError` hierarchy raised by `graph_ops`. | @@ -129,7 +130,10 @@ Implementation Notes below for the Decision Log status of this call.)* ## Executable linear pipelines -`compile_target` turns a validated graph into an explicit-column BigQuery `SELECT`. It supports +`compile_target` turns a validated graph into one explicit-column sqlglot relational AST and keeps +the current `query` property as its BigQuery rendering. `CompiledTarget.query_ast` returns an +isolated copy, and `render()` can render semantically compatible graphs for BigQuery, Snowflake, +Redshift, or PostgreSQL without recompiling nodes. It supports linear mappings plus two-input joins whose output is a distinct transform node, JSON constants, scalar expressions, target casts, and the built-in custom-transform registry. Expressions are parsed with sqlglot, must reference exactly their declared `inputs`, may use only allow-listed @@ -140,6 +144,12 @@ passed to Python `eval`. incremental, or replace writer and a `WriteTarget`. Constructing it has no network side effect; calling its `write()` method performs the configured BigQuery write. +Rendering is intentionally fail-closed. Existing graph `cast_to` means null-on-failure casting; +Snowflake and PostgreSQL are rejected until Dander proves an exact mapping for that operation. +PostgreSQL rendering drops the connected database/catalog qualifier and keeps schema/relation +coordinates. A rendered dialect is not an executable provider claim: graph runtime dispatch and +writers remain BigQuery-only in this slice. + An executable join is authored on `TransformNodeConfig.join`: `left_input` and `right_input` must match its two incoming edges, `keys` name equality columns on those inputs, and the transform node is the output. The older `Edge.join` shape remains readable for backward compatibility but fails @@ -149,7 +159,7 @@ closed during compilation because it makes the edge target both a right input an An executable `transform` node may declare an ordered `config.operations` list. Each operation runs after the node's edge mappings have produced its declared fields and before downstream nodes -read them. `dander run` compiles the list to explicit BigQuery CTEs after normal connector +read them. `dander run` compiles the list to relational CTE nodes, then renders BigQuery after normal connector ingestion, so raw-table schemas, extraction cursors, and replay behavior remain unchanged. ```yaml @@ -592,8 +602,9 @@ This package combines declarative modeling with a bounded execution bridge. It d - execute graph field tests or publish graph metadata to Dataplex; - combine multiple connector YAML files in one hosted graph pipeline. -`compile_target` parses allow-listed scalar expressions and emits explicit-column SQL for the -supported graph subset. `prepare_target_writer` constructs a concrete writer; only a later call to +`compile_target` parses allow-listed scalar expressions and emits one explicit-column relational +AST plus the compatible BigQuery rendering for the supported graph subset. `prepare_target_writer` +constructs a concrete writer; only a later call to that writer's `write()` method performs I/O. Ingestion, transform submission, hosted scheduling, and write execution otherwise remain in their named modules. diff --git a/src/dander/pipeline/__init__.py b/src/dander/pipeline/__init__.py index 3187541f..396885b3 100644 --- a/src/dander/pipeline/__init__.py +++ b/src/dander/pipeline/__init__.py @@ -4,9 +4,9 @@ (duplicate field names, unresolved mapping/transformation/join/operation field references), and derived graph algorithms (adjacency, topological order) build on top of these models in -`dander.pipeline.graph_ops`. `compile_target` compiles the supported graph subset to explicit -BigQuery SQL, while `prepare_target_writer` binds a configured target to a concrete writer without -performing a network call. +`dander.pipeline.graph_ops`. `compile_target` compiles the supported graph subset to a relational +AST and its compatible BigQuery SQL, while `prepare_target_writer` binds a configured target to a +concrete writer without performing a network call. """ from __future__ import annotations @@ -17,6 +17,7 @@ PreparedTargetWriter, compile_target, prepare_target_writer, + render_graph_query, ) from dander.pipeline.errors import ( DanglingEdgeError, @@ -120,6 +121,7 @@ "load_graph_from_yaml", "topological_order", "prepare_target_writer", + "render_graph_query", "validate", "validate_field_wiring", ] diff --git a/src/dander/pipeline/compiler.py b/src/dander/pipeline/compiler.py index 36deef2b..2315c739 100644 --- a/src/dander/pipeline/compiler.py +++ b/src/dander/pipeline/compiler.py @@ -1,9 +1,10 @@ -"""Compile the executable subset of a visual pipeline graph to BigQuery SQL.""" +"""Compile the executable visual-graph subset to a provider-neutral relational AST.""" from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, field +from functools import reduce from typing import TYPE_CHECKING, Any, cast import sqlglot @@ -26,6 +27,8 @@ TrimWhitespaceParams, TruncateStringParams, ) +from dander.transform.model import SqlDialect +from dander.warehouse import RelationRef from dander.writer import ( BigQueryIncrementalWriter, BigQueryReplaceWriter, @@ -104,6 +107,18 @@ class CompiledTarget: query: str write_mode: WriteMode target: WriteTarget + _query_ast: exp.Query = field(repr=False, compare=False) + + @property + def query_ast(self) -> exp.Query: + """Return an isolated copy of the provider-neutral relational plan.""" + copied = self._query_ast.copy() + assert isinstance(copied, exp.Query) + return copied + + def render(self, target_dialect: SqlDialect | str) -> str: + """Render the graph AST for one warehouse without recompiling graph nodes.""" + return render_graph_query(self._query_ast, target_dialect=target_dialect) @dataclass(frozen=True) @@ -158,12 +173,12 @@ def compile_target( raise PipelineCompileError( f"Target node {target_node_id!r} must set destination.project for compilation" ) - query = ( - "WITH\n" - + ",\n".join(compiler.ctes) - + f"\nSELECT {', '.join(_quote(field.name) for field in target.fields)}\n" - + f"FROM {_quote(final_alias)}" + query_ast = exp.Select( + expressions=[_column(field.name) for field in target.fields], + from_=exp.From(this=_cte_table(final_alias)), ) + query_ast.set("with_", exp.With(expressions=compiler.ctes)) + query = render_graph_query(query_ast, target_dialect=SqlDialect.BIGQUERY) return CompiledTarget( node_id=target_node_id, query=query, @@ -178,9 +193,42 @@ def compile_target( for field in target.fields ), ), + _query_ast=query_ast, ) +def render_graph_query( + expression: exp.Query, + *, + target_dialect: SqlDialect | str, +) -> str: + """Render a compiled graph AST, rejecting semantics a target cannot preserve.""" + try: + target = SqlDialect(target_dialect) + except ValueError as error: + raise PipelineCompileError(f"Unknown graph SQL target: {target_dialect}") from error + if target is SqlDialect.PORTABLE: + raise PipelineCompileError("portable is a graph contract, not a render target") + if target in {SqlDialect.SNOWFLAKE, SqlDialect.POSTGRES} and expression.find(exp.TryCast): + raise PipelineCompileError( + f"Graph safe-cast semantics are not yet exact for {target.value}" + ) + rendered = expression.copy() + if target is SqlDialect.POSTGRES: + transformed = rendered.transform(_postgres_relation) + assert isinstance(transformed, exp.Query) + rendered = transformed + return rendered.sql(dialect=target.value, pretty=True) + + +def _postgres_relation(node: exp.Expression) -> exp.Expression: + if isinstance(node, exp.Table) and node.catalog: + updated = node.copy() + updated.set("catalog", None) + return updated + return node + + class _GraphSqlCompiler: """Stateful recursive CTE builder for one target's dependency subgraph.""" @@ -195,7 +243,7 @@ def __init__( self._incoming = incoming self._source_relations = source_relations self._aliases: dict[str, str] = {} - self.ctes: list[str] = [] + self.ctes: list[exp.CTE] = [] def compile_node(self, node_id: str) -> str: """Compile dependencies first and return this node's CTE alias.""" @@ -223,7 +271,7 @@ def compile_node(self, node_id: str) -> str: return self._append_projection( node, [(edges[0], "source")], - from_sql=f"{_quote(upstream)} AS source", + from_table=_cte_table(upstream, alias="source"), ) if ( len(edges) == 2 @@ -245,11 +293,13 @@ def _compile_source(self, node: Node, edges: list[Edge]) -> str: raise PipelineCompileError( f"Source relation for node {node.id!r} must be project.dataset.table" ) - columns = ", ".join(_compile_source_field(field) for field in node.fields) + columns = [_compile_source_field(field) for field in node.fields] if not columns: raise PipelineCompileError(f"Source node {node.id!r} must declare fields") alias = self._next_alias() - self.ctes.append(f"{_quote(alias)} AS (\n SELECT {columns} FROM `{relation}`\n)") + relation_ref = _parse_relation(relation) + query = exp.Select(expressions=columns, from_=exp.From(this=_relation_table(relation_ref))) + self.ctes.append(_cte(alias, query)) self._aliases[node.id] = alias return alias @@ -274,18 +324,22 @@ def _compile_join(self, node: Node, edges: list[Edge]) -> str: ) left_alias = self.compile_node(join.left_input) right_alias = self.compile_node(join.right_input) - join_keyword = { - ExecutableJoinType.INNER: "INNER JOIN", - ExecutableJoinType.LEFT: "LEFT JOIN", - ExecutableJoinType.RIGHT: "RIGHT JOIN", - ExecutableJoinType.FULL: "FULL OUTER JOIN", + side, kind = { + ExecutableJoinType.INNER: (None, "INNER"), + ExecutableJoinType.LEFT: ("LEFT", None), + ExecutableJoinType.RIGHT: ("RIGHT", None), + ExecutableJoinType.FULL: ("FULL", "OUTER"), }[join.type] - condition = " AND ".join( - f"lhs.{_quote(key.left)} = rhs.{_quote(key.right)}" for key in join.keys - ) - from_sql = ( - f"{_quote(left_alias)} AS lhs\n" - f" {join_keyword} {_quote(right_alias)} AS rhs ON {condition}" + conditions: list[exp.Expression] = [ + exp.EQ(this=_column(key.left, table="lhs"), expression=_column(key.right, table="rhs")) + for key in join.keys + ] + condition = reduce(_and, conditions) + join_expression = exp.Join( + this=_cte_table(right_alias, alias="rhs"), + side=side, + kind=kind, + on=condition, ) return self._append_projection( node, @@ -293,7 +347,8 @@ def _compile_join(self, node: Node, edges: list[Edge]) -> str: (by_source[join.left_input], "lhs"), (by_source[join.right_input], "rhs"), ], - from_sql=from_sql, + from_table=_cte_table(left_alias, alias="lhs"), + joins=[join_expression], ) def _append_projection( @@ -301,7 +356,8 @@ def _append_projection( node: Node, edge_aliases: list[tuple[Edge, str]], *, - from_sql: str, + from_table: exp.Table, + joins: list[exp.Join] | None = None, ) -> str: mappings: dict[str, tuple[FieldMapping, str]] = {} mapping_count = 0 @@ -328,9 +384,13 @@ def _append_projection( ) for field in node.fields ] - select_list = ",\n ".join(projected) alias = self._next_alias() - self.ctes.append(f"{_quote(alias)} AS (\n SELECT\n {select_list}\n FROM {from_sql}\n)") + query = exp.Select( + expressions=projected, + from_=exp.From(this=from_table), + joins=joins or [], + ) + self.ctes.append(_cte(alias, query)) self._aliases[node.id] = alias if isinstance(node.config, TransformNodeConfig) and node.config.operations: return self._append_operations(node, alias, node.config.operations) @@ -348,15 +408,14 @@ def _append_operations( for operation in operations: params = operation.params if isinstance(params, FilterRowsParams): - select_list = ",\n ".join( - f"source.{_quote(field.name)} AS {_quote(field.name)}" for field in node.fields - ) + select_list = [ + _alias(_column(field.name, table="source"), field.name) for field in node.fields + ] predicate = _compile_filter(params) - cte_sql = ( - "SELECT\n" - f" {select_list}\n" - f" FROM {_quote(alias)} AS source\n" - f" WHERE {predicate}" + query = exp.Select( + expressions=select_list, + from_=exp.From(this=_cte_table(alias, alias="source")), + where=exp.Where(this=predicate), ) else: if not isinstance( @@ -366,29 +425,40 @@ def _append_operations( raise AssertionError(f"Unexpected operation params: {type(params).__name__}") field_name = params.field field = fields[field_name] - source = f"source.{_quote(field_name)}" + source = _column(field_name, table="source") if isinstance(params, TrimWhitespaceParams): _require_string_operation(node.id, field_name, field.cast_to or field.type) - replacement = f"TRIM({source})" + replacement: exp.Expression = exp.Trim(this=source) elif isinstance(params, TruncateStringParams): _require_string_operation(node.id, field_name, field.cast_to or field.type) - replacement = f"SUBSTR({source}, 1, {params.max_length})" + replacement = exp.Substring( + this=source, + start=exp.Literal.number(1), + length=exp.Literal.number(params.max_length), + ) elif isinstance(params, DefaultValueParams): data_type = _operation_type(node.id, field_name, field.cast_to or field.type) - replacement = ( - f"COALESCE({source}, CAST({_literal(params.default)} AS {data_type}))" + replacement = exp.Coalesce( + this=source, + expressions=[ + exp.Cast(this=exp.convert(params.default), to=_data_type(data_type)) + ], ) - select_list = ",\n ".join( - ( - f"{replacement} AS {_quote(declared.name)}" + select_list = [ + _alias( + replacement if declared.name == field_name - else f"source.{_quote(declared.name)} AS {_quote(declared.name)}" + else _column(declared.name, table="source"), + declared.name, ) for declared in node.fields + ] + query = exp.Select( + expressions=select_list, + from_=exp.From(this=_cte_table(alias, alias="source")), ) - cte_sql = f"SELECT\n {select_list}\n FROM {_quote(alias)} AS source" alias = self._next_alias() - self.ctes.append(f"{_quote(alias)} AS (\n {cte_sql}\n)") + self.ctes.append(_cte(alias, query)) self._aliases[node.id] = alias return alias @@ -487,14 +557,19 @@ def prepare_target_writer( ) -def _compile_mapping(mapping: FieldMapping, cast_to: str | None, *, alias: str) -> str: +def _compile_mapping( + mapping: FieldMapping, + cast_to: str | None, + *, + alias: str, +) -> exp.Expression: transformation = mapping.transformation if transformation is None or transformation.kind is TransformationKind.DIRECT: if mapping.source is None: raise PipelineCompileError("A direct mapping must name its source field") - expression = f"{alias}.{_quote(mapping.source)}" + expression: exp.Expression = _column(mapping.source, table=alias) elif transformation.kind is TransformationKind.CONSTANT: - expression = exp.convert(transformation.constant).sql(dialect="bigquery") + expression = cast("exp.Expression", exp.convert(transformation.constant)) elif transformation.kind is TransformationKind.EXPRESSION: expression = _compile_expression(transformation, alias=alias) else: @@ -502,20 +577,23 @@ def _compile_mapping(mapping: FieldMapping, cast_to: str | None, *, alias: str) if cast_to is not None: if not _TYPE.fullmatch(cast_to): raise PipelineCompileError(f"Unsupported target cast type {cast_to!r}") - expression = f"SAFE_CAST({expression} AS {cast_to.upper()})" - return f"{expression} AS {_quote(mapping.target)}" + expression = exp.TryCast(this=expression, to=_data_type(cast_to)) + return _alias(expression, mapping.target) -def _compile_source_field(field: NodeField) -> str: - column = _quote(field.name) +def _compile_source_field(field: NodeField) -> exp.Expression: + column = _column(field.name) if field.cast_to is None: return column if not _TYPE.fullmatch(field.cast_to): raise PipelineCompileError(f"Unsupported source cast type {field.cast_to!r}") - return f"SAFE_CAST({column} AS {field.cast_to.upper()}) AS {column}" + return _alias( + exp.TryCast(this=column, to=_data_type(field.cast_to)), + field.name, + ) -def _compile_expression(transformation: Transformation, *, alias: str) -> str: +def _compile_expression(transformation: Transformation, *, alias: str) -> exp.Expression: assert transformation.expression is not None try: parsed = sqlglot.parse_one(transformation.expression, read="bigquery") @@ -541,25 +619,25 @@ def _compile_expression(transformation: Transformation, *, alias: str) -> str: qualified = parsed.transform( lambda node: exp.column(node.name, table=alias) if isinstance(node, exp.Column) else node ) - return qualified.sql(dialect="bigquery") + return cast("exp.Expression", qualified) -def _compile_custom(transformation: Transformation, *, alias: str) -> str: - inputs = [f"{alias}.{_quote(name)}" for name in transformation.inputs] +def _compile_custom(transformation: Transformation, *, alias: str) -> exp.Expression: + inputs = [_column(name, table=alias) for name in transformation.inputs] arguments = [ - exp.convert(value).sql(dialect="bigquery") for value in transformation.arguments.values() + cast("exp.Expression", exp.convert(value)) for value in transformation.arguments.values() ] - values = [*inputs, *arguments] + values: list[exp.Expression] = [*inputs, *arguments] match transformation.function: case "transforms.lower": _require_arity(transformation.function, values, 1) - return f"LOWER({values[0]})" + return exp.Lower(this=values[0]) case "transforms.upper": _require_arity(transformation.function, values, 1) - return f"UPPER({values[0]})" + return exp.Upper(this=values[0]) case "transforms.trim": _require_arity(transformation.function, values, 1) - return f"TRIM({values[0]})" + return exp.Trim(this=values[0]) case "transforms.normalize_phone": if len(inputs) != 1: raise PipelineCompileError( @@ -569,43 +647,53 @@ def _compile_custom(transformation: Transformation, *, alias: str) -> str: raise PipelineCompileError( "Custom transform 'transforms.normalize_phone' does not accept arguments" ) - return f"REGEXP_REPLACE(CAST({inputs[0]} AS STRING), r'[^0-9+]', '')" + return exp.RegexpReplace( + this=exp.Cast(this=inputs[0], to=_data_type("STRING")), + expression=exp.Literal.string("[^0-9+]"), + replacement=exp.Literal.string(""), + ) case _: raise PipelineCompileError( f"Custom transform {transformation.function!r} is not allow-listed" ) -def _compile_filter(params: FilterRowsParams) -> str: - joiner = " AND " if params.logic is MatchLogic.ALL else " OR " - return joiner.join(f"({_compile_condition(condition)})" for condition in params.conditions) +def _compile_filter(params: FilterRowsParams) -> exp.Expression: + combinator = _and if params.logic is MatchLogic.ALL else _or + return reduce(combinator, (_compile_condition(condition) for condition in params.conditions)) -def _compile_condition(condition: FieldCondition) -> str: - field = f"source.{_quote(condition.field)}" +def _compile_condition(condition: FieldCondition) -> exp.Expression: + field = _column(condition.field, table="source") if condition.op is ComparisonOperator.IS_NULL: - return f"{field} IS NULL" + return exp.Is(this=field, expression=exp.Null()) if condition.op is ComparisonOperator.IS_NOT_NULL: - return f"{field} IS NOT NULL" + return exp.Not(this=exp.Is(this=field, expression=exp.Null())) if condition.op in {ComparisonOperator.IN, ComparisonOperator.NOT_IN}: assert isinstance(condition.value, list) - keyword = "IN" if condition.op is ComparisonOperator.IN else "NOT IN" - values = ", ".join(_literal(value) for value in condition.value) - return f"{field} {keyword} ({values})" + contained: exp.Expression = exp.In( + this=field, + expressions=[exp.convert(value) for value in condition.value], + ) + return exp.Not(this=contained) if condition.op is ComparisonOperator.NOT_IN else contained assert condition.value is not None and not isinstance(condition.value, list) - operator = { - ComparisonOperator.EQ: "=", - ComparisonOperator.NE: "!=", - ComparisonOperator.GT: ">", - ComparisonOperator.GTE: ">=", - ComparisonOperator.LT: "<", - ComparisonOperator.LTE: "<=", + operator: type[exp.Binary] = { + ComparisonOperator.EQ: exp.EQ, + ComparisonOperator.NE: exp.NEQ, + ComparisonOperator.GT: exp.GT, + ComparisonOperator.GTE: exp.GTE, + ComparisonOperator.LT: exp.LT, + ComparisonOperator.LTE: exp.LTE, }[condition.op] - return f"{field} {operator} {_literal(condition.value)}" + return cast("exp.Expression", operator(this=field, expression=exp.convert(condition.value))) + + +def _and(left: exp.Expression, right: exp.Expression) -> exp.Expression: + return exp.And(this=left, expression=right) -def _literal(value: object) -> str: - return exp.convert(value).sql(dialect="bigquery") +def _or(left: exp.Expression, right: exp.Expression) -> exp.Expression: + return exp.Or(this=left, expression=right) def _require_string_operation(node_id: str, field_name: str, data_type: str) -> None: @@ -624,12 +712,59 @@ def _operation_type(node_id: str, field_name: str, data_type: str) -> str: return data_type.upper() -def _require_arity(function: str | None, values: list[str], count: int) -> None: +def _require_arity(function: str | None, values: list[exp.Expression], count: int) -> None: if len(values) != count: raise PipelineCompileError(f"Custom transform {function!r} requires {count} argument") -def _quote(identifier: str) -> str: +def _identifier(identifier: str, *, quoted: bool = True) -> exp.Identifier: if not _IDENTIFIER.fullmatch(identifier): - raise PipelineCompileError(f"Unsafe BigQuery identifier {identifier!r}") - return f"`{identifier}`" + raise PipelineCompileError(f"Unsafe graph identifier {identifier!r}") + return exp.to_identifier(identifier, quoted=quoted) + + +def _column(name: str, *, table: str | None = None) -> exp.Column: + return exp.Column( + this=_identifier(name), + table=_identifier(table, quoted=False) if table is not None else None, + ) + + +def _alias(expression: exp.Expression, name: str) -> exp.Alias: + return exp.Alias(this=expression, alias=_identifier(name)) + + +def _cte_table(name: str, *, alias: str | None = None) -> exp.Table: + return exp.Table( + this=_identifier(name), + alias=( + exp.TableAlias(this=_identifier(alias, quoted=False)) if alias is not None else None + ), + ) + + +def _relation_table(relation: RelationRef) -> exp.Table: + return exp.Table( + this=_identifier(relation.name), + db=_identifier(relation.namespace), + catalog=exp.to_identifier(relation.catalog, quoted=True), + ) + + +def _cte(name: str, query: exp.Query) -> exp.CTE: + return exp.CTE( + this=query, + alias=exp.TableAlias(this=_identifier(name)), + ) + + +def _parse_relation(value: str) -> RelationRef: + catalog, namespace, name = value.split(".", maxsplit=2) + return RelationRef(catalog=catalog, namespace=namespace, name=name) + + +def _data_type(value: str) -> exp.DataType: + try: + return exp.DataType.build(value, dialect="bigquery") + except (TypeError, ValueError) as error: + raise PipelineCompileError(f"Unsupported graph type {value!r}") from error diff --git a/tests/pipeline/test_compiler.py b/tests/pipeline/test_compiler.py index 938e9dd0..4fd99e35 100644 --- a/tests/pipeline/test_compiler.py +++ b/tests/pipeline/test_compiler.py @@ -6,6 +6,7 @@ import pytest import sqlglot +from sqlglot import exp from dander.pipeline import PipelineCompileError, compile_target, prepare_target_writer from dander.pipeline.graph import ( @@ -153,11 +154,11 @@ def test_compiles_linear_graph_to_explicit_bigquery_sql() -> None: assert compiled.write_mode is WriteMode.SCD1 assert compiled.target.business_key == ("person_id",) - assert "FROM `dander-test.raw.people`" in compiled.query + assert "FROM `dander-test`.`raw`.`people`" in compiled.query assert "SAFE_CAST(`id` AS STRING) AS `id`" in compiled.query assert "CONCAT(source.first_name, ' ', source.last_name) AS `full_name`" in compiled.query assert ( - "REGEXP_REPLACE(CAST(source.`phone` AS STRING), r'[^0-9+]', '') AS `phone_normalized`" + "REGEXP_REPLACE(CAST(source.`phone` AS STRING), '[^0-9+]', '') AS `phone_normalized`" ) in compiled.query assert "SAFE_CAST(source.`status` AS STRING) AS `status`" in compiled.query assert "SELECT *" not in compiled.query @@ -202,19 +203,79 @@ def test_compiles_ordered_schema_preserving_transform_operations() -> None: assert "TRIM(source.`full_name`) AS `full_name`" in query assert "FROM `_node_1` AS source" in query - assert "SUBSTR(source.`full_name`, 1, 80) AS `full_name`" in query + assert "SUBSTRING(source.`full_name`, 1, 80) AS `full_name`" in query assert "FROM `_node_2` AS source" in query assert "COALESCE(source.`status`, CAST('unknown' AS STRING)) AS `status`" in query assert "FROM `_node_3` AS source" in query - assert ( - "WHERE (source.`status` NOT IN ('archived', 'deleted')) AND " - "(source.`full_name` IS NOT NULL)" - ) in query + assert "NOT source.`status` IN ('archived', 'deleted')" in query + assert "NOT source.`full_name` IS NULL" in query assert "FROM `_node_4` AS source" in query assert "SELECT *" not in query assert sqlglot.parse_one(query, read="bigquery") is not None +def test_graph_compiles_to_one_provider_neutral_relational_ast() -> None: + compiled = compile_target( + _linear_graph(), + "target", + source_relations={"source": "dander-test.raw.people"}, + ) + + assert isinstance(compiled.query_ast, exp.Query) + assert len(tuple(compiled.query_ast.find_all(exp.CTE))) == 3 + assert len(tuple(compiled.query_ast.find_all(exp.TryCast))) == 2 + assert compiled.render("bigquery") == compiled.query + assert 'FROM "dander-test"."raw"."people"' in compiled.render("redshift") + + +def test_cast_free_graph_ast_renders_for_all_declared_targets() -> None: + graph = _linear_graph() + graph.nodes[0].fields[0].cast_to = None + graph.nodes[-1].fields[-1].cast_to = None + compiled = compile_target( + graph, + "target", + source_relations={"source": "dander-test.raw.people"}, + ) + + rendered = { + target: compiled.render(target) + for target in ("bigquery", "snowflake", "redshift", "postgres") + } + + assert "`dander-test`.`raw`.`people`" in rendered["bigquery"] + assert '"dander-test"."raw"."people"' in rendered["snowflake"] + assert '"dander-test"."raw"."people"' in rendered["redshift"] + assert '"raw"."people"' in rendered["postgres"] + for target, query in rendered.items(): + assert sqlglot.parse_one(query, read=target) is not None + + +def test_returned_graph_ast_is_an_isolated_copy() -> None: + compiled = compile_target( + _linear_graph(), + "target", + source_relations={"source": "dander-test.raw.people"}, + ) + external = compiled.query_ast + external.set("with_", None) + + assert compiled.query_ast.args.get("with_") is not None + assert compiled.render("bigquery") == compiled.query + + +@pytest.mark.parametrize("target", ["snowflake", "postgres"]) +def test_graph_render_fails_when_target_cannot_preserve_safe_cast(target: str) -> None: + compiled = compile_target( + _linear_graph(), + "target", + source_relations={"source": "dander-test.raw.people"}, + ) + + with pytest.raises(PipelineCompileError, match="safe-cast semantics"): + compiled.render(target) + + def test_string_operation_rejects_non_string_declared_field() -> None: graph = _linear_graph() config = graph.nodes[1].config diff --git a/tests/pipeline/test_runtime_bridge.py b/tests/pipeline/test_runtime_bridge.py index 993ffc41..e65fb5b1 100644 --- a/tests/pipeline/test_runtime_bridge.py +++ b/tests/pipeline/test_runtime_bridge.py @@ -142,7 +142,7 @@ def test_plan_binds_only_declared_connector_endpoint() -> None: assert plan.bindings.endpoint_names == ("jobs",) assert plan.bindings.source_relations == {"jobs": "unit-project.raw.greenhouse_job_board_jobs"} assert plan.targets[0].target.table == "graph_jobs" - assert "unit-project.raw.greenhouse_job_board_jobs" in plan.targets[0].query + assert "`unit-project`.`raw`.`greenhouse_job_board_jobs`" in plan.targets[0].query def test_plan_compiles_operations_for_the_post_ingestion_transform_stage() -> None: @@ -234,9 +234,9 @@ def test_plan_compiles_operations_for_the_post_ingestion_transform_stage() -> No ) query = plan.targets[0].query - assert "FROM `unit-project.raw.greenhouse_job_board_jobs`" in query + assert "FROM `unit-project`.`raw`.`greenhouse_job_board_jobs`" in query assert "TRIM(source.`title`) AS `title`" in query - assert "WHERE (source.`title` IS NOT NULL)" in query + assert "NOT source.`title` IS NULL" in query def test_plan_fails_closed_for_writer_mode_not_yet_executable() -> None: diff --git a/tickets/DANDER-82-graph-relational-ast.md b/tickets/DANDER-82-graph-relational-ast.md new file mode 100644 index 00000000..9f22d3c7 --- /dev/null +++ b/tickets/DANDER-82-graph-relational-ast.md @@ -0,0 +1,41 @@ +--- +id: DANDER-82 +title: Compile graph operations through the relational AST +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-81] +created: 2026-08-07 +--- + +## Context + +The graph compiler assembled BigQuery CTE, join, cast, filter, and operation strings directly. +Models now use a provider-neutral sqlglot boundary, so graph nodes need the same relational shape. + +## Acceptance Criteria + +- [x] Graph source relations remain structured catalog/namespace/name coordinates. +- [x] Sources, CTEs, projections, joins, conditions, casts, and operations compile as AST nodes. +- [x] Scalar expression validation returns the parsed expression instead of rendered SQL. +- [x] BigQuery remains the default rendering and current writer runtime remains unchanged. +- [x] The returned AST is isolated from caller mutation. +- [x] Cast-free graphs render and parse for BigQuery, Snowflake, Redshift, and PostgreSQL. +- [x] Targets that cannot preserve safe-cast behavior fail clearly before rendering. +- [x] Existing graph bridge and compiler tests remain green. +- [x] Full local validation passes. +- [ ] Protected CI passes. + +## Design + +Use sqlglot's expression tree as the shared relational representation rather than adding a second +custom AST. Keep provider execution dispatch out of this ticket; rendering syntax is not support. + +## Implementation Notes + +Graph type tokens and expression input remain BigQuery-compatible during migration. Generic target +writers, provider type mapping, and runtime dispatch are separate adapter work. + +## Review Log + +Pending protected review. From 564b9c8b7932e1f0cfe1f035f954a16272b55f3e Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 22:17:54 -0400 Subject: [PATCH 032/163] Add provider-neutral run telemetry --- CHANGELOG.md | 5 + HANDOFF.md | 37 ++--- docs/decisions.md | 9 + docs/oci-runtime-contract.md | 11 ++ src/dander/cli/runtime_command.py | 19 ++- src/dander/executor.py | 33 +++- src/dander/runtime_contract.py | 10 +- src/dander/state/failure.py | 8 +- src/dander/telemetry.py | 182 +++++++++++++++++++++ tests/cli/test_runtime_cli.py | 2 + tests/state/test_failure.py | 48 ++++++ tests/test_executor.py | 16 +- tests/test_runtime_contract.py | 29 +++- tests/test_telemetry.py | 124 ++++++++++++++ tickets/DANDER-83-generic-run-telemetry.md | 43 +++++ 15 files changed, 548 insertions(+), 28 deletions(-) create mode 100644 src/dander/telemetry.py create mode 100644 tests/test_telemetry.py create mode 100644 tickets/DANDER-83-generic-run-telemetry.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a21c0c56..a3884ee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ capabilities enter through the next minor release. ### Added +- Add a provider-neutral run telemetry contract for duration, rows, bytes, retries, provider + query/job correlation, and explicit decimal cost attribution; emit it on every terminal OCI + runtime event without admitting arbitrary provider payloads. +- Make durable failure summaries and structured pipeline terminal logs cloud-neutral while + retaining existing stable failure codes and retry decisions. - Compile executable graph mappings, joins, transformations, and ordered operations into one provider-neutral relational AST before dialect rendering; preserve BigQuery runtime output and fail when another target cannot retain graph safe-cast semantics. diff --git a/HANDOFF.md b/HANDOFF.md index 5e5bba66..51184260 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,37 @@ ## Finished -- Replaced graph SQL-fragment construction with one provider-neutral sqlglot relational AST. -- Compiled relations, CTEs, joins, mappings, conditions, casts, and operations as expressions. -- Preserved `CompiledTarget.query` as the compatible BigQuery runtime rendering. -- Added isolated AST access and four-dialect rendering for semantically compatible graphs. -- Added fail-closed safe-cast target checks and graph/runtime-bridge regression coverage. +- Added immutable provider-neutral run, operation, and cost telemetry contracts. +- Emitted normalized telemetry for successful, skipped, and failed OCI runtime terminal events. +- Measured executor and launcher duration with monotonic clocks. +- Added cloud-neutral structured terminal logs and operator failure summaries. +- Preserved existing runtime-v1 row metrics, stable failure codes, and retry behavior. ## Try It -Call `compile_target(...)` as before. Inspect `compiled.query_ast` or call -`compiled.render("postgres")`; safe-cast graphs reject targets that cannot preserve semantics. +Run `dander runtime execute ...` and inspect `outputs.telemetry` in the terminal JSON event. Future +warehouse adapters can attach ordered `OperationTelemetry` values to `PipelineExecutionResult`. ## Checks -- All 892 tests, Ruff, formatting, and strict mypy across 199 source files passed. -- Wheel/sdist build, inspection, source-free installs, generated project, all four Terraform roots, - and local non-root container conformance passed. +- All 896 tests, Ruff, formatting, strict mypy across 201 files, and dependency audit passed. +- Wheel/sdist build and source-free installs, generation, validation, and Terraform passed. +- All four Terraform roots and local non-root/read-only container conformance passed. - The isolated GCP plan reported `No changes`; both schedules stayed paused and no apply ran. ## Decisions -- Reuse sqlglot rather than creating another relational AST. -- Preserve BigQuery runtime behavior while provider execution remains separate. -- Reject lossy target rendering; syntax generation is not a support claim. +- Use one closed value contract rather than an exporter or observability subsystem. +- Record decimal costs exactly and distinguish estimates; never infer missing billing data. +- Let concrete provider slices populate operation detail after their adapters exist. ## Remaining -- Complete local and isolated GCP validation. -- Open and merge the focused graph-AST PR after protected CI passes. -- Add normalized telemetry, dependency extras, and runtime adapter assembly next. +- Open and merge the focused telemetry PR after protected CI passes. +- Add provider dependency extras and full runtime-image assembly next. ## Review First -- `src/dander/pipeline/compiler.py` -- `tests/pipeline/test_compiler.py` -- `tests/pipeline/test_runtime_bridge.py` +- `src/dander/telemetry.py` +- `src/dander/runtime_contract.py` +- `src/dander/executor.py` diff --git a/docs/decisions.md b/docs/decisions.md index c43597a6..94b1ece4 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-07 — Runtime telemetry is normalized before provider reporting + +- **Contract:** Terminal runtime events carry one validated `RunTelemetry` shape with whole-run + duration and ordered row, byte, retry, query/job, and monetary-cost operation statistics. +- **Safety:** The closed payload admits only bounded identifiers and numeric aggregates; arbitrary + provider metadata, SQL, URLs, request bodies, rows, and exception strings are not telemetry. +- **Scope:** Core currently measures elapsed time. Concrete warehouse adapters populate detailed + operations in their own vertical slices, and zero values do not imply provider billing data. + ## 2026-08-07 — Graph nodes compile to relational expressions before SQL - **Boundary:** Source relations, CTEs, joins, projections, scalar transforms, and ordered graph diff --git a/docs/oci-runtime-contract.md b/docs/oci-runtime-contract.md index beb695ac..b306cbb7 100644 --- a/docs/oci-runtime-contract.md +++ b/docs/oci-runtime-contract.md @@ -40,6 +40,17 @@ exactly one `runtime.completed` when the process can terminate normally. The ter retryability decision. Cursor values, record contents, credential material, and unrestricted exception text are excluded. +Every terminal event includes `outputs.telemetry`. The provider-neutral shape records whole-run +`duration_ms` and ordered operation statistics for retries, rows, bytes, provider query/job IDs, +and cost attribution. Fields not reported by the selected adapter remain zero or absent; Dander +does not infer provider billing. Cost values are decimal strings with an explicit currency and +`estimated` marker. Query and job IDs are correlation identifiers only—adapters must never put +SQL, request bodies, record contents, URLs, or credentials in telemetry. + +Successful ingestion row counts remain in `outputs.metrics` and `outputs.endpoints` for runtime-v1 +compatibility. Detailed operation telemetry is additive and does not change the stable process +exit or retry contract. + Invalid contract or identifier input fails before `runtime.started`. `SIGTERM` and `SIGINT` request bounded normal cleanup and produce `interrupted_run` when time remains. `SIGKILL` cannot produce a terminal event; lease expiry and the next owner reconcile that run. diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index b53d2686..bab41b7e 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -3,6 +3,7 @@ from __future__ import annotations import tempfile +import time from pathlib import Path from typing import Annotated @@ -96,6 +97,7 @@ def execute_runtime( except RuntimeContractError as error: raise typer.BadParameter(str(error)) from error + started_ns = time.monotonic_ns() typer.echo( RuntimeEvent.started( context=context, @@ -139,6 +141,7 @@ def execute_runtime( stage="cancelled", failure_code="interrupted_run", retryable=True, + duration_ms=_elapsed_ms(started_ns), ).to_json() ) raise typer.Exit(code=RuntimeExitCode.CANCELLED) from None @@ -154,6 +157,7 @@ def execute_runtime( stage="runtime", failure_code=failure.code, retryable=retryable, + duration_ms=_elapsed_ms(started_ns), ).to_json() ) code = ( @@ -170,6 +174,7 @@ def execute_runtime( stage="configuration", failure_code="invalid_configuration", retryable=False, + duration_ms=_elapsed_ms(started_ns), ).to_json() ) raise typer.Exit(code=RuntimeExitCode.INVALID_INVOCATION) from None @@ -184,10 +189,22 @@ def execute_runtime( stage="runtime", failure_code=failure.code, retryable=retryable, + duration_ms=_elapsed_ms(started_ns), ).to_json() ) code = RuntimeExitCode.RETRYABLE_FAILURE if retryable else RuntimeExitCode.PERMANENT_FAILURE raise typer.Exit(code=code) from None assert result is not None - typer.echo(RuntimeEvent.completed(result, context=context, platform=platform).to_json()) + typer.echo( + RuntimeEvent.completed( + result, + context=context, + platform=platform, + duration_ms=_elapsed_ms(started_ns), + ).to_json() + ) + + +def _elapsed_ms(started_ns: int) -> int: + return max(0, (time.monotonic_ns() - started_ns) // 1_000_000) diff --git a/src/dander/executor.py b/src/dander/executor.py index 53ea9d25..ad53a650 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -3,13 +3,15 @@ from __future__ import annotations import logging -from dataclasses import dataclass +import time +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Protocol from uuid import uuid4 from dander.catalog import MetadataSpine, SemanticRegistryPublisher from dander.runtime import PipelineRunResult from dander.state import LeaseHeartbeat, RunStage, RunStatus, classify_failure +from dander.telemetry import RunTelemetry from dander.transform import TransformProject, TransformRunResult if TYPE_CHECKING: @@ -61,6 +63,7 @@ class PipelineExecutionResult: assertions: int assets: int skipped: bool = False + telemetry: RunTelemetry = field(default_factory=RunTelemetry) class PipelineExecutor: @@ -101,6 +104,7 @@ def __init__( def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: """Execute every enabled stage and record one truthful terminal outcome.""" + started_ns = time.monotonic_ns() run_id = run_id or uuid4().hex stage = RunStage.INGEST endpoints = extracted = affected = models = assertions = assets = 0 @@ -141,6 +145,7 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: assertions=0, assets=0, skipped=True, + telemetry=RunTelemetry(duration_ms=_elapsed_ms(started_ns)), ) heartbeat = LeaseHeartbeat(self._leases, lease) heartbeat.__enter__() @@ -259,10 +264,31 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: "run_id": run_id, }, ) + _LOGGER.warning( + "pipeline_failed", + extra={ + "dander_event": "pipeline_failed", + "pipeline_id": self._pipeline_id, + "run_id": run_id, + "stage": stage.value, + "failure_code": failure.code, + "duration_ms": _elapsed_ms(started_ns), + }, + ) raise finally: if heartbeat is not None: heartbeat.__exit__() + telemetry = RunTelemetry(duration_ms=_elapsed_ms(started_ns)) + _LOGGER.info( + "pipeline_completed", + extra={ + "dander_event": "pipeline_completed", + "pipeline_id": self._pipeline_id, + "run_id": run_id, + "duration_ms": telemetry.duration_ms, + }, + ) return PipelineExecutionResult( run_id=run_id, pipeline_id=self._pipeline_id, @@ -270,6 +296,7 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: models=transform_result.models, assertions=transform_result.assertions, assets=len(compiled_assets), + telemetry=telemetry, ) @property @@ -303,3 +330,7 @@ def _checkpoint( models=models, assertions=assertions, ) + + +def _elapsed_ms(started_ns: int) -> int: + return max(0, (time.monotonic_ns() - started_ns) // 1_000_000) diff --git a/src/dander/runtime_contract.py b/src/dander/runtime_contract.py index 716e27e2..314a0bda 100644 --- a/src/dander/runtime_contract.py +++ b/src/dander/runtime_contract.py @@ -13,6 +13,8 @@ from typing import TYPE_CHECKING, Any, Literal from uuid import uuid4 +from dander.telemetry import RunTelemetry + if TYPE_CHECKING: from collections.abc import Callable, Iterator, Mapping from types import FrameType @@ -183,6 +185,7 @@ def completed( *, context: LauncherContext, platform: str, + duration_ms: int | None = None, ) -> RuntimeEvent: """Build the terminal success/overlap record from non-sensitive aggregates.""" endpoints = [ @@ -194,6 +197,9 @@ def completed( } for endpoint in result.ingestion.endpoints ] + telemetry = ( + result.telemetry if duration_ms is None else result.telemetry.with_duration(duration_ms) + ) return cls( event="runtime.completed", timestamp=_timestamp(), @@ -207,6 +213,7 @@ def completed( "source": result.ingestion.source, "endpoints": endpoints, "models": list(result.models), + "telemetry": telemetry.to_payload(), "metrics": { "endpoints": len(endpoints), "extracted_rows": sum( @@ -233,6 +240,7 @@ def failed( stage: str, failure_code: str, retryable: bool, + duration_ms: int = 0, ) -> RuntimeEvent: """Build a terse terminal failure without copying exception text.""" return cls( @@ -244,7 +252,7 @@ def failed( stage=stage, dimensions=context.dimensions(), status="failed", - outputs={}, + outputs={"telemetry": RunTelemetry(duration_ms=duration_ms).to_payload()}, failure_code=failure_code, retryable=retryable, ) diff --git a/src/dander/state/failure.py b/src/dander/state/failure.py index 150b3897..8663c2f4 100644 --- a/src/dander/state/failure.py +++ b/src/dander/state/failure.py @@ -39,7 +39,7 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu if 403 in statuses or names & {"Forbidden", "PermissionDenied"}: return _details( "permission_denied", - "Permission was denied. Verify the provider and GCP permissions for this pipeline.", + "Permission was denied. Verify the selected provider permissions for this pipeline.", ) if 429 in statuses or names & {"TooManyRequests", "ResourceExhausted"}: return _details( @@ -56,10 +56,10 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu "source_schema_failed", f"A source record did not match the declared schema. Inspect logs for run {run_id}.", ) - if "BigQueryWriteError" in names: + if names & {"BigQueryWriteError", "DestinationWriteError", "WarehouseWriteError"}: return _details( "destination_write_failed", - f"BigQuery rejected a destination write. Inspect logs for run {run_id}.", + f"The destination rejected a write. Inspect logs for run {run_id}.", ) if names & {"TransformProjectError", "TransformRunError", "GraphRuntimeError"}: return _details( @@ -99,7 +99,7 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu ) return _details( "unexpected_error", - f"The {stage.value} stage failed unexpectedly. Inspect Cloud Run logs for run {run_id}.", + f"The {stage.value} stage failed unexpectedly. Inspect launcher logs for run {run_id}.", ) diff --git a/src/dander/telemetry.py b/src/dander/telemetry.py new file mode 100644 index 00000000..37ce0f2e --- /dev/null +++ b/src/dander/telemetry.py @@ -0,0 +1,182 @@ +"""Provider-neutral, non-sensitive telemetry for one Dander pipeline run.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, replace +from decimal import Decimal +from enum import StrEnum + +_SAFE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,126}$") +_SAFE_CURRENCY = re.compile(r"^[A-Z]{3}$") +_MAX_OPAQUE_LENGTH = 256 + + +class TelemetryOperation(StrEnum): + """Closed operation kinds shared by warehouse and runtime adapters.""" + + EXTRACT = "extract" + LOAD = "load" + QUERY = "query" + TRANSFORM = "transform" + TEST = "test" + CATALOG = "catalog" + STATE = "state" + + +@dataclass(frozen=True, slots=True) +class CostAttribution: + """One provider-reported or estimated monetary cost for an operation.""" + + provider: str + service: str + amount: Decimal + currency: str = "USD" + estimated: bool = True + + def __post_init__(self) -> None: + _require_name(self.provider, label="cost provider") + _require_name(self.service, label="cost service") + if not isinstance(self.amount, Decimal) or not self.amount.is_finite(): + raise ValueError("cost amount must be a finite Decimal") + if self.amount < 0: + raise ValueError("cost amount must be non-negative") + if not _SAFE_CURRENCY.fullmatch(self.currency): + raise ValueError("cost currency must be a three-letter uppercase code") + if not isinstance(self.estimated, bool): + raise ValueError("cost estimated must be a boolean") + + def to_payload(self) -> dict[str, object]: + """Return deterministic JSON-compatible cost metadata.""" + return { + "provider": self.provider, + "service": self.service, + "amount": str(self.amount), + "currency": self.currency, + "estimated": self.estimated, + } + + +@dataclass(frozen=True, slots=True) +class OperationTelemetry: + """Normalized statistics for one bounded provider operation.""" + + provider: str + operation: TelemetryOperation + duration_ms: int = 0 + retry_count: int = 0 + rows_read: int = 0 + rows_written: int = 0 + rows_affected: int = 0 + bytes_read: int = 0 + bytes_written: int = 0 + bytes_processed: int = 0 + bytes_billed: int = 0 + query_id: str | None = None + job_id: str | None = None + costs: tuple[CostAttribution, ...] = () + + def __post_init__(self) -> None: + _require_name(self.provider, label="telemetry provider") + if not isinstance(self.operation, TelemetryOperation): + raise ValueError("telemetry operation must be a TelemetryOperation") + for label, value in self._numeric_values().items(): + _require_nonnegative_integer(value, label=label) + _require_opaque(self.query_id, label="query id") + _require_opaque(self.job_id, label="job id") + if not isinstance(self.costs, tuple) or not all( + isinstance(cost, CostAttribution) for cost in self.costs + ): + raise ValueError("telemetry costs must be a tuple of CostAttribution values") + + def to_payload(self) -> dict[str, object]: + """Return deterministic JSON-compatible operation statistics.""" + payload: dict[str, object] = { + "provider": self.provider, + "operation": self.operation.value, + **self._numeric_values(), + } + if self.query_id is not None: + payload["query_id"] = self.query_id + if self.job_id is not None: + payload["job_id"] = self.job_id + if self.costs: + payload["costs"] = [cost.to_payload() for cost in self.costs] + return payload + + def _numeric_values(self) -> dict[str, int]: + return { + "duration_ms": self.duration_ms, + "retry_count": self.retry_count, + "rows_read": self.rows_read, + "rows_written": self.rows_written, + "rows_affected": self.rows_affected, + "bytes_read": self.bytes_read, + "bytes_written": self.bytes_written, + "bytes_processed": self.bytes_processed, + "bytes_billed": self.bytes_billed, + } + + +@dataclass(frozen=True, slots=True) +class RunTelemetry: + """Whole-run duration plus ordered provider-operation telemetry.""" + + duration_ms: int = 0 + operations: tuple[OperationTelemetry, ...] = () + + def __post_init__(self) -> None: + _require_nonnegative_integer(self.duration_ms, label="duration_ms") + if not isinstance(self.operations, tuple) or not all( + isinstance(operation, OperationTelemetry) for operation in self.operations + ): + raise ValueError("telemetry operations must be a tuple of OperationTelemetry values") + + def with_duration(self, duration_ms: int) -> RunTelemetry: + """Return a copy using a launcher-observed whole-process duration.""" + return replace(self, duration_ms=duration_ms) + + def to_payload(self) -> dict[str, object]: + """Return stable totals and ordered details without arbitrary extension data.""" + totals = { + key: sum(operation._numeric_values()[key] for operation in self.operations) + for key in ( + "retry_count", + "rows_read", + "rows_written", + "rows_affected", + "bytes_read", + "bytes_written", + "bytes_processed", + "bytes_billed", + ) + } + return { + "duration_ms": self.duration_ms, + **totals, + "operations": [operation.to_payload() for operation in self.operations], + } + + +def _require_name(value: str, *, label: str) -> None: + if not isinstance(value, str) or not _SAFE_NAME.fullmatch(value): + raise ValueError(f"{label} must use lowercase portable name syntax") + + +def _require_nonnegative_integer(value: int, *, label: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{label} must be a non-negative integer") + + +def _require_opaque(value: str | None, *, label: str) -> None: + if value is None: + return + if ( + not isinstance(value, str) + or not value + or len(value) > _MAX_OPAQUE_LENGTH + or not value.isprintable() + ): + raise ValueError( + f"{label} must be 1-{_MAX_OPAQUE_LENGTH} characters without control characters" + ) diff --git a/tests/cli/test_runtime_cli.py b/tests/cli/test_runtime_cli.py index 51d6d08a..858fa1b2 100644 --- a/tests/cli/test_runtime_cli.py +++ b/tests/cli/test_runtime_cli.py @@ -95,6 +95,7 @@ def execute( assert [event["event"] for event in events] == ["runtime.started", "runtime.completed"] assert events[-1]["run_id"] == "cloud-run:execution-42" assert events[-1]["outputs"]["metrics"]["extracted_rows"] == 4 + assert events[-1]["outputs"]["telemetry"]["duration_ms"] >= 0 options = cast("RunOptions", captured["options"]) assert options.pipeline_or_source == "greenhouse_jobs" assert options.project == "unit-project" @@ -119,6 +120,7 @@ def fail(*args: object, **kwargs: object) -> None: terminal = json.loads(result.output.splitlines()[-1]) assert terminal["failure_code"] == "rate_limited" assert terminal["retryable"] is True + assert terminal["outputs"]["telemetry"]["duration_ms"] >= 0 assert "credential-value" not in result.output diff --git a/tests/state/test_failure.py b/tests/state/test_failure.py index 73879357..75fd173a 100644 --- a/tests/state/test_failure.py +++ b/tests/state/test_failure.py @@ -43,6 +43,16 @@ def test_unknown_failure_is_stage_specific_and_points_to_run_logs() -> None: ) assert "customer payload" not in failure.summary + unexpected = classify_failure( + RuntimeError("customer payload must never persist"), + stage=RunStage.INGEST, + run_id="run-123", + ) + assert unexpected.summary == ( + "The ingest stage failed unexpectedly. Inspect launcher logs for run run-123." + ) + assert "Cloud Run" not in unexpected.summary + def test_runtime_cancellation_is_safe_and_retryable_by_a_fresh_run() -> None: class RuntimeCancelledError(RuntimeError): @@ -81,3 +91,41 @@ class CatalogPublishError(RuntimeError): assert catalog.code == "catalog_failed" assert "private" not in transform.summary assert "private" not in catalog.summary + + +def test_provider_specific_failures_use_cloud_neutral_summaries() -> None: + class PermissionDenied(RuntimeError): # noqa: N818 - matches provider SDK class name + pass + + class BigQueryWriteError(RuntimeError): + pass + + class DestinationWriteError(RuntimeError): + pass + + permission = classify_failure( + PermissionDenied("private permission detail"), + stage=RunStage.INGEST, + run_id="run-123", + ) + destination = classify_failure( + BigQueryWriteError("private row detail"), + stage=RunStage.INGEST, + run_id="run-123", + ) + portable_destination = classify_failure( + DestinationWriteError("private row detail"), + stage=RunStage.INGEST, + run_id="run-123", + ) + + assert permission.summary == ( + "Permission was denied. Verify the selected provider permissions for this pipeline." + ) + assert destination.summary == ( + "The destination rejected a write. Inspect logs for run run-123." + ) + assert portable_destination.code == "destination_write_failed" + assert portable_destination.summary == destination.summary + assert "GCP" not in permission.summary + assert "BigQuery" not in destination.summary diff --git a/tests/test_executor.py b/tests/test_executor.py index 85c612f5..ddde0f2a 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING import pytest @@ -220,10 +221,15 @@ def _executor( ) -def test_executor_records_one_truthful_complete_lifecycle(tmp_path: Path) -> None: +def test_executor_records_one_truthful_complete_lifecycle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: _models(tmp_path) history = _History() metadata = _Metadata() + clock = iter((1_000_000, 8_500_000)) + monkeypatch.setattr("dander.executor.time.monotonic_ns", lambda: next(clock)) result = _executor(tmp_path, history=history, metadata=metadata).execute() @@ -232,6 +238,7 @@ def test_executor_records_one_truthful_complete_lifecycle(tmp_path: Path) -> Non assert history.finished == (RunStatus.SUCCEEDED, None, 1, 2, 1) assert result.ingestion.run_id == result.run_id assert result.assets == 1 + assert result.telemetry.duration_ms == 7 assert metadata.manifest is not None assert metadata.manifest["pipeline_id"] == "example_pipeline" assets = metadata.manifest["assets"] @@ -253,9 +260,11 @@ def test_executor_preserves_launcher_supplied_run_id(tmp_path: Path) -> None: def test_executor_marks_transform_failure_without_claiming_ingestion_only_success( tmp_path: Path, + caplog: pytest.LogCaptureFixture, ) -> None: _models(tmp_path) history = _History() + caplog.set_level(logging.WARNING, logger="dander.executor") with pytest.raises(RuntimeError, match="transform failed"): _executor( @@ -270,6 +279,11 @@ def test_executor_marks_transform_failure_without_claiming_ingestion_only_succes assert history.failure is not None assert history.failure[0] == "transform_failed" assert "Inspect logs for run" in (history.failure[1] or "") + terminal = next(record for record in caplog.records if record.msg == "pipeline_failed") + assert terminal.__dict__["dander_event"] == "pipeline_failed" + assert terminal.__dict__["pipeline_id"] == "example_pipeline" + assert terminal.__dict__["failure_code"] == "transform_failed" + assert not hasattr(terminal, "exception") def test_executor_records_active_overlap_as_skipped_without_running_pipeline( diff --git a/tests/test_runtime_contract.py b/tests/test_runtime_contract.py index 04c2dfa5..991ffc0b 100644 --- a/tests/test_runtime_contract.py +++ b/tests/test_runtime_contract.py @@ -18,6 +18,7 @@ graceful_signal_handlers, validate_runtime_contract, ) +from dander.telemetry import OperationTelemetry, RunTelemetry, TelemetryOperation def _context() -> LauncherContext: @@ -55,6 +56,17 @@ def _result(*, skipped: bool = False) -> PipelineExecutionResult: assertions=2, assets=1, skipped=skipped, + telemetry=RunTelemetry( + duration_ms=17, + operations=( + OperationTelemetry( + provider="bigquery", + operation=TelemetryOperation.LOAD, + rows_written=2, + bytes_written=128, + ), + ), + ), ) @@ -81,6 +93,9 @@ def test_runtime_events_are_json_lines_without_cursor_values() -> None: assert completed["event"] == "runtime.completed" assert completed["status"] == "succeeded" assert completed["outputs"]["metrics"]["extracted_rows"] == 3 + assert completed["outputs"]["telemetry"]["duration_ms"] == 17 + assert completed["outputs"]["telemetry"]["rows_written"] == 2 + assert completed["outputs"]["telemetry"]["operations"][0]["provider"] == "bigquery" assert completed["outputs"]["endpoints"][0]["cursor_committed"] is True assert "2026-08-06" not in completed_json @@ -96,13 +111,25 @@ def test_runtime_event_distinguishes_overlap_and_sanitized_failure() -> None: stage="runtime", failure_code="authentication_failed", retryable=False, + duration_ms=9, ).to_json() failed = json.loads(failed_json) assert skipped["status"] == "skipped" assert failed["status"] == "failed" assert failed["failure_code"] == "authentication_failed" - assert failed["outputs"] == {} + assert failed["outputs"]["telemetry"] == { + "duration_ms": 9, + "retry_count": 0, + "rows_read": 0, + "rows_written": 0, + "rows_affected": 0, + "bytes_read": 0, + "bytes_written": 0, + "bytes_processed": 0, + "bytes_billed": 0, + "operations": [], + } assert "credential-value" not in failed_json diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 00000000..c585520f --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,124 @@ +"""Provider-neutral telemetry validation and deterministic serialization.""" + +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from dander.telemetry import ( + CostAttribution, + OperationTelemetry, + RunTelemetry, + TelemetryOperation, +) + + +def _operation() -> OperationTelemetry: + return OperationTelemetry( + provider="bigquery", + operation=TelemetryOperation.LOAD, + duration_ms=42, + retry_count=1, + rows_read=7, + rows_written=6, + rows_affected=5, + bytes_read=400, + bytes_written=300, + bytes_processed=500, + bytes_billed=1_000, + query_id="query/abc-123", + job_id="job:abc-123", + costs=( + CostAttribution( + provider="gcp", + service="bigquery", + amount=Decimal("0.000125"), + ), + ), + ) + + +def test_run_telemetry_serializes_normalized_totals_and_operation_details() -> None: + telemetry = RunTelemetry(duration_ms=50, operations=(_operation(),)) + + payload = telemetry.to_payload() + + assert payload == { + "duration_ms": 50, + "retry_count": 1, + "rows_read": 7, + "rows_written": 6, + "rows_affected": 5, + "bytes_read": 400, + "bytes_written": 300, + "bytes_processed": 500, + "bytes_billed": 1_000, + "operations": [ + { + "provider": "bigquery", + "operation": "load", + "duration_ms": 42, + "retry_count": 1, + "rows_read": 7, + "rows_written": 6, + "rows_affected": 5, + "bytes_read": 400, + "bytes_written": 300, + "bytes_processed": 500, + "bytes_billed": 1_000, + "query_id": "query/abc-123", + "job_id": "job:abc-123", + "costs": [ + { + "provider": "gcp", + "service": "bigquery", + "amount": "0.000125", + "currency": "USD", + "estimated": True, + } + ], + } + ], + } + assert json.dumps(payload, separators=(",", ":"), sort_keys=True) == json.dumps( + telemetry.to_payload(), separators=(",", ":"), sort_keys=True + ) + + +def test_telemetry_rejects_unsafe_or_ambiguous_values() -> None: + with pytest.raises(ValueError, match="non-negative"): + OperationTelemetry( + provider="bigquery", + operation=TelemetryOperation.QUERY, + retry_count=-1, + ) + with pytest.raises(ValueError, match="control characters"): + OperationTelemetry( + provider="bigquery", + operation=TelemetryOperation.QUERY, + query_id="query\nsecret", + ) + with pytest.raises(ValueError, match="finite Decimal"): + CostAttribution( + provider="gcp", + service="bigquery", + amount=Decimal("NaN"), + ) + with pytest.raises(TypeError): + OperationTelemetry( # type: ignore[call-arg] + provider="bigquery", + operation=TelemetryOperation.QUERY, + unrestricted_details={"token": "secret"}, + ) + + +def test_whole_run_duration_can_be_replaced_without_losing_operations() -> None: + original = RunTelemetry(duration_ms=10, operations=(_operation(),)) + + replaced = original.with_duration(25) + + assert replaced.duration_ms == 25 + assert replaced.operations == original.operations + assert original.duration_ms == 10 diff --git a/tickets/DANDER-83-generic-run-telemetry.md b/tickets/DANDER-83-generic-run-telemetry.md new file mode 100644 index 00000000..19068296 --- /dev/null +++ b/tickets/DANDER-83-generic-run-telemetry.md @@ -0,0 +1,43 @@ +--- +id: DANDER-83 +title: Normalize provider-neutral run telemetry +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-82] +created: 2026-08-07 +--- + +## Context + +The OCI runtime emitted aggregate ingestion counts but had no shared shape for elapsed time, +provider operation statistics, or cost attribution. Some persisted failure guidance also named +Cloud Run, GCP, or BigQuery even when the stable code was generic. + +## Acceptance Criteria + +- [x] Whole-run duration is measured with a monotonic clock. +- [x] Operation telemetry normalizes row, byte, duration, retry, and query/job identifiers. +- [x] Cost attribution uses finite non-negative decimals, currency, and an estimated marker. +- [x] Telemetry rejects arbitrary extension payloads and unsafe identifiers. +- [x] Successful, skipped, and failed runtime terminal events include telemetry. +- [x] Existing runtime row metrics and retry/exit behavior remain compatible. +- [x] Failure summaries and structured terminal logs do not assume one cloud or warehouse. +- [x] Focused tests cover validation, aggregation, event output, duration, and redaction. +- [x] Full local validation passes. +- [ ] Protected CI passes. + +## Design + +Keep telemetry as a small immutable value contract. Core measures elapsed time; provider adapters +will contribute detailed operations when each concrete vertical slice moves behind its capability +boundary. Do not introduce a telemetry service, exporter, or billing estimator. + +## Implementation Notes + +The runtime-v1 event schema receives additive `outputs.telemetry`; existing metrics remain in +place. Detailed zeroes mean “not reported,” not “free.” + +## Review Log + +Pending protected review. From 016ed7ef3f89808a782eae6b870fecfdc84dda8f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Fri, 7 Aug 2026 22:51:56 -0400 Subject: [PATCH 033/163] Assemble provider runtime dependencies --- .github/workflows/ci.yml | 7 +- CHANGELOG.md | 3 + Dockerfile | 6 +- HANDOFF.md | 39 +- docs/decisions.md | 12 + docs/known-limitations.md | 6 + docs/oci-artifact-contract.md | 17 + docs/platform-profiles.md | 6 + pyproject.toml | 42 ++ scripts/check_distribution.py | 19 + src/dander/providers/__init__.py | 14 + src/dander/providers/dependencies.py | 85 ++++ src/dander/templates/project/Dockerfile | 6 +- tests/project/test_scaffold.py | 3 + tests/providers/test_dependencies.py | 97 ++++ .../DANDER-84-provider-dependency-assembly.md | 46 ++ uv.lock | 463 +++++++++++++++++- 17 files changed, 848 insertions(+), 23 deletions(-) create mode 100644 src/dander/providers/dependencies.py create mode 100644 tests/providers/test_dependencies.py create mode 100644 tickets/DANDER-84-provider-dependency-assembly.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90c016fc..9f89873b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: run: uv run pytest - name: Dependency audit run: | - uv export --frozen --format requirements.txt --no-dev --no-emit-project \ + uv export --frozen --extra runtime-all --format requirements.txt --no-dev --no-emit-project \ --output-file /tmp/dander-requirements.txt uv run --with pip-audit pip-audit --strict -r /tmp/dander-requirements.txt @@ -111,6 +111,11 @@ jobs: ) INDEX=$((INDEX + 1)) done + FULL_ENV="$TEMP_DIR/venv-runtime-all" + uv venv "$FULL_ENV" + uv pip install --python "$FULL_ENV/bin/python" "dist/dander_platform-${VERSION}-py3-none-any.whl[runtime-all]" + "$FULL_ENV/bin/python" -c \ + 'from dander.providers.dependencies import require_full_runtime; require_full_runtime()' terraform -chdir="$TEMP_DIR/project-0/infra" init -backend=false -input=false terraform -chdir="$TEMP_DIR/project-0/infra" validate diff --git a/CHANGELOG.md b/CHANGELOG.md index a3884ee3..274618f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Expose provider dependency extras for BigQuery, Snowflake, Redshift, PostgreSQL, GCP, AWS, Azure, + and OCI, plus a validated `runtime-all` union used by repository and generated source-free OCI + images without expanding the supported-adapter capability manifest. - Add a provider-neutral run telemetry contract for duration, rows, bytes, retries, provider query/job correlation, and explicit decimal cost attribution; emit it on every terminal OCI runtime event without admitting arbitrary provider payloads. diff --git a/Dockerfile b/Dockerfile index 3c6ad622..ea7ae952 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,9 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +RUN apt-get update \ + && apt-get install --yes --no-install-recommends libpq5 \ + && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir uv==0.12.0 COPY pyproject.toml uv.lock README.md LICENSE ./ @@ -31,7 +34,8 @@ COPY graphs ./graphs COPY models ./models COPY examples ./examples COPY infra ./infra -RUN uv sync --frozen --no-dev --no-editable +RUN uv sync --frozen --no-dev --no-editable --extra runtime-all \ + && .venv/bin/python -c "from dander.providers.dependencies import require_full_runtime; require_full_runtime()" COPY dander.yaml ./dander.yaml diff --git a/HANDOFF.md b/HANDOFF.md index 51184260..1eecb6bb 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,38 @@ ## Finished -- Added immutable provider-neutral run, operation, and cost telemetry contracts. -- Emitted normalized telemetry for successful, skipped, and failed OCI runtime terminal events. -- Measured executor and launcher duration with monotonic clocks. -- Added cloud-neutral structured terminal logs and operator failure summaries. -- Preserved existing runtime-v1 row metrics, stable failure codes, and retry behavior. +- Added eight public provider extras and a deterministic 12-distribution `runtime-all` union. +- Made repository and generated source-free images install and verify the full dependency set. +- Added metadata-only dependency inspection that never imports provider SDKs. +- Preserved the capability manifest as the support boundary; only GCP/BigQuery remains advertised. +- Added package, scaffold, missing-dependency, and lazy-inspection regression coverage. ## Try It -Run `dander runtime execute ...` and inspect `outputs.telemetry` in the terminal JSON event. Future -warehouse adapters can attach ordered `OperationTelemetry` values to `PipelineExecutionResult`. +Install one SDK set with `pip install 'dander-platform[postgres]'`. Official images use +`dander-platform[runtime-all]`; package presence does not make an adapter supported. ## Checks -- All 896 tests, Ruff, formatting, strict mypy across 201 files, and dependency audit passed. -- Wheel/sdist build and source-free installs, generation, validation, and Terraform passed. -- All four Terraform roots and local non-root/read-only container conformance passed. -- The isolated GCP plan reported `No changes`; both schedules stayed paused and no apply ran. +- All 900 tests, Ruff, formatting, strict mypy, and the full dependency audit passed. +- Wheel/sdist source-free installs, generation, validation, and full-runtime installation passed. +- Linux amd64/arm64 builds and local non-root/read-only container conformance passed. +- Trivy reported zero high/critical findings; the branch secret scan passed. +- All Terraform roots passed; GCP reported `No changes` and both schedules stayed paused. ## Decisions -- Use one closed value contract rather than an exporter or observability subsystem. -- Record decimal costs exactly and distinguish estimates; never infer missing billing data. -- Let concrete provider slices populate operation detail after their adapters exist. +- Keep base installation backward compatible while making provider SDK groups explicit. +- Build one checked full image; keep implementation loading lazy. +- Reserve OCI without its currently incompatible SDK; never weaken the dependency audit. ## Remaining -- Open and merge the focused telemetry PR after protected CI passes. -- Add provider dependency extras and full runtime-image assembly next. +- Open and merge the focused dependency-assembly PR after protected CI passes. +- Implement the portable BigQuery vertical slice next. ## Review First -- `src/dander/telemetry.py` -- `src/dander/runtime_contract.py` -- `src/dander/executor.py` +- `pyproject.toml` +- `src/dander/providers/dependencies.py` +- `src/dander/templates/project/Dockerfile` diff --git a/docs/decisions.md b/docs/decisions.md index 94b1ece4..c63ed5e5 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,17 @@ # Engineering Decisions +## 2026-08-07 — Provider dependencies are assembled separately from support claims + +- **Local installs:** Public extras group BigQuery, Snowflake, Redshift, PostgreSQL, GCP, AWS, and + Azure SDK dependencies without importing or registering an adapter. The OCI name is reserved but + empty while Oracle's SDK requires a cryptography version with a known fixed-in-50 advisory. +- **Release image:** `runtime-all` is their checked union; repository and generated Dockerfiles + install and validate it so one immutable multi-platform image has deterministic dependencies. + PostgreSQL uses pure-Python Psycopg with Debian's maintained `libpq5`, avoiding a bundled wheel + whose embedded native-library SBOM fails the release image scan. +- **Support boundary:** `runtime-capabilities.json`, concrete adapter conformance, and live profile + gates remain authoritative. A present SDK or package extra never makes a provider supported. + ## 2026-08-07 — Runtime telemetry is normalized before provider reporting - **Contract:** Terminal runtime events carry one validated `RunTelemetry` shape with whole-run diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 66aba3b3..dc311fa3 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -6,6 +6,12 @@ evaluate these limits before using it for an unattended system containing busine - AWS ECS/Fargate is not yet a supported Dander launcher. Phase 1B contains an isolated artifact and keyless-identity feasibility stack only; launcher lifecycle, scheduling, alerts, rollback, and parity remain gated on the later portable BigQuery vertical slice. +- Provider package extras and the full runtime image contain planned provider SDK dependencies, + but do not by themselves implement or qualify Snowflake, Redshift, PostgreSQL, AWS, Azure, or + OCI profiles. The packaged capability manifest is the support boundary. +- The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version + below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or + use a reviewed direct signed-HTTP client before it can enter the full image. ## Ingestion and schemas diff --git a/docs/oci-artifact-contract.md b/docs/oci-artifact-contract.md index 8a515afb..d1d97ecf 100644 --- a/docs/oci-artifact-contract.md +++ b/docs/oci-artifact-contract.md @@ -22,3 +22,20 @@ platform-manifest map; registries that rewrite the index fail verification. The runtime executes as UID/GID `65532:65532`. CI runs the local conformance probe with a read-only root filesystem and a temporary `/tmp`, so runtime code must declare durable state through an adapter rather than writing into the image filesystem. + +## Provider dependency assembly + +Python installations expose `bigquery`, `snowflake`, `redshift`, `postgres`, `gcp`, `aws`, +`azure`, and `oci` extras. These install provider SDKs only; an extra is not an adapter or a support +claim. Provider implementation imports remain lazy until that provider is selected. The `oci` +extra is currently an empty reserved name: Oracle's available SDK constrains `cryptography` below +the audited fixed version, so it is excluded until a compatible SDK or direct signed-HTTP adapter +passes its own review. + +The `runtime-all` extra is the deterministic union of those dependency sets. Repository and +generated source-free Dockerfiles install it, then validate every required distribution from +package metadata without importing an SDK. This catches an incomplete build before an image can be +published. The packaged capability manifest remains authoritative about adapters actually present +and supported in that image; it continues to list only the proven GCP/BigQuery composition today. +The image uses the maintained Debian `libpq5` package with pure-Python Psycopg instead of bundling +an opaque database client library inside a wheel. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 09fbc2a1..33ac8dd2 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -50,3 +50,9 @@ adapter routing arrives in later, separate portability changes. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; the selected provider remains responsible for physical identifier rendering and type mapping. + +Provider SDKs can be installed independently with extras such as +`dander-platform[postgres]` or `dander-platform[aws]`. The official OCI runtime installs +`dander-platform[runtime-all]` so one immutable image can later serve any first-class adapter. +Installing an extra does not make its provider selectable; configuration remains gated by the +registered adapter and capability manifest. diff --git a/pyproject.toml b/pyproject.toml index f31c06ca..cda69949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,48 @@ dander = "dander.cli.main:app" dander-synthetic-api = "dander.dev.synthetic_vendor:main" [project.optional-dependencies] +bigquery = [ + "dlt[bigquery]>=1.0", + "google-cloud-bigquery>=3.25", + "google-cloud-bigquery-storage>=2.30", +] +snowflake = [ + "snowflake-connector-python>=3.12,<5", +] +redshift = [ + "boto3>=1.35,<2", + "redshift-connector>=2.1,<3", +] +postgres = [ + "psycopg[pool]>=3.2,<4", +] +gcp = [ + "google-auth>=2.35", + "google-cloud-dataplex>=2.20,<3", + "google-cloud-secret-manager>=2.20", +] +aws = [ + "boto3>=1.35,<2", +] +azure = [ + "azure-identity>=1.19,<2", + "azure-keyvault-secrets>=4.9,<5", +] +oci = [] +runtime-all = [ + "azure-identity>=1.19,<2", + "azure-keyvault-secrets>=4.9,<5", + "boto3>=1.35,<2", + "dlt[bigquery]>=1.0", + "google-auth>=2.35", + "google-cloud-bigquery>=3.25", + "google-cloud-bigquery-storage>=2.30", + "google-cloud-dataplex>=2.20,<3", + "google-cloud-secret-manager>=2.20", + "psycopg[pool]>=3.2,<4", + "redshift-connector>=2.1,<3", + "snowflake-connector-python>=3.12,<5", +] dev = [ "fastapi>=0.115", "ruff>=0.6", diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 3363fb19..e9d248ba 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -69,6 +69,17 @@ def _check_wheel(path: Path, *, expected_name: str, expected_version: str) -> No expected_name=expected_name, expected_version=expected_version, ) + starter_dockerfile = archive.read("dander/templates/project/Dockerfile").decode("utf-8") + required_runtime_markers = { + "dander-platform[runtime-all]==${DANDER_VERSION}", + "require_full_runtime", + } + if missing := sorted( + marker for marker in required_runtime_markers if marker not in starter_dockerfile + ): + raise ValueError( + "Starter Dockerfile does not install the full runtime: " + ", ".join(missing) + ) def _check_sdist(path: Path, *, expected_name: str, expected_version: str) -> None: @@ -124,6 +135,14 @@ def _check_metadata( version = metadata["Version"] if name != expected_name or version != expected_version: raise ValueError(f"Unexpected distribution identity: {name} {version}") + config = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + expected_extras = set(config["project"]["optional-dependencies"]) + provided_extras = set(metadata.get_all("Provides-Extra") or ()) + if provided_extras != expected_extras: + raise ValueError( + "Distribution extras differ from pyproject.toml: " + f"expected {sorted(expected_extras)}, received {sorted(provided_extras)}" + ) description = metadata.get_payload() required_description = { f"uv tool install dander-platform=={expected_version}", diff --git a/src/dander/providers/__init__.py b/src/dander/providers/__init__.py index 95deae62..86e5b962 100644 --- a/src/dander/providers/__init__.py +++ b/src/dander/providers/__init__.py @@ -1,5 +1,13 @@ """Cloud-provider registration and construction contracts.""" +from dander.providers.dependencies import ( + FULL_RUNTIME_DISTRIBUTIONS, + PROVIDER_DEPENDENCY_SETS, + ProviderDependencySet, + RuntimeDependencyError, + installed_runtime_dependencies, + require_full_runtime, +) from dander.providers.registry import ( PROVIDER_API_VERSION, ProviderFactory, @@ -10,10 +18,16 @@ ) __all__ = [ + "FULL_RUNTIME_DISTRIBUTIONS", "PROVIDER_API_VERSION", + "PROVIDER_DEPENDENCY_SETS", + "ProviderDependencySet", "ProviderFactory", "ProviderFactoryError", "ProviderKind", "ProviderRegistry", + "RuntimeDependencyError", + "installed_runtime_dependencies", "lazy_provider_factory", + "require_full_runtime", ] diff --git a/src/dander/providers/dependencies.py b/src/dander/providers/dependencies.py new file mode 100644 index 00000000..28dd0333 --- /dev/null +++ b/src/dander/providers/dependencies.py @@ -0,0 +1,85 @@ +"""Distribution-only dependency assembly for Dander provider adapters.""" + +from __future__ import annotations + +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _distribution_version + + +class RuntimeDependencyError(RuntimeError): + """The full runtime image is missing one or more declared provider dependencies.""" + + +@dataclass(frozen=True, slots=True) +class ProviderDependencySet: + """One public package extra and the distributions it installs.""" + + extra: str + distributions: tuple[str, ...] + + +PROVIDER_DEPENDENCY_SETS = ( + ProviderDependencySet( + extra="bigquery", + distributions=("dlt", "google-cloud-bigquery", "google-cloud-bigquery-storage"), + ), + ProviderDependencySet( + extra="snowflake", + distributions=("snowflake-connector-python",), + ), + ProviderDependencySet( + extra="redshift", + distributions=("boto3", "redshift-connector"), + ), + ProviderDependencySet( + extra="postgres", + distributions=("psycopg",), + ), + ProviderDependencySet( + extra="gcp", + distributions=( + "google-auth", + "google-cloud-dataplex", + "google-cloud-secret-manager", + ), + ), + ProviderDependencySet(extra="aws", distributions=("boto3",)), + ProviderDependencySet( + extra="azure", + distributions=("azure-identity", "azure-keyvault-secrets"), + ), + ProviderDependencySet(extra="oci", distributions=()), +) + +FULL_RUNTIME_DISTRIBUTIONS = tuple( + sorted( + { + distribution + for dependency_set in PROVIDER_DEPENDENCY_SETS + for distribution in dependency_set.distributions + } + ) +) + + +def installed_runtime_dependencies() -> dict[str, str | None]: + """Inspect distribution metadata without importing any provider SDK.""" + installed: dict[str, str | None] = {} + for distribution in FULL_RUNTIME_DISTRIBUTIONS: + try: + installed[distribution] = _distribution_version(distribution) + except PackageNotFoundError: + installed[distribution] = None + return installed + + +def require_full_runtime() -> dict[str, str]: + """Fail a full-image build when its declared provider dependency set is incomplete.""" + inspected = installed_runtime_dependencies() + missing = tuple(name for name, version in inspected.items() if version is None) + if missing: + raise RuntimeDependencyError( + "Full runtime is missing provider distributions: " + ", ".join(missing) + ) + return {name: version for name, version in inspected.items() if version is not None} diff --git a/src/dander/templates/project/Dockerfile b/src/dander/templates/project/Dockerfile index 69f39d1d..0d723026 100644 --- a/src/dander/templates/project/Dockerfile +++ b/src/dander/templates/project/Dockerfile @@ -20,7 +20,11 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app -RUN pip install --no-cache-dir "dander-platform==${DANDER_VERSION}" +RUN apt-get update \ + && apt-get install --yes --no-install-recommends libpq5 \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir "dander-platform[runtime-all]==${DANDER_VERSION}" \ + && python -c "from dander.providers.dependencies import require_full_runtime; require_full_runtime()" COPY --chown=65532:65532 dander.yaml ./dander.yaml COPY --chown=65532:65532 dander.platforms.yaml ./dander.platforms.yaml diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index 3fd4b11e..af953b70 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -25,6 +25,9 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: assert manifest.deployment_name == "gcp_cloud_run" dockerfile = (project / "Dockerfile").read_text(encoding="utf-8") assert f"ARG DANDER_VERSION={__version__}" in dockerfile + assert "apt-get install --yes --no-install-recommends libpq5" in dockerfile + assert "dander-platform[runtime-all]==${DANDER_VERSION}" in dockerfile + assert "require_full_runtime" in dockerfile assert "RUN dander plugins install --config dander.yaml" in dockerfile assert "COPY --chown=65532:65532 dander.yaml ./dander.yaml" in dockerfile assert "COPY --chown=65532:65532 dander.platforms.yaml ./dander.platforms.yaml" in dockerfile diff --git a/tests/providers/test_dependencies.py b/tests/providers/test_dependencies.py new file mode 100644 index 00000000..4491396d --- /dev/null +++ b/tests/providers/test_dependencies.py @@ -0,0 +1,97 @@ +"""Public provider extras and full-runtime dependency assembly.""" + +from __future__ import annotations + +import re +import tomllib +from importlib import metadata +from pathlib import Path + +import pytest + +import dander.providers.dependencies as dependency_module +from dander.providers import ( + FULL_RUNTIME_DISTRIBUTIONS, + PROVIDER_DEPENDENCY_SETS, + RuntimeDependencyError, + installed_runtime_dependencies, + require_full_runtime, +) + +_DISTRIBUTION = re.compile(r"^[A-Za-z0-9._-]+") + + +def _distribution(requirement: str) -> str: + matched = _DISTRIBUTION.match(requirement) + assert matched is not None + return matched.group(0).lower().replace("_", "-") + + +def test_public_provider_extras_match_the_full_runtime_union() -> None: + root = Path(__file__).resolve().parents[2] + project = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))["project"] + extras = project["optional-dependencies"] + + expected_names = { + "bigquery", + "snowflake", + "redshift", + "postgres", + "gcp", + "aws", + "azure", + "oci", + } + assert {dependency_set.extra for dependency_set in PROVIDER_DEPENDENCY_SETS} == expected_names + for dependency_set in PROVIDER_DEPENDENCY_SETS: + assert {_distribution(requirement) for requirement in extras[dependency_set.extra]} == set( + dependency_set.distributions + ) + assert {_distribution(requirement) for requirement in extras["runtime-all"]} == set( + FULL_RUNTIME_DISTRIBUTIONS + ) + + +def test_dependency_inspection_uses_metadata_without_importing_provider_sdks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + inspected_names: list[str] = [] + + def version(distribution: str) -> str: + inspected_names.append(distribution) + return "1.2.3" + + monkeypatch.setattr(dependency_module, "_distribution_version", version) + + assert installed_runtime_dependencies() == { + distribution: "1.2.3" for distribution in FULL_RUNTIME_DISTRIBUTIONS + } + assert tuple(inspected_names) == FULL_RUNTIME_DISTRIBUTIONS + + +def test_full_runtime_requirement_reports_only_missing_distribution_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def version(distribution: str) -> str: + if distribution in {"azure-identity", "snowflake-connector-python"}: + raise metadata.PackageNotFoundError(distribution) + return "1.2.3" + + monkeypatch.setattr(dependency_module, "_distribution_version", version) + + with pytest.raises(RuntimeDependencyError) as raised: + require_full_runtime() + + assert str(raised.value) == ( + "Full runtime is missing provider distributions: azure-identity, snowflake-connector-python" + ) + + +def test_full_runtime_requirement_returns_installed_versions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(dependency_module, "_distribution_version", lambda _name: "9.8.7") + + assert require_full_runtime() == { + distribution: "9.8.7" for distribution in FULL_RUNTIME_DISTRIBUTIONS + } diff --git a/tickets/DANDER-84-provider-dependency-assembly.md b/tickets/DANDER-84-provider-dependency-assembly.md new file mode 100644 index 00000000..6f1b76cb --- /dev/null +++ b/tickets/DANDER-84-provider-dependency-assembly.md @@ -0,0 +1,46 @@ +--- +id: DANDER-84 +title: Assemble optional provider dependencies in the full runtime image +status: in-review +component: packaging +epic: cloud-portability +depends_on: [DANDER-83] +created: 2026-08-07 +--- + +## Context + +Provider factories load implementations lazily, but Python packages and the official OCI image +need deterministic SDK dependency sets before concrete adapters can be qualified independently. + +## Acceptance Criteria + +- [x] Public extras exist for BigQuery, Snowflake, Redshift, PostgreSQL, GCP, AWS, Azure, and OCI. +- [x] `runtime-all` is the exact union of provider dependency distributions. +- [x] The OCI extra remains reserved and excludes the SDK that conflicts with the audited + cryptography line. +- [x] Repository and generated source-free Dockerfiles install `runtime-all`. +- [x] Full-image assembly fails when a declared distribution is missing. +- [x] Dependency validation reads package metadata and does not import provider SDKs. +- [x] The capability manifest still advertises only implemented, proven adapters. +- [x] Tests prevent drift between public extras, the runtime union, and generated Dockerfiles. +- [x] The full image builds and runs conformance on linux/amd64 and linux/arm64. +- [x] Full local validation and dependency/security audits pass. +- [ ] Protected CI passes. + +## Design + +Keep dependency availability separate from provider registration. Extras install SDKs; lazy +factories select adapters; the capability manifest and live qualification define support. + +## Implementation Notes + +The default dependency set remains backward compatible. `runtime-all` deliberately prepares one +release image for later adapters without adding provider configuration or runtime dispatch here. +OCI has no first-class adapter and its reserved empty extra avoids an unsafe dependency override. +PostgreSQL uses pure-Python Psycopg with the maintained Debian `libpq5` runtime package; this keeps +the image scan clean without suppressing findings from a wheel-bundled native library. + +## Review Log + +Pending protected review. diff --git a/uv.lock b/uv.lock index f43d8aa9..cdbd3305 100644 --- a/uv.lock +++ b/uv.lock @@ -173,6 +173,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, ] +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + [[package]] name = "ast-serialize" version = "0.6.0" @@ -223,6 +232,90 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "azure-core" +version = "1.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.25.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/0e/3a63efb48aa4a5ae2cfca61ee152fbcb668092134d3eb8bfda472dd5c617/azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6", size = 286304, upload-time = "2026-03-13T01:12:20.892Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138, upload-time = "2026-03-13T01:12:22.951Z" }, +] + +[[package]] +name = "azure-keyvault-secrets" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "isodate" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/b8/03c7b4edd1e3355ad5fffb70e68af70cd09542963f45cf3a2aa9fb3930b5/azure_keyvault_secrets-4.11.0.tar.gz", hash = "sha256:ac14727b9159cca353173ec5a454d8d7b192a6f2f5e7eb540f9fbcf914fa0ca0", size = 112291, upload-time = "2026-04-17T00:52:12.422Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/76/41c513917c0e8dc68de0c33578c6a0668b4a09eb4dc3e2782d382dc2947f/azure_keyvault_secrets-4.11.0-py3-none-any.whl", hash = "sha256:d8543b710423569bd00ada2a2533fe83d52d8e1fe026e1c47f41a3bc0fc73ef5", size = 103148, upload-time = "2026-04-17T00:52:13.796Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.67" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/bf/ef5de9b55523bc2141d072fbe6614627088e3e6f97b47850216e99de6a1c/boto3-1.43.67.tar.gz", hash = "sha256:75fe983b70d39cfdc274dc51f9bb02b8a0a104bdad4fa073c1af85a35e707c91", size = 112665, upload-time = "2026-08-07T19:30:20.418Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/0f/b41f8968452dc6bf3b83f15f5e9f76a27fd93e246906aeb2e0555f494375/boto3-1.43.67-py3-none-any.whl", hash = "sha256:082cf9df068168cb44028a1703822374c1eb7e48fa49470d7e8a76f0c977d0bc", size = 140025, upload-time = "2026-08-07T19:30:18.49Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.67" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/1c/3a75deae60e36bd0ee5c27d040384756b2ea1c90bd7c8c9658335a18b4f5/botocore-1.43.67.tar.gz", hash = "sha256:6fe5cfa0c8676ba809efe505b618ec00f30d1af2d014bf316a7aa4ee86accb20", size = 15889514, upload-time = "2026-08-07T19:30:15.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/be/38af8e96f3d200c9d34eab8e9f69a4468388ed6030ea04ff012974e5af5a/botocore-1.43.67-py3-none-any.whl", hash = "sha256:48ab8e9fac26fbc2a700d57010251003e6a5f731cf74d8540fb796bc8f3fc0ef", size = 15575924, upload-time = "2026-08-07T19:30:12.116Z" }, +] + [[package]] name = "certifi" version = "2026.6.17" @@ -542,6 +635,18 @@ dependencies = [ ] [package.optional-dependencies] +aws = [ + { name = "boto3" }, +] +azure = [ + { name = "azure-identity" }, + { name = "azure-keyvault-secrets" }, +] +bigquery = [ + { name = "dlt", extra = ["bigquery"] }, + { name = "google-cloud-bigquery" }, + { name = "google-cloud-bigquery-storage" }, +] dev = [ { name = "fastapi" }, { name = "mypy" }, @@ -552,27 +657,81 @@ dev = [ { name = "types-pyyaml" }, { name = "uvicorn" }, ] +gcp = [ + { name = "google-auth" }, + { name = "google-cloud-dataplex" }, + { name = "google-cloud-secret-manager" }, +] +postgres = [ + { name = "psycopg", extra = ["pool"] }, +] +redshift = [ + { name = "boto3" }, + { name = "redshift-connector" }, +] +runtime-all = [ + { name = "azure-identity" }, + { name = "azure-keyvault-secrets" }, + { name = "boto3" }, + { name = "dlt", extra = ["bigquery"] }, + { name = "google-auth" }, + { name = "google-cloud-bigquery" }, + { name = "google-cloud-bigquery-storage" }, + { name = "google-cloud-dataplex" }, + { name = "google-cloud-secret-manager" }, + { name = "psycopg", extra = ["pool"] }, + { name = "redshift-connector" }, + { name = "snowflake-connector-python" }, +] +snowflake = [ + { name = "snowflake-connector-python" }, +] [package.metadata] requires-dist = [ + { name = "azure-identity", marker = "extra == 'azure'", specifier = ">=1.19,<2" }, + { name = "azure-identity", marker = "extra == 'runtime-all'", specifier = ">=1.19,<2" }, + { name = "azure-keyvault-secrets", marker = "extra == 'azure'", specifier = ">=4.9,<5" }, + { name = "azure-keyvault-secrets", marker = "extra == 'runtime-all'", specifier = ">=4.9,<5" }, + { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.35,<2" }, + { name = "boto3", marker = "extra == 'redshift'", specifier = ">=1.35,<2" }, + { name = "boto3", marker = "extra == 'runtime-all'", specifier = ">=1.35,<2" }, { name = "dlt", extras = ["bigquery"], specifier = ">=1.0" }, + { name = "dlt", extras = ["bigquery"], marker = "extra == 'bigquery'", specifier = ">=1.0" }, + { name = "dlt", extras = ["bigquery"], marker = "extra == 'runtime-all'", specifier = ">=1.0" }, { name = "fastapi", marker = "extra == 'dev'", specifier = ">=0.115" }, { name = "google-auth", specifier = ">=2.35" }, + { name = "google-auth", marker = "extra == 'gcp'", specifier = ">=2.35" }, + { name = "google-auth", marker = "extra == 'runtime-all'", specifier = ">=2.35" }, { name = "google-cloud-bigquery", specifier = ">=3.25" }, + { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.25" }, + { name = "google-cloud-bigquery", marker = "extra == 'runtime-all'", specifier = ">=3.25" }, { name = "google-cloud-bigquery-storage", specifier = ">=2.30" }, + { name = "google-cloud-bigquery-storage", marker = "extra == 'bigquery'", specifier = ">=2.30" }, + { name = "google-cloud-bigquery-storage", marker = "extra == 'runtime-all'", specifier = ">=2.30" }, { name = "google-cloud-dataplex", specifier = ">=2.20,<3" }, + { name = "google-cloud-dataplex", marker = "extra == 'gcp'", specifier = ">=2.20,<3" }, + { name = "google-cloud-dataplex", marker = "extra == 'runtime-all'", specifier = ">=2.20,<3" }, { name = "google-cloud-secret-manager", specifier = ">=2.20" }, + { name = "google-cloud-secret-manager", marker = "extra == 'gcp'", specifier = ">=2.20" }, + { name = "google-cloud-secret-manager", marker = "extra == 'runtime-all'", specifier = ">=2.20" }, { name = "httpx", specifier = ">=0.27" }, { name = "jinja2", specifier = ">=3.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11" }, { name = "packaging", specifier = ">=24.0" }, + { name = "psycopg", extras = ["pool"], marker = "extra == 'postgres'", specifier = ">=3.2,<4" }, + { name = "psycopg", extras = ["pool"], marker = "extra == 'runtime-all'", specifier = ">=3.2,<4" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pydantic-settings", specifier = ">=2.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "redshift-connector", marker = "extra == 'redshift'", specifier = ">=2.1,<3" }, + { name = "redshift-connector", marker = "extra == 'runtime-all'", specifier = ">=2.1,<3" }, { name = "requests", specifier = ">=2.32" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6" }, + { name = "snowflake-connector-python", marker = "extra == 'runtime-all'", specifier = ">=3.12,<5" }, + { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=3.12,<5" }, { name = "sqlglot", specifier = ">=25.0" }, { name = "tenacity", specifier = ">=8.5" }, { name = "typer", specifier = ">=0.12" }, @@ -580,7 +739,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'" }, { name = "uvicorn", marker = "extra == 'dev'", specifier = ">=0.30" }, ] -provides-extras = ["dev"] +provides-extras = ["bigquery", "snowflake", "redshift", "postgres", "gcp", "aws", "azure", "oci", "runtime-all", "dev"] [[package]] name = "db-dtypes" @@ -668,6 +827,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] +[[package]] +name = "filelock" +version = "3.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -1171,6 +1339,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1183,6 +1360,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "jsonpath-ng" version = "1.7.0" @@ -1257,6 +1443,86 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, ] +[[package]] +name = "lxml" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/6e/c4add832b6fc1e887125b96f880d7b9b70aae5248718e046b1704bcac4b9/lxml-6.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7", size = 8570821, upload-time = "2026-05-18T19:17:42.068Z" }, + { url = "https://files.pythonhosted.org/packages/22/00/ff3009c88e65de8011630acf8ab5a09cb2becd2aaf47fba2f3449f6224e9/lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1", size = 4624252, upload-time = "2026-05-18T19:17:47.897Z" }, + { url = "https://files.pythonhosted.org/packages/42/95/bb63f0fd62e554fe078e1fb3c8fe9083c14ddc7ad7fa178d10e57e071ac7/lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc", size = 4930746, upload-time = "2026-05-18T19:18:29.637Z" }, + { url = "https://files.pythonhosted.org/packages/eb/99/0013e8d9b5960f4f041cf0b73e2f80c23eb5205b1f7bfb20203243651359/lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc", size = 5093723, upload-time = "2026-05-18T19:18:34.168Z" }, + { url = "https://files.pythonhosted.org/packages/29/91/317b332636bfc7bddcff828d41b3307f50043f4b237e40849c333d80fa1a/lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383", size = 5005557, upload-time = "2026-05-18T19:18:39.798Z" }, + { url = "https://files.pythonhosted.org/packages/42/2f/cc9bf06afe70f9c9093ae60855d9759da9db601ec4080f7473319666ffd7/lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b", size = 5631036, upload-time = "2026-05-18T19:18:44.858Z" }, + { url = "https://files.pythonhosted.org/packages/08/f6/af32e23e563971ffb0fb86be52bc5be5c2c118858ffc119bf6a9039b173d/lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818", size = 5240367, upload-time = "2026-05-18T19:18:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/78/83/8555d40948b09ce86f1bd0c68a7ac31d07b1929f92cc1b074006c97ef2d2/lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740", size = 5350171, upload-time = "2026-05-18T19:18:52.779Z" }, + { url = "https://files.pythonhosted.org/packages/63/75/5d92da93729b7bad783689e6496049fa40927b45bec7bf183c981de3ca70/lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f", size = 4694874, upload-time = "2026-05-18T19:18:55.139Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/3aad415a9a25b822e783f15deeb4dffccf5113030f1afa2222dd929313d9/lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2", size = 5244492, upload-time = "2026-05-18T19:19:01.28Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a1/5fcf7eb9904b80086aa47dcf0027de07b1bb990afad2e6823144c368ae04/lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635", size = 5048232, upload-time = "2026-05-18T19:18:12.67Z" }, + { url = "https://files.pythonhosted.org/packages/77/74/1f601b63c7a69fcdf10fa9b148c81da8442204194f6c55509cc485c786b9/lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf", size = 4777023, upload-time = "2026-05-18T19:18:15.928Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b9/7a78f51aec95b1bf780d78e12705a9f6533284f8693dc5c0e6724fa53d3f/lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc", size = 5645773, upload-time = "2026-05-18T19:18:23.223Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6e/98a7b7ad54e4e74fa1f20fff776913980619d0ebe5558232d7da6580bdd8/lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955", size = 5233088, upload-time = "2026-05-18T19:18:31.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/d1/bc0ed2427bf609f2ee10da303a6a226f9c8bce94f945dc29a32ce55de6e4/lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a", size = 5260995, upload-time = "2026-05-18T19:18:37.091Z" }, + { url = "https://files.pythonhosted.org/packages/69/8b/6772e1a4b513fc50a8d931f19edde0e13ae6918510a1e13ff67864f3e5ed/lxml-6.1.1-cp312-cp312-win32.whl", hash = "sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a", size = 3596382, upload-time = "2026-05-18T19:17:18.37Z" }, + { url = "https://files.pythonhosted.org/packages/1b/89/45198e9624762af2dfd2cb8782598477ceb29f6e59caab560388ae1f4ec1/lxml-6.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77", size = 3997255, upload-time = "2026-05-18T19:17:56.781Z" }, + { url = "https://files.pythonhosted.org/packages/90/a9/7a54b6834088d9ae528a7b780584ba6a39a9457b0ac330479f20ffbc9449/lxml-6.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f", size = 3659610, upload-time = "2026-05-19T19:22:50.843Z" }, + { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" }, + { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" }, + { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" }, + { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" }, + { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" }, + { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" }, + { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" }, + { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -1341,6 +1607,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "msal" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/99/d840198ecf6e8057bbc937f129ae940404485d736cda73253bbff9537f01/msal-1.37.0.tar.gz", hash = "sha256:1b1672a33ee467c1d70b341bb16cafd51bb3c817147a95b93263794b03971bec", size = 182444, upload-time = "2026-05-29T19:49:05.561Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/b0/d807279f4b55d16d1f120d5ac4344c6e39b56732e2a224d40bded7fd67ad/msal-1.37.0-py3-none-any.whl", hash = "sha256:dd17e95a7c71bce75e8108113438ba7c4a086b3bcad4f57a8c09b7af3d753c2d", size = 123725, upload-time = "2026-05-29T19:49:04.335Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -1729,6 +2021,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/fb/d65db067a67df7252f18b0cb7420dda84078b9e8bfb375215469c14a50be/pendulum-3.2.0-py3-none-any.whl", hash = "sha256:f3a9c18a89b4d9ef39c5fa6a78722aaff8d5be2597c129a3b16b9f40a561acf3", size = 114111, upload-time = "2026-01-30T11:22:22.361Z" }, ] +[[package]] +name = "platformdirs" +version = "4.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/0a/062135c9a98dac804265073cc3afdbec5ae1aa37980bb354f461bafe81b4/platformdirs-4.11.1.tar.gz", hash = "sha256:bb1af68078f25e2f3e111e2d43b8d536df41b73c8a684b40bb018223b66fae27", size = 32396, upload-time = "2026-08-07T23:06:48.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/85/9b31b44296cfa3bb56cddb35e6a0f6578bab0b490c0806c0245e32c6110c/platformdirs-4.11.1-py3-none-any.whl", hash = "sha256:2efd27d363e8dd2e661639ffb398865a5e0a46442a11d266bf375a0e0c10e386", size = 23261, upload-time = "2026-08-07T23:06:47.219Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1868,6 +2169,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/82/7a23d26039827ecd4ebe93905651029ddd307c5182ad59296dfb6f67b528/psycopg_pool-3.3.1.tar.gz", hash = "sha256:b10b10b7a175d5cc1592147dc5b7eec8a9e0834eb3ed2c4a92c858e2f51eb63c", size = 31661, upload-time = "2026-05-01T23:31:59.809Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/ed/89c2c620af0e1660354cd8aabf9f5b21f911597ce22acb37c805d6c86bc8/psycopg_pool-3.3.1-py3-none-any.whl", hash = "sha256:2af5b432941c4c9ad5c87b3fa410aec910ec8f7c122855897983a06c45f2e4b5", size = 40023, upload-time = "2026-05-01T23:31:53.136Z" }, +] + [[package]] name = "pyarrow" version = "24.0.0" @@ -2054,6 +2385,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pyopenssl" +version = "26.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -2179,6 +2537,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "redshift-connector" +version = "2.1.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "lxml" }, + { name = "packaging" }, + { name = "pytz" }, + { name = "requests" }, + { name = "scramp" }, + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/5a/cb88fb9fd5c5aedbe47e4c54bd6c73360ec2362865eb56f21d8404dcfa07/redshift_connector-2.1.16-py3-none-any.whl", hash = "sha256:6d50ef7c19bf48e0895f8e81636f182bd8d0f3b447f02e9c924e9c52ca8e2cb6", size = 160339, upload-time = "2026-08-03T23:56:49.178Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -2269,6 +2646,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, +] + +[[package]] +name = "scramp" +version = "1.4.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/76/6db02f36db58a7d009e90f51e961bcc3c44a1c930a744f026e30e791989b/scramp-1.4.17.tar.gz", hash = "sha256:28970f29ebc33df47f9975c805e5e5a360effe5b31045e607d64b3b60370dba1", size = 21291, upload-time = "2026-08-07T17:19:40.827Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/99/0e372781210cd36b2f2727e5be3ea93066edad7edd6fa2dfdec3b3e28845/scramp-1.4.17-py3-none-any.whl", hash = "sha256:a4e3fd2e8169461a28a13777a166d3da94274454f0714a7d3023fee124474ac8", size = 16131, upload-time = "2026-08-07T17:19:39.591Z" }, +] + [[package]] name = "semver" version = "3.0.4" @@ -2367,6 +2768,66 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] +[[package]] +name = "snowflake-connector-python" +version = "4.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "cryptography" }, + { name = "filelock" }, + { name = "idna" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "pyjwt" }, + { name = "pyopenssl" }, + { name = "pytz" }, + { name = "requests" }, + { name = "sortedcontainers" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/73/e6e45343dd133207a6f710a88e0675ae69f8c12c23efe7f192448bbdace9/snowflake_connector_python-4.7.2.tar.gz", hash = "sha256:c261b73b8dbe3f3d5b9bfdc19196f38e3eade42cb2f24322b7a2bb60fef76534", size = 954059, upload-time = "2026-08-07T13:59:54.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/17/a3f4c38a4535b137139a353e60874c83e22eea535b37ce43651191ac5d1f/snowflake_connector_python-4.7.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4d19ab07f57ec14d074086d7a4eadb51b32e75b37c91e9893ad3702ecd2e3505", size = 1181036, upload-time = "2026-08-07T14:00:02.642Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2b/c9446d830f33961ea7c86b4b5343a0fefda3882a31f577d74025ee8d26f5/snowflake_connector_python-4.7.2-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:1d94f27d67520e6729709114ee00916d89defa909d4cd456d53fe6106dc14df1", size = 1193269, upload-time = "2026-08-07T14:00:04.337Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/0cee4d1d6ddd268c2443b2c3e8e2a98139198746525bcf73a1da876d940e/snowflake_connector_python-4.7.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:243c4e2621a58a34909a7863bbffaa886870c9fe19e88804a608660fb775f025", size = 2915086, upload-time = "2026-08-07T13:59:45.446Z" }, + { url = "https://files.pythonhosted.org/packages/29/2b/a52387d5abd5988498deeee502c3889b576f21e486f128115ce548006517/snowflake_connector_python-4.7.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:24e7a787cae7745756cb25c829e4fef1b828d2872dd70e0e31ff37a9323a62c7", size = 2947242, upload-time = "2026-08-07T13:59:46.904Z" }, + { url = "https://files.pythonhosted.org/packages/e5/29/4cc9353e21ec19570ac2916ed15a237d9cd42675ad70d7b9aa3dd2be6ea7/snowflake_connector_python-4.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:d7b1eabecdeaff6b0df8f583dc1b7a8bbd92e996e29212c324013e4ad7662c75", size = 5419590, upload-time = "2026-08-07T14:00:16.379Z" }, + { url = "https://files.pythonhosted.org/packages/93/c5/12defd8c0022ded5a11d3e2204ee52c0484654a869db3d2bf9027c7d2384/snowflake_connector_python-4.7.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9cb47d1d4881fe7338af91a38029c68926fab016b58fb6963954492257e2a1d9", size = 1179960, upload-time = "2026-08-07T14:00:05.802Z" }, + { url = "https://files.pythonhosted.org/packages/9f/70/b5ccddf8b3689455a3652f71eba16b35753dd90dc6936a1f0905c41a4db9/snowflake_connector_python-4.7.2-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:06281d6f66e857b4d8362fc0738600dafd635da1c1cf91537f0cd57f658ec1ec", size = 1192786, upload-time = "2026-08-07T14:00:07.207Z" }, + { url = "https://files.pythonhosted.org/packages/3a/0f/839bcabc5231daf009db958b11838fd991f68f7de137bb3918f8e5368de4/snowflake_connector_python-4.7.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0771229b11862b0eac60d8a771c66af79b5c3248b5200ad2898217ab4c7d385b", size = 2886123, upload-time = "2026-08-07T13:59:48.314Z" }, + { url = "https://files.pythonhosted.org/packages/e6/77/85754d5a5f3ba7f11a3f0aa569b2edd2429aa96f2342486b1da4530329c6/snowflake_connector_python-4.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:eb1e56efaaf258bb8a28968bbebea82488a02a2ffda354e65ea7921935289a19", size = 2918812, upload-time = "2026-08-07T13:59:49.747Z" }, + { url = "https://files.pythonhosted.org/packages/e8/01/9015c0918f4146281c2d36f72069a9ec538b4165bddb2b5cd3bc9ff4f24c/snowflake_connector_python-4.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:17f9cee8985b726afe027920b2842c41b0c52280747d37efa1643c3f77fd5505", size = 5419347, upload-time = "2026-08-07T14:00:18.013Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9e/d768063750fd8e155311414580b79f35552e95b72f1bcbb702f9e577a5bc/snowflake_connector_python-4.7.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:9b5a3860bf4350786c35ffaac0ba60f7b13ef72d6296d23b28e356be1fe938c9", size = 1180582, upload-time = "2026-08-07T14:00:08.808Z" }, + { url = "https://files.pythonhosted.org/packages/01/40/fcff393b5b6de392931e43c5ca17298e59aaf8fd89b34189748af8166bc9/snowflake_connector_python-4.7.2-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:89e19d070ce884a370a2fa6273fcba8c8e1d7fe7c57861ba15ef92be603dfe19", size = 1192907, upload-time = "2026-08-07T14:00:10.44Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/b2e0676e2637d612e45e711eb3cbd17be3ffa60ea64ff5d7fea8f9ef6261/snowflake_connector_python-4.7.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ebae2ff7311e9f383ce805d790b6b85cb9a30fe81b1b5dc56d96fe92cd6b5df", size = 2885786, upload-time = "2026-08-07T13:59:51.11Z" }, + { url = "https://files.pythonhosted.org/packages/ec/cb/a8918acef070f833800c1d3c8221610efffe2c8f0d087e69c81d01c33d39/snowflake_connector_python-4.7.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3751daec1a0f05470d6419ddf068f11e3b4d464d683ee4d0b8749a9a0c0605b8", size = 2914542, upload-time = "2026-08-07T13:59:52.624Z" }, + { url = "https://files.pythonhosted.org/packages/2f/0f/e1556f2501b538875c7d3f656f91b5c1ea66d6b88c33ec611b835c4933c5/snowflake_connector_python-4.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:b06b910ff3965bad9c389e1b423837503c5e29f0bc51a9cec7ba8a3df92e6ca8", size = 5478102, upload-time = "2026-08-07T14:00:19.752Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/99/a6ca3beb3ccacb41fb3321d8a60e5566f9e6467601ef8eba6a17e1b89778/soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74", size = 122445, upload-time = "2026-08-07T00:57:24.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823", size = 37370, upload-time = "2026-08-07T00:57:23.524Z" }, +] + [[package]] name = "sqlglot" version = "30.12.0" From a5fb2ece5b1d41c59fb7c87aac8b66fe15b83006 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 05:32:08 -0400 Subject: [PATCH 034/163] Route BigQuery through warehouse capabilities --- CHANGELOG.md | 3 + HANDOFF.md | 38 ++-- docs/decisions.md | 9 + docs/platform-profiles.md | 7 +- src/dander/cli/run_command.py | 79 ++++--- src/dander/project/config.py | 1 + src/dander/project/portable_config.py | 14 +- src/dander/providers/__init__.py | 2 + src/dander/providers/bigquery/__init__.py | 5 + src/dander/providers/bigquery/config.py | 14 ++ src/dander/providers/bigquery/runtime.py | 192 ++++++++++++++++++ src/dander/providers/defaults.py | 18 ++ src/dander/warehouse/__init__.py | 20 ++ src/dander/warehouse/runtime.py | 147 ++++++++++++++ tests/cli/test_run_command.py | 35 +++- tests/project/test_portable_config.py | 2 + tests/providers/test_bigquery_runtime.py | 148 ++++++++++++++ .../DANDER-85-bigquery-warehouse-runtime.md | 40 ++++ 18 files changed, 712 insertions(+), 62 deletions(-) create mode 100644 src/dander/providers/bigquery/__init__.py create mode 100644 src/dander/providers/bigquery/config.py create mode 100644 src/dander/providers/bigquery/runtime.py create mode 100644 src/dander/providers/defaults.py create mode 100644 src/dander/warehouse/runtime.py create mode 100644 tests/providers/test_bigquery_runtime.py create mode 100644 tickets/DANDER-85-bigquery-warehouse-runtime.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 274618f9..6c711af4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Route BigQuery writer and transform construction through one lazily selected, typed warehouse + runtime that also exposes canonical schema, relation rendering, target fencing, normalized job + telemetry, and explicit capabilities without changing current GCP behavior. - Expose provider dependency extras for BigQuery, Snowflake, Redshift, PostgreSQL, GCP, AWS, Azure, and OCI, plus a validated `runtime-all` union used by repository and generated source-free OCI images without expanding the supported-adapter capability manifest. diff --git a/HANDOFF.md b/HANDOFF.md index 1eecb6bb..60dda247 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Added eight public provider extras and a deterministic 12-distribution `runtime-all` union. -- Made repository and generated source-free images install and verify the full dependency set. -- Added metadata-only dependency inspection that never imports provider SDKs. -- Preserved the capability manifest as the support boundary; only GCP/BigQuery remains advertised. -- Added package, scaffold, missing-dependency, and lazy-inspection regression coverage. +- Added a typed `WarehouseRuntime` composed from small provider capabilities. +- Registered BigQuery through a dependency-light config and lazy runtime factory. +- Routed hosted/sandbox writers and model/graph runners through the selected warehouse. +- Exposed BigQuery relation, canonical schema, fencing, telemetry, and capability adapters. +- Preserved implicit v1 and explicit v2 BigQuery selection without moving other providers. ## Try It -Install one SDK set with `pip install 'dander-platform[postgres]'`. Official images use -`dander-platform[runtime-all]`; package presence does not make an adapter supported. +Run an existing v1 or v2 BigQuery project normally. `dander run` now selects the BigQuery +warehouse runtime internally; public commands and configuration are unchanged. ## Checks -- All 900 tests, Ruff, formatting, strict mypy, and the full dependency audit passed. -- Wheel/sdist source-free installs, generation, validation, and full-runtime installation passed. -- Linux amd64/arm64 builds and local non-root/read-only container conformance passed. -- Trivy reported zero high/critical findings; the branch secret scan passed. +- All 903 tests, Ruff, formatting, and strict mypy across 209 files passed. +- Wheel/sdist build, metadata/distribution validation, and dependency audit passed. +- The non-root/read-only full runtime image passed conformance and its BigQuery factory probe. +- Trivy found no high/critical issues; the focused secret scan passed. - All Terraform roots passed; GCP reported `No changes` and both schedules stayed paused. ## Decisions -- Keep base installation backward compatible while making provider SDK groups explicit. -- Build one checked full image; keep implementation loading lazy. -- Reserve OCI without its currently incompatible SDK; never weaken the dependency audit. +- Compose small capabilities instead of adding a large warehouse-provider interface. +- Keep the BigQuery implementation lazy and behaviorally unchanged behind the new factory. +- Move state, catalog, secrets, and launchers only in their own focused tickets. ## Remaining -- Open and merge the focused dependency-assembly PR after protected CI passes. -- Implement the portable BigQuery vertical slice next. +- Open and merge the focused BigQuery runtime PR after protected CI passes. +- Route BigQuery durable state through the shared state contract next. ## Review First -- `pyproject.toml` -- `src/dander/providers/dependencies.py` -- `src/dander/templates/project/Dockerfile` +- `src/dander/warehouse/runtime.py` +- `src/dander/providers/bigquery/runtime.py` +- `src/dander/cli/run_command.py` diff --git a/docs/decisions.md b/docs/decisions.md index c63ed5e5..20cd1a90 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-08 — BigQuery enters portability through one composed runtime + +- **Selection:** Version 1 projects retain an implicit BigQuery warehouse; version 2 resolution + carries the selected provider and location into execution. +- **Composition:** One lazily built `WarehouseRuntime` owns small relation, schema, writer, + transform, fence, telemetry, and capability surfaces; the CLI does not branch on BigQuery. +- **Scope:** Existing BigQuery classes remain behaviorally intact. State, catalog, secrets, and + launchers move through their own focused Phase 3 changes. + ## 2026-08-07 — Provider dependencies are assembled separately from support claims - **Local installs:** Public extras group BigQuery, Snowflake, Redshift, PostgreSQL, GCP, AWS, and diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 33ac8dd2..3021a122 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -43,9 +43,10 @@ is represented in two files. The current runtime continues to support only the GCP compatibility composition. The internal factory contract provides one API-v1 registry across warehouse, state, catalog, secret, and launcher categories. Registration loads only a small configuration model; selecting and building -a provider loads its implementation and SDK dependencies. The current GCP resolver is not routed -through that boundary yet. The registry is a construction contract, not a support claim; concrete -adapter routing arrives in later, separate portability changes. +a provider loads its implementation and SDK dependencies. BigQuery writer and transform +construction now use that boundary. BigQuery state, Dataplex, GCP secrets, and Cloud Run remain on +their proven direct paths until their separate Phase 3 changes. The registry is a construction +contract, not a support claim. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 7a5565cd..472d18f0 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -22,7 +22,6 @@ from dander.executor import PipelineExecutionResult, PipelineExecutor from dander.ingestion import Endpoint, Source, SourceConfig, load_source_config from dander.pipeline.runtime import ( - BigQueryGraphRunner, GraphExecutionPlan, GraphRuntimeError, load_graph_for_execution, @@ -34,6 +33,11 @@ load_connector_plugins, ) from dander.project import ProjectConfigError, load_project_config +from dander.providers import ( + ProviderFactoryError, + ProviderKind, + default_provider_registry, +) from dander.runtime import PipelineRunner from dander.sandbox import GuardedFreeTierVerifier, SandboxDataset, SandboxSafetyError from dander.security import ( @@ -58,12 +62,9 @@ SqliteRunHistoryStore, SqliteWatermarkStore, ) -from dander.transform import ( - BigQueryTransformRunner, - TransformProjectError, - TransformRunError, -) -from dander.writer import BigQueryReplaceWriter, BigQueryScd1Writer, SchemaEvolution +from dander.transform import TransformProjectError, TransformRunError +from dander.warehouse import WarehouseRuntime, WarehouseTransformRunner +from dander.writer import SchemaEvolution if TYPE_CHECKING: from collections.abc import Sequence @@ -111,6 +112,8 @@ class _ResolvedRun: selected_models: tuple[str, ...] | None build_models: bool publish_dataplex: bool + warehouse_provider: str + warehouse_location: str @dataclass(frozen=True, slots=True) @@ -161,6 +164,8 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: ) build_models = options.build_models publish_dataplex = options.publish_dataplex + warehouse_provider = "bigquery" + warehouse_location = "US" plugin_registry: ConnectorPluginRegistry | None = None if options.project_config.is_file(): @@ -183,6 +188,8 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: selected_models = tuple(pipeline.models) build_models = build_models or pipeline.build_models publish_dataplex = publish_dataplex or pipeline.publish_dataplex + warehouse_provider = manifest.warehouse_provider + warehouse_location = manifest.platform.bigquery_location except (ConnectorPluginError, ProjectConfigError) as error: raise ClickException(str(error)) from error @@ -225,6 +232,8 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: selected_models=selected_models, build_models=build_models, publish_dataplex=publish_dataplex, + warehouse_provider=warehouse_provider, + warehouse_location=warehouse_location, ) @@ -294,8 +303,9 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec if resolved.publish_dataplex else None ) - ingestion = _build_ingestion_runner(options, resolved, source_adapter) - transform_runner = _build_transform_runner(resolved) + warehouse = _build_warehouse_runtime(resolved) + ingestion = _build_ingestion_runner(options, resolved, source_adapter, warehouse) + transform_runner = _build_transform_runner(resolved, warehouse) return PipelineExecutor( pipeline_id=resolved.pipeline_id, source_config=resolved.source_config, @@ -344,17 +354,14 @@ def _build_ingestion_runner( options: RunOptions, resolved: _ResolvedRun, source: Source, + warehouse: WarehouseRuntime, ) -> PipelineRunner: - writer = ( - BigQueryReplaceWriter(project=resolved.project, max_batch_rows=options.batch_rows) - if options.sandbox - else BigQueryScd1Writer( - project=resolved.project, - max_batch_rows=options.batch_rows, - schema_evolution=( - SchemaEvolution.ADDITIVE if resolved.project_pipeline else SchemaEvolution.STRICT - ), - ) + writer = warehouse.writers.build_ingestion_writer( + sandbox=options.sandbox, + batch_rows=options.batch_rows, + schema_evolution=( + SchemaEvolution.ADDITIVE if resolved.project_pipeline else SchemaEvolution.STRICT + ), ) watermarks = ( SqliteWatermarkStore(options.state_path) @@ -377,12 +384,34 @@ def _build_ingestion_runner( def _build_transform_runner( resolved: _ResolvedRun, -) -> BigQueryGraphRunner | BigQueryTransformRunner | None: - if resolved.graph_plan is not None: - return BigQueryGraphRunner(plan=resolved.graph_plan, project=resolved.project) - if resolved.build_models: - return BigQueryTransformRunner(project=resolved.project) - return None + warehouse: WarehouseRuntime, +) -> WarehouseTransformRunner | None: + return warehouse.transforms.build_transform_runner( + graph_plan=resolved.graph_plan, + build_models=resolved.build_models, + ) + + +def _build_warehouse_runtime(resolved: _ResolvedRun) -> WarehouseRuntime: + registry = default_provider_registry() + try: + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": resolved.warehouse_provider, + "location": resolved.warehouse_location, + }, + ) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={"project": resolved.project}, + ) + except ProviderFactoryError as error: + raise ClickException(str(error)) from error + if not isinstance(runtime, WarehouseRuntime): + raise ClickException("Selected warehouse provider returned an invalid runtime") + return runtime def build_source_adapter( diff --git a/src/dander/project/config.py b/src/dander/project/config.py index ae5e0163..08a26814 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -203,6 +203,7 @@ class DanderProject(BaseModel): pipelines: dict[str, PipelineSpec] = Field(min_length=1) platform_name: str = Field(default="gcp", exclude=True) deployment_name: str = Field(default="gcp_cloud_run", exclude=True) + warehouse_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 334b3a94..a2060be6 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -23,6 +23,7 @@ ProjectConfigError, _load_yaml_mapping, ) +from dander.providers.bigquery import BigQueryWarehouseConfig # noqa: TC001 _NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") _PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") @@ -109,15 +110,6 @@ def validate_pipeline_ids( return values -class BigQueryWarehouseSpec(BaseModel): - """BigQuery connection identity owned by one named platform profile.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["bigquery"] - location: str = Field(default="US", min_length=1) - - class BigQueryStateSpec(BaseModel): """BigQuery durable-state selection for the compatibility profile.""" @@ -172,7 +164,7 @@ class PlatformProfileSpec(BaseModel): model_config = ConfigDict(extra="forbid") - warehouse: BigQueryWarehouseSpec + warehouse: BigQueryWarehouseConfig state: BigQueryStateSpec catalog: CatalogSpec secrets: SecretProviderSpec @@ -390,6 +382,7 @@ def _resolve( pipelines=resolved_pipelines, platform_name=selected.platform, deployment_name=selected_name, + warehouse_provider=profile.warehouse.provider, deployed_pipeline_ids=tuple(sorted(selected.pipelines)), ) @@ -476,6 +469,7 @@ def _platforms_document(project: DanderProject) -> dict[str, object]: def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: return ( legacy.platform == migrated.platform + and legacy.warehouse_provider == migrated.warehouse_provider and legacy.plugins == migrated.plugins and legacy.pipelines == migrated.pipelines and legacy.terraform_pipelines() == migrated.terraform_pipelines() diff --git a/src/dander/providers/__init__.py b/src/dander/providers/__init__.py index 86e5b962..714c8a8d 100644 --- a/src/dander/providers/__init__.py +++ b/src/dander/providers/__init__.py @@ -1,5 +1,6 @@ """Cloud-provider registration and construction contracts.""" +from dander.providers.defaults import default_provider_registry from dander.providers.dependencies import ( FULL_RUNTIME_DISTRIBUTIONS, PROVIDER_DEPENDENCY_SETS, @@ -28,6 +29,7 @@ "ProviderRegistry", "RuntimeDependencyError", "installed_runtime_dependencies", + "default_provider_registry", "lazy_provider_factory", "require_full_runtime", ] diff --git a/src/dander/providers/bigquery/__init__.py b/src/dander/providers/bigquery/__init__.py new file mode 100644 index 00000000..8340cb7d --- /dev/null +++ b/src/dander/providers/bigquery/__init__.py @@ -0,0 +1,5 @@ +"""BigQuery provider configuration; runtime implementation remains lazily loaded.""" + +from dander.providers.bigquery.config import BigQueryWarehouseConfig + +__all__ = ["BigQueryWarehouseConfig"] diff --git a/src/dander/providers/bigquery/config.py b/src/dander/providers/bigquery/config.py new file mode 100644 index 00000000..d2fd32d4 --- /dev/null +++ b/src/dander/providers/bigquery/config.py @@ -0,0 +1,14 @@ +"""Dependency-light BigQuery warehouse configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class BigQueryWarehouseConfig(BaseModel): + """BigQuery location selected by one named platform profile.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["bigquery"] + location: str = Field(default="US", min_length=1) diff --git a/src/dander/providers/bigquery/runtime.py b/src/dander/providers/bigquery/runtime.py new file mode 100644 index 00000000..f3ef005c --- /dev/null +++ b/src/dander/providers/bigquery/runtime.py @@ -0,0 +1,192 @@ +"""BigQuery implementation of the provider-neutral warehouse runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, cast + +from dander.concurrency import FencingToken, fenced_dml, fencing_job_config +from dander.pipeline.runtime import BigQueryGraphRunner, GraphExecutionPlan +from dander.providers.bigquery.config import BigQueryWarehouseConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.telemetry import OperationTelemetry, TelemetryOperation +from dander.transform import BigQueryTransformRunner +from dander.warehouse.bigquery_compat import canonical_schema_from_bigquery +from dander.warehouse.runtime import ( + PreparedWarehouseStatement, + WarehouseCapabilities, + WarehouseRuntime, + WarehouseTransformRunner, +) +from dander.writer import ( + BigQueryReplaceWriter, + BigQueryScd1Writer, + SchemaEvolution, + WriteMode, + WritePattern, + WriteTransport, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from pydantic import BaseModel + + from dander.warehouse.bigquery_compat import BigQueryFieldLike + from dander.warehouse.contracts import RelationRef, RelationSchema + + +@dataclass(frozen=True, slots=True) +class BigQueryRelationCodec: + """Render validated canonical coordinates as quoted BigQuery identifiers.""" + + provider_id: str = "bigquery" + + def render(self, relation: RelationRef) -> str: + """Render ``project.dataset.table`` without accepting raw SQL fragments.""" + return ".".join(f"`{coordinate}`" for coordinate in relation.coordinates) + + +@dataclass(frozen=True, slots=True) +class BigQuerySchemaMapper: + """Expose the existing strict BigQuery compatibility mapper as a capability.""" + + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + """Map a complete legacy BigQuery declaration to canonical schema v1.""" + return canonical_schema_from_bigquery(cast("Sequence[BigQueryFieldLike]", fields)) + + +@dataclass(frozen=True, slots=True) +class BigQueryWriterFactory: + """Construct current BigQuery ingestion writers without changing behavior.""" + + project: str + + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + ) -> WritePattern: + """Use replace for local sandbox execution and SCD1 for hosted execution.""" + if sandbox: + return BigQueryReplaceWriter(project=self.project, max_batch_rows=batch_rows) + return BigQueryScd1Writer( + project=self.project, + max_batch_rows=batch_rows, + schema_evolution=schema_evolution, + ) + + +@dataclass(frozen=True, slots=True) +class BigQueryTransformFactory: + """Construct the existing BigQuery model or graph runner.""" + + project: str + + def build_transform_runner( + self, + *, + graph_plan: object | None, + build_models: bool, + ) -> WarehouseTransformRunner | None: + """Preserve graph precedence and model-build behavior.""" + if graph_plan is not None: + if not isinstance(graph_plan, GraphExecutionPlan): + raise TypeError("BigQuery graph plan has the wrong type") + return BigQueryGraphRunner(plan=graph_plan, project=self.project) + if build_models: + return BigQueryTransformRunner(project=self.project) + return None + + +@dataclass(frozen=True, slots=True) +class BigQueryTargetFence: + """Bind BigQuery lease ownership inside the destination transaction.""" + + def prepare_dml(self, statement: str, fence: object) -> PreparedWarehouseStatement: + """Return the existing conditional DML touch and parameter bindings.""" + if not isinstance(fence, FencingToken): + raise TypeError("BigQuery target fencing requires a FencingToken") + return PreparedWarehouseStatement( + sql=fenced_dml(statement, fence), + options=fencing_job_config(fence), + ) + + +@dataclass(frozen=True, slots=True) +class BigQueryTelemetry: + """Normalize stable BigQuery job counters and identifiers.""" + + def operation( + self, + job: object, + *, + operation: TelemetryOperation, + duration_ms: int = 0, + retry_count: int = 0, + ) -> OperationTelemetry: + """Read only approved scalar attributes from a completed BigQuery job.""" + return OperationTelemetry( + provider="bigquery", + operation=operation, + duration_ms=duration_ms, + retry_count=retry_count, + rows_written=_nonnegative_attribute(job, "output_rows"), + rows_affected=_nonnegative_attribute(job, "num_dml_affected_rows"), + bytes_processed=_nonnegative_attribute(job, "total_bytes_processed"), + bytes_billed=_nonnegative_attribute(job, "total_bytes_billed"), + job_id=_optional_identifier(job, "job_id"), + ) + + +BIGQUERY_CAPABILITIES = WarehouseCapabilities( + provider_id="bigquery", + schema_contract_version=1, + write_modes=frozenset(WriteMode), + transports=frozenset(WriteTransport), + supports_transforms=True, + supports_graphs=True, + supports_target_fencing=True, +) + + +def build_bigquery_warehouse( + config: BaseModel, + context: Mapping[str, object], +) -> WarehouseRuntime: + """Build the selected BigQuery runtime after provider validation.""" + if not isinstance(config, BigQueryWarehouseConfig): + raise TypeError("BigQuery warehouse factory received the wrong configuration") + project = context.get("project") + if not isinstance(project, str) or not project: + raise ValueError("BigQuery warehouse factory requires a project context") + return WarehouseRuntime( + provider_id="bigquery", + relation_codec=BigQueryRelationCodec(), + schema_mapper=BigQuerySchemaMapper(), + writers=BigQueryWriterFactory(project), + transforms=BigQueryTransformFactory(project), + target_fence=BigQueryTargetFence(), + telemetry=BigQueryTelemetry(), + capabilities=BIGQUERY_CAPABILITIES, + ) + + +BIGQUERY_WAREHOUSE_FACTORY: ProviderFactory[WarehouseRuntime] = ProviderFactory( + kind=ProviderKind.WAREHOUSE, + provider_id="bigquery", + api_version=PROVIDER_API_VERSION, + build=build_bigquery_warehouse, +) + + +def _nonnegative_attribute(job: object, name: str) -> int: + value = getattr(job, name, None) + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0 + + +def _optional_identifier(job: object, name: str) -> str | None: + value = getattr(job, name, None) + return value if isinstance(value, str) and value else None diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py new file mode 100644 index 00000000..b888d567 --- /dev/null +++ b/src/dander/providers/defaults.py @@ -0,0 +1,18 @@ +"""Built-in provider registrations with lazy implementation loading.""" + +from dander.providers.bigquery import BigQueryWarehouseConfig +from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory + + +def default_provider_registry() -> ProviderRegistry: + """Return fresh built-in registrations without importing provider SDK modules.""" + registry = ProviderRegistry() + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="bigquery", + config_model=BigQueryWarehouseConfig, + load_factory=lazy_provider_factory( + "dander.providers.bigquery.runtime:BIGQUERY_WAREHOUSE_FACTORY" + ), + ) + return registry diff --git a/src/dander/warehouse/__init__.py b/src/dander/warehouse/__init__.py index 7716060c..3fdd279e 100644 --- a/src/dander/warehouse/__init__.py +++ b/src/dander/warehouse/__init__.py @@ -15,6 +15,17 @@ RelationRef, RelationSchema, ) +from dander.warehouse.runtime import ( + PreparedWarehouseStatement, + WarehouseCapabilities, + WarehouseRuntime, + WarehouseSchemaMapper, + WarehouseTargetFence, + WarehouseTelemetry, + WarehouseTransformFactory, + WarehouseTransformRunner, + WarehouseWriterFactory, +) __all__ = [ "BigQuerySchemaCompatibilityError", @@ -23,9 +34,18 @@ "FieldCardinality", "LogicalTypeKind", "ProviderExtension", + "PreparedWarehouseStatement", "RelationCodec", "RelationRef", "RelationSchema", + "WarehouseCapabilities", + "WarehouseRuntime", + "WarehouseSchemaMapper", + "WarehouseTargetFence", + "WarehouseTelemetry", + "WarehouseTransformFactory", + "WarehouseTransformRunner", + "WarehouseWriterFactory", "canonical_field_from_bigquery", "canonical_schema_from_bigquery", ] diff --git a/src/dander/warehouse/runtime.py b/src/dander/warehouse/runtime.py new file mode 100644 index 00000000..3416e6d3 --- /dev/null +++ b/src/dander/warehouse/runtime.py @@ -0,0 +1,147 @@ +"""Small composable capabilities used by one selected warehouse runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from pathlib import Path + + from dander.concurrency import OwnershipGuard + from dander.telemetry import OperationTelemetry, TelemetryOperation + from dander.transform import TransformRunResult + from dander.warehouse.contracts import RelationCodec, RelationSchema + from dander.writer.base import SchemaEvolution, WriteMode, WritePattern, WriteTransport + + +@dataclass(frozen=True, slots=True) +class PreparedWarehouseStatement: + """Provider statement plus opaque provider-native execution options.""" + + sql: str + options: object | None = None + + def __post_init__(self) -> None: + if not self.sql.strip(): + raise ValueError("prepared warehouse statement must not be blank") + + +@dataclass(frozen=True, slots=True) +class WarehouseCapabilities: + """Closed support declaration for one concrete warehouse implementation.""" + + provider_id: str + schema_contract_version: int + write_modes: frozenset[WriteMode] + transports: frozenset[WriteTransport] + supports_transforms: bool + supports_graphs: bool + supports_target_fencing: bool + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("warehouse capability provider_id must not be blank") + if self.schema_contract_version != 1: + raise ValueError("warehouse schema contract version must be 1") + if not self.write_modes: + raise ValueError("warehouse capabilities must declare at least one write mode") + + +@runtime_checkable +class WarehouseSchemaMapper(Protocol): + """Map a legacy provider declaration into canonical schema v1.""" + + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + """Return the lossless canonical representation of ``fields``.""" + ... + + +@runtime_checkable +class WarehouseWriterFactory(Protocol): + """Construct the ingestion writer selected by the execution context.""" + + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + ) -> WritePattern: + """Build one writer without branching in the cloud-neutral CLI.""" + ... + + +@runtime_checkable +class WarehouseTransformRunner(Protocol): + """Provider transform runner consumed by ``PipelineExecutor``.""" + + def build( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Build selected transformations and execute their assertions.""" + ... + + +@runtime_checkable +class WarehouseTransformFactory(Protocol): + """Construct a provider transform or graph runner.""" + + def build_transform_runner( + self, + *, + graph_plan: object | None, + build_models: bool, + ) -> WarehouseTransformRunner | None: + """Return the selected provider runner or ``None`` when transforms are disabled.""" + ... + + +@runtime_checkable +class WarehouseTargetFence(Protocol): + """Prepare one destination-side fenced DML finalizer.""" + + def prepare_dml(self, statement: str, fence: object) -> PreparedWarehouseStatement: + """Bind the provider fencing statement and its execution options.""" + ... + + +@runtime_checkable +class WarehouseTelemetry(Protocol): + """Normalize one provider job without leaking provider response payloads.""" + + def operation( + self, + job: object, + *, + operation: TelemetryOperation, + duration_ms: int = 0, + retry_count: int = 0, + ) -> OperationTelemetry: + """Return provider-neutral telemetry for one completed operation.""" + ... + + +@dataclass(frozen=True, slots=True) +class WarehouseRuntime: + """Selected warehouse composition consumed by Dander execution.""" + + provider_id: str + relation_codec: RelationCodec + schema_mapper: WarehouseSchemaMapper + writers: WarehouseWriterFactory + transforms: WarehouseTransformFactory + target_fence: WarehouseTargetFence + telemetry: WarehouseTelemetry + capabilities: WarehouseCapabilities + + def __post_init__(self) -> None: + if self.provider_id != self.relation_codec.provider_id: + raise ValueError("warehouse relation codec provider does not match runtime") + if self.provider_id != self.capabilities.provider_id: + raise ValueError("warehouse capabilities provider does not match runtime") diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index cde9fe9b..0876c010 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Protocol, cast from typer.testing import CliRunner @@ -32,6 +32,11 @@ class _Built: kwargs: dict[str, object] +class _ResolvedWarehouse(Protocol): + warehouse_provider: str + warehouse_location: str + + def test_hosted_project_run_wires_runtime_without_network( monkeypatch: MonkeyPatch, ) -> None: @@ -86,16 +91,36 @@ def execute(self) -> PipelineExecutionResult: assets=1, ) + class _WriterFactory: + def build_ingestion_writer(self, **kwargs: object) -> _Built: + component = _Built(name="writer", args=(), kwargs=kwargs) + captured["writer"] = component + return component + + class _TransformFactory: + def build_transform_runner(self, **kwargs: object) -> _Built: + component = _Built(name="transform", args=(), kwargs=kwargs) + captured["transform"] = component + return component + + class _Warehouse: + writers = _WriterFactory() + transforms = _TransformFactory() + + def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: + assert resolved.warehouse_provider == "bigquery" + assert resolved.warehouse_location == "US" + return _Warehouse() + monkeypatch.setattr(run_module, "DefaultSecretStore", recording_factory("secrets")) monkeypatch.setattr(run_module, "build_auth", build_auth) monkeypatch.setattr(run_module, "build_source_adapter", build_source) monkeypatch.setattr(run_module, "BigQueryRunHistoryStore", recording_factory("history")) monkeypatch.setattr(run_module, "BigQueryLeaseStore", recording_factory("leases")) monkeypatch.setattr(run_module, "BigQueryMetadataStore", recording_factory("metadata")) - monkeypatch.setattr(run_module, "BigQueryScd1Writer", recording_factory("writer")) monkeypatch.setattr(run_module, "BigQueryWatermarkStore", recording_factory("watermarks")) monkeypatch.setattr(run_module, "PipelineRunner", recording_factory("ingestion")) - monkeypatch.setattr(run_module, "BigQueryTransformRunner", recording_factory("transform")) + monkeypatch.setattr(run_module, "_build_warehouse_runtime", build_warehouse) monkeypatch.setattr(run_module, "PipelineExecutor", _Executor) result = CliRunner().invoke( @@ -136,8 +161,8 @@ def execute(self) -> PipelineExecutionResult: assert ingestion.kwargs["watermarks"] == captured["watermarks"] writer = cast("_Built", ingestion.kwargs["writer"]) assert writer.kwargs == { - "project": "unit-project", - "max_batch_rows": 10_000, + "sandbox": False, + "batch_rows": 10_000, "schema_evolution": SchemaEvolution.ADDITIVE, } history = cast("_Built", captured["history"]) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index af41b011..0b2137d5 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -66,6 +66,8 @@ def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( assert migrated.version == 2 assert migrated.platform_name == "gcp" assert migrated.deployment_name == "gcp_cloud_run" + assert original.warehouse_provider == "bigquery" + assert migrated.warehouse_provider == "bigquery" assert migrated.platform == original.platform assert migrated.plugins == original.plugins assert migrated.pipelines == original.pipelines diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py new file mode 100644 index 00000000..41652e9f --- /dev/null +++ b/tests/providers/test_bigquery_runtime.py @@ -0,0 +1,148 @@ +"""BigQuery's first provider-selected warehouse composition.""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from google.cloud import bigquery + +from dander.concurrency import FencingToken +from dander.providers import ProviderKind, default_provider_registry +from dander.telemetry import TelemetryOperation +from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTransport + +if TYPE_CHECKING: + from pytest import MonkeyPatch + + +def _runtime() -> WarehouseRuntime: + registry = default_provider_registry() + config = registry.parse( + ProviderKind.WAREHOUSE, + {"provider": "bigquery", "location": "US"}, + ) + built = registry.build( + ProviderKind.WAREHOUSE, + config, + context={"project": "unit-project"}, + ) + assert isinstance(built, WarehouseRuntime) + return built + + +def test_default_registry_loads_bigquery_runtime_only_after_selection() -> None: + module_name = "dander.providers.bigquery.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + + config = registry.parse( + ProviderKind.WAREHOUSE, + {"provider": "bigquery", "location": "EU"}, + ) + + assert module_name not in sys.modules + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={"project": "unit-project"}, + ) + assert isinstance(runtime, WarehouseRuntime) + assert module_name in sys.modules + assert runtime.capabilities.write_modes == frozenset(WriteMode) + assert runtime.capabilities.transports == frozenset(WriteTransport) + + +def test_bigquery_runtime_exposes_codec_schema_fence_and_telemetry() -> None: + runtime = _runtime() + relation = RelationRef(catalog="unit-project", namespace="raw", name="records") + + schema = runtime.schema_mapper.canonical_schema( + [WriteField(name="id", data_type="STRING", mode="REQUIRED")] + ) + fence = FencingToken( + lease_table="unit-project.dander_meta._dander_leases", + pipeline_id="records", + run_id="run-123", + token=7, + ) + prepared = runtime.target_fence.prepare_dml("DELETE FROM `target` WHERE TRUE", fence) + + @dataclass + class _Job: + job_id: str = "job-123" + output_rows: int = 11 + num_dml_affected_rows: int = 9 + total_bytes_processed: int = 1_024 + total_bytes_billed: int = 2_048 + + telemetry = runtime.telemetry.operation( + _Job(), + operation=TelemetryOperation.LOAD, + duration_ms=45, + retry_count=1, + ) + + assert runtime.relation_codec.render(relation) == "`unit-project`.`raw`.`records`" + assert schema.fields[0].data_type.kind is LogicalTypeKind.STRING + assert "ASSERT @@row_count = 1" in prepared.sql + assert isinstance(prepared.options, bigquery.QueryJobConfig) + assert telemetry.to_payload() == { + "provider": "bigquery", + "operation": "load", + "duration_ms": 45, + "retry_count": 1, + "rows_read": 0, + "rows_written": 11, + "rows_affected": 9, + "bytes_read": 0, + "bytes_written": 0, + "bytes_processed": 1_024, + "bytes_billed": 2_048, + "job_id": "job-123", + } + + +def test_bigquery_runtime_constructs_writers_through_capability( + monkeypatch: MonkeyPatch, +) -> None: + import dander.providers.bigquery.runtime as bigquery_runtime + + captured: list[tuple[str, dict[str, object]]] = [] + + def replace(**kwargs: object) -> object: + captured.append(("replace", kwargs)) + return object() + + def scd1(**kwargs: object) -> object: + captured.append(("scd1", kwargs)) + return object() + + monkeypatch.setattr(bigquery_runtime, "BigQueryReplaceWriter", replace) + monkeypatch.setattr(bigquery_runtime, "BigQueryScd1Writer", scd1) + factory = _runtime().writers + + factory.build_ingestion_writer( + sandbox=True, + batch_rows=50, + schema_evolution=SchemaEvolution.STRICT, + ) + factory.build_ingestion_writer( + sandbox=False, + batch_rows=75, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + assert captured == [ + ("replace", {"project": "unit-project", "max_batch_rows": 50}), + ( + "scd1", + { + "project": "unit-project", + "max_batch_rows": 75, + "schema_evolution": SchemaEvolution.ADDITIVE, + }, + ), + ] diff --git a/tickets/DANDER-85-bigquery-warehouse-runtime.md b/tickets/DANDER-85-bigquery-warehouse-runtime.md new file mode 100644 index 00000000..32ba2bfb --- /dev/null +++ b/tickets/DANDER-85-bigquery-warehouse-runtime.md @@ -0,0 +1,40 @@ +--- +id: DANDER-85 +title: Route BigQuery execution through warehouse capabilities +status: in-review +component: warehouse +epic: cloud-portability +depends_on: [DANDER-84] +created: 2026-08-08 +--- + +## Context + +Named profiles and lazy provider factories existed, but `dander run` still constructed BigQuery +writers and transform runners directly. The first Phase 3 slice must select the proven warehouse +through the provider boundary without mixing state, catalog, secrets, or launcher migration into +the same review. + +## Acceptance Criteria + +- [x] BigQuery configuration remains dependency-light and its implementation loads only on build. +- [x] One typed runtime composes relation, schema, writer, transform, fence, telemetry, and declared + capability surfaces. +- [x] Hosted SCD1, sandbox replace, model, and graph runner construction preserve existing behavior. +- [x] Version 1 projects resolve implicitly to BigQuery and version 2 profiles retain their + selected warehouse ID and location. +- [x] The CLI no longer constructs a BigQuery writer or transform runner directly. +- [x] Focused provider, migration, warehouse, and CLI composition tests pass. +- [x] Full local validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Use one small `WarehouseRuntime` composition rather than a large provider interface. Existing +BigQuery implementations remain intact behind factories, limiting this PR to construction and +contract movement. State, Dataplex, GCP secrets, and Cloud Run stay on their proven paths until +their own Phase 3 tickets. + +## Review Log + +Pending protected review. From ef54c1c1a22e19c97904ae5561e9b1c1cb025906 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 08:29:53 -0400 Subject: [PATCH 035/163] Route BigQuery state through provider runtime --- CHANGELOG.md | 3 + HANDOFF.md | 38 ++-- docs/decisions.md | 9 + docs/platform-profiles.md | 9 +- src/dander/catalog/store.py | 4 + src/dander/cli/run_command.py | 85 +++++--- src/dander/project/config.py | 1 + src/dander/project/portable_config.py | 14 +- src/dander/providers/bigquery/__init__.py | 4 +- src/dander/providers/bigquery/config.py | 8 + src/dander/providers/bigquery/state.py | 201 ++++++++++++++++++ src/dander/providers/defaults.py | 10 +- src/dander/state/__init__.py | 10 + src/dander/state/run_history.py | 4 + src/dander/state/runtime.py | 84 ++++++++ src/dander/state/watermark.py | 4 + tests/cli/test_run_command.py | 62 +++++- tests/project/test_portable_config.py | 2 + .../providers/test_bigquery_state_runtime.py | 146 +++++++++++++ .../DANDER-85-bigquery-warehouse-runtime.md | 6 +- tickets/DANDER-86-bigquery-state-runtime.md | 41 ++++ 21 files changed, 666 insertions(+), 79 deletions(-) create mode 100644 src/dander/providers/bigquery/state.py create mode 100644 src/dander/state/runtime.py create mode 100644 tests/providers/test_bigquery_state_runtime.py create mode 100644 tickets/DANDER-86-bigquery-state-runtime.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c711af4..7528c0e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Route BigQuery leases, watermarks, run history, and metadata snapshots through one lazily selected + durable-state runtime with an explicit versioned migration ledger while preserving all existing + table identities and correctness behavior. - Route BigQuery writer and transform construction through one lazily selected, typed warehouse runtime that also exposes canonical schema, relation rendering, target fencing, normalized job telemetry, and explicit capabilities without changing current GCP behavior. diff --git a/HANDOFF.md b/HANDOFF.md index 60dda247..b2c6a9c8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Added a typed `WarehouseRuntime` composed from small provider capabilities. -- Registered BigQuery through a dependency-light config and lazy runtime factory. -- Routed hosted/sandbox writers and model/graph runners through the selected warehouse. -- Exposed BigQuery relation, canonical schema, fencing, telemetry, and capability adapters. -- Preserved implicit v1 and explicit v2 BigQuery selection without moving other providers. +- Added a typed `StateRuntime` for leases, watermarks, run history, metadata, and migrations. +- Registered BigQuery state through a dependency-light config and lazy provider factory. +- Routed hosted state construction through the selected runtime; SQLite sandbox stays unchanged. +- Added schema ledger version 1 while preserving every existing BigQuery table identity. +- Kept server-time leases, fencing, cursor CAS, and interrupted-run reconciliation intact. ## Try It -Run an existing v1 or v2 BigQuery project normally. `dander run` now selects the BigQuery -warehouse runtime internally; public commands and configuration are unchanged. +Run an existing v1 or v2 BigQuery project normally. `dander run` selects and migrates the BigQuery +state runtime internally; public commands and authored configuration are unchanged. ## Checks -- All 903 tests, Ruff, formatting, and strict mypy across 209 files passed. -- Wheel/sdist build, metadata/distribution validation, and dependency audit passed. -- The non-root/read-only full runtime image passed conformance and its BigQuery factory probe. -- Trivy found no high/critical issues; the focused secret scan passed. -- All Terraform roots passed; GCP reported `No changes` and both schedules stayed paused. +- All 907 tests, Ruff, formatting, and strict mypy across 212 files passed. +- Wheel/sdist inspection, source-free installs, runtime-all assembly, and dependency audit passed. +- The non-root/read-only full runtime image passed conformance and its bundled-asset checks. +- Trivy and Gitleaks passed; all Terraform roots validated successfully. +- Isolated GCP reported `No changes`; Salesforce and ServiceNow schedules remain paused. ## Decisions -- Compose small capabilities instead of adding a large warehouse-provider interface. -- Keep the BigQuery implementation lazy and behaviorally unchanged behind the new factory. -- Move state, catalog, secrets, and launchers only in their own focused tickets. +- Preserve current BigQuery stores and table identities behind a small composed runtime. +- Record a migration only after every shared state table is ready. +- Keep per-pipeline lease-table creation lazy rather than inventing a data migration. ## Remaining -- Open and merge the focused BigQuery runtime PR after protected CI passes. -- Route BigQuery durable state through the shared state contract next. +- Open and merge the focused durable-state PR after protected CI passes. +- Route Dataplex catalog publication through its provider boundary next. ## Review First -- `src/dander/warehouse/runtime.py` -- `src/dander/providers/bigquery/runtime.py` +- `src/dander/state/runtime.py` +- `src/dander/providers/bigquery/state.py` - `src/dander/cli/run_command.py` diff --git a/docs/decisions.md b/docs/decisions.md index 20cd1a90..2710741c 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-08 — Durable state is composed without relocating existing data + +- **Selection:** Version 1 projects retain implicit BigQuery state; version 2 resolution carries + the named state provider into the same lazy API-v1 factory registry as the warehouse. +- **Migration:** One `_dander_state_schema` ledger records completed provider schema versions only + after the existing watermark, history, and catalog tables are ready. Re-entry is idempotent. +- **Compatibility:** Existing table identities, per-pipeline lease tables, server-time leases, + fencing, watermark CAS, and interrupted-run reconciliation remain unchanged. + ## 2026-08-08 — BigQuery enters portability through one composed runtime - **Selection:** Version 1 projects retain an implicit BigQuery warehouse; version 2 resolution diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 3021a122..e5930b41 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -43,10 +43,11 @@ is represented in two files. The current runtime continues to support only the GCP compatibility composition. The internal factory contract provides one API-v1 registry across warehouse, state, catalog, secret, and launcher categories. Registration loads only a small configuration model; selecting and building -a provider loads its implementation and SDK dependencies. BigQuery writer and transform -construction now use that boundary. BigQuery state, Dataplex, GCP secrets, and Cloud Run remain on -their proven direct paths until their separate Phase 3 changes. The registry is a construction -contract, not a support claim. +a provider loads its implementation and SDK dependencies. BigQuery writer, transform, lease, +watermark, run-history, and metadata-store construction now use that boundary. State migrations +are explicit and versioned, but retain the existing BigQuery table names and semantics. Dataplex, +GCP secrets, and Cloud Run remain on their proven direct paths until their separate Phase 3 +changes. The registry is a construction contract, not a support claim. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; diff --git a/src/dander/catalog/store.py b/src/dander/catalog/store.py index e498d7ad..81149046 100644 --- a/src/dander/catalog/store.py +++ b/src/dander/catalog/store.py @@ -76,6 +76,10 @@ def __init__(self, *, project: str, dataset: str, client: _Client | None = None) self._client = client or cast("_Client", bigquery.Client(project=project)) self._ready = False + def migrate(self) -> None: + """Idempotently ensure the current metadata schema exists.""" + self._ensure_table() + def publish( self, *, diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 472d18f0..46f2c1ce 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -11,7 +11,6 @@ from rich.table import Table from dander.catalog import ( - BigQueryMetadataStore, CatalogPublishError, DataplexCatalogPublisher, MetadataStore, @@ -53,14 +52,13 @@ OAuth2JWT, ) from dander.state import ( - BigQueryLeaseStore, - BigQueryRunHistoryStore, - BigQueryWatermarkStore, LeaseStore, RunHistoryStore, SqliteLeaseStore, SqliteRunHistoryStore, SqliteWatermarkStore, + StateRuntime, + WatermarkStore, ) from dander.transform import TransformProjectError, TransformRunError from dander.warehouse import WarehouseRuntime, WarehouseTransformRunner @@ -114,12 +112,14 @@ class _ResolvedRun: publish_dataplex: bool warehouse_provider: str warehouse_location: str + state_provider: str @dataclass(frozen=True, slots=True) class _ControlStores: history: RunHistoryStore leases: LeaseStore + watermarks: WatermarkStore metadata: MetadataStore | None @@ -166,6 +166,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: publish_dataplex = options.publish_dataplex warehouse_provider = "bigquery" warehouse_location = "US" + state_provider = "bigquery" plugin_registry: ConnectorPluginRegistry | None = None if options.project_config.is_file(): @@ -190,6 +191,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: publish_dataplex = publish_dataplex or pipeline.publish_dataplex warehouse_provider = manifest.warehouse_provider warehouse_location = manifest.platform.bigquery_location + state_provider = manifest.state_provider except (ConnectorPluginError, ProjectConfigError) as error: raise ClickException(str(error)) from error @@ -234,6 +236,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: publish_dataplex=publish_dataplex, warehouse_provider=warehouse_provider, warehouse_location=warehouse_location, + state_provider=state_provider, ) @@ -304,7 +307,7 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec else None ) warehouse = _build_warehouse_runtime(resolved) - ingestion = _build_ingestion_runner(options, resolved, source_adapter, warehouse) + ingestion = _build_ingestion_runner(options, resolved, source_adapter, warehouse, stores) transform_runner = _build_transform_runner(resolved, warehouse) return PipelineExecutor( pipeline_id=resolved.pipeline_id, @@ -326,28 +329,25 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec def _build_control_stores(options: RunOptions, resolved: _ResolvedRun) -> _ControlStores: - control_dataset = resolved.metadata_dataset if resolved.project_pipeline else resolved.dataset - history: RunHistoryStore = ( - SqliteRunHistoryStore(options.state_path) - if options.sandbox - else BigQueryRunHistoryStore(project=resolved.project, dataset=control_dataset) - ) - leases: LeaseStore = ( - SqliteLeaseStore(options.state_path) - if options.sandbox - else BigQueryLeaseStore(project=resolved.project, dataset=control_dataset) - ) - metadata: MetadataStore | None = None - if resolved.project_pipeline and resolved.graph_plan is None: - metadata = ( - SqliteMetadataStore(options.state_path) - if options.sandbox - else BigQueryMetadataStore( - project=resolved.project, - dataset=resolved.metadata_dataset, - ) + if options.sandbox: + return _ControlStores( + history=SqliteRunHistoryStore(options.state_path), + leases=SqliteLeaseStore(options.state_path), + watermarks=SqliteWatermarkStore(options.state_path), + metadata=( + SqliteMetadataStore(options.state_path) + if resolved.project_pipeline and resolved.graph_plan is None + else None + ), ) - return _ControlStores(history=history, leases=leases, metadata=metadata) + runtime = _build_state_runtime(resolved) + runtime.migrator.migrate() + return _ControlStores( + history=runtime.history, + leases=runtime.leases, + watermarks=runtime.watermarks, + metadata=runtime.metadata, + ) def _build_ingestion_runner( @@ -355,6 +355,7 @@ def _build_ingestion_runner( resolved: _ResolvedRun, source: Source, warehouse: WarehouseRuntime, + stores: _ControlStores, ) -> PipelineRunner: writer = warehouse.writers.build_ingestion_writer( sandbox=options.sandbox, @@ -363,15 +364,10 @@ def _build_ingestion_runner( SchemaEvolution.ADDITIVE if resolved.project_pipeline else SchemaEvolution.STRICT ), ) - watermarks = ( - SqliteWatermarkStore(options.state_path) - if options.sandbox - else BigQueryWatermarkStore(project=resolved.project, dataset=resolved.dataset) - ) return PipelineRunner( source=source, writer=writer, - watermarks=watermarks, + watermarks=stores.watermarks, project=resolved.project, dataset=resolved.dataset, resume_from_watermark=not options.sandbox, @@ -414,6 +410,31 @@ def _build_warehouse_runtime(resolved: _ResolvedRun) -> WarehouseRuntime: return runtime +def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: + registry = default_provider_registry() + try: + config = registry.parse( + ProviderKind.STATE, + {"provider": resolved.state_provider}, + ) + runtime = registry.build( + ProviderKind.STATE, + config, + context={ + "project": resolved.project, + "raw_dataset": resolved.dataset, + "metadata_dataset": resolved.metadata_dataset, + "project_pipeline": resolved.project_pipeline, + "metadata_enabled": (resolved.project_pipeline and resolved.graph_plan is None), + }, + ) + except ProviderFactoryError as error: + raise ClickException(str(error)) from error + if not isinstance(runtime, StateRuntime): + raise ClickException("Selected state provider returned an invalid runtime") + return runtime + + def build_source_adapter( config: SourceConfig, auth: AuthStrategy, diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 08a26814..d5f93301 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -204,6 +204,7 @@ class DanderProject(BaseModel): platform_name: str = Field(default="gcp", exclude=True) deployment_name: str = Field(default="gcp_cloud_run", exclude=True) warehouse_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) + state_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index a2060be6..dbd2dbe2 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -23,7 +23,7 @@ ProjectConfigError, _load_yaml_mapping, ) -from dander.providers.bigquery import BigQueryWarehouseConfig # noqa: TC001 +from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig # noqa: TC001 _NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") _PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") @@ -110,14 +110,6 @@ def validate_pipeline_ids( return values -class BigQueryStateSpec(BaseModel): - """BigQuery durable-state selection for the compatibility profile.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["bigquery"] - - class DataplexCatalogSpec(BaseModel): """Dataplex catalog publication selection.""" @@ -165,7 +157,7 @@ class PlatformProfileSpec(BaseModel): model_config = ConfigDict(extra="forbid") warehouse: BigQueryWarehouseConfig - state: BigQueryStateSpec + state: BigQueryStateConfig catalog: CatalogSpec secrets: SecretProviderSpec @@ -383,6 +375,7 @@ def _resolve( platform_name=selected.platform, deployment_name=selected_name, warehouse_provider=profile.warehouse.provider, + state_provider=profile.state.provider, deployed_pipeline_ids=tuple(sorted(selected.pipelines)), ) @@ -470,6 +463,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: return ( legacy.platform == migrated.platform and legacy.warehouse_provider == migrated.warehouse_provider + and legacy.state_provider == migrated.state_provider and legacy.plugins == migrated.plugins and legacy.pipelines == migrated.pipelines and legacy.terraform_pipelines() == migrated.terraform_pipelines() diff --git a/src/dander/providers/bigquery/__init__.py b/src/dander/providers/bigquery/__init__.py index 8340cb7d..e3ff6f31 100644 --- a/src/dander/providers/bigquery/__init__.py +++ b/src/dander/providers/bigquery/__init__.py @@ -1,5 +1,5 @@ """BigQuery provider configuration; runtime implementation remains lazily loaded.""" -from dander.providers.bigquery.config import BigQueryWarehouseConfig +from dander.providers.bigquery.config import BigQueryStateConfig, BigQueryWarehouseConfig -__all__ = ["BigQueryWarehouseConfig"] +__all__ = ["BigQueryStateConfig", "BigQueryWarehouseConfig"] diff --git a/src/dander/providers/bigquery/config.py b/src/dander/providers/bigquery/config.py index d2fd32d4..14c84da1 100644 --- a/src/dander/providers/bigquery/config.py +++ b/src/dander/providers/bigquery/config.py @@ -12,3 +12,11 @@ class BigQueryWarehouseConfig(BaseModel): provider: Literal["bigquery"] location: str = Field(default="US", min_length=1) + + +class BigQueryStateConfig(BaseModel): + """BigQuery durable-state selection for one named platform profile.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["bigquery"] diff --git a/src/dander/providers/bigquery/state.py b/src/dander/providers/bigquery/state.py new file mode 100644 index 00000000..28b17e13 --- /dev/null +++ b/src/dander/providers/bigquery/state.py @@ -0,0 +1,201 @@ +"""BigQuery durable-state runtime and explicit schema migrations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +from google.cloud import bigquery + +from dander.catalog import BigQueryMetadataStore +from dander.providers.bigquery.config import BigQueryStateConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.state import ( + BigQueryLeaseStore, + BigQueryRunHistoryStore, + BigQueryWatermarkStore, + StateCapabilities, + StateMigration, + StateRuntime, +) + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + + from pydantic import BaseModel + +_STATE_SCHEMA_VERSION = 1 +_MIGRATIONS = (StateMigration(version=1, name="existing_control_tables"),) + + +class _QueryJob(Protocol): + def result(self) -> Iterable[Mapping[str, Any]]: + """Wait for query completion and return projected rows.""" + + +class _BigQueryClient(Protocol): + def query( + self, + query: str, + *, + job_config: bigquery.QueryJobConfig | None = None, + ) -> _QueryJob: + """Execute one Standard SQL statement.""" + + +class BigQueryStateMigrator: + """Version existing BigQuery control tables without changing their identities.""" + + def __init__( + self, + *, + project: str, + dataset: str, + client: _BigQueryClient, + watermarks: BigQueryWatermarkStore, + history: BigQueryRunHistoryStore, + metadata: BigQueryMetadataStore | None, + ) -> None: + self._table = f"{project}.{dataset}._dander_state_schema" + self._client = client + self._watermarks = watermarks + self._history = history + self._metadata = metadata + self._ledger_ready = False + + @property + def migrations(self) -> tuple[StateMigration, ...]: + return _MIGRATIONS + + def current_version(self) -> int: + """Return the latest recorded migration, creating only the ledger if absent.""" + self._ensure_ledger() + rows = list( + self._client.query( + f"SELECT COALESCE(MAX(version), 0) AS version FROM `{self._table}`" + ).result() + ) + if len(rows) != 1: + raise RuntimeError("BigQuery state migration ledger returned an invalid result") + version = rows[0]["version"] + if isinstance(version, bool) or not isinstance(version, int) or version < 0: + raise RuntimeError("BigQuery state migration ledger contains an invalid version") + if version > _STATE_SCHEMA_VERSION: + raise RuntimeError( + f"BigQuery state schema version {version} is newer than this Dander runtime" + ) + return version + + def migrate(self) -> int: + """Apply outstanding idempotent migrations and record only completed work.""" + version = self.current_version() + if version >= _STATE_SCHEMA_VERSION: + return version + + self._watermarks.migrate() + self._history.migrate() + if self._metadata is not None: + self._metadata.migrate() + self._record(_MIGRATIONS[0]) + return _STATE_SCHEMA_VERSION + + def _ensure_ledger(self) -> None: + if self._ledger_ready: + return + self._client.query( + f"CREATE TABLE IF NOT EXISTS `{self._table}` (" + "version INT64 NOT NULL, name STRING NOT NULL, applied_at TIMESTAMP NOT NULL) " + "CLUSTER BY version" + ).result() + self._ledger_ready = True + + def _record(self, migration: StateMigration) -> None: + config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter("version", "INT64", migration.version), + bigquery.ScalarQueryParameter("name", "STRING", migration.name), + ] + ) + self._client.query( + f"MERGE `{self._table}` AS target " + "USING (SELECT @version AS version, @name AS name) AS incoming " + "ON target.version = incoming.version " + "WHEN NOT MATCHED THEN INSERT (version, name, applied_at) " + "VALUES (incoming.version, incoming.name, CURRENT_TIMESTAMP())", + job_config=config, + ).result() + + +def _build_bigquery_state( + config: BaseModel, + context: Mapping[str, object], +) -> StateRuntime: + if not isinstance(config, BigQueryStateConfig): + raise TypeError("BigQuery state factory received the wrong configuration") + project = _required_text(context, "project") + raw_dataset = _required_text(context, "raw_dataset") + metadata_dataset = _required_text(context, "metadata_dataset") + project_pipeline = _required_bool(context, "project_pipeline") + metadata_enabled = _required_bool(context, "metadata_enabled") + control_dataset = metadata_dataset if project_pipeline else raw_dataset + supplied_client = context.get("client") + client: Any = ( + supplied_client if supplied_client is not None else bigquery.Client(project=project) + ) + + watermarks = BigQueryWatermarkStore(project=project, dataset=raw_dataset, client=client) + history = BigQueryRunHistoryStore(project=project, dataset=control_dataset, client=client) + leases = BigQueryLeaseStore(project=project, dataset=control_dataset, client=client) + metadata = ( + BigQueryMetadataStore(project=project, dataset=metadata_dataset, client=client) + if metadata_enabled + else None + ) + migrator = BigQueryStateMigrator( + project=project, + dataset=control_dataset, + client=client, + watermarks=watermarks, + history=history, + metadata=metadata, + ) + return StateRuntime( + provider_id="bigquery", + leases=leases, + watermarks=watermarks, + history=history, + metadata=metadata, + migrator=migrator, + capabilities=StateCapabilities( + provider_id="bigquery", + schema_version=_STATE_SCHEMA_VERSION, + server_time=True, + atomic_leases=True, + monotonic_fencing=True, + atomic_watermark_cas=True, + interrupted_run_reconciliation=True, + ), + ) + + +def _required_text(context: Mapping[str, object], name: str) -> str: + value = context.get(name) + if not isinstance(value, str) or not value: + raise ValueError(f"BigQuery state runtime requires a non-empty {name}") + return value + + +def _required_bool(context: Mapping[str, object], name: str) -> bool: + value = context.get(name) + if not isinstance(value, bool): + raise ValueError(f"BigQuery state runtime requires a boolean {name}") + return value + + +BIGQUERY_STATE_FACTORY = ProviderFactory[StateRuntime]( + kind=ProviderKind.STATE, + provider_id="bigquery", + api_version=PROVIDER_API_VERSION, + build=_build_bigquery_state, +) + +__all__ = ["BIGQUERY_STATE_FACTORY", "BigQueryStateMigrator"] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index b888d567..01a9853b 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -1,6 +1,6 @@ """Built-in provider registrations with lazy implementation loading.""" -from dander.providers.bigquery import BigQueryWarehouseConfig +from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -15,4 +15,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.bigquery.runtime:BIGQUERY_WAREHOUSE_FACTORY" ), ) + registry.register( + kind=ProviderKind.STATE, + provider_id="bigquery", + config_model=BigQueryStateConfig, + load_factory=lazy_provider_factory( + "dander.providers.bigquery.state:BIGQUERY_STATE_FACTORY" + ), + ) return registry diff --git a/src/dander/state/__init__.py b/src/dander/state/__init__.py index b525a355..2af5b2cf 100644 --- a/src/dander/state/__init__.py +++ b/src/dander/state/__init__.py @@ -17,6 +17,12 @@ RunStatus, SqliteRunHistoryStore, ) +from dander.state.runtime import ( + StateCapabilities, + StateMigration, + StateMigrator, + StateRuntime, +) from dander.state.watermark import BigQueryWatermarkStore, SqliteWatermarkStore, WatermarkStore __all__ = [ @@ -35,6 +41,10 @@ "SqliteRunHistoryStore", "SqliteLeaseStore", "SqliteWatermarkStore", + "StateCapabilities", + "StateMigration", + "StateMigrator", + "StateRuntime", "WatermarkStore", "classify_failure", ] diff --git a/src/dander/state/run_history.py b/src/dander/state/run_history.py index 038019b2..8ffb52fe 100644 --- a/src/dander/state/run_history.py +++ b/src/dander/state/run_history.py @@ -150,6 +150,10 @@ def __init__( self._initialize_on_read = initialize_on_read self._ready = False + def migrate(self) -> None: + """Idempotently ensure the current run-history schema exists.""" + self._ensure_table() + def start(self, run_id: str, source: str, *, pipeline_id: str | None = None) -> None: self._ensure_table() config = bigquery.QueryJobConfig( diff --git a/src/dander/state/runtime.py b/src/dander/state/runtime.py new file mode 100644 index 00000000..a891d2e4 --- /dev/null +++ b/src/dander/state/runtime.py @@ -0,0 +1,84 @@ +"""Provider-neutral durable-state runtime composition.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from dander.catalog import MetadataStore + from dander.state.lease import LeaseStore + from dander.state.run_history import RunHistoryStore + from dander.state.watermark import WatermarkStore + +_MIGRATION_NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") + + +@dataclass(frozen=True, slots=True) +class StateMigration: + """One monotonically versioned durable-state schema change.""" + + version: int + name: str + + def __post_init__(self) -> None: + if self.version <= 0: + raise ValueError("state migration version must be positive") + if not _MIGRATION_NAME.fullmatch(self.name): + raise ValueError("state migration name must be a safe lowercase identifier") + + +class StateMigrator(Protocol): + """Apply a provider's ordered, idempotent durable-state migrations.""" + + @property + def migrations(self) -> tuple[StateMigration, ...]: + """Return every migration understood by this runtime.""" + + def current_version(self) -> int: + """Return the greatest successfully applied migration version.""" + + def migrate(self) -> int: + """Apply outstanding migrations and return the resulting schema version.""" + + +@dataclass(frozen=True, slots=True) +class StateCapabilities: + """Correctness guarantees exposed by one durable-state provider.""" + + provider_id: str + schema_version: int + server_time: bool + atomic_leases: bool + monotonic_fencing: bool + atomic_watermark_cas: bool + interrupted_run_reconciliation: bool + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("state capabilities require a provider id") + if self.schema_version <= 0: + raise ValueError("state schema version must be positive") + + +@dataclass(frozen=True, slots=True) +class StateRuntime: + """The durable control-plane stores selected by one platform profile.""" + + provider_id: str + leases: LeaseStore + watermarks: WatermarkStore + history: RunHistoryStore + metadata: MetadataStore | None + migrator: StateMigrator + capabilities: StateCapabilities + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("state runtime requires a provider id") + if self.capabilities.provider_id != self.provider_id: + raise ValueError("state runtime and capabilities provider ids must match") + latest = max(migration.version for migration in self.migrator.migrations) + if latest != self.capabilities.schema_version: + raise ValueError("state runtime migration and capability versions must match") diff --git a/src/dander/state/watermark.py b/src/dander/state/watermark.py index 2df47e11..31da290a 100644 --- a/src/dander/state/watermark.py +++ b/src/dander/state/watermark.py @@ -83,6 +83,10 @@ def __init__( self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) self._table_ready = False + def migrate(self) -> None: + """Idempotently ensure the current watermark schema exists.""" + self._ensure_table() + def get(self, source: str, entity: str) -> str | None: """Return the most recently committed cursor for `(source, entity)`.""" self._ensure_table() diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 0876c010..85745cb3 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -13,6 +13,7 @@ from dander.executor import PipelineExecutionResult from dander.runtime import EndpointRunResult, PipelineRunResult from dander.security import NoAuth +from dander.state import StateCapabilities, StateMigration, StateRuntime from dander.writer import SchemaEvolution if TYPE_CHECKING: @@ -20,7 +21,9 @@ from pytest import MonkeyPatch + from dander.catalog import MetadataStore from dander.ingestion import SourceConfig + from dander.state import LeaseStore, RunHistoryStore, WatermarkStore _REPO_ROOT = Path(__file__).parents[2] @@ -37,6 +40,27 @@ class _ResolvedWarehouse(Protocol): warehouse_location: str +class _ResolvedState(Protocol): + state_provider: str + project: str + dataset: str + metadata_dataset: str + + +class _Migrator: + migrations = (StateMigration(version=1, name="existing_control_tables"),) + + def current_version(self) -> int: + return 0 + + def migrate(self) -> int: + self.calls += 1 + return 1 + + def __init__(self) -> None: + self.calls = 0 + + def test_hosted_project_run_wires_runtime_without_network( monkeypatch: MonkeyPatch, ) -> None: @@ -112,15 +136,39 @@ def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: assert resolved.warehouse_location == "US" return _Warehouse() + def build_state(resolved: _ResolvedState) -> StateRuntime: + assert resolved.state_provider == "bigquery" + assert resolved.project == "unit-project" + assert resolved.dataset == "raw" + assert resolved.metadata_dataset == "dander_meta" + migrator = _Migrator() + captured["migrator"] = migrator + for name in ("history", "leases", "metadata", "watermarks"): + captured[name] = _Built(name=name, args=(), kwargs={}) + return StateRuntime( + provider_id="bigquery", + leases=cast("LeaseStore", captured["leases"]), + watermarks=cast("WatermarkStore", captured["watermarks"]), + history=cast("RunHistoryStore", captured["history"]), + metadata=cast("MetadataStore", captured["metadata"]), + migrator=migrator, + capabilities=StateCapabilities( + provider_id="bigquery", + schema_version=1, + server_time=True, + atomic_leases=True, + monotonic_fencing=True, + atomic_watermark_cas=True, + interrupted_run_reconciliation=True, + ), + ) + monkeypatch.setattr(run_module, "DefaultSecretStore", recording_factory("secrets")) monkeypatch.setattr(run_module, "build_auth", build_auth) monkeypatch.setattr(run_module, "build_source_adapter", build_source) - monkeypatch.setattr(run_module, "BigQueryRunHistoryStore", recording_factory("history")) - monkeypatch.setattr(run_module, "BigQueryLeaseStore", recording_factory("leases")) - monkeypatch.setattr(run_module, "BigQueryMetadataStore", recording_factory("metadata")) - monkeypatch.setattr(run_module, "BigQueryWatermarkStore", recording_factory("watermarks")) monkeypatch.setattr(run_module, "PipelineRunner", recording_factory("ingestion")) monkeypatch.setattr(run_module, "_build_warehouse_runtime", build_warehouse) + monkeypatch.setattr(run_module, "_build_state_runtime", build_state) monkeypatch.setattr(run_module, "PipelineExecutor", _Executor) result = CliRunner().invoke( @@ -165,7 +213,5 @@ def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: "batch_rows": 10_000, "schema_evolution": SchemaEvolution.ADDITIVE, } - history = cast("_Built", captured["history"]) - metadata = cast("_Built", captured["metadata"]) - assert history.kwargs["dataset"] == "dander_meta" - assert metadata.kwargs["dataset"] == "dander_meta" + migrator = cast("_Migrator", captured["migrator"]) + assert migrator.calls == 1 diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 0b2137d5..f76e900f 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -68,6 +68,8 @@ def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( assert migrated.deployment_name == "gcp_cloud_run" assert original.warehouse_provider == "bigquery" assert migrated.warehouse_provider == "bigquery" + assert original.state_provider == "bigquery" + assert migrated.state_provider == "bigquery" assert migrated.platform == original.platform assert migrated.plugins == original.plugins assert migrated.pipelines == original.pipelines diff --git a/tests/providers/test_bigquery_state_runtime.py b/tests/providers/test_bigquery_state_runtime.py new file mode 100644 index 00000000..fb144944 --- /dev/null +++ b/tests/providers/test_bigquery_state_runtime.py @@ -0,0 +1,146 @@ +"""BigQuery state provider composition and migration coverage.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any + +import pytest + +from dander.catalog import BigQueryMetadataStore +from dander.providers import ProviderKind, default_provider_registry +from dander.state import ( + BigQueryLeaseStore, + BigQueryRunHistoryStore, + BigQueryWatermarkStore, + StateRuntime, +) + +if TYPE_CHECKING: + from pytest import MonkeyPatch + + +class _Job: + num_dml_affected_rows = 1 + + def __init__(self, rows: list[dict[str, Any]] | None = None) -> None: + self._rows = rows or [] + + def result(self) -> list[dict[str, Any]]: + return self._rows + + +class _Client: + def __init__(self, *, version: int = 0) -> None: + self.version = version + self.queries: list[str] = [] + + def query(self, query: str, *, job_config: object | None = None) -> _Job: + del job_config + self.queries.append(query) + if "SELECT COALESCE(MAX(version), 0)" in query: + return _Job([{"version": self.version}]) + if query.startswith("MERGE `") and "._dander_state_schema`" in query: + self.version = 1 + return _Job() + + +def _build(client: _Client, *, metadata_enabled: bool = True) -> StateRuntime: + registry = default_provider_registry() + config = registry.parse(ProviderKind.STATE, {"provider": "bigquery"}) + runtime = registry.build( + ProviderKind.STATE, + config, + context={ + "project": "unit-project", + "raw_dataset": "raw", + "metadata_dataset": "dander_meta", + "project_pipeline": True, + "metadata_enabled": metadata_enabled, + "client": client, + }, + ) + assert isinstance(runtime, StateRuntime) + return runtime + + +def test_default_registry_loads_bigquery_state_only_after_selection() -> None: + module_name = "dander.providers.bigquery.state" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + + config = registry.parse(ProviderKind.STATE, {"provider": "bigquery"}) + + assert module_name not in sys.modules + runtime = registry.build( + ProviderKind.STATE, + config, + context={ + "project": "unit-project", + "raw_dataset": "raw", + "metadata_dataset": "dander_meta", + "project_pipeline": True, + "metadata_enabled": True, + "client": _Client(), + }, + ) + assert isinstance(runtime, StateRuntime) + assert module_name in sys.modules + + +def test_bigquery_state_runtime_preserves_existing_table_locations() -> None: + runtime = _build(_Client()) + + assert isinstance(runtime.leases, BigQueryLeaseStore) + assert isinstance(runtime.watermarks, BigQueryWatermarkStore) + assert isinstance(runtime.history, BigQueryRunHistoryStore) + assert isinstance(runtime.metadata, BigQueryMetadataStore) + assert runtime.leases._table_prefix == "unit-project.dander_meta._dander_lease_" + assert runtime.watermarks._table_id == "unit-project.raw._dander_watermarks" + assert runtime.history._table == "unit-project.dander_meta._dander_runs" + assert runtime.metadata._table == "unit-project.dander_meta._dander_catalog" + assert runtime.capabilities.server_time is True + assert runtime.capabilities.atomic_watermark_cas is True + assert runtime.capabilities.interrupted_run_reconciliation is True + + +def test_bigquery_state_migration_is_versioned_and_idempotent() -> None: + client = _Client() + runtime = _build(client) + + assert runtime.migrator.migrate() == 1 + first_queries = tuple(client.queries) + + assert client.version == 1 + assert "._dander_state_schema`" in first_queries[0] + assert any("._dander_watermarks`" in query for query in first_queries) + assert any("._dander_runs`" in query for query in first_queries) + assert any("._dander_catalog`" in query for query in first_queries) + assert first_queries[-1].startswith("MERGE `") + + before_second = len(client.queries) + assert runtime.migrator.migrate() == 1 + second_queries = client.queries[before_second:] + assert len(second_queries) == 1 + assert "SELECT COALESCE(MAX(version), 0)" in second_queries[0] + + +def test_failed_state_migration_is_not_recorded( + monkeypatch: MonkeyPatch, +) -> None: + client = _Client() + runtime = _build(client) + + def fail() -> None: + raise RuntimeError("history migration failed") + + monkeypatch.setattr(runtime.history, "migrate", fail) + + with pytest.raises(RuntimeError, match="history migration failed"): + runtime.migrator.migrate() + + assert client.version == 0 + assert not any( + query.startswith("MERGE `") and "._dander_state_schema`" in query + for query in client.queries + ) diff --git a/tickets/DANDER-85-bigquery-warehouse-runtime.md b/tickets/DANDER-85-bigquery-warehouse-runtime.md index 32ba2bfb..39e2cfcf 100644 --- a/tickets/DANDER-85-bigquery-warehouse-runtime.md +++ b/tickets/DANDER-85-bigquery-warehouse-runtime.md @@ -1,7 +1,7 @@ --- id: DANDER-85 title: Route BigQuery execution through warehouse capabilities -status: in-review +status: completed component: warehouse epic: cloud-portability depends_on: [DANDER-84] @@ -26,7 +26,7 @@ the same review. - [x] The CLI no longer constructs a BigQuery writer or transform runner directly. - [x] Focused provider, migration, warehouse, and CLI composition tests pass. - [x] Full local validation and isolated GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -37,4 +37,4 @@ their own Phase 3 tickets. ## Review Log -Pending protected review. +Merged through protected PR #116 as `5478e1ba5ad7aaa66c4258f23ea01949942221d1`. diff --git a/tickets/DANDER-86-bigquery-state-runtime.md b/tickets/DANDER-86-bigquery-state-runtime.md new file mode 100644 index 00000000..e6f5a1f9 --- /dev/null +++ b/tickets/DANDER-86-bigquery-state-runtime.md @@ -0,0 +1,41 @@ +--- +id: DANDER-86 +title: Route BigQuery durable state through provider capabilities +status: in-review +component: state +epic: cloud-portability +depends_on: [DANDER-85] +created: 2026-08-08 +--- + +## Context + +BigQuery warehouse execution used the provider registry, but hosted runs still constructed leases, +watermarks, run history, and metadata stores directly in the CLI. Durable state needs one selected +runtime and explicit schema versions before another backend can satisfy the same correctness +contract. + +## Acceptance Criteria + +- [x] Version 1 projects retain implicit BigQuery state and version 2 profiles carry the selection. +- [x] The BigQuery state implementation loads only after its factory is selected. +- [x] One runtime composes leases, watermarks, history, optional metadata, migrations, and declared + correctness capabilities. +- [x] Existing datasets and table identities remain unchanged. +- [x] Schema version 1 is recorded only after shared tables migrate successfully. +- [x] Re-running a completed migration performs no table-schema mutations. +- [x] Hosted CLI composition uses the selected runtime; SQLite sandbox behavior remains unchanged. +- [x] Focused state, project migration, and CLI composition tests pass. +- [x] Full local validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Keep the established BigQuery stores and expose their existing idempotent table setup as explicit +migrations. A small `_dander_state_schema` ledger records only completed migration versions. Lease +tables remain lazy and per pipeline because changing their identity would add migration risk with +no portability benefit in this slice. + +## Review Log + +Pending protected review. From 56d3d581fb03c36b5f8a7c665ba62548a30c69ae Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 13:12:57 -0400 Subject: [PATCH 036/163] Route Dataplex through provider runtime --- CHANGELOG.md | 3 + HANDOFF.md | 36 ++++---- docs/decisions.md | 10 +++ docs/platform-profiles.md | 10 ++- src/dander/catalog/__init__.py | 37 +++++++-- src/dander/catalog/dataplex.py | 6 +- src/dander/catalog/publisher.py | 4 + src/dander/catalog/runtime.py | 38 +++++++++ src/dander/cli/main.py | 8 +- src/dander/cli/provider_runtime.py | 30 +++++++ src/dander/cli/run_command.py | 11 ++- src/dander/project/config.py | 1 + src/dander/project/portable_config.py | 22 ++--- src/dander/providers/dataplex/__init__.py | 5 ++ src/dander/providers/dataplex/config.py | 13 +++ src/dander/providers/dataplex/runtime.py | 57 +++++++++++++ src/dander/providers/defaults.py | 18 ++++ src/dander/providers/no_catalog/__init__.py | 5 ++ src/dander/providers/no_catalog/config.py | 13 +++ src/dander/providers/no_catalog/runtime.py | 44 ++++++++++ tests/cli/test_catalog_cli.py | 9 +- tests/cli/test_hosted_tail.py | 6 +- tests/project/test_portable_config.py | 18 ++++ tests/providers/test_catalog_runtime.py | 82 +++++++++++++++++++ tickets/DANDER-86-bigquery-state-runtime.md | 6 +- tickets/DANDER-87-dataplex-catalog-runtime.md | 38 +++++++++ 26 files changed, 468 insertions(+), 62 deletions(-) create mode 100644 src/dander/catalog/runtime.py create mode 100644 src/dander/cli/provider_runtime.py create mode 100644 src/dander/providers/dataplex/__init__.py create mode 100644 src/dander/providers/dataplex/config.py create mode 100644 src/dander/providers/dataplex/runtime.py create mode 100644 src/dander/providers/no_catalog/__init__.py create mode 100644 src/dander/providers/no_catalog/config.py create mode 100644 src/dander/providers/no_catalog/runtime.py create mode 100644 tests/providers/test_catalog_runtime.py create mode 100644 tickets/DANDER-87-dataplex-catalog-runtime.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7528c0e4..ea48a5ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Select Dataplex or explicit no-catalog behavior through the lazy provider registry; retain + Dataplex aspect-only publication and readback semantics while avoiding its SDK and credentials + entirely when catalog publication is disabled. - Route BigQuery leases, watermarks, run history, and metadata snapshots through one lazily selected durable-state runtime with an explicit versioned migration ledger while preserving all existing table identities and correctness behavior. diff --git a/HANDOFF.md b/HANDOFF.md index b2c6a9c8..ce48f9ad 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Added a typed `StateRuntime` for leases, watermarks, run history, metadata, and migrations. -- Registered BigQuery state through a dependency-light config and lazy provider factory. -- Routed hosted state construction through the selected runtime; SQLite sandbox stays unchanged. -- Added schema ledger version 1 while preserving every existing BigQuery table identity. -- Kept server-time leases, fencing, cursor CAS, and interrupted-run reconciliation intact. +- Added a typed `CatalogRuntime` with explicit publication capabilities. +- Registered Dataplex and `none` through dependency-light, lazy provider factories. +- Routed all CLI Dataplex publisher construction through the selected provider boundary. +- Preserved current aspect-only updates, required-schema exclusion, and normalized readback. +- Kept local registry and state snapshots independent; `none` loads no Dataplex implementation. ## Try It -Run an existing v1 or v2 BigQuery project normally. `dander run` selects and migrates the BigQuery -state runtime internally; public commands and authored configuration are unchanged. +Run an existing v1 project or v2 `catalog.provider: dataplex` project normally. Select +`catalog.provider: none` to keep local metadata without external catalog publication. ## Checks -- All 907 tests, Ruff, formatting, and strict mypy across 212 files passed. +- All 911 tests, Ruff, formatting, and strict mypy across 221 files passed. - Wheel/sdist inspection, source-free installs, runtime-all assembly, and dependency audit passed. -- The non-root/read-only full runtime image passed conformance and its bundled-asset checks. -- Trivy and Gitleaks passed; all Terraform roots validated successfully. +- The non-root/read-only full runtime image passed conformance and bundled-asset checks. +- Trivy found no high/critical findings, Gitleaks found no leaks, and all Terraform roots validated. - Isolated GCP reported `No changes`; Salesforce and ServiceNow schedules remain paused. ## Decisions -- Preserve current BigQuery stores and table identities behind a small composed runtime. -- Record a migration only after every shared state table is ready. -- Keep per-pipeline lease-table creation lazy rather than inventing a data migration. +- Treat local registry/state snapshots separately from optional external catalog publication. +- Preserve Dataplex public imports lazily for compatibility. +- Defer canonical non-BigQuery catalog assets and Glue to their dedicated slices. ## Remaining -- Open and merge the focused durable-state PR after protected CI passes. -- Route Dataplex catalog publication through its provider boundary next. +- Open and merge the focused Dataplex provider PR after protected CI passes. +- Route GCP Secret Manager through the provider boundary next. ## Review First -- `src/dander/state/runtime.py` -- `src/dander/providers/bigquery/state.py` -- `src/dander/cli/run_command.py` +- `src/dander/catalog/runtime.py` +- `src/dander/providers/dataplex/runtime.py` +- `src/dander/cli/provider_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 2710741c..8f2ea319 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-08 — External catalog publication is selected independently + +- **Selection:** Version 1 retains implicit Dataplex behavior; version 2 carries `dataplex` or + `none` into the API-v1 catalog factory while explicit legacy CLI publication still selects + Dataplex. +- **Dataplex parity:** The existing first-party BigQuery entry naming, optional aspect-only updates, + unrelated-field preservation, required-schema exclusion, and normalized readback remain intact. +- **No catalog:** The selected runtime has no publisher and imports no Dataplex implementation; + local semantic manifests and durable metadata snapshots remain separate, unchanged stages. + ## 2026-08-08 — Durable state is composed without relocating existing data - **Selection:** Version 1 projects retain implicit BigQuery state; version 2 resolution carries diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index e5930b41..e1879e69 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -44,10 +44,12 @@ The current runtime continues to support only the GCP compatibility composition. factory contract provides one API-v1 registry across warehouse, state, catalog, secret, and launcher categories. Registration loads only a small configuration model; selecting and building a provider loads its implementation and SDK dependencies. BigQuery writer, transform, lease, -watermark, run-history, and metadata-store construction now use that boundary. State migrations -are explicit and versioned, but retain the existing BigQuery table names and semantics. Dataplex, -GCP secrets, and Cloud Run remain on their proven direct paths until their separate Phase 3 -changes. The registry is a construction contract, not a support claim. +watermark, run-history, metadata-store, and external catalog-publisher construction now use that +boundary. State migrations are explicit and versioned, but retain the existing BigQuery table +names and semantics. Dataplex keeps aspect-only updates and normalized readback; selecting `none` +loads no Dataplex implementation or credentials. GCP secrets and Cloud Run remain on their proven +direct paths until their separate Phase 3 changes. The registry is a construction contract, not a +support claim. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; diff --git a/src/dander/catalog/__init__.py b/src/dander/catalog/__init__.py index b5bc2754..26bc6680 100644 --- a/src/dander/catalog/__init__.py +++ b/src/dander/catalog/__init__.py @@ -1,13 +1,13 @@ -"""Catalog module: the metadata spine (Dataplex aspects + semantic/agent registry).""" +"""Catalog module: the metadata spine (cloud catalog + semantic/agent registry).""" -from dander.catalog.dataplex import ( - CatalogPublishError, - DataplexAspectGenerator, - DataplexCatalogPublisher, - GeneratedAspect, -) -from dander.catalog.publisher import CatalogPublisher +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING + +from dander.catalog.publisher import CatalogPublisher, CatalogPublishError from dander.catalog.registry import SemanticRegistryError, SemanticRegistryPublisher +from dander.catalog.runtime import CatalogCapabilities, CatalogRuntime from dander.catalog.spine import ( CatalogAsset, CatalogColumn, @@ -22,11 +22,32 @@ SqliteMetadataStore, ) +if TYPE_CHECKING: + from dander.catalog.dataplex import ( + DataplexAspectGenerator, + DataplexCatalogPublisher, + GeneratedAspect, + ) + +_DATAPLEX_EXPORTS = frozenset( + {"DataplexAspectGenerator", "DataplexCatalogPublisher", "GeneratedAspect"} +) + + +def __getattr__(name: str) -> object: + """Load legacy Dataplex exports only when a caller explicitly requests them.""" + if name in _DATAPLEX_EXPORTS: + return getattr(import_module("dander.catalog.dataplex"), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + __all__ = [ "CatalogAsset", "CatalogColumn", "CatalogPublishError", "CatalogPublisher", + "CatalogCapabilities", + "CatalogRuntime", "BigQueryMetadataStore", "DataplexAspectGenerator", "DataplexCatalogPublisher", diff --git a/src/dander/catalog/dataplex.py b/src/dander/catalog/dataplex.py index ed4686b7..2317a532 100644 --- a/src/dander/catalog/dataplex.py +++ b/src/dander/catalog/dataplex.py @@ -10,6 +10,8 @@ from google.api_core.exceptions import GoogleAPICallError from google.cloud import dataplex_v1 +from dander.catalog.publisher import CatalogPublishError + if TYPE_CHECKING: from dander.catalog.spine import CatalogAsset @@ -22,10 +24,6 @@ _BIGQUERY_MANAGED_REQUIRED_ASPECTS = frozenset({_SCHEMA}) -class CatalogPublishError(RuntimeError): - """Raised when a catalog request is invalid or cannot be completed.""" - - class _CatalogClient(Protocol): def modify_entry(self, request: dataplex_v1.ModifyEntryRequest) -> object: """Modify a first-party catalog entry through source-system permissions.""" diff --git a/src/dander/catalog/publisher.py b/src/dander/catalog/publisher.py index 101e3fa8..83a8f6ca 100644 --- a/src/dander/catalog/publisher.py +++ b/src/dander/catalog/publisher.py @@ -13,6 +13,10 @@ from dander.catalog.spine import CatalogAsset +class CatalogPublishError(RuntimeError): + """Raised when a catalog request is invalid or cannot be completed.""" + + class CatalogPublisher(Protocol): """Publishes generated metadata for a catalog asset.""" diff --git a/src/dander/catalog/runtime.py b/src/dander/catalog/runtime.py new file mode 100644 index 00000000..1a1eaf10 --- /dev/null +++ b/src/dander/catalog/runtime.py @@ -0,0 +1,38 @@ +"""Provider-neutral cloud-catalog runtime composition.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dander.catalog.publisher import CatalogPublisher + + +@dataclass(frozen=True, slots=True) +class CatalogCapabilities: + """Publication guarantees exposed by one cloud-catalog provider.""" + + provider_id: str + readback: bool + preserves_unrelated_fields: bool + first_party_entries: bool + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("catalog capabilities require a provider id") + + +@dataclass(frozen=True, slots=True) +class CatalogRuntime: + """A selected cloud catalog and its optional asset publisher.""" + + provider_id: str + publisher: CatalogPublisher | None + capabilities: CatalogCapabilities + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("catalog runtime requires a provider id") + if self.capabilities.provider_id != self.provider_id: + raise ValueError("catalog runtime and capabilities provider ids must match") diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index ed0a67e3..0e525b2a 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -36,7 +36,6 @@ from dander.catalog import ( BigQueryMetadataStore, CatalogPublishError, - DataplexCatalogPublisher, MetadataSnapshot, MetadataSpine, MetadataStore, @@ -51,6 +50,7 @@ execute_platform_bootstrap, resolve_platform_config, ) +from dander.cli.provider_runtime import build_catalog_publisher from dander.cli.run_command import ( RunOptions, build_auth, @@ -1389,7 +1389,8 @@ def _run_post_ingestion( if catalog_output is not None: SemanticRegistryPublisher().publish(manifest, catalog_output) if publish_dataplex: - publisher = DataplexCatalogPublisher( + publisher = build_catalog_publisher( + provider_id="dataplex", project=project, location=dataplex_location, ) @@ -1521,7 +1522,8 @@ def catalog( budget_name=budget_name, ) try: - publisher = DataplexCatalogPublisher( + publisher = build_catalog_publisher( + provider_id="dataplex", project=resolved_project, location=location, ) diff --git a/src/dander/cli/provider_runtime.py b/src/dander/cli/provider_runtime.py new file mode 100644 index 00000000..77f76acc --- /dev/null +++ b/src/dander/cli/provider_runtime.py @@ -0,0 +1,30 @@ +"""Small CLI composition helpers for selected provider runtimes.""" + +from click import ClickException + +from dander.catalog import CatalogPublisher, CatalogRuntime +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry + + +def build_catalog_publisher( + *, + provider_id: str, + project: str, + location: str, +) -> CatalogPublisher: + """Build the selected external catalog publisher or fail when it is disabled.""" + registry = default_provider_registry() + try: + config = registry.parse(ProviderKind.CATALOG, {"provider": provider_id}) + runtime = registry.build( + ProviderKind.CATALOG, + config, + context={"project": project, "location": location}, + ) + except ProviderFactoryError as error: + raise ClickException(str(error)) from error + if not isinstance(runtime, CatalogRuntime): + raise ClickException("Selected catalog provider returned an invalid runtime") + if runtime.publisher is None: + raise ClickException(f"Catalog provider {provider_id!r} does not publish assets") + return runtime.publisher diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 46f2c1ce..73b98e87 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -12,11 +12,11 @@ from dander.catalog import ( CatalogPublishError, - DataplexCatalogPublisher, MetadataStore, SemanticRegistryError, SqliteMetadataStore, ) +from dander.cli.provider_runtime import build_catalog_publisher from dander.core.config import Settings from dander.executor import PipelineExecutionResult, PipelineExecutor from dander.ingestion import Endpoint, Source, SourceConfig, load_source_config @@ -113,6 +113,7 @@ class _ResolvedRun: warehouse_provider: str warehouse_location: str state_provider: str + catalog_provider: str @dataclass(frozen=True, slots=True) @@ -167,6 +168,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_provider = "bigquery" warehouse_location = "US" state_provider = "bigquery" + catalog_provider = "dataplex" plugin_registry: ConnectorPluginRegistry | None = None if options.project_config.is_file(): @@ -192,6 +194,9 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_provider = manifest.warehouse_provider warehouse_location = manifest.platform.bigquery_location state_provider = manifest.state_provider + catalog_provider = ( + "dataplex" if options.publish_dataplex else manifest.catalog_provider + ) except (ConnectorPluginError, ProjectConfigError) as error: raise ClickException(str(error)) from error @@ -237,6 +242,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_provider=warehouse_provider, warehouse_location=warehouse_location, state_provider=state_provider, + catalog_provider=catalog_provider, ) @@ -299,7 +305,8 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec stores = _build_control_stores(options, resolved) dataplex_publisher = ( - DataplexCatalogPublisher( + build_catalog_publisher( + provider_id=resolved.catalog_provider, project=resolved.project, location=options.dataplex_location, ) diff --git a/src/dander/project/config.py b/src/dander/project/config.py index d5f93301..97226236 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -205,6 +205,7 @@ class DanderProject(BaseModel): deployment_name: str = Field(default="gcp_cloud_run", exclude=True) warehouse_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) state_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) + catalog_provider: Literal["dataplex", "none"] = Field(default="dataplex", exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index dbd2dbe2..3e871652 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -24,6 +24,8 @@ _load_yaml_mapping, ) from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig # noqa: TC001 +from dander.providers.dataplex import DataplexCatalogConfig # noqa: TC001 +from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 _NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") _PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") @@ -110,23 +112,7 @@ def validate_pipeline_ids( return values -class DataplexCatalogSpec(BaseModel): - """Dataplex catalog publication selection.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["dataplex"] - - -class NoCatalogSpec(BaseModel): - """Explicit cloud-catalog no-op selection.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["none"] - - -CatalogSpec = Annotated[DataplexCatalogSpec | NoCatalogSpec, Field(discriminator="provider")] +CatalogSpec = Annotated[DataplexCatalogConfig | NoCatalogConfig, Field(discriminator="provider")] class GcpSecretManagerSpec(BaseModel): @@ -376,6 +362,7 @@ def _resolve( deployment_name=selected_name, warehouse_provider=profile.warehouse.provider, state_provider=profile.state.provider, + catalog_provider=profile.catalog.provider, deployed_pipeline_ids=tuple(sorted(selected.pipelines)), ) @@ -464,6 +451,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: legacy.platform == migrated.platform and legacy.warehouse_provider == migrated.warehouse_provider and legacy.state_provider == migrated.state_provider + and legacy.catalog_provider == migrated.catalog_provider and legacy.plugins == migrated.plugins and legacy.pipelines == migrated.pipelines and legacy.terraform_pipelines() == migrated.terraform_pipelines() diff --git a/src/dander/providers/dataplex/__init__.py b/src/dander/providers/dataplex/__init__.py new file mode 100644 index 00000000..db6ca4cf --- /dev/null +++ b/src/dander/providers/dataplex/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light Dataplex provider configuration.""" + +from dander.providers.dataplex.config import DataplexCatalogConfig + +__all__ = ["DataplexCatalogConfig"] diff --git a/src/dander/providers/dataplex/config.py b/src/dander/providers/dataplex/config.py new file mode 100644 index 00000000..cd3e3d5a --- /dev/null +++ b/src/dander/providers/dataplex/config.py @@ -0,0 +1,13 @@ +"""Dependency-light Dataplex catalog configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class DataplexCatalogConfig(BaseModel): + """Select Dataplex catalog publication for one platform profile.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["dataplex"] diff --git a/src/dander/providers/dataplex/runtime.py b/src/dander/providers/dataplex/runtime.py new file mode 100644 index 00000000..9c948422 --- /dev/null +++ b/src/dander/providers/dataplex/runtime.py @@ -0,0 +1,57 @@ +"""Dataplex cloud-catalog runtime.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dander.catalog.dataplex import DataplexCatalogPublisher +from dander.catalog.runtime import CatalogCapabilities, CatalogRuntime +from dander.providers.dataplex.config import DataplexCatalogConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind + +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + + from pydantic import BaseModel + + +def build_dataplex_catalog( + config: BaseModel, + context: Mapping[str, object], +) -> CatalogRuntime: + """Build Dataplex only after its catalog provider is selected.""" + if not isinstance(config, DataplexCatalogConfig): + raise TypeError("Dataplex catalog factory received the wrong configuration") + project = context.get("project") + location = context.get("location") + if not isinstance(project, str) or not project: + raise ValueError("Dataplex catalog factory requires a project context") + if not isinstance(location, str) or not location: + raise ValueError("Dataplex catalog factory requires a location context") + client: Any = context.get("client") + publisher = DataplexCatalogPublisher( + project=project, + location=location, + client=client, + ) + return CatalogRuntime( + provider_id="dataplex", + publisher=publisher, + capabilities=CatalogCapabilities( + provider_id="dataplex", + readback=True, + preserves_unrelated_fields=True, + first_party_entries=True, + ), + ) + + +DATAPLEX_CATALOG_FACTORY: ProviderFactory[CatalogRuntime] = ProviderFactory( + kind=ProviderKind.CATALOG, + provider_id="dataplex", + api_version=PROVIDER_API_VERSION, + build=build_dataplex_catalog, +) + +__all__ = ["DATAPLEX_CATALOG_FACTORY"] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 01a9853b..3523e102 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -1,6 +1,8 @@ """Built-in provider registrations with lazy implementation loading.""" from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig +from dander.providers.dataplex import DataplexCatalogConfig +from dander.providers.no_catalog import NoCatalogConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -23,4 +25,20 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.bigquery.state:BIGQUERY_STATE_FACTORY" ), ) + registry.register( + kind=ProviderKind.CATALOG, + provider_id="dataplex", + config_model=DataplexCatalogConfig, + load_factory=lazy_provider_factory( + "dander.providers.dataplex.runtime:DATAPLEX_CATALOG_FACTORY" + ), + ) + registry.register( + kind=ProviderKind.CATALOG, + provider_id="none", + config_model=NoCatalogConfig, + load_factory=lazy_provider_factory( + "dander.providers.no_catalog.runtime:NO_CATALOG_FACTORY" + ), + ) return registry diff --git a/src/dander/providers/no_catalog/__init__.py b/src/dander/providers/no_catalog/__init__.py new file mode 100644 index 00000000..49aa5b46 --- /dev/null +++ b/src/dander/providers/no_catalog/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light disabled-catalog provider configuration.""" + +from dander.providers.no_catalog.config import NoCatalogConfig + +__all__ = ["NoCatalogConfig"] diff --git a/src/dander/providers/no_catalog/config.py b/src/dander/providers/no_catalog/config.py new file mode 100644 index 00000000..7654b54e --- /dev/null +++ b/src/dander/providers/no_catalog/config.py @@ -0,0 +1,13 @@ +"""Explicit disabled-catalog configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class NoCatalogConfig(BaseModel): + """Select no external cloud-catalog publication.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["none"] diff --git a/src/dander/providers/no_catalog/runtime.py b/src/dander/providers/no_catalog/runtime.py new file mode 100644 index 00000000..a3dd1130 --- /dev/null +++ b/src/dander/providers/no_catalog/runtime.py @@ -0,0 +1,44 @@ +"""Explicit no-op cloud-catalog runtime.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dander.catalog.runtime import CatalogCapabilities, CatalogRuntime +from dander.providers.no_catalog.config import NoCatalogConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind + +if TYPE_CHECKING: + from collections.abc import Mapping + + from pydantic import BaseModel + + +def build_no_catalog( + config: BaseModel, + context: Mapping[str, object], +) -> CatalogRuntime: + """Build an explicit runtime that performs no external publication.""" + del context + if not isinstance(config, NoCatalogConfig): + raise TypeError("No-catalog factory received the wrong configuration") + return CatalogRuntime( + provider_id="none", + publisher=None, + capabilities=CatalogCapabilities( + provider_id="none", + readback=False, + preserves_unrelated_fields=True, + first_party_entries=False, + ), + ) + + +NO_CATALOG_FACTORY: ProviderFactory[CatalogRuntime] = ProviderFactory( + kind=ProviderKind.CATALOG, + provider_id="none", + api_version=PROVIDER_API_VERSION, + build=build_no_catalog, +) + +__all__ = ["NO_CATALOG_FACTORY"] diff --git a/tests/cli/test_catalog_cli.py b/tests/cli/test_catalog_cli.py index 0991571d..ab96f4f3 100644 --- a/tests/cli/test_catalog_cli.py +++ b/tests/cli/test_catalog_cli.py @@ -30,6 +30,11 @@ def publish(self, asset: CatalogAsset) -> str: return f"entries/{asset.name}" +def _build_publisher(*, provider_id: str, project: str, location: str) -> _FakeDataplexPublisher: + assert provider_id == "dataplex" + return _FakeDataplexPublisher(project=project, location=location) + + def test_catalog_compiles_locally_without_cloud_publication(tmp_path: Path) -> None: output = tmp_path / "catalog.json" @@ -62,8 +67,8 @@ def test_catalog_only_publishes_with_explicit_flag( ) -> None: _FakeDataplexPublisher.published.clear() monkeypatch.setattr( - "dander.cli.main.DataplexCatalogPublisher", - _FakeDataplexPublisher, + "dander.cli.main.build_catalog_publisher", + _build_publisher, ) result = CliRunner().invoke( diff --git a/tests/cli/test_hosted_tail.py b/tests/cli/test_hosted_tail.py index ab879e32..30aa4db8 100644 --- a/tests/cli/test_hosted_tail.py +++ b/tests/cli/test_hosted_tail.py @@ -70,11 +70,15 @@ def publish(self, asset: object) -> str: events.append("dataplex") return "published" + def build_publisher(*, provider_id: str, project: str, location: str) -> _Dataplex: + assert provider_id == "dataplex" + return _Dataplex(project=project, location=location) + monkeypatch.setattr("dander.cli.main.BigQueryTransformRunner", _Runner) monkeypatch.setattr("dander.cli.main.TransformProject", _Project) monkeypatch.setattr("dander.cli.main.MetadataSpine", _Spine) monkeypatch.setattr("dander.cli.main.SemanticRegistryPublisher", _Registry) - monkeypatch.setattr("dander.cli.main.DataplexCatalogPublisher", _Dataplex) + monkeypatch.setattr("dander.cli.main.build_catalog_publisher", build_publisher) _run_post_ingestion( project="unit-project", diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index f76e900f..25481843 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -70,6 +70,8 @@ def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( assert migrated.warehouse_provider == "bigquery" assert original.state_provider == "bigquery" assert migrated.state_provider == "bigquery" + assert original.catalog_provider == "dataplex" + assert migrated.catalog_provider == "dataplex" assert migrated.platform == original.platform assert migrated.plugins == original.plugins assert migrated.pipelines == original.pipelines @@ -128,6 +130,22 @@ def test_version_two_rejects_unknown_provider_and_unknown_pipeline(tmp_path: Pat load_project_config(project_path) +def test_version_two_none_catalog_disables_external_publication(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["platforms"]["gcp"]["catalog"]["provider"] = "none" + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path) + + assert resolved.catalog_provider == "none" + assert resolved.pipelines["example_records"].publish_dataplex is False + + def test_config_migrate_check_is_read_only_then_write_is_atomic(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_catalog_runtime.py b/tests/providers/test_catalog_runtime.py new file mode 100644 index 00000000..0bea1e63 --- /dev/null +++ b/tests/providers/test_catalog_runtime.py @@ -0,0 +1,82 @@ +"""Cloud-catalog provider selection and composition coverage.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +import pytest +from click import ClickException + +from dander.catalog import CatalogRuntime +from dander.cli.provider_runtime import build_catalog_publisher +from dander.providers import ProviderKind, default_provider_registry + +if TYPE_CHECKING: + from google.cloud import dataplex_v1 + + +class _Client: + def modify_entry(self, request: dataplex_v1.ModifyEntryRequest) -> object: + del request + return object() + + def get_entry(self, request: dataplex_v1.GetEntryRequest) -> dataplex_v1.Entry: + del request + raise AssertionError("readback is not used by this composition test") + + +def test_default_registry_loads_dataplex_only_after_selection() -> None: + module_name = "dander.providers.dataplex.runtime" + implementation_module = "dander.catalog.dataplex" + sys.modules.pop(module_name, None) + sys.modules.pop(implementation_module, None) + registry = default_provider_registry() + + config = registry.parse(ProviderKind.CATALOG, {"provider": "dataplex"}) + + assert module_name not in sys.modules + assert implementation_module not in sys.modules + runtime = registry.build( + ProviderKind.CATALOG, + config, + context={"project": "unit-project", "location": "us", "client": _Client()}, + ) + assert isinstance(runtime, CatalogRuntime) + assert module_name in sys.modules + assert implementation_module in sys.modules + assert runtime.publisher is not None + assert type(runtime.publisher).__name__ == "DataplexCatalogPublisher" + assert runtime.capabilities.readback is True + assert runtime.capabilities.preserves_unrelated_fields is True + assert runtime.capabilities.first_party_entries is True + + +def test_no_catalog_runtime_has_no_publisher_or_dataplex_import() -> None: + dataplex_module = "dander.providers.dataplex.runtime" + implementation_module = "dander.catalog.dataplex" + no_catalog_module = "dander.providers.no_catalog.runtime" + sys.modules.pop(dataplex_module, None) + sys.modules.pop(implementation_module, None) + sys.modules.pop(no_catalog_module, None) + registry = default_provider_registry() + config = registry.parse(ProviderKind.CATALOG, {"provider": "none"}) + + runtime = registry.build(ProviderKind.CATALOG, config) + + assert isinstance(runtime, CatalogRuntime) + assert runtime.provider_id == "none" + assert runtime.publisher is None + assert runtime.capabilities.readback is False + assert no_catalog_module in sys.modules + assert dataplex_module not in sys.modules + assert implementation_module not in sys.modules + + +def test_cli_rejects_publication_when_catalog_is_disabled() -> None: + with pytest.raises(ClickException, match="does not publish assets"): + build_catalog_publisher( + provider_id="none", + project="unit-project", + location="us", + ) diff --git a/tickets/DANDER-86-bigquery-state-runtime.md b/tickets/DANDER-86-bigquery-state-runtime.md index e6f5a1f9..18978b91 100644 --- a/tickets/DANDER-86-bigquery-state-runtime.md +++ b/tickets/DANDER-86-bigquery-state-runtime.md @@ -1,7 +1,7 @@ --- id: DANDER-86 title: Route BigQuery durable state through provider capabilities -status: in-review +status: completed component: state epic: cloud-portability depends_on: [DANDER-85] @@ -27,7 +27,7 @@ contract. - [x] Hosted CLI composition uses the selected runtime; SQLite sandbox behavior remains unchanged. - [x] Focused state, project migration, and CLI composition tests pass. - [x] Full local validation and isolated GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -38,4 +38,4 @@ no portability benefit in this slice. ## Review Log -Pending protected review. +Merged through protected PR #117 as `825472095efd1a8d5127ac3db16bdc3f2dbfdb2a`. diff --git a/tickets/DANDER-87-dataplex-catalog-runtime.md b/tickets/DANDER-87-dataplex-catalog-runtime.md new file mode 100644 index 00000000..6d178213 --- /dev/null +++ b/tickets/DANDER-87-dataplex-catalog-runtime.md @@ -0,0 +1,38 @@ +--- +id: DANDER-87 +title: Route Dataplex publication through catalog capabilities +status: in-review +component: catalog +epic: cloud-portability +depends_on: [DANDER-86] +created: 2026-08-08 +--- + +## Context + +Named profiles declared Dataplex or no cloud catalog, but every CLI publication path still +constructed Dataplex directly and importing the catalog package loaded its Google SDK. Catalog +selection must become independent without changing the accepted metadata spine or GCP behavior. + +## Acceptance Criteria + +- [x] Version 1 retains implicit Dataplex selection and version 2 carries `dataplex` or `none`. +- [x] Dataplex and no-catalog runtimes use the existing lazy API-v1 provider registry. +- [x] Dataplex implementation and SDK modules load only after Dataplex is selected. +- [x] Every CLI Dataplex publication path constructs its publisher through the provider boundary. +- [x] Existing aspect generation, publication, unrelated-field preservation, and readback pass. +- [x] No-catalog selection has no publisher and performs no external publication. +- [x] Local semantic-registry and durable metadata-snapshot behavior remains unchanged. +- [x] Focused provider, Dataplex, CLI, and project-resolution tests pass. +- [x] Full local validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Compose one small `CatalogRuntime` containing an optional publisher and explicit capabilities. +Keep `CatalogPublishError` provider-neutral and lazily expose legacy Dataplex imports. This is a +construction change only; canonical non-BigQuery catalog assets and Glue remain later tickets. + +## Review Log + +Pending protected review. From 7f47fd9f4c36b1af36e1af3259bf25ba58c4e33f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 13:30:22 -0400 Subject: [PATCH 037/163] Route GCP secrets through provider runtime --- CHANGELOG.md | 3 + HANDOFF.md | 34 ++++---- docs/decisions.md | 10 +++ docs/platform-profiles.md | 12 +-- src/dander/cli/main.py | 7 +- src/dander/cli/provider_runtime.py | 15 ++++ src/dander/cli/run_command.py | 14 ++-- src/dander/project/config.py | 4 + src/dander/project/portable_config.py | 30 +++---- src/dander/providers/defaults.py | 18 ++++ .../providers/environment_secrets/__init__.py | 5 ++ .../providers/environment_secrets/config.py | 13 +++ .../providers/environment_secrets/runtime.py | 48 +++++++++++ .../providers/gcp_secret_manager/__init__.py | 5 ++ .../providers/gcp_secret_manager/config.py | 13 +++ .../providers/gcp_secret_manager/runtime.py | 57 +++++++++++++ src/dander/security/__init__.py | 3 + src/dander/security/runtime.py | 40 +++++++++ src/dander/security/secret_manager.py | 11 ++- tests/cli/test_run_command.py | 6 +- tests/project/test_portable_config.py | 16 ++++ tests/providers/test_secret_runtime.py | 83 +++++++++++++++++++ tickets/DANDER-87-dataplex-catalog-runtime.md | 6 +- tickets/DANDER-88-gcp-secret-runtime.md | 40 +++++++++ 24 files changed, 437 insertions(+), 56 deletions(-) create mode 100644 src/dander/providers/environment_secrets/__init__.py create mode 100644 src/dander/providers/environment_secrets/config.py create mode 100644 src/dander/providers/environment_secrets/runtime.py create mode 100644 src/dander/providers/gcp_secret_manager/__init__.py create mode 100644 src/dander/providers/gcp_secret_manager/config.py create mode 100644 src/dander/providers/gcp_secret_manager/runtime.py create mode 100644 src/dander/security/runtime.py create mode 100644 tests/providers/test_secret_runtime.py create mode 100644 tickets/DANDER-88-gcp-secret-runtime.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ea48a5ff..befb86ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Select GCP Secret Manager and local environment resolvers through the lazy provider registry; + preserve hosted environment-to-resource-name indirection while preventing environment-only + secrets from being selected for Cloud Run. - Select Dataplex or explicit no-catalog behavior through the lazy provider registry; retain Dataplex aspect-only publication and readback semantics while avoiding its SDK and credentials entirely when catalog publication is disabled. diff --git a/HANDOFF.md b/HANDOFF.md index ce48f9ad..a5425acd 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Added a typed `CatalogRuntime` with explicit publication capabilities. -- Registered Dataplex and `none` through dependency-light, lazy provider factories. -- Routed all CLI Dataplex publisher construction through the selected provider boundary. -- Preserved current aspect-only updates, required-schema exclusion, and normalized readback. -- Kept local registry and state snapshots independent; `none` loads no Dataplex implementation. +- Added a typed `SecretRuntime` with explicit resolution capabilities. +- Registered GCP Secret Manager and environment stores through lazy provider factories. +- Routed hosted, sandbox, and connector-capability secret construction through provider selection. +- Preserved GCP resource names, environment indirection, auditing, IAM, and Terraform behavior. +- Kept environment-only secrets unavailable to the Cloud Run compatibility profile. ## Try It -Run an existing v1 project or v2 `catalog.provider: dataplex` project normally. Select -`catalog.provider: none` to keep local metadata without external catalog publication. +Run an existing v1 project or v2 GCP profile normally. Sandbox runs select environment variables; +hosted Cloud Run runs retain GCP Secret Manager through the selected provider runtime. ## Checks -- All 911 tests, Ruff, formatting, and strict mypy across 221 files passed. +- All 915 tests, Ruff, formatting, and strict mypy across 229 files passed. - Wheel/sdist inspection, source-free installs, runtime-all assembly, and dependency audit passed. - The non-root/read-only full runtime image passed conformance and bundled-asset checks. -- Trivy found no high/critical findings, Gitleaks found no leaks, and all Terraform roots validated. +- Trivy found no high/critical findings, Gitleaks found no leaks, and every Terraform root validated. - Isolated GCP reported `No changes`; Salesforce and ServiceNow schedules remain paused. ## Decisions -- Treat local registry/state snapshots separately from optional external catalog publication. -- Preserve Dataplex public imports lazily for compatibility. -- Defer canonical non-BigQuery catalog assets and Glue to their dedicated slices. +- Keep the existing `DefaultSecretStore` compatibility behavior behind the GCP factory. +- Load the Google SDK only when a real GCP client is first needed. +- Defer AWS secret resolution and explicit URI parsing to the Fargate slice. ## Remaining -- Open and merge the focused Dataplex provider PR after protected CI passes. -- Route GCP Secret Manager through the provider boundary next. +- Open and merge the focused GCP secret-provider PR after protected CI passes. +- Route Cloud Run through the launcher provider boundary next. ## Review First -- `src/dander/catalog/runtime.py` -- `src/dander/providers/dataplex/runtime.py` -- `src/dander/cli/provider_runtime.py` +- `src/dander/security/runtime.py` +- `src/dander/providers/gcp_secret_manager/runtime.py` +- `src/dander/cli/run_command.py` diff --git a/docs/decisions.md b/docs/decisions.md index 8f2ea319..93367275 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-08 — Secret resolution uses an explicit provider runtime + +- **Compatibility:** Version 1 and the Cloud Run profile retain GCP Secret Manager plus the + existing environment-variable indirection; secret names, resource paths, and audit events do + not change. +- **Selection:** Hosted and connector-capability paths build the manifest-selected API-v1 secret + runtime; sandbox execution explicitly selects the environment provider. +- **Boundary:** Environment-only secrets remain limited to local or future operator-managed + Kubernetes execution, and the Google Secret Manager SDK loads only on actual GCP access. + ## 2026-08-08 — External catalog publication is selected independently - **Selection:** Version 1 retains implicit Dataplex behavior; version 2 carries `dataplex` or diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index e1879e69..392401b9 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -44,11 +44,13 @@ The current runtime continues to support only the GCP compatibility composition. factory contract provides one API-v1 registry across warehouse, state, catalog, secret, and launcher categories. Registration loads only a small configuration model; selecting and building a provider loads its implementation and SDK dependencies. BigQuery writer, transform, lease, -watermark, run-history, metadata-store, and external catalog-publisher construction now use that -boundary. State migrations are explicit and versioned, but retain the existing BigQuery table -names and semantics. Dataplex keeps aspect-only updates and normalized readback; selecting `none` -loads no Dataplex implementation or credentials. GCP secrets and Cloud Run remain on their proven -direct paths until their separate Phase 3 changes. The registry is a construction contract, not a +watermark, run-history, metadata-store, external catalog-publisher, and secret-store construction +now use that boundary. State migrations are explicit and versioned, but retain the existing +BigQuery table names and semantics. Dataplex keeps aspect-only updates and normalized readback; +selecting `none` loads no Dataplex implementation or credentials. GCP Secret Manager preserves its +existing environment indirection and audit behavior; environment-only resolution remains local +because Cloud Run requires managed secret references. Cloud Run itself remains on its proven +direct path until its separate Phase 3 change. The registry is a construction contract, not a support claim. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 0e525b2a..c4f2c5f8 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -50,7 +50,7 @@ execute_platform_bootstrap, resolve_platform_config, ) -from dander.cli.provider_runtime import build_catalog_publisher +from dander.cli.provider_runtime import build_catalog_publisher, build_secret_store from dander.cli.run_command import ( RunOptions, build_auth, @@ -95,7 +95,6 @@ ) from dander.sandbox import GuardedFreeTierVerifier, SandboxSafetyError from dander.security import ( - DefaultSecretStore, OAuthTokenError, SecretResolutionError, ) @@ -1802,10 +1801,12 @@ def _load_connector_capabilities( ) -> tuple[SourceConfig, SourceCapabilities]: """Resolve one configured source and inspect its optional operations without provider I/O.""" source = source_or_pipeline + secret_provider = "gcp_secret_manager" try: registry = load_connector_plugins({}) if project_config.is_file(): manifest = load_project_config(project_config) + secret_provider = manifest.secret_provider registry = load_connector_plugins(manifest.plugins) pipeline = manifest.pipelines.get(source_or_pipeline) if pipeline is not None: @@ -1819,7 +1820,7 @@ def _load_connector_capabilities( raise ConnectorConfigError( f"Connector file declares source {config.name!r}, expected {source!r}" ) - auth = _build_auth(config, DefaultSecretStore()) + auth = _build_auth(config, build_secret_store(secret_provider)) return config, registry.build_capabilities(config, auth) except (ConnectorConfigError, ConnectorPluginError, ProjectConfigError) as error: raise ClickException(str(error)) from error diff --git a/src/dander/cli/provider_runtime.py b/src/dander/cli/provider_runtime.py index 77f76acc..dbd05cc2 100644 --- a/src/dander/cli/provider_runtime.py +++ b/src/dander/cli/provider_runtime.py @@ -3,7 +3,9 @@ from click import ClickException from dander.catalog import CatalogPublisher, CatalogRuntime +from dander.core.interfaces import SecretStoreProvider from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry +from dander.security import SecretRuntime def build_catalog_publisher( @@ -28,3 +30,16 @@ def build_catalog_publisher( if runtime.publisher is None: raise ClickException(f"Catalog provider {provider_id!r} does not publish assets") return runtime.publisher + + +def build_secret_store(provider_id: str) -> SecretStoreProvider: + """Build the selected secret resolver through the shared provider registry.""" + registry = default_provider_registry() + try: + config = registry.parse(ProviderKind.SECRETS, {"provider": provider_id}) + runtime = registry.build(ProviderKind.SECRETS, config) + except ProviderFactoryError as error: + raise ClickException(str(error)) from error + if not isinstance(runtime, SecretRuntime): + raise ClickException("Selected secret provider returned an invalid runtime") + return runtime.store diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 73b98e87..5c7d2f96 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -16,7 +16,7 @@ SemanticRegistryError, SqliteMetadataStore, ) -from dander.cli.provider_runtime import build_catalog_publisher +from dander.cli.provider_runtime import build_catalog_publisher, build_secret_store from dander.core.config import Settings from dander.executor import PipelineExecutionResult, PipelineExecutor from dander.ingestion import Endpoint, Source, SourceConfig, load_source_config @@ -44,8 +44,6 @@ ApiKeyBearer, AuthStrategy, ClientCredentialPlacement, - DefaultSecretStore, - EnvironmentSecretStore, NoAuth, OAuth1TBA, OAuth2ClientCredentials, @@ -70,6 +68,8 @@ from rich.console import Console + from dander.core.interfaces import SecretStoreProvider + _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") @@ -114,6 +114,7 @@ class _ResolvedRun: warehouse_location: str state_provider: str catalog_provider: str + secret_provider: str @dataclass(frozen=True, slots=True) @@ -169,6 +170,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_location = "US" state_provider = "bigquery" catalog_provider = "dataplex" + secret_provider = "gcp_secret_manager" plugin_registry: ConnectorPluginRegistry | None = None if options.project_config.is_file(): @@ -197,6 +199,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: catalog_provider = ( "dataplex" if options.publish_dataplex else manifest.catalog_provider ) + secret_provider = manifest.secret_provider except (ConnectorPluginError, ProjectConfigError) as error: raise ClickException(str(error)) from error @@ -243,6 +246,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_location=warehouse_location, state_provider=state_provider, catalog_provider=catalog_provider, + secret_provider=secret_provider, ) @@ -292,7 +296,7 @@ def _verify_safety(options: RunOptions, resolved: _ResolvedRun) -> None: def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExecutor: - secrets = EnvironmentSecretStore() if options.sandbox else DefaultSecretStore() + secrets = build_secret_store("environment" if options.sandbox else resolved.secret_provider) auth = build_auth(resolved.source_config, secrets) try: source_adapter = build_source_adapter( @@ -455,7 +459,7 @@ def build_source_adapter( def build_auth( config: SourceConfig, - secrets: DefaultSecretStore | EnvironmentSecretStore, + secrets: SecretStoreProvider, ) -> AuthStrategy: """Construct a supported authentication strategy from connector metadata.""" if config.auth_strategy == "none": diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 97226236..6832012f 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -206,6 +206,10 @@ class DanderProject(BaseModel): warehouse_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) state_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) catalog_provider: Literal["dataplex", "none"] = Field(default="dataplex", exclude=True) + secret_provider: Literal["gcp_secret_manager", "environment"] = Field( + default="gcp_secret_manager", + exclude=True, + ) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 3e871652..35af9169 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -25,6 +25,8 @@ ) from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig # noqa: TC001 from dander.providers.dataplex import DataplexCatalogConfig # noqa: TC001 +from dander.providers.environment_secrets import EnvironmentSecretConfig # noqa: TC001 +from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 _NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") @@ -115,24 +117,8 @@ def validate_pipeline_ids( CatalogSpec = Annotated[DataplexCatalogConfig | NoCatalogConfig, Field(discriminator="provider")] -class GcpSecretManagerSpec(BaseModel): - """GCP Secret Manager runtime resolution selection.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["gcp_secret_manager"] - - -class EnvironmentSecretSpec(BaseModel): - """Environment-only runtime secret resolution selection.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["environment"] - - SecretProviderSpec = Annotated[ - GcpSecretManagerSpec | EnvironmentSecretSpec, + GcpSecretManagerConfig | EnvironmentSecretConfig, Field(discriminator="provider"), ] @@ -322,9 +308,13 @@ def _resolve( selected_name = _select_deployment(platforms, deployment) selected = platforms.deployments[selected_name] profile = platforms.platforms[selected.platform] - if profile.secrets.provider != "gcp_secret_manager": + if ( + selected.launcher.provider == "cloud_run" + and profile.secrets.provider != "gcp_secret_manager" + ): raise ProjectConfigError( - "The GCP compatibility runtime currently requires secrets.provider='gcp_secret_manager'" + "Cloud Run requires secrets.provider='gcp_secret_manager'; " + "environment secrets are local or operator-managed Kubernetes only" ) unknown_pipelines = sorted(set(selected.pipelines) - set(logical.pipelines)) if unknown_pipelines: @@ -363,6 +353,7 @@ def _resolve( warehouse_provider=profile.warehouse.provider, state_provider=profile.state.provider, catalog_provider=profile.catalog.provider, + secret_provider=profile.secrets.provider, deployed_pipeline_ids=tuple(sorted(selected.pipelines)), ) @@ -452,6 +443,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: and legacy.warehouse_provider == migrated.warehouse_provider and legacy.state_provider == migrated.state_provider and legacy.catalog_provider == migrated.catalog_provider + and legacy.secret_provider == migrated.secret_provider and legacy.plugins == migrated.plugins and legacy.pipelines == migrated.pipelines and legacy.terraform_pipelines() == migrated.terraform_pipelines() diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 3523e102..c68240bb 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -2,6 +2,8 @@ from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig from dander.providers.dataplex import DataplexCatalogConfig +from dander.providers.environment_secrets import EnvironmentSecretConfig +from dander.providers.gcp_secret_manager import GcpSecretManagerConfig from dander.providers.no_catalog import NoCatalogConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -41,4 +43,20 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.no_catalog.runtime:NO_CATALOG_FACTORY" ), ) + registry.register( + kind=ProviderKind.SECRETS, + provider_id="gcp_secret_manager", + config_model=GcpSecretManagerConfig, + load_factory=lazy_provider_factory( + "dander.providers.gcp_secret_manager.runtime:GCP_SECRET_MANAGER_FACTORY" + ), + ) + registry.register( + kind=ProviderKind.SECRETS, + provider_id="environment", + config_model=EnvironmentSecretConfig, + load_factory=lazy_provider_factory( + "dander.providers.environment_secrets.runtime:ENVIRONMENT_SECRET_FACTORY" + ), + ) return registry diff --git a/src/dander/providers/environment_secrets/__init__.py b/src/dander/providers/environment_secrets/__init__.py new file mode 100644 index 00000000..081edd00 --- /dev/null +++ b/src/dander/providers/environment_secrets/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light environment secret-provider configuration.""" + +from dander.providers.environment_secrets.config import EnvironmentSecretConfig + +__all__ = ["EnvironmentSecretConfig"] diff --git a/src/dander/providers/environment_secrets/config.py b/src/dander/providers/environment_secrets/config.py new file mode 100644 index 00000000..328a1389 --- /dev/null +++ b/src/dander/providers/environment_secrets/config.py @@ -0,0 +1,13 @@ +"""Dependency-light environment secret-provider configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class EnvironmentSecretConfig(BaseModel): + """Select direct environment-variable secret resolution.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["environment"] diff --git a/src/dander/providers/environment_secrets/runtime.py b/src/dander/providers/environment_secrets/runtime.py new file mode 100644 index 00000000..6a6b60b7 --- /dev/null +++ b/src/dander/providers/environment_secrets/runtime.py @@ -0,0 +1,48 @@ +"""Environment-only secret runtime selected through the provider registry.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from dander.providers.environment_secrets.config import EnvironmentSecretConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.security.runtime import SecretCapabilities, SecretRuntime +from dander.security.secret_manager import EnvironmentSecretStore + +if TYPE_CHECKING: + from collections.abc import Mapping + + from pydantic import BaseModel + + +def build_environment_secrets( + config: BaseModel, + context: Mapping[str, object], +) -> SecretRuntime: + """Build direct environment resolution without loading a cloud SDK.""" + if not isinstance(config, EnvironmentSecretConfig): + raise TypeError("Environment secret factory received the wrong configuration") + environment_value = context.get("environment") + store = EnvironmentSecretStore( + cast("Mapping[str, str]", environment_value) if environment_value is not None else None + ) + return SecretRuntime( + provider_id="environment", + store=store, + capabilities=SecretCapabilities( + provider_id="environment", + reference_forms=frozenset({"environment_name"}), + environment_indirection=False, + audited_access=True, + ), + ) + + +ENVIRONMENT_SECRET_FACTORY: ProviderFactory[SecretRuntime] = ProviderFactory( + kind=ProviderKind.SECRETS, + provider_id="environment", + api_version=PROVIDER_API_VERSION, + build=build_environment_secrets, +) + +__all__ = ["ENVIRONMENT_SECRET_FACTORY"] diff --git a/src/dander/providers/gcp_secret_manager/__init__.py b/src/dander/providers/gcp_secret_manager/__init__.py new file mode 100644 index 00000000..0cf78da5 --- /dev/null +++ b/src/dander/providers/gcp_secret_manager/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light GCP Secret Manager provider configuration.""" + +from dander.providers.gcp_secret_manager.config import GcpSecretManagerConfig + +__all__ = ["GcpSecretManagerConfig"] diff --git a/src/dander/providers/gcp_secret_manager/config.py b/src/dander/providers/gcp_secret_manager/config.py new file mode 100644 index 00000000..54e74cdd --- /dev/null +++ b/src/dander/providers/gcp_secret_manager/config.py @@ -0,0 +1,13 @@ +"""Dependency-light GCP Secret Manager configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class GcpSecretManagerConfig(BaseModel): + """Select GCP Secret Manager with legacy environment indirection.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["gcp_secret_manager"] diff --git a/src/dander/providers/gcp_secret_manager/runtime.py b/src/dander/providers/gcp_secret_manager/runtime.py new file mode 100644 index 00000000..b6b2c077 --- /dev/null +++ b/src/dander/providers/gcp_secret_manager/runtime.py @@ -0,0 +1,57 @@ +"""GCP Secret Manager runtime selected through the provider registry.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from dander.providers.gcp_secret_manager.config import GcpSecretManagerConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.security.runtime import SecretCapabilities, SecretRuntime +from dander.security.secret_manager import ( + DefaultSecretStore, + EnvironmentSecretStore, + GcpSecretStore, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + + from pydantic import BaseModel + + +def build_gcp_secret_manager( + config: BaseModel, + context: Mapping[str, object], +) -> SecretRuntime: + """Build the compatibility resolver without eagerly constructing a Google client.""" + if not isinstance(config, GcpSecretManagerConfig): + raise TypeError("GCP Secret Manager factory received the wrong configuration") + environment_value = context.get("environment") + client = context.get("client") + environment = ( + EnvironmentSecretStore(cast("Mapping[str, str]", environment_value)) + if environment_value is not None + else None + ) + gcp = GcpSecretStore(cast("Any", client)) if client is not None else None + return SecretRuntime( + provider_id="gcp_secret_manager", + store=DefaultSecretStore(environment=environment, gcp=gcp), + capabilities=SecretCapabilities( + provider_id="gcp_secret_manager", + reference_forms=frozenset({"environment_name", "gcp_resource_name"}), + environment_indirection=True, + audited_access=True, + ), + ) + + +GCP_SECRET_MANAGER_FACTORY: ProviderFactory[SecretRuntime] = ProviderFactory( + kind=ProviderKind.SECRETS, + provider_id="gcp_secret_manager", + api_version=PROVIDER_API_VERSION, + build=build_gcp_secret_manager, +) + +__all__ = ["GCP_SECRET_MANAGER_FACTORY"] diff --git a/src/dander/security/__init__.py b/src/dander/security/__init__.py index a1fac690..dfbd281c 100644 --- a/src/dander/security/__init__.py +++ b/src/dander/security/__init__.py @@ -11,6 +11,7 @@ ) from dander.security.oauth1 import OAuth1TBA from dander.security.oauth_jwt import OAuth2JWT +from dander.security.runtime import SecretCapabilities, SecretRuntime from dander.security.secret_manager import ( DefaultSecretStore, EnvironmentSecretStore, @@ -32,4 +33,6 @@ "OAuth1TBA", "OAuthTokenError", "SecretResolutionError", + "SecretCapabilities", + "SecretRuntime", ] diff --git a/src/dander/security/runtime.py b/src/dander/security/runtime.py new file mode 100644 index 00000000..7f4c24d4 --- /dev/null +++ b/src/dander/security/runtime.py @@ -0,0 +1,40 @@ +"""Provider-neutral secret-resolution runtime composition.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from dander.core.interfaces import SecretStoreProvider + + +@dataclass(frozen=True, slots=True) +class SecretCapabilities: + """Resolution behavior exposed by one secret provider.""" + + provider_id: str + reference_forms: frozenset[str] + environment_indirection: bool + audited_access: bool + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("secret capabilities require a provider id") + if not self.reference_forms: + raise ValueError("secret capabilities require at least one reference form") + + +@dataclass(frozen=True, slots=True) +class SecretRuntime: + """A selected secret store and its declared behavior.""" + + provider_id: str + store: SecretStoreProvider + capabilities: SecretCapabilities + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("secret runtime requires a provider id") + if self.capabilities.provider_id != self.provider_id: + raise ValueError("secret runtime and capabilities provider ids must match") + if not isinstance(self.store, SecretStoreProvider): + raise TypeError("secret runtime store must implement SecretStoreProvider") diff --git a/src/dander/security/secret_manager.py b/src/dander/security/secret_manager.py index a774d9d2..a05dfe84 100644 --- a/src/dander/security/secret_manager.py +++ b/src/dander/security/secret_manager.py @@ -10,12 +10,12 @@ import logging import os +from importlib import import_module from typing import TYPE_CHECKING, Protocol, cast -from google.cloud import secretmanager - if TYPE_CHECKING: from collections.abc import Mapping + from typing import Any _LOGGER = logging.getLogger(__name__) _GCP_REFERENCE_PREFIX = "projects/" @@ -55,7 +55,12 @@ class GcpSecretStore: """Resolve secrets from GCP Secret Manager by resource name.""" def __init__(self, client: _SecretManagerClient | None = None) -> None: - self._client = client or secretmanager.SecretManagerServiceClient() + if client is None: + client_type = cast( + "Any", import_module("google.cloud.secretmanager") + ).SecretManagerServiceClient + client = cast("_SecretManagerClient", client_type()) + self._client = client def get_secret(self, reference: str) -> str: """Return a secret value from a fully-qualified Secret Manager reference. diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 85745cb3..14326544 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -163,7 +163,11 @@ def build_state(resolved: _ResolvedState) -> StateRuntime: ), ) - monkeypatch.setattr(run_module, "DefaultSecretStore", recording_factory("secrets")) + def build_secrets(provider_id: str) -> _Built: + assert provider_id == "gcp_secret_manager" + return recording_factory("secrets")() + + monkeypatch.setattr(run_module, "build_secret_store", build_secrets) monkeypatch.setattr(run_module, "build_auth", build_auth) monkeypatch.setattr(run_module, "build_source_adapter", build_source) monkeypatch.setattr(run_module, "PipelineRunner", recording_factory("ingestion")) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 25481843..04a614f5 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -72,6 +72,8 @@ def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( assert migrated.state_provider == "bigquery" assert original.catalog_provider == "dataplex" assert migrated.catalog_provider == "dataplex" + assert original.secret_provider == "gcp_secret_manager" + assert migrated.secret_provider == "gcp_secret_manager" assert migrated.platform == original.platform assert migrated.plugins == original.plugins assert migrated.pipelines == original.pipelines @@ -146,6 +148,20 @@ def test_version_two_none_catalog_disables_external_publication(tmp_path: Path) assert resolved.pipelines["example_records"].publish_dataplex is False +def test_cloud_run_rejects_environment_only_secrets(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["platforms"]["gcp"]["secrets"]["provider"] = "environment" + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + with pytest.raises(ProjectConfigError, match="Cloud Run requires"): + load_project_config(project_path) + + def test_config_migrate_check_is_read_only_then_write_is_atomic(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_secret_runtime.py b/tests/providers/test_secret_runtime.py new file mode 100644 index 00000000..d688a22e --- /dev/null +++ b/tests/providers/test_secret_runtime.py @@ -0,0 +1,83 @@ +"""Secret-provider selection and composition coverage.""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander.cli.provider_runtime import build_secret_store +from dander.providers import ProviderKind, default_provider_registry +from dander.security import SecretRuntime + +if TYPE_CHECKING: + from collections.abc import Mapping + + +@dataclass +class _Payload: + data: bytes + + +@dataclass +class _Response: + payload: _Payload + + +class _Client: + def __init__(self) -> None: + self.requests: list[dict[str, str]] = [] + + def access_secret_version(self, *, request: Mapping[str, str]) -> _Response: + self.requests.append(dict(request)) + return _Response(payload=_Payload(data=b"resolved-value")) + + +def test_environment_provider_loads_without_gcp_runtime_or_sdk() -> None: + gcp_runtime = "dander.providers.gcp_secret_manager.runtime" + google_sdk = "google.cloud.secretmanager" + environment_runtime = "dander.providers.environment_secrets.runtime" + sys.modules.pop(gcp_runtime, None) + sys.modules.pop(google_sdk, None) + sys.modules.pop(environment_runtime, None) + registry = default_provider_registry() + config = registry.parse(ProviderKind.SECRETS, {"provider": "environment"}) + + runtime = registry.build( + ProviderKind.SECRETS, + config, + context={"environment": {"DANDER_TEST_TOKEN": "resolved-value"}}, + ) + + assert isinstance(runtime, SecretRuntime) + assert runtime.store.get_secret("DANDER_TEST_TOKEN") == "resolved-value" + assert runtime.capabilities.environment_indirection is False + assert environment_runtime in sys.modules + assert gcp_runtime not in sys.modules + assert google_sdk not in sys.modules + + +def test_gcp_provider_preserves_environment_indirection_without_sdk_eagerness() -> None: + google_sdk = "google.cloud.secretmanager" + sys.modules.pop(google_sdk, None) + reference = "projects/unit/secrets/runtime/versions/latest" + client = _Client() + registry = default_provider_registry() + config = registry.parse(ProviderKind.SECRETS, {"provider": "gcp_secret_manager"}) + + runtime = registry.build( + ProviderKind.SECRETS, + config, + context={"environment": {"DANDER_TEST_TOKEN": reference}, "client": client}, + ) + + assert isinstance(runtime, SecretRuntime) + assert runtime.store.get_secret("DANDER_TEST_TOKEN") == "resolved-value" + assert client.requests == [{"name": reference}] + assert runtime.capabilities.environment_indirection is True + assert google_sdk not in sys.modules + + +def test_cli_builds_environment_store_through_provider_registry() -> None: + store = build_secret_store("environment") + assert type(store).__name__ == "EnvironmentSecretStore" diff --git a/tickets/DANDER-87-dataplex-catalog-runtime.md b/tickets/DANDER-87-dataplex-catalog-runtime.md index 6d178213..74d654da 100644 --- a/tickets/DANDER-87-dataplex-catalog-runtime.md +++ b/tickets/DANDER-87-dataplex-catalog-runtime.md @@ -1,7 +1,7 @@ --- id: DANDER-87 title: Route Dataplex publication through catalog capabilities -status: in-review +status: completed component: catalog epic: cloud-portability depends_on: [DANDER-86] @@ -25,7 +25,7 @@ selection must become independent without changing the accepted metadata spine o - [x] Local semantic-registry and durable metadata-snapshot behavior remains unchanged. - [x] Focused provider, Dataplex, CLI, and project-resolution tests pass. - [x] Full local validation and isolated GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -35,4 +35,4 @@ construction change only; canonical non-BigQuery catalog assets and Glue remain ## Review Log -Pending protected review. +Merged through protected PR #118 as `7c6c09ccb19a4e2126f3d204dba7b3dbfdd4d2a5`. diff --git a/tickets/DANDER-88-gcp-secret-runtime.md b/tickets/DANDER-88-gcp-secret-runtime.md new file mode 100644 index 00000000..2907f1c5 --- /dev/null +++ b/tickets/DANDER-88-gcp-secret-runtime.md @@ -0,0 +1,40 @@ +--- +id: DANDER-88 +title: Route GCP secret resolution through provider capabilities +status: in-review +component: security +epic: cloud-portability +depends_on: [DANDER-87] +created: 2026-08-08 +--- + +## Context + +Named profiles declared a secret provider, but execution constructed GCP and environment stores +directly and importing the implementation eagerly loaded the Google SDK. Secret selection must use +the shared provider boundary without changing the accepted GCP runtime or Terraform resources. + +## Acceptance Criteria + +- [x] Version 1 and migrated version 2 projects retain GCP Secret Manager selection. +- [x] GCP and environment runtimes use the existing lazy API-v1 provider registry. +- [x] Hosted execution and connector capability operations use the selected secret runtime. +- [x] Sandbox execution explicitly selects environment-only resolution. +- [x] GCP resource-name and environment-indirection behavior remains unchanged. +- [x] Credential-access auditing remains unchanged and never records a value. +- [x] Environment selection loads neither the GCP runtime nor Google Secret Manager SDK. +- [x] Cloud Run continues to reject environment-only secret profiles. +- [x] Focused provider, resolution, CLI, and project-profile tests pass. +- [x] Full local validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Compose one small `SecretRuntime` containing the existing `SecretStoreProvider` protocol plus +explicit capabilities. Keep legacy public classes available, make Google client construction lazy, +and leave secret references, IAM, Terraform, and launcher projection unchanged. + +## Review Log + +Full local validation and the isolated retained-project no-drift proof passed. Pending protected +review. From cf4e196985c5fad5943c83d4c8928b8916bafbc4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 13:45:01 -0400 Subject: [PATCH 038/163] Route Cloud Run through launcher provider --- CHANGELOG.md | 2 + HANDOFF.md | 32 ++++---- docs/decisions.md | 9 +++ docs/execution-projection.md | 3 +- docs/platform-profiles.md | 6 +- src/dander/bootstrap/terraform.py | 26 +++++- src/dander/cli/init_command.py | 3 + src/dander/deployment/__init__.py | 3 + src/dander/deployment/runtime.py | 53 +++++++++++++ src/dander/project/config.py | 1 + src/dander/project/portable_config.py | 14 +--- src/dander/providers/cloud_run/__init__.py | 5 ++ src/dander/providers/cloud_run/config.py | 14 ++++ src/dander/providers/cloud_run/runtime.py | 79 +++++++++++++++++++ src/dander/providers/defaults.py | 9 +++ tests/bootstrap/test_terraform.py | 31 ++++++++ tests/project/test_portable_config.py | 2 + tests/providers/test_launcher_runtime.py | 73 +++++++++++++++++ tickets/DANDER-88-gcp-secret-runtime.md | 8 +- .../DANDER-89-cloud-run-launcher-runtime.md | 38 +++++++++ 20 files changed, 374 insertions(+), 37 deletions(-) create mode 100644 src/dander/deployment/runtime.py create mode 100644 src/dander/providers/cloud_run/__init__.py create mode 100644 src/dander/providers/cloud_run/config.py create mode 100644 src/dander/providers/cloud_run/runtime.py create mode 100644 tests/providers/test_launcher_runtime.py create mode 100644 tickets/DANDER-89-cloud-run-launcher-runtime.md diff --git a/CHANGELOG.md b/CHANGELOG.md index befb86ff..951e78f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ capabilities enter through the next minor release. ### Added +- Select Cloud Run execution-template construction through the lazy launcher-provider registry + while preserving the existing Terraform projection, resource addresses, and runtime behavior. - Select GCP Secret Manager and local environment resolvers through the lazy provider registry; preserve hosted environment-to-resource-name indirection while preventing environment-only secrets from being selected for Cloud Run. diff --git a/HANDOFF.md b/HANDOFF.md index a5425acd..62eded56 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,20 +2,20 @@ ## Finished -- Added a typed `SecretRuntime` with explicit resolution capabilities. -- Registered GCP Secret Manager and environment stores through lazy provider factories. -- Routed hosted, sandbox, and connector-capability secret construction through provider selection. -- Preserved GCP resource names, environment indirection, auditing, IAM, and Terraform behavior. -- Kept environment-only secrets unavailable to the Cloud Run compatibility profile. +- Added a typed launcher runtime and execution-template factory boundary. +- Registered Cloud Run through the lazy API-v1 provider registry. +- Routed Terraform bootstrap through the selected launcher without changing projection output. +- Preserved version 1 and migrated version 2 Cloud Run selection. +- Added exact projection-parity and pre-Terraform rejection coverage. ## Try It -Run an existing v1 project or v2 GCP profile normally. Sandbox runs select environment variables; -hosted Cloud Run runs retain GCP Secret Manager through the selected provider runtime. +Run an existing GCP project normally. Its Cloud Run templates now come through the provider +registry while the rendered Terraform remains unchanged. ## Checks -- All 915 tests, Ruff, formatting, and strict mypy across 229 files passed. +- All 917 tests, Ruff, formatting, and strict mypy across 234 files passed. - Wheel/sdist inspection, source-free installs, runtime-all assembly, and dependency audit passed. - The non-root/read-only full runtime image passed conformance and bundled-asset checks. - Trivy found no high/critical findings, Gitleaks found no leaks, and every Terraform root validated. @@ -23,17 +23,17 @@ hosted Cloud Run runs retain GCP Secret Manager through the selected provider ru ## Decisions -- Keep the existing `DefaultSecretStore` compatibility behavior behind the GCP factory. -- Load the Google SDK only when a real GCP client is first needed. -- Defer AWS secret resolution and explicit URI parsing to the Fargate slice. +- Keep the accepted Cloud Run projector as the implementation behind the provider boundary. +- Carry launcher selection as resolved internal configuration without changing manifest v1. +- Leave Fargate infrastructure and lifecycle to the next isolated vertical slice. ## Remaining -- Open and merge the focused GCP secret-provider PR after protected CI passes. -- Route Cloud Run through the launcher provider boundary next. +- Open and merge the focused Cloud Run launcher PR after protected CI passes. +- Begin the Fargate vertical slice using the ready AWS account. ## Review First -- `src/dander/security/runtime.py` -- `src/dander/providers/gcp_secret_manager/runtime.py` -- `src/dander/cli/run_command.py` +- `src/dander/deployment/runtime.py` +- `src/dander/providers/cloud_run/runtime.py` +- `src/dander/bootstrap/terraform.py` diff --git a/docs/decisions.md b/docs/decisions.md index 93367275..c4d2bb3f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-08 — Cloud Run construction uses the launcher provider boundary + +- **Selection:** Version 1 and migrated version 2 projects retain `cloud_run`; Terraform bootstrap + builds its launcher runtime through the shared lazy API-v1 provider registry. +- **Parity:** The Cloud Run factory delegates to the accepted execution projector, preserving + template values, Terraform addresses, schedules, IAM, alerts, and runtime behavior. +- **Scope:** This change creates a construction seam only. Fargate lifecycle and infrastructure + remain a separate vertical slice with their own planning and acceptance. + ## 2026-08-08 — Secret resolution uses an explicit provider runtime - **Compatibility:** Version 1 and the Cloud Run profile retain GCP Secret Manager plus the diff --git a/docs/execution-projection.md b/docs/execution-projection.md index ba668203..5b3bee0b 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -22,7 +22,8 @@ existing GCP/BigQuery compatibility profile. A later configuration migration wil pipeline intent from environment-specific deployment settings; it does not change this runtime boundary. -The GCP plan path supplies these templates to Terraform. Cloud Run consumes their image, command, +The selected Cloud Run provider supplies these templates to Terraform. Cloud Run consumes their +image, command, resources, schedule, environment, secret references, identity, labels, and observability contract. Run-specific IDs and attempts continue to come from Cloud Run's native execution context and are normalized by the OCI runtime contract. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 392401b9..5e91b1d1 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -49,9 +49,9 @@ now use that boundary. State migrations are explicit and versioned, but retain t BigQuery table names and semantics. Dataplex keeps aspect-only updates and normalized readback; selecting `none` loads no Dataplex implementation or credentials. GCP Secret Manager preserves its existing environment indirection and audit behavior; environment-only resolution remains local -because Cloud Run requires managed secret references. Cloud Run itself remains on its proven -direct path until its separate Phase 3 change. The registry is a construction contract, not a -support claim. +because Cloud Run requires managed secret references. Cloud Run execution templates now pass +through the same lazy provider boundary while delegating to the accepted projection unchanged. +The registry is a construction contract, not a support claim. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; diff --git a/src/dander/bootstrap/terraform.py b/src/dander/bootstrap/terraform.py index f5da7811..8ba6298b 100644 --- a/src/dander/bootstrap/terraform.py +++ b/src/dander/bootstrap/terraform.py @@ -8,7 +8,8 @@ from json import dumps from typing import TYPE_CHECKING -from dander.deployment import ExecutionProjectionError, build_gcp_execution_templates +from dander.deployment import ExecutionProjectionError, LauncherRuntime +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry if TYPE_CHECKING: from collections.abc import Mapping @@ -52,6 +53,7 @@ def execute( state_prefix: str, bootstrap_service_account: str, apply: bool, + launcher_provider: str = "cloud_run", region: str = "us-central1", bigquery_location: str = "US", runtime_cpu: int = 1, @@ -82,6 +84,7 @@ def execute( state_prefix: Object prefix for the Dander state. bootstrap_service_account: Existing service account that Terraform must impersonate. apply: Whether to apply the saved plan after planning. + launcher_provider: Selected launcher provider ID from the deployment profile. region: GCP region for regional resources. bigquery_location: BigQuery dataset location. runtime_cpu: Cloud Run CPU count shared by hosted jobs. @@ -171,9 +174,13 @@ def execute( execution_projections: dict[str, dict[str, object]] = {} if enable_runtime: try: + launcher = _build_launcher_runtime( + provider_id=launcher_provider, + region=region, + ) execution_projections = { pipeline_id: template.as_dict() - for pipeline_id, template in build_gcp_execution_templates( + for pipeline_id, template in launcher.templates.build( expanded_pipelines, image=container_image, project=project, @@ -186,7 +193,7 @@ def execute( alert_target=failure_alert_email or None, ).items() } - except ExecutionProjectionError as error: + except (ExecutionProjectionError, ProviderFactoryError) as error: raise TerraformBootstrapError(str(error)) from error if billing_account_id and not (enable_runtime or enable_cost_guard): raise TerraformBootstrapError( @@ -322,6 +329,19 @@ def _run(self, *args: str) -> None: ) from error +def _build_launcher_runtime(*, provider_id: str, region: str) -> LauncherRuntime: + """Build one launcher through the shared lazy provider registry.""" + registry = default_provider_registry() + config = registry.parse( + ProviderKind.LAUNCHER, + {"provider": provider_id, "region": region}, + ) + runtime = registry.build(ProviderKind.LAUNCHER, config) + if not isinstance(runtime, LauncherRuntime): + raise ProviderFactoryError("Selected launcher provider returned an invalid runtime") + return runtime + + def _validate_pipelines(pipelines: Mapping[str, Mapping[str, object]]) -> None: """Defensively validate the expanded project manifest before invoking Terraform.""" required = { diff --git a/src/dander/cli/init_command.py b/src/dander/cli/init_command.py index 0534aee5..8b11873d 100644 --- a/src/dander/cli/init_command.py +++ b/src/dander/cli/init_command.py @@ -174,6 +174,7 @@ def execute_init( state_prefix=options.state_prefix, bootstrap_service_account=bootstrap_account, apply=options.apply, + launcher_provider=manifest.launcher_provider, region=platform.region, bigquery_location=platform.bigquery_location, runtime_cpu=runtime.cpu, @@ -315,6 +316,7 @@ def execute_platform_bootstrap( state_prefix: str, bootstrap_service_account: str, apply: bool, + launcher_provider: str, region: str, bigquery_location: str, runtime_cpu: int, @@ -347,6 +349,7 @@ def execute_platform_bootstrap( state_prefix=state_prefix, bootstrap_service_account=bootstrap_service_account, apply=apply, + launcher_provider=launcher_provider, region=region, bigquery_location=bigquery_location, runtime_cpu=runtime_cpu, diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index 11356227..ee7bcfef 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -16,6 +16,7 @@ build_gcp_v1_execution_templates, validate_launcher_projection, ) +from dander.deployment.runtime import ExecutionTemplateFactory, LauncherRuntime __all__ = [ "CLOUD_RUN_CAPABILITIES", @@ -23,7 +24,9 @@ "ExecutionProjectionError", "ExecutionRequest", "ExecutionTemplate", + "ExecutionTemplateFactory", "LauncherCapabilities", + "LauncherRuntime", "NetworkPlacement", "ObservabilityProjection", "ResourceProjection", diff --git a/src/dander/deployment/runtime.py b/src/dander/deployment/runtime.py new file mode 100644 index 00000000..17a3e4df --- /dev/null +++ b/src/dander/deployment/runtime.py @@ -0,0 +1,53 @@ +"""Provider-neutral launcher runtime composition.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from collections.abc import Mapping + + from dander.deployment.projection import ExecutionTemplate, LauncherCapabilities + + +@runtime_checkable +class ExecutionTemplateFactory(Protocol): + """Build launcher-specific templates from validated deployment inputs.""" + + def build( + self, + pipelines: Mapping[str, Mapping[str, object]], + *, + image: str, + project: str, + cpu: int, + memory: str, + deadline_seconds: int, + launcher_retry_count: int, + batch_rows: int, + require_guarded_free_tier: bool, + alert_target: str | None, + ) -> dict[str, ExecutionTemplate]: + """Return validated execution templates keyed by pipeline ID.""" + ... + + +@dataclass(frozen=True, slots=True) +class LauncherRuntime: + """A selected launcher projection factory and its declared limits.""" + + provider_id: str + region: str + templates: ExecutionTemplateFactory + capabilities: LauncherCapabilities + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("launcher runtime requires a provider id") + if not self.region: + raise ValueError("launcher runtime requires a region") + if self.capabilities.launcher != self.provider_id: + raise ValueError("launcher runtime and capabilities provider ids must match") + if not isinstance(self.templates, ExecutionTemplateFactory): + raise TypeError("launcher template factory has the wrong type") diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 6832012f..90f760ee 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -210,6 +210,7 @@ class DanderProject(BaseModel): default="gcp_secret_manager", exclude=True, ) + launcher_provider: Literal["cloud_run"] = Field(default="cloud_run", exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 35af9169..5d427750 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -24,6 +24,7 @@ _load_yaml_mapping, ) from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig # noqa: TC001 +from dander.providers.cloud_run import CloudRunLauncherConfig # noqa: TC001 from dander.providers.dataplex import DataplexCatalogConfig # noqa: TC001 from dander.providers.environment_secrets import EnvironmentSecretConfig # noqa: TC001 from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 @@ -134,15 +135,6 @@ class PlatformProfileSpec(BaseModel): secrets: SecretProviderSpec -class CloudRunLauncherSpec(BaseModel): - """Cloud Run deployment projection for the existing GCP path.""" - - model_config = ConfigDict(extra="forbid") - - provider: Literal["cloud_run"] - region: str = Field(default="us-central1", min_length=1) - - class DeploymentPipelineSpec(BaseModel): """Environment-specific projection for one logical pipeline.""" @@ -180,7 +172,7 @@ class DeploymentSpec(BaseModel): model_config = ConfigDict(extra="forbid") platform: str = Field(pattern=_NAME.pattern) - launcher: CloudRunLauncherSpec + launcher: CloudRunLauncherConfig runtime: PlatformRuntimeSpec = Field(default_factory=PlatformRuntimeSpec) safety: PlatformSafetySpec = Field(default_factory=PlatformSafetySpec) pipelines: dict[str, DeploymentPipelineSpec] = Field(min_length=1) @@ -354,6 +346,7 @@ def _resolve( state_provider=profile.state.provider, catalog_provider=profile.catalog.provider, secret_provider=profile.secrets.provider, + launcher_provider=selected.launcher.provider, deployed_pipeline_ids=tuple(sorted(selected.pipelines)), ) @@ -444,6 +437,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: and legacy.state_provider == migrated.state_provider and legacy.catalog_provider == migrated.catalog_provider and legacy.secret_provider == migrated.secret_provider + and legacy.launcher_provider == migrated.launcher_provider and legacy.plugins == migrated.plugins and legacy.pipelines == migrated.pipelines and legacy.terraform_pipelines() == migrated.terraform_pipelines() diff --git a/src/dander/providers/cloud_run/__init__.py b/src/dander/providers/cloud_run/__init__.py new file mode 100644 index 00000000..8b9afcd7 --- /dev/null +++ b/src/dander/providers/cloud_run/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light Cloud Run launcher configuration.""" + +from dander.providers.cloud_run.config import CloudRunLauncherConfig + +__all__ = ["CloudRunLauncherConfig"] diff --git a/src/dander/providers/cloud_run/config.py b/src/dander/providers/cloud_run/config.py new file mode 100644 index 00000000..6e770fce --- /dev/null +++ b/src/dander/providers/cloud_run/config.py @@ -0,0 +1,14 @@ +"""Dependency-light Cloud Run launcher configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class CloudRunLauncherConfig(BaseModel): + """Select Cloud Run Jobs in one GCP region.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["cloud_run"] + region: str = Field(default="us-central1", min_length=1) diff --git a/src/dander/providers/cloud_run/runtime.py b/src/dander/providers/cloud_run/runtime.py new file mode 100644 index 00000000..a0ba2f79 --- /dev/null +++ b/src/dander/providers/cloud_run/runtime.py @@ -0,0 +1,79 @@ +"""Cloud Run execution projection selected through the provider registry.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander.deployment.projection import ( + CLOUD_RUN_CAPABILITIES, + ExecutionTemplate, + build_gcp_execution_templates, +) +from dander.deployment.runtime import LauncherRuntime +from dander.providers.cloud_run.config import CloudRunLauncherConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind + +if TYPE_CHECKING: + from collections.abc import Mapping + + from pydantic import BaseModel + + +@dataclass(frozen=True, slots=True) +class CloudRunTemplateFactory: + """Preserve the accepted GCP template projection behind a launcher boundary.""" + + def build( + self, + pipelines: Mapping[str, Mapping[str, object]], + *, + image: str, + project: str, + cpu: int, + memory: str, + deadline_seconds: int, + launcher_retry_count: int, + batch_rows: int, + require_guarded_free_tier: bool, + alert_target: str | None, + ) -> dict[str, ExecutionTemplate]: + """Build byte-equivalent Cloud Run templates through the existing projector.""" + return build_gcp_execution_templates( + pipelines, + image=image, + project=project, + cpu=cpu, + memory=memory, + deadline_seconds=deadline_seconds, + launcher_retry_count=launcher_retry_count, + batch_rows=batch_rows, + require_guarded_free_tier=require_guarded_free_tier, + alert_target=alert_target, + ) + + +def build_cloud_run_launcher( + config: BaseModel, + context: Mapping[str, object], +) -> LauncherRuntime: + """Build Cloud Run projection behavior only after launcher selection.""" + del context + if not isinstance(config, CloudRunLauncherConfig): + raise TypeError("Cloud Run launcher factory received the wrong configuration") + return LauncherRuntime( + provider_id="cloud_run", + region=config.region, + templates=CloudRunTemplateFactory(), + capabilities=CLOUD_RUN_CAPABILITIES, + ) + + +CLOUD_RUN_LAUNCHER_FACTORY: ProviderFactory[LauncherRuntime] = ProviderFactory( + kind=ProviderKind.LAUNCHER, + provider_id="cloud_run", + api_version=PROVIDER_API_VERSION, + build=build_cloud_run_launcher, +) + +__all__ = ["CLOUD_RUN_LAUNCHER_FACTORY"] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index c68240bb..a224105f 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -1,6 +1,7 @@ """Built-in provider registrations with lazy implementation loading.""" from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig +from dander.providers.cloud_run import CloudRunLauncherConfig from dander.providers.dataplex import DataplexCatalogConfig from dander.providers.environment_secrets import EnvironmentSecretConfig from dander.providers.gcp_secret_manager import GcpSecretManagerConfig @@ -59,4 +60,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.environment_secrets.runtime:ENVIRONMENT_SECRET_FACTORY" ), ) + registry.register( + kind=ProviderKind.LAUNCHER, + provider_id="cloud_run", + config_model=CloudRunLauncherConfig, + load_factory=lazy_provider_factory( + "dander.providers.cloud_run.runtime:CLOUD_RUN_LAUNCHER_FACTORY" + ), + ) return registry diff --git a/tests/bootstrap/test_terraform.py b/tests/bootstrap/test_terraform.py index 62e720bb..5fe24cf2 100644 --- a/tests/bootstrap/test_terraform.py +++ b/tests/bootstrap/test_terraform.py @@ -204,6 +204,37 @@ def fake_run( assert any(argument.startswith("-var=pipelines=") for argument in commands[1]) +def test_bootstrap_rejects_an_unknown_launcher_before_terraform( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls = 0 + + def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: + del args, kwargs + nonlocal calls + calls += 1 + return subprocess.CompletedProcess((), 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(TerraformBootstrapError, match="Unknown launcher provider"): + TerraformBootstrap(tmp_path).execute( + project="unit-project", + state_bucket="unit-state", + state_prefix="dander/state", + bootstrap_service_account=("dander-bootstrap@unit-project.iam.gserviceaccount.com"), + apply=False, + launcher_provider="missing", + require_guarded_free_tier=False, + enable_runtime=True, + container_image=f"example.invalid/dander@sha256:{'a' * 64}", + pipelines=_pipelines(), + ) + + assert calls == 0 + + def test_bootstrap_passes_simulation_first_cost_guard( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 04a614f5..8d6fbb70 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -74,6 +74,8 @@ def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( assert migrated.catalog_provider == "dataplex" assert original.secret_provider == "gcp_secret_manager" assert migrated.secret_provider == "gcp_secret_manager" + assert original.launcher_provider == "cloud_run" + assert migrated.launcher_provider == "cloud_run" assert migrated.platform == original.platform assert migrated.plugins == original.plugins assert migrated.pipelines == original.pipelines diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py new file mode 100644 index 00000000..15346dd4 --- /dev/null +++ b/tests/providers/test_launcher_runtime.py @@ -0,0 +1,73 @@ +"""Launcher-provider selection and projection parity coverage.""" + +from __future__ import annotations + +import sys + +from dander.deployment import LauncherRuntime, build_gcp_execution_templates +from dander.providers import ProviderKind, default_provider_registry + +_IMAGE = "us-central1-docker.pkg.dev/unit-project/dander/dander@sha256:" + "a" * 64 +_PIPELINES: dict[str, dict[str, object]] = { + "greenhouse_jobs": { + "runtime_service_account_id": "dander-runtime", + "build_models": True, + "schedule": "0 9 * * *", + "time_zone": "America/New_York", + "paused": True, + "secret_env": {}, + } +} + + +def _templates(runtime: LauncherRuntime) -> dict[str, object]: + return { + pipeline_id: template.as_dict() + for pipeline_id, template in runtime.templates.build( + _PIPELINES, + image=_IMAGE, + project="unit-project", + cpu=1, + memory="512Mi", + deadline_seconds=300, + launcher_retry_count=1, + batch_rows=10_000, + require_guarded_free_tier=False, + alert_target="operator@example.invalid", + ).items() + } + + +def test_cloud_run_factory_is_lazy_and_matches_the_accepted_projection() -> None: + module_name = "dander.providers.cloud_run.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.LAUNCHER, + {"provider": "cloud_run", "region": "us-central1"}, + ) + + assert module_name not in sys.modules + runtime = registry.build(ProviderKind.LAUNCHER, config) + + assert isinstance(runtime, LauncherRuntime) + assert runtime.provider_id == "cloud_run" + assert runtime.region == "us-central1" + assert runtime.capabilities.supports_schedules is True + assert module_name in sys.modules + expected = { + pipeline_id: template.as_dict() + for pipeline_id, template in build_gcp_execution_templates( + _PIPELINES, + image=_IMAGE, + project="unit-project", + cpu=1, + memory="512Mi", + deadline_seconds=300, + launcher_retry_count=1, + batch_rows=10_000, + require_guarded_free_tier=False, + alert_target="operator@example.invalid", + ).items() + } + assert _templates(runtime) == expected diff --git a/tickets/DANDER-88-gcp-secret-runtime.md b/tickets/DANDER-88-gcp-secret-runtime.md index 2907f1c5..ea89d6be 100644 --- a/tickets/DANDER-88-gcp-secret-runtime.md +++ b/tickets/DANDER-88-gcp-secret-runtime.md @@ -1,7 +1,7 @@ --- id: DANDER-88 title: Route GCP secret resolution through provider capabilities -status: in-review +status: completed component: security epic: cloud-portability depends_on: [DANDER-87] @@ -26,7 +26,7 @@ the shared provider boundary without changing the accepted GCP runtime or Terraf - [x] Cloud Run continues to reject environment-only secret profiles. - [x] Focused provider, resolution, CLI, and project-profile tests pass. - [x] Full local validation and isolated GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -36,5 +36,5 @@ and leave secret references, IAM, Terraform, and launcher projection unchanged. ## Review Log -Full local validation and the isolated retained-project no-drift proof passed. Pending protected -review. +Full local validation, isolated retained-project no-drift, and protected CI passed. PR #119 merged +as `c57c948dd4af83f51ec3385487429e0472049dd5`. diff --git a/tickets/DANDER-89-cloud-run-launcher-runtime.md b/tickets/DANDER-89-cloud-run-launcher-runtime.md new file mode 100644 index 00000000..bd12e684 --- /dev/null +++ b/tickets/DANDER-89-cloud-run-launcher-runtime.md @@ -0,0 +1,38 @@ +--- +id: DANDER-89 +title: Route Cloud Run projection through the launcher provider boundary +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-88] +created: 2026-08-08 +--- + +## Context + +The shared provider registry declared launcher support, but Terraform bootstrap still constructed +Cloud Run templates directly. Cloud Run must use the provider boundary without changing any +accepted template value, Terraform address, or live resource. + +## Acceptance Criteria + +- [x] Version 1 and migrated version 2 projects retain Cloud Run selection. +- [x] The Cloud Run implementation loads only after launcher selection. +- [x] Terraform bootstrap obtains execution templates from the selected launcher runtime. +- [x] Provider output matches the accepted Cloud Run projector exactly. +- [x] An unknown launcher fails before Terraform runs. +- [x] Existing direct projection APIs remain compatible. +- [x] Focused launcher, bootstrap, project, and projection tests pass. +- [x] Full local validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Add one small `LauncherRuntime` containing the existing execution-template construction protocol +and declared launcher capabilities. Register Cloud Run lazily and delegate to the proven GCP +projector. Keep Terraform modules, resource addresses, schedules, IAM, and runtime commands intact. + +## Review Log + +Full local validation and the isolated retained-project no-drift proof passed. Protected review +remains pending. From 019209a728d68feeb57fa3105524466cdc22f6f6 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 14:01:10 -0400 Subject: [PATCH 039/163] Add fail-closed Fargate projection --- CHANGELOG.md | 3 + HANDOFF.md | 33 +-- docs/decisions.md | 9 + docs/execution-projection.md | 6 + src/dander/deployment/__init__.py | 2 + src/dander/deployment/projection.py | 24 ++ src/dander/providers/defaults.py | 9 + src/dander/providers/fargate/__init__.py | 5 + src/dander/providers/fargate/config.py | 44 ++++ src/dander/providers/fargate/runtime.py | 216 ++++++++++++++++++ tests/providers/test_launcher_runtime.py | 115 +++++++++- .../DANDER-89-cloud-run-launcher-runtime.md | 8 +- .../DANDER-90-fargate-launcher-projection.md | 40 ++++ 13 files changed, 493 insertions(+), 21 deletions(-) create mode 100644 src/dander/providers/fargate/__init__.py create mode 100644 src/dander/providers/fargate/config.py create mode 100644 src/dander/providers/fargate/runtime.py create mode 100644 tickets/DANDER-90-fargate-launcher-projection.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 951e78f6..ff666fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Add a lazy ECS/Fargate launcher factory that projects the existing BigQuery data-plane runtime + onto immutable ECR images, task-role identity, explicit networking, and valid Fargate resource + pairs without yet claiming deployable Fargate support. - Select Cloud Run execution-template construction through the lazy launcher-provider registry while preserving the existing Terraform projection, resource addresses, and runtime behavior. - Select GCP Secret Manager and local environment resolvers through the lazy provider registry; diff --git a/HANDOFF.md b/HANDOFF.md index 62eded56..7d8c6d78 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,20 +2,20 @@ ## Finished -- Added a typed launcher runtime and execution-template factory boundary. -- Registered Cloud Run through the lazy API-v1 provider registry. -- Routed Terraform bootstrap through the selected launcher without changing projection output. -- Preserved version 1 and migrated version 2 Cloud Run selection. -- Added exact projection-parity and pre-Terraform rejection coverage. +- Added dependency-light Fargate configuration and lazy launcher registration. +- Projected immutable ECR images, task-role identity, `awsvpc` placement, and CloudWatch intent. +- Preserved the BigQuery/GCP runtime command and GCP Secret Manager references. +- Rejected invalid Fargate sizing and the unsupported guarded-free-tier path before planning. +- Kept Fargate out of the supported runtime-capability manifest. ## Try It -Run an existing GCP project normally. Its Cloud Run templates now come through the provider -registry while the rendered Terraform remains unchanged. +Build the `fargate` launcher through the provider registry with a disposable AWS account, existing +subnet/security-group IDs, and a valid ECR digest. It returns an execution template only. ## Checks -- All 917 tests, Ruff, formatting, and strict mypy across 234 files passed. +- All 920 tests, Ruff, formatting, and strict mypy across 237 files passed. - Wheel/sdist inspection, source-free installs, runtime-all assembly, and dependency audit passed. - The non-root/read-only full runtime image passed conformance and bundled-asset checks. - Trivy found no high/critical findings, Gitleaks found no leaks, and every Terraform root validated. @@ -23,17 +23,18 @@ registry while the rendered Terraform remains unchanged. ## Decisions -- Keep the accepted Cloud Run projector as the implementation behind the provider boundary. -- Carry launcher selection as resolved internal configuration without changing manifest v1. -- Leave Fargate infrastructure and lifecycle to the next isolated vertical slice. +- Keep this PR to projection and validation; infrastructure and lifecycle are separate slices. +- Reuse each pipeline's stable runtime identity name as its future AWS task-role name. +- Reject GCP's guarded-free-tier preflight on Fargate instead of weakening it. ## Remaining -- Open and merge the focused Cloud Run launcher PR after protected CI passes. -- Begin the Fargate vertical slice using the ready AWS account. +- Merge the focused projection PR after protected CI passes. +- Add AWS secret resolution and the keyless BigQuery credential runtime. +- Add reviewed Fargate infrastructure and controller lifecycle. ## Review First -- `src/dander/deployment/runtime.py` -- `src/dander/providers/cloud_run/runtime.py` -- `src/dander/bootstrap/terraform.py` +- `src/dander/providers/fargate/config.py` +- `src/dander/providers/fargate/runtime.py` +- `tests/providers/test_launcher_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index c4d2bb3f..cdbb4944 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-08 — Fargate projection precedes infrastructure support + +- **Boundary:** The lazy Fargate factory projects the existing BigQuery/GCP runtime onto an + immutable ECR image, AWS task role, `awsvpc` placement, and declared CloudWatch destinations. +- **Fail closed:** Unsupported CPU/memory pairs, guarded-free-tier execution, mutable images, and + invalid account/network identifiers fail before any infrastructure operation. +- **Support:** Factory registration is not a support claim. Terraform, controller lifecycle, + keyless identity, and live comparison remain separate promotion gates. + ## 2026-08-08 — Cloud Run construction uses the launcher provider boundary - **Selection:** Version 1 and migrated version 2 projects retain `cloud_run`; Terraform bootstrap diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 5b3bee0b..54538a4b 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -27,3 +27,9 @@ image, command, resources, schedule, environment, secret references, identity, labels, and observability contract. Run-specific IDs and attempts continue to come from Cloud Run's native execution context and are normalized by the OCI runtime contract. + +The Fargate provider can build the same BigQuery data-plane intent with an immutable ECR image, +AWS task-role identity, `awsvpc` placement, CloudWatch destinations, and explicit Fargate resource +limits. This provider projection is an internal construction gate: provisioning, controller +lifecycle, AWS-to-Google credential assembly, and live acceptance must pass before Fargate is a +supported launcher. diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index ee7bcfef..da84a3b7 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -3,6 +3,7 @@ from dander.deployment.projection import ( CLOUD_RUN_CAPABILITIES, EXECUTION_PROJECTION_SCHEMA, + FARGATE_CAPABILITIES, ExecutionProjectionError, ExecutionRequest, ExecutionTemplate, @@ -21,6 +22,7 @@ __all__ = [ "CLOUD_RUN_CAPABILITIES", "EXECUTION_PROJECTION_SCHEMA", + "FARGATE_CAPABILITIES", "ExecutionProjectionError", "ExecutionRequest", "ExecutionTemplate", diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index c99454ec..7736744a 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -339,6 +339,30 @@ class LauncherCapabilities: ) +FARGATE_CAPABILITIES = LauncherCapabilities( + launcher="fargate", + cpu_millis=frozenset({1_000, 2_000, 4_000, 8_000, 16_000}), + minimum_memory_mib=512, + maximum_memory_mib=122_880, + maximum_deadline_seconds=86_400, + maximum_launcher_retries=10, + maximum_task_count=1, + maximum_parallelism=1, + supports_ephemeral_storage=True, + supports_schedules=True, + supports_time_zones=True, + supports_network_placement=True, + extension_names=frozenset( + { + "fargate_architecture", + "fargate_assign_public_ip", + "fargate_security_group_ids", + "fargate_subnet_ids", + } + ), +) + + def validate_launcher_projection( template: ExecutionTemplate, capabilities: LauncherCapabilities, diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index a224105f..4c1bf8c4 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -4,6 +4,7 @@ from dander.providers.cloud_run import CloudRunLauncherConfig from dander.providers.dataplex import DataplexCatalogConfig from dander.providers.environment_secrets import EnvironmentSecretConfig +from dander.providers.fargate import FargateLauncherConfig from dander.providers.gcp_secret_manager import GcpSecretManagerConfig from dander.providers.no_catalog import NoCatalogConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -68,4 +69,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.cloud_run.runtime:CLOUD_RUN_LAUNCHER_FACTORY" ), ) + registry.register( + kind=ProviderKind.LAUNCHER, + provider_id="fargate", + config_model=FargateLauncherConfig, + load_factory=lazy_provider_factory( + "dander.providers.fargate.runtime:FARGATE_LAUNCHER_FACTORY" + ), + ) return registry diff --git a/src/dander/providers/fargate/__init__.py b/src/dander/providers/fargate/__init__.py new file mode 100644 index 00000000..f41b498d --- /dev/null +++ b/src/dander/providers/fargate/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light ECS/Fargate launcher configuration.""" + +from dander.providers.fargate.config import FargateLauncherConfig + +__all__ = ["FargateLauncherConfig"] diff --git a/src/dander/providers/fargate/config.py b/src/dander/providers/fargate/config.py new file mode 100644 index 00000000..a35abf16 --- /dev/null +++ b/src/dander/providers/fargate/config.py @@ -0,0 +1,44 @@ +"""Validated ECS/Fargate launcher configuration without AWS SDK imports.""" + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +_SUBNET_ID = re.compile(r"^subnet-(?:[0-9a-f]{8}|[0-9a-f]{17})$") +_SECURITY_GROUP_ID = re.compile(r"^sg-(?:[0-9a-f]{8}|[0-9a-f]{17})$") + + +class FargateLauncherConfig(BaseModel): + """Select one existing AWS network for Fargate tasks.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["fargate"] + region: str = Field(pattern=r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") + aws_account_id: str = Field(pattern=r"^[0-9]{12}$") + subnet_ids: tuple[str, ...] = Field(min_length=1) + security_group_ids: tuple[str, ...] = Field(min_length=1) + architecture: Literal["ARM64", "X86_64"] = "ARM64" + assign_public_ip: bool = False + ephemeral_storage_mib: int = Field(default=20_480, ge=20_480, le=204_800) + + @field_validator("subnet_ids") + @classmethod + def validate_subnets(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Require deterministic, unique subnet identifiers.""" + if values != tuple(sorted(set(values))) or any( + _SUBNET_ID.fullmatch(value) is None for value in values + ): + raise ValueError("subnet_ids must be unique, sorted AWS subnet IDs") + return values + + @field_validator("security_group_ids") + @classmethod + def validate_security_groups(cls, values: tuple[str, ...]) -> tuple[str, ...]: + """Require deterministic, unique security-group identifiers.""" + if values != tuple(sorted(set(values))) or any( + _SECURITY_GROUP_ID.fullmatch(value) is None for value in values + ): + raise ValueError("security_group_ids must be unique, sorted AWS security-group IDs") + return values diff --git a/src/dander/providers/fargate/runtime.py b/src/dander/providers/fargate/runtime.py new file mode 100644 index 00000000..d956866d --- /dev/null +++ b/src/dander/providers/fargate/runtime.py @@ -0,0 +1,216 @@ +"""ECS/Fargate execution projection selected through the provider registry.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander import __version__ +from dander.deployment.projection import ( + EXECUTION_PROJECTION_SCHEMA, + FARGATE_CAPABILITIES, + ExecutionProjectionError, + ExecutionTemplate, + NetworkPlacement, + ObservabilityProjection, + ResourceProjection, + ScheduleProjection, + SecretReference, + validate_launcher_projection, +) +from dander.deployment.runtime import LauncherRuntime +from dander.providers.fargate.config import FargateLauncherConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.runtime_contract import RUNTIME_CONTRACT + +if TYPE_CHECKING: + from pydantic import BaseModel + +_ROLE_NAME = re.compile(r"^[A-Za-z0-9+=,.@_-]{1,64}$") +_GCP_PROJECT = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") + + +@dataclass(frozen=True, slots=True) +class FargateTemplateFactory: + """Build the GCP data-plane profile for an AWS Fargate launcher.""" + + config: FargateLauncherConfig + + def build( + self, + pipelines: Mapping[str, Mapping[str, object]], + *, + image: str, + project: str, + cpu: int, + memory: str, + deadline_seconds: int, + launcher_retry_count: int, + batch_rows: int, + require_guarded_free_tier: bool, + alert_target: str | None, + ) -> dict[str, ExecutionTemplate]: + """Build fail-closed Fargate templates for the portable BigQuery proof.""" + if require_guarded_free_tier: + raise ExecutionProjectionError("Fargate cannot run the GCP guarded-free-tier preflight") + if _GCP_PROJECT.fullmatch(project) is None: + raise ExecutionProjectionError("invalid GCP project identifier") + memory_mib = _memory_mib(memory) + cpu_millis = cpu * 1_000 + _validate_fargate_size(cpu_millis=cpu_millis, memory_mib=memory_mib) + templates: dict[str, ExecutionTemplate] = {} + for pipeline_id, pipeline in sorted(pipelines.items()): + role_name = str(pipeline["runtime_service_account_id"]) + if _ROLE_NAME.fullmatch(role_name) is None: + raise ExecutionProjectionError("invalid Fargate task role name") + identity = f"arn:aws:iam::{self.config.aws_account_id}:role/{role_name}" + secret_env = pipeline["secret_env"] + if not isinstance(secret_env, Mapping): + raise ExecutionProjectionError("pipeline secret bindings are invalid") + command: tuple[str, ...] = ( + "runtime", + "execute", + "--contract", + RUNTIME_CONTRACT, + "--pipeline", + pipeline_id, + "--platform", + "gcp", + "--config", + "/app/dander.yaml", + "--models-dir", + "/app/models", + "--batch-rows", + str(batch_rows), + ) + if bool(pipeline["build_models"]): + command = (*command, "--catalog-output", "/tmp/dander-catalog.json") + template = ExecutionTemplate( + schema=EXECUTION_PROJECTION_SCHEMA, + contract=RUNTIME_CONTRACT, + pipeline_id=pipeline_id, + profile_id="gcp", + launcher="fargate", + image=image, + command=command, + configuration_reference="/app/dander.yaml", + environment=tuple( + sorted( + { + "AWS_DEFAULT_REGION": self.config.region, + "AWS_REGION": self.config.region, + "BQ_DATASET_METADATA": "dander_meta", + "BQ_DATASET_RAW": "raw", + "DANDER_IMAGE_DIGEST": image.rsplit("@", maxsplit=1)[-1], + "DANDER_LAUNCHER": "fargate", + "DANDER_PRINCIPAL": identity, + "GCP_PROJECT_ID": project, + }.items() + ) + ), + secret_bindings=tuple( + ( + str(environment_name), + SecretReference( + provider="gcp_secret_manager", + reference=( + f"gcp-sm://projects/{project}/secrets/{secret_id}/versions/latest" + ), + ), + ) + for environment_name, secret_id in sorted(secret_env.items()) + ), + workload_identity=identity, + resources=ResourceProjection( + cpu_millis=cpu_millis, + memory_mib=memory_mib, + ephemeral_storage_mib=self.config.ephemeral_storage_mib, + deadline_seconds=deadline_seconds, + runtime_retry_count=0, + launcher_retry_count=launcher_retry_count, + ), + schedule=ScheduleProjection( + task_count=1, + maximum_parallelism=1, + expression=str(pipeline["schedule"]), + time_zone=str(pipeline["time_zone"]), + paused=bool(pipeline["paused"]), + ), + network=NetworkPlacement( + placement="awsvpc", + extensions=( + ("fargate_security_group_ids", ",".join(self.config.security_group_ids)), + ("fargate_subnet_ids", ",".join(self.config.subnet_ids)), + ), + ), + labels=( + ("dander_version", __version__), + ("image_digest", image.rsplit("@", maxsplit=1)[-1]), + ("pipeline", pipeline_id), + ("profile", "gcp"), + ), + observability=ObservabilityProjection( + log_destination="cloudwatch_logs", + metric_namespace="Dander", + alert_target=alert_target, + retention_days=30, + ), + extensions=( + ("fargate_architecture", self.config.architecture), + ( + "fargate_assign_public_ip", + "enabled" if self.config.assign_public_ip else "disabled", + ), + ), + ) + validate_launcher_projection(template, FARGATE_CAPABILITIES) + templates[pipeline_id] = template + return templates + + +def build_fargate_launcher( + config: BaseModel, + context: Mapping[str, object], +) -> LauncherRuntime: + """Build Fargate projection behavior only after launcher selection.""" + del context + if not isinstance(config, FargateLauncherConfig): + raise TypeError("Fargate launcher factory received the wrong configuration") + return LauncherRuntime( + provider_id="fargate", + region=config.region, + templates=FargateTemplateFactory(config), + capabilities=FARGATE_CAPABILITIES, + ) + + +def _memory_mib(value: str) -> int: + match = re.fullmatch(r"([1-9][0-9]*)(Mi|Gi)", value) + if match is None: + raise ExecutionProjectionError("runtime memory must use Mi or Gi") + quantity = int(match.group(1)) + return quantity if match.group(2) == "Mi" else quantity * 1_024 + + +def _validate_fargate_size(*, cpu_millis: int, memory_mib: int) -> None: + ranges = { + 1_000: range(2_048, 8_193, 1_024), + 2_000: range(4_096, 16_385, 1_024), + 4_000: range(8_192, 30_721, 1_024), + 8_000: range(16_384, 61_441, 4_096), + 16_000: range(32_768, 122_881, 8_192), + } + if memory_mib not in ranges.get(cpu_millis, range(0)): + raise ExecutionProjectionError("Fargate does not support the requested CPU/memory pair") + + +FARGATE_LAUNCHER_FACTORY: ProviderFactory[LauncherRuntime] = ProviderFactory( + kind=ProviderKind.LAUNCHER, + provider_id="fargate", + api_version=PROVIDER_API_VERSION, + build=build_fargate_launcher, +) + +__all__ = ["FARGATE_LAUNCHER_FACTORY"] diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py index 15346dd4..0251c688 100644 --- a/tests/providers/test_launcher_runtime.py +++ b/tests/providers/test_launcher_runtime.py @@ -4,10 +4,17 @@ import sys -from dander.deployment import LauncherRuntime, build_gcp_execution_templates +import pytest + +from dander.deployment import ( + ExecutionProjectionError, + LauncherRuntime, + build_gcp_execution_templates, +) from dander.providers import ProviderKind, default_provider_registry _IMAGE = "us-central1-docker.pkg.dev/unit-project/dander/dander@sha256:" + "a" * 64 +_FARGATE_IMAGE = "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "b" * 64 _PIPELINES: dict[str, dict[str, object]] = { "greenhouse_jobs": { "runtime_service_account_id": "dander-runtime", @@ -71,3 +78,109 @@ def test_cloud_run_factory_is_lazy_and_matches_the_accepted_projection() -> None ).items() } assert _templates(runtime) == expected + + +def _fargate_runtime() -> LauncherRuntime: + registry = default_provider_registry() + config = registry.parse( + ProviderKind.LAUNCHER, + { + "provider": "fargate", + "region": "us-east-1", + "aws_account_id": "184463061564", + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + "architecture": "ARM64", + "assign_public_ip": True, + }, + ) + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + return runtime + + +def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> None: + module_name = "dander.providers.fargate.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.LAUNCHER, + { + "provider": "fargate", + "region": "us-east-1", + "aws_account_id": "184463061564", + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + }, + ) + + assert module_name not in sys.modules + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + assert module_name in sys.modules + pipelines = { + "salesforce_crm": { + **_PIPELINES["greenhouse_jobs"], + "runtime_service_account_id": "dander-runtime-salesforce", + "secret_env": {"SALESFORCE_KEY": "salesforce-private-key"}, + } + } + template = runtime.templates.build( + pipelines, + image=_FARGATE_IMAGE, + project="unit-project", + cpu=1, + memory="2Gi", + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=1_000, + require_guarded_free_tier=False, + alert_target="arn:aws:sns:us-east-1:184463061564:dander-failures", + )["salesforce_crm"] + + assert template.launcher == "fargate" + assert template.workload_identity == ( + "arn:aws:iam::184463061564:role/dander-runtime-salesforce" + ) + assert template.resources.ephemeral_storage_mib == 20_480 + assert template.network.placement == "awsvpc" + assert dict(template.network.extensions) == { + "fargate_security_group_ids": "sg-0123456789abcdef0", + "fargate_subnet_ids": "subnet-0123456789abcdef0", + } + assert dict(template.extensions) == { + "fargate_architecture": "ARM64", + "fargate_assign_public_ip": "disabled", + } + secret = dict(template.secret_bindings)["SALESFORCE_KEY"] + assert secret.reference.endswith("/secrets/salesforce-private-key/versions/latest") + serialized = repr(template.as_dict()) + assert "PRIVATE KEY" not in serialized + assert "AWS_SECRET_ACCESS_KEY" not in serialized + + +@pytest.mark.parametrize( + ("memory", "guarded", "message"), + [ + ("512Mi", False, "CPU/memory"), + ("2Gi", True, "guarded-free-tier"), + ], +) +def test_fargate_rejects_unhonored_runtime_intent( + memory: str, + guarded: bool, + message: str, +) -> None: + with pytest.raises(ExecutionProjectionError, match=message): + _fargate_runtime().templates.build( + _PIPELINES, + image=_FARGATE_IMAGE, + project="unit-project", + cpu=1, + memory=memory, + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=1_000, + require_guarded_free_tier=guarded, + alert_target=None, + ) diff --git a/tickets/DANDER-89-cloud-run-launcher-runtime.md b/tickets/DANDER-89-cloud-run-launcher-runtime.md index bd12e684..c42ec5bb 100644 --- a/tickets/DANDER-89-cloud-run-launcher-runtime.md +++ b/tickets/DANDER-89-cloud-run-launcher-runtime.md @@ -1,7 +1,7 @@ --- id: DANDER-89 title: Route Cloud Run projection through the launcher provider boundary -status: in-review +status: completed component: deployment epic: cloud-portability depends_on: [DANDER-88] @@ -24,7 +24,7 @@ accepted template value, Terraform address, or live resource. - [x] Existing direct projection APIs remain compatible. - [x] Focused launcher, bootstrap, project, and projection tests pass. - [x] Full local validation and isolated GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -34,5 +34,5 @@ projector. Keep Terraform modules, resource addresses, schedules, IAM, and runti ## Review Log -Full local validation and the isolated retained-project no-drift proof passed. Protected review -remains pending. +Full local validation, isolated retained-project no-drift, and protected CI passed. PR #120 merged +as `2d8837d4a59c45c9b33b3ca211c9303b8fb1a16f`. diff --git a/tickets/DANDER-90-fargate-launcher-projection.md b/tickets/DANDER-90-fargate-launcher-projection.md new file mode 100644 index 00000000..3a4d0f09 --- /dev/null +++ b/tickets/DANDER-90-fargate-launcher-projection.md @@ -0,0 +1,40 @@ +--- +id: DANDER-90 +title: Project the BigQuery runtime onto Fargate +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-89] +created: 2026-08-08 +--- + +## Context + +Phase 1B proved immutable artifact copy and keyless Fargate-to-Google identity in isolation. The +first production-shaped step is a typed Fargate launcher projection that consumes the shared +execution contract without implying that infrastructure or lifecycle support already exists. + +## Acceptance Criteria + +- [x] Fargate configuration validates AWS account, region, network, architecture, and storage. +- [x] The implementation loads only after Fargate selection. +- [x] Templates use immutable ECR images and distinct AWS task-role identity. +- [x] BigQuery environment and GCP Secret Manager references contain no credential values. +- [x] CPU/memory pairs and Fargate resource limits fail closed. +- [x] Guarded-free-tier execution is rejected because its GCP billing preflight is unsupported. +- [x] Schedule, CloudWatch, `awsvpc`, architecture, and public-IP intent remain explicit. +- [x] Fargate remains absent from the supported runtime-capability manifest. +- [x] Focused projection and registry tests pass. +- [x] Full local validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Register one dependency-light `FargateLauncherConfig` and lazy factory. Reuse the shared immutable +execution template, carry provider-specific intent only in declared extensions, and keep AWS +Terraform, scheduler/controller behavior, credential assembly, and live execution out of this PR. + +## Review Log + +Full local validation and the isolated retained-project no-drift proof passed. Protected review +remains pending. From d43dedccb498ce460a4b76a8cbdc738a8117e5d4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 14:28:35 -0400 Subject: [PATCH 040/163] Prepare keyless AWS runtime identity --- CHANGELOG.md | 5 + HANDOFF.md | 41 +++-- docs/decisions.md | 10 ++ docs/execution-projection.md | 11 +- docs/known-limitations.md | 6 +- scripts/portability/wif_bigquery_probe.py | 67 +------- src/dander/cli/runtime_command.py | 2 + src/dander/deployment/projection.py | 2 +- src/dander/identity/__init__.py | 15 ++ src/dander/identity/aws_google.py | 160 ++++++++++++++++++ .../providers/aws_secrets_manager/__init__.py | 5 + .../providers/aws_secrets_manager/config.py | 14 ++ .../providers/aws_secrets_manager/runtime.py | 128 ++++++++++++++ src/dander/providers/defaults.py | 9 + src/dander/providers/fargate/config.py | 6 + src/dander/providers/fargate/runtime.py | 6 + src/dander/security/secret_manager.py | 6 +- src/dander/state/failure.py | 8 +- tests/cli/test_runtime_cli.py | 29 ++++ tests/identity/test_aws_google.py | 116 +++++++++++++ tests/portability/test_phase1b_tools.py | 15 +- tests/providers/test_launcher_runtime.py | 27 +++ tests/providers/test_secret_runtime.py | 83 ++++++++- .../DANDER-90-fargate-launcher-projection.md | 8 +- .../DANDER-91-aws-secrets-keyless-runtime.md | 43 +++++ 25 files changed, 720 insertions(+), 102 deletions(-) create mode 100644 src/dander/identity/__init__.py create mode 100644 src/dander/identity/aws_google.py create mode 100644 src/dander/providers/aws_secrets_manager/__init__.py create mode 100644 src/dander/providers/aws_secrets_manager/config.py create mode 100644 src/dander/providers/aws_secrets_manager/runtime.py create mode 100644 tests/identity/test_aws_google.py create mode 100644 tickets/DANDER-91-aws-secrets-keyless-runtime.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ff666fbf..0da748da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ capabilities enter through the next minor release. ### Added +- Add a lazy AWS Secrets Manager runtime that accepts only full, region-matching secret ARNs, + retains audited environment indirection, and never loads the AWS SDK until a secret is read. +- Prepare Fargate's keyless Google identity before runtime construction by adapting only temporary + ECS task-role credentials from the fixed link-local endpoint into a non-secret external-account + configuration; keep Fargate runs bounded to one hour until renewable credentials are supported. - Add a lazy ECS/Fargate launcher factory that projects the existing BigQuery data-plane runtime onto immutable ECR images, task-role identity, explicit networking, and valid Fargate resource pairs without yet claiming deployable Fargate support. diff --git a/HANDOFF.md b/HANDOFF.md index 7d8c6d78..59efe8e3 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,38 @@ ## Finished -- Added dependency-light Fargate configuration and lazy launcher registration. -- Projected immutable ECR images, task-role identity, `awsvpc` placement, and CloudWatch intent. -- Preserved the BigQuery/GCP runtime command and GCP Secret Manager references. -- Rejected invalid Fargate sizing and the unsupported guarded-free-tier path before planning. -- Kept Fargate out of the supported runtime-capability manifest. +- Added lazy, audited AWS Secrets Manager resolution for full region-matching ARNs. +- Prepared keyless Fargate-to-Google identity before provider clients are constructed. +- Reused the hardened Phase 1B ECS credential adapter instead of maintaining a second copy. +- Sanitized launcher identity failures and bounded Fargate projections to one hour. +- Preserved the unsupported Fargate capability boundary and all existing GCP behavior. ## Try It -Build the `fargate` launcher through the provider registry with a disposable AWS account, existing -subnet/security-group IDs, and a valid ECR digest. It returns an execution template only. +Build the `aws_secret_manager` provider with a synthetic client or build a Fargate execution +template with a valid WIF audience. Neither operation contacts or changes cloud infrastructure. ## Checks -- All 920 tests, Ruff, formatting, and strict mypy across 237 files passed. -- Wheel/sdist inspection, source-free installs, runtime-all assembly, and dependency audit passed. -- The non-root/read-only full runtime image passed conformance and bundled-asset checks. -- Trivy found no high/critical findings, Gitleaks found no leaks, and every Terraform root validated. -- Isolated GCP reported `No changes`; Salesforce and ServiceNow schedules remain paused. +- All 933 tests, Ruff, formatting, and strict mypy across 243 source files passed. +- Wheel, sdist, source-free installs, runtime-all assembly, and dependency audit passed. +- Container conformance, non-root/read-only checks, and bundled assets passed. +- Trivy found no high/critical findings; all Terraform roots validated. +- Isolated GCP reported `No changes`; AWS identity was confirmed read-only in `us-east-1`. ## Decisions -- Keep this PR to projection and validation; infrastructure and lifecycle are separate slices. -- Reuse each pipeline's stable runtime identity name as its future AWS task-role name. -- Reject GCP's guarded-free-tier preflight on Fargate instead of weakening it. +- Accept only temporary ECS task-role credentials from the fixed link-local endpoint. +- Keep the generated Google external-account file non-secret and its impersonated token at 600s. +- Cap Fargate at one hour until the task-role session can be renewed in-process. ## Remaining -- Merge the focused projection PR after protected CI passes. -- Add AWS secret resolution and the keyless BigQuery credential runtime. -- Add reviewed Fargate infrastructure and controller lifecycle. +- Open the focused PR and let protected CI repeat validation. +- Add Fargate infrastructure and controller lifecycle in a separate slice. ## Review First -- `src/dander/providers/fargate/config.py` -- `src/dander/providers/fargate/runtime.py` -- `tests/providers/test_launcher_runtime.py` +- `src/dander/identity/aws_google.py` +- `src/dander/providers/aws_secrets_manager/runtime.py` +- `src/dander/cli/runtime_command.py` diff --git a/docs/decisions.md b/docs/decisions.md index cdbb4944..4a4cc9e7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-08 — Fargate identity remains keyless and bounded + +- **Identity:** Fargate accepts only temporary `ASIA` task-role credentials from the fixed ECS + link-local endpoint. Preconfigured AWS access keys, unsafe endpoints, and nearly expired sessions + fail before provider clients are constructed. +- **Google federation:** Dander writes a non-secret external-account configuration with a 600-second + impersonated token lifetime; task credentials remain in process memory and are never logged. +- **Boundary:** Fargate deadlines are capped at one hour until renewable ECS credential supply and + launcher lifecycle are delivered. This remains construction readiness, not a support claim. + ## 2026-08-08 — Fargate projection precedes infrastructure support - **Boundary:** The lazy Fargate factory projects the existing BigQuery/GCP runtime onto an diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 54538a4b..f202742a 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -30,6 +30,11 @@ normalized by the OCI runtime contract. The Fargate provider can build the same BigQuery data-plane intent with an immutable ECR image, AWS task-role identity, `awsvpc` placement, CloudWatch destinations, and explicit Fargate resource -limits. This provider projection is an internal construction gate: provisioning, controller -lifecycle, AWS-to-Google credential assembly, and live acceptance must pass before Fargate is a -supported launcher. +limits. At runtime, Dander accepts only temporary credentials obtained from the fixed ECS task-role +endpoint, then prepares a non-secret Google external-account configuration before any provider +client is constructed. The Google impersonated token is limited to 600 seconds. Because this first +runtime bridge does not yet renew the copied ECS session, Fargate projections are limited to one +hour. + +This provider projection is an internal construction gate: provisioning, controller lifecycle, +renewable credential supply, and live acceptance must pass before Fargate is a supported launcher. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index dc311fa3..f137ca5d 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -4,8 +4,10 @@ Dander `0.7.x` is beta and proves a focused GCP-native vertical slice. It remain evaluate these limits before using it for an unattended system containing business-critical data. - AWS ECS/Fargate is not yet a supported Dander launcher. Phase 1B contains an isolated artifact - and keyless-identity feasibility stack only; launcher lifecycle, scheduling, alerts, rollback, - and parity remain gated on the later portable BigQuery vertical slice. + and keyless-identity feasibility stack only. The current runtime bridge copies one temporary ECS + task-role session into process memory and therefore caps a Fargate projection at one hour; + renewable credentials, launcher lifecycle, scheduling, alerts, rollback, and parity remain gated + on the later portable BigQuery vertical slice. - Provider package extras and the full runtime image contain planned provider SDK dependencies, but do not by themselves implement or qualify Snowflake, Redshift, PostgreSQL, AWS, Azure, or OCI profiles. The packaged capability manifest is the support boundary. diff --git a/scripts/portability/wif_bigquery_probe.py b/scripts/portability/wif_bigquery_probe.py index 6ebb5fcc..ae8260b1 100644 --- a/scripts/portability/wif_bigquery_probe.py +++ b/scripts/portability/wif_bigquery_probe.py @@ -4,27 +4,24 @@ import argparse import json -import os import platform import re import time -import urllib.request from datetime import UTC, datetime from typing import TYPE_CHECKING, Protocol import google.auth from google.cloud import bigquery +from dander.identity import prepare_fargate_task_credentials + if TYPE_CHECKING: - from collections.abc import Callable, MutableMapping + from collections.abc import Callable _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,1023}$") _PROJECT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") _SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) _SCHEMA = "io.dander.portability.identity-probe/v1" -_ECS_CREDENTIALS_PATH = re.compile(r"^/v2/credentials/[A-Za-z0-9-]{16,128}$") -_ECS_CREDENTIALS_ORIGIN = "http://169.254.170.2" -_TEMPORARY_ACCESS_KEY = re.compile(r"^ASIA[A-Z0-9]{16}$") class ProbeError(RuntimeError): @@ -40,62 +37,6 @@ def query_count(self, *, project: str, dataset: str, table: str) -> int: """Return a bounded count from the proof relation.""" -class CredentialFetcher(Protocol): - def __call__(self, url: str) -> object: - """Return one parsed ECS task-credential response.""" - - -def _fetch_ecs_credentials(url: str) -> object: - request = urllib.request.Request(url, headers={"Accept": "application/json"}) - with urllib.request.urlopen(request, timeout=5) as response: # noqa: S310 - if response.status != 200: - raise ProbeError("ECS task credentials were unavailable") - return json.load(response) - - -def prepare_fargate_task_credentials( - *, - environ: MutableMapping[str, str] = os.environ, - fetch: CredentialFetcher = _fetch_ecs_credentials, -) -> bool: - """Expose short-lived ECS task credentials to Google Auth without persisting them.""" - existing = tuple( - environ.get(name) - for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN") - ) - if all(existing): - return False - if any(existing): - raise ProbeError("AWS credential environment is incomplete") - - relative_uri = environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") - if relative_uri is None: - return False - if _ECS_CREDENTIALS_PATH.fullmatch(relative_uri) is None: - raise ProbeError("ECS task credential endpoint is invalid") - - document = fetch(f"{_ECS_CREDENTIALS_ORIGIN}{relative_uri}") - if not isinstance(document, dict): - raise ProbeError("ECS task credentials were invalid") - access_key = document.get("AccessKeyId") - secret_key = document.get("SecretAccessKey") - session_token = document.get("Token") - if ( - not isinstance(access_key, str) - or _TEMPORARY_ACCESS_KEY.fullmatch(access_key) is None - or not isinstance(secret_key, str) - or not secret_key - or not isinstance(session_token, str) - or not session_token - ): - raise ProbeError("ECS task credentials were invalid") - - environ["AWS_ACCESS_KEY_ID"] = access_key - environ["AWS_SECRET_ACCESS_KEY"] = secret_key - environ["AWS_SESSION_TOKEN"] = session_token - return True - - class BigQueryProbeClient: """Small adapter that keeps the exact proof query out of logs and task configuration.""" @@ -225,7 +166,7 @@ def main() -> int: refresh_margin_seconds=args.refresh_margin_seconds, emit=_emit, ) - except Exception as error: # noqa: BLE001 - the probe must sanitize every library failure + except Exception as error: # noqa: BLE001 _emit( { "schema": _SCHEMA, diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index bab41b7e..ec9c175c 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -12,6 +12,7 @@ from rich.console import Console from dander.cli.run_command import RunOptions, execute_run +from dander.identity import prepare_launcher_identity from dander.runtime_contract import ( RUNTIME_CONTRACT, LauncherContext, @@ -126,6 +127,7 @@ def execute_runtime( ) try: with graceful_signal_handlers(): + prepare_launcher_identity(context) result = execute_run( options, console=_CONSOLE, diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index 7736744a..cb4df525 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -344,7 +344,7 @@ class LauncherCapabilities: cpu_millis=frozenset({1_000, 2_000, 4_000, 8_000, 16_000}), minimum_memory_mib=512, maximum_memory_mib=122_880, - maximum_deadline_seconds=86_400, + maximum_deadline_seconds=3_600, maximum_launcher_retries=10, maximum_task_count=1, maximum_parallelism=1, diff --git a/src/dander/identity/__init__.py b/src/dander/identity/__init__.py new file mode 100644 index 00000000..39f40816 --- /dev/null +++ b/src/dander/identity/__init__.py @@ -0,0 +1,15 @@ +"""Launcher identity preparation without long-lived cloud credentials.""" + +from dander.identity.aws_google import ( + FargateIdentityError, + prepare_fargate_google_identity, + prepare_fargate_task_credentials, + prepare_launcher_identity, +) + +__all__ = [ + "FargateIdentityError", + "prepare_fargate_google_identity", + "prepare_fargate_task_credentials", + "prepare_launcher_identity", +] diff --git a/src/dander/identity/aws_google.py b/src/dander/identity/aws_google.py new file mode 100644 index 00000000..055b5d8e --- /dev/null +++ b/src/dander/identity/aws_google.py @@ -0,0 +1,160 @@ +"""Adapt short-lived Fargate task-role credentials for Google workload federation.""" + +from __future__ import annotations + +import json +import os +import re +import urllib.request +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from collections.abc import MutableMapping + + from dander.runtime_contract import LauncherContext + +_ECS_CREDENTIALS_PATH = re.compile(r"^/v2/credentials/[A-Za-z0-9-]{16,128}$") +_ECS_CREDENTIALS_ORIGIN = "http://169.254.170.2" +_TEMPORARY_ACCESS_KEY = re.compile(r"^ASIA[A-Z0-9]{16}$") +_AWS_CREDENTIAL_NAMES = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", +) +_WIF_AUDIENCE = re.compile( + r"^//iam\.googleapis\.com/projects/[0-9]{6,20}/locations/global/" + r"workloadIdentityPools/[a-z][a-z0-9-]{3,31}/providers/[a-z][a-z0-9-]{3,31}$" +) +_SERVICE_ACCOUNT = re.compile( + r"^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z][a-z0-9-]{4,28}[a-z0-9]" + r"\.iam\.gserviceaccount\.com$" +) +_DEFAULT_CREDENTIAL_PATH = Path("/tmp/dander-gcp-wif.json") + + +class FargateIdentityError(RuntimeError): + """Fargate task identity is missing, stale, or unsafe for federation.""" + + +class CredentialFetcher(Protocol): + def __call__(self, url: str) -> object: + """Return one parsed ECS task-credential response.""" + + +def _fetch_ecs_credentials(url: str) -> object: + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=5) as response: # noqa: S310 + if response.status != 200: + raise FargateIdentityError("Fargate task credentials were unavailable") + return json.load(response) + + +def prepare_launcher_identity(context: LauncherContext) -> None: + """Prepare only the selected launcher's ambient identity before provider clients start.""" + if context.launcher == "fargate": + prepare_fargate_google_identity() + + +def prepare_fargate_google_identity( + *, + environ: MutableMapping[str, str] = os.environ, + fetch: CredentialFetcher = _fetch_ecs_credentials, + credential_path: Path = _DEFAULT_CREDENTIAL_PATH, + now: datetime | None = None, +) -> None: + """Prepare task-role credentials and a non-secret Google external-account file.""" + audience = environ.get("DANDER_GCP_WIF_AUDIENCE", "") + service_account = environ.get("DANDER_GCP_SERVICE_ACCOUNT", "") + if ( + _WIF_AUDIENCE.fullmatch(audience) is None + or _SERVICE_ACCOUNT.fullmatch(service_account) is None + ): + raise FargateIdentityError("Fargate Google workload identity is invalid") + prepare_fargate_task_credentials(environ=environ, fetch=fetch, now=now) + config: dict[str, object] = { + "audience": audience, + "credential_source": { + "environment_id": "aws1", + "region_url": ("http://169.254.169.254/latest/meta-data/placement/availability-zone"), + "regional_cred_verification_url": ( + "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" + ), + "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials", + }, + "service_account_impersonation": {"token_lifetime_seconds": 600}, + "service_account_impersonation_url": ( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" + f"{service_account}:generateAccessToken" + ), + "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", + "token_url": "https://sts.googleapis.com/v1/token", + "type": "external_account", + } + temporary = credential_path.with_suffix(credential_path.suffix + ".tmp") + try: + temporary.write_text( + json.dumps(config, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + temporary.chmod(0o600) + os.replace(temporary, credential_path) + except OSError as error: + raise FargateIdentityError( + "Fargate Google workload identity configuration could not be prepared" + ) from error + finally: + temporary.unlink(missing_ok=True) + environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(credential_path) + + +def prepare_fargate_task_credentials( + *, + environ: MutableMapping[str, str] = os.environ, + fetch: CredentialFetcher = _fetch_ecs_credentials, + now: datetime | None = None, +) -> bool: + """Expose one bounded ECS task-role session to Google Auth in the current process.""" + existing = tuple(environ.get(name) for name in _AWS_CREDENTIAL_NAMES) + if any(existing): + raise FargateIdentityError("Fargate does not accept preconfigured AWS credentials") + + relative_uri = environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if relative_uri is None or _ECS_CREDENTIALS_PATH.fullmatch(relative_uri) is None: + raise FargateIdentityError("Fargate task credential endpoint is unavailable") + document = fetch(f"{_ECS_CREDENTIALS_ORIGIN}{relative_uri}") + if not isinstance(document, dict): + raise FargateIdentityError("Fargate task credentials were invalid") + access_key = document.get("AccessKeyId") + secret_key = document.get("SecretAccessKey") + session_token = document.get("Token") + expiration = _expiration(document.get("Expiration")) + current = (now or datetime.now(UTC)).astimezone(UTC) + if ( + not isinstance(access_key, str) + or _TEMPORARY_ACCESS_KEY.fullmatch(access_key) is None + or not isinstance(secret_key, str) + or not secret_key + or not isinstance(session_token, str) + or not session_token + or expiration <= current + timedelta(minutes=5) + ): + raise FargateIdentityError("Fargate task credentials were invalid") + + environ["AWS_SECRET_ACCESS_KEY"] = secret_key + environ["AWS_SESSION_TOKEN"] = session_token + environ["AWS_ACCESS_KEY_ID"] = access_key + return True + + +def _expiration(value: object) -> datetime: + if not isinstance(value, str): + raise FargateIdentityError("Fargate task credentials were invalid") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise FargateIdentityError("Fargate task credentials were invalid") from error + if parsed.tzinfo is None: + raise FargateIdentityError("Fargate task credentials were invalid") + return parsed.astimezone(UTC) diff --git a/src/dander/providers/aws_secrets_manager/__init__.py b/src/dander/providers/aws_secrets_manager/__init__.py new file mode 100644 index 00000000..c5f58d63 --- /dev/null +++ b/src/dander/providers/aws_secrets_manager/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light AWS Secrets Manager configuration.""" + +from dander.providers.aws_secrets_manager.config import AwsSecretsManagerConfig + +__all__ = ["AwsSecretsManagerConfig"] diff --git a/src/dander/providers/aws_secrets_manager/config.py b/src/dander/providers/aws_secrets_manager/config.py new file mode 100644 index 00000000..376bc140 --- /dev/null +++ b/src/dander/providers/aws_secrets_manager/config.py @@ -0,0 +1,14 @@ +"""Validated AWS Secrets Manager provider configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class AwsSecretsManagerConfig(BaseModel): + """Select AWS Secrets Manager in one AWS region.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["aws_secret_manager"] + region: str = Field(pattern=r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") diff --git a/src/dander/providers/aws_secrets_manager/runtime.py b/src/dander/providers/aws_secrets_manager/runtime.py new file mode 100644 index 00000000..f63e4506 --- /dev/null +++ b/src/dander/providers/aws_secrets_manager/runtime.py @@ -0,0 +1,128 @@ +"""AWS Secrets Manager resolution selected through the provider registry.""" + +from __future__ import annotations + +import re +from importlib import import_module +from typing import TYPE_CHECKING, Protocol, cast + +from dander.providers.aws_secrets_manager.config import AwsSecretsManagerConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.security.runtime import SecretCapabilities, SecretRuntime +from dander.security.secret_manager import ( + EnvironmentSecretStore, + SecretResolutionError, + audit_secret_access, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + + from pydantic import BaseModel + +_AWS_REFERENCE = re.compile( + r"^aws-sm://arn:(?:aws|aws-us-gov):secretsmanager:(?P[a-z0-9-]+):[0-9]{12}:" + r"secret:[A-Za-z0-9/_+=.@-]{1,512}$" +) + + +class _SecretsManagerClient(Protocol): + def get_secret_value(self, *, SecretId: str) -> Mapping[str, object]: # noqa: N803 + """Return one AWS Secrets Manager value.""" + + +class AwsSecretStore: + """Resolve text secrets by fully qualified AWS Secrets Manager ARN.""" + + def __init__(self, *, region: str, client: _SecretsManagerClient | None = None) -> None: + self._region = region + self._client = client + + def get_secret(self, reference: str) -> str: + """Resolve an `aws-sm://` ARN without accepting ambiguous secret names.""" + match = _AWS_REFERENCE.fullmatch(reference) + if match is None: + raise SecretResolutionError("AWS secret references must use a full aws-sm:// ARN") + if match.group("region") != self._region: + raise SecretResolutionError("AWS secret reference region does not match the provider") + arn = reference.removeprefix("aws-sm://") + response = self._secrets_client().get_secret_value(SecretId=arn) + value = response.get("SecretString") + if not isinstance(value, str) or not value: + raise SecretResolutionError("AWS secret value is missing or is not text") + audit_secret_access(reference, "aws_secret_manager") + return value + + def _secrets_client(self) -> _SecretsManagerClient: + if self._client is None: + boto3 = cast("Any", import_module("boto3")) + self._client = cast( + "_SecretsManagerClient", + boto3.client("secretsmanager", region_name=self._region), + ) + return self._client + + +class AwsSecretRuntimeStore: + """Resolve direct AWS ARNs or environment names that contain an AWS ARN.""" + + def __init__( + self, + *, + aws: AwsSecretStore, + environment: EnvironmentSecretStore | None = None, + ) -> None: + self._aws = aws + self._environment = environment or EnvironmentSecretStore() + + def get_secret(self, reference: str) -> str: + """Resolve one AWS secret reference with legacy environment indirection.""" + if reference.startswith("aws-sm://"): + return self._aws.get_secret(reference) + value = self._environment.get_secret(reference) + if value.startswith("aws-sm://"): + return self._aws.get_secret(value) + return value + + +def build_aws_secret_manager( + config: BaseModel, + context: Mapping[str, object], +) -> SecretRuntime: + """Build AWS resolution without loading boto3 or making a provider call.""" + if not isinstance(config, AwsSecretsManagerConfig): + raise TypeError("AWS Secrets Manager factory received the wrong configuration") + environment_value = context.get("environment") + client = context.get("client") + environment = ( + EnvironmentSecretStore(cast("Mapping[str, str]", environment_value)) + if environment_value is not None + else None + ) + return SecretRuntime( + provider_id="aws_secret_manager", + store=AwsSecretRuntimeStore( + aws=AwsSecretStore( + region=config.region, + client=cast("_SecretsManagerClient", client) if client is not None else None, + ), + environment=environment, + ), + capabilities=SecretCapabilities( + provider_id="aws_secret_manager", + reference_forms=frozenset({"aws_secret_arn", "environment_name"}), + environment_indirection=True, + audited_access=True, + ), + ) + + +AWS_SECRET_MANAGER_FACTORY: ProviderFactory[SecretRuntime] = ProviderFactory( + kind=ProviderKind.SECRETS, + provider_id="aws_secret_manager", + api_version=PROVIDER_API_VERSION, + build=build_aws_secret_manager, +) + +__all__ = ["AWS_SECRET_MANAGER_FACTORY"] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 4c1bf8c4..2d1b1c4a 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -1,5 +1,6 @@ """Built-in provider registrations with lazy implementation loading.""" +from dander.providers.aws_secrets_manager import AwsSecretsManagerConfig from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig from dander.providers.cloud_run import CloudRunLauncherConfig from dander.providers.dataplex import DataplexCatalogConfig @@ -61,6 +62,14 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.environment_secrets.runtime:ENVIRONMENT_SECRET_FACTORY" ), ) + registry.register( + kind=ProviderKind.SECRETS, + provider_id="aws_secret_manager", + config_model=AwsSecretsManagerConfig, + load_factory=lazy_provider_factory( + "dander.providers.aws_secrets_manager.runtime:AWS_SECRET_MANAGER_FACTORY" + ), + ) registry.register( kind=ProviderKind.LAUNCHER, provider_id="cloud_run", diff --git a/src/dander/providers/fargate/config.py b/src/dander/providers/fargate/config.py index a35abf16..2647df3d 100644 --- a/src/dander/providers/fargate/config.py +++ b/src/dander/providers/fargate/config.py @@ -17,6 +17,12 @@ class FargateLauncherConfig(BaseModel): provider: Literal["fargate"] region: str = Field(pattern=r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") aws_account_id: str = Field(pattern=r"^[0-9]{12}$") + google_workload_identity_audience: str = Field( + pattern=( + r"^//iam\.googleapis\.com/projects/[0-9]{6,20}/locations/global/" + r"workloadIdentityPools/[a-z][a-z0-9-]{3,31}/providers/[a-z][a-z0-9-]{3,31}$" + ) + ) subnet_ids: tuple[str, ...] = Field(min_length=1) security_group_ids: tuple[str, ...] = Field(min_length=1) architecture: Literal["ARM64", "X86_64"] = "ARM64" diff --git a/src/dander/providers/fargate/runtime.py b/src/dander/providers/fargate/runtime.py index d956866d..a2212652 100644 --- a/src/dander/providers/fargate/runtime.py +++ b/src/dander/providers/fargate/runtime.py @@ -104,6 +104,12 @@ def build( "BQ_DATASET_METADATA": "dander_meta", "BQ_DATASET_RAW": "raw", "DANDER_IMAGE_DIGEST": image.rsplit("@", maxsplit=1)[-1], + "DANDER_GCP_SERVICE_ACCOUNT": ( + f"{role_name}@{project}.iam.gserviceaccount.com" + ), + "DANDER_GCP_WIF_AUDIENCE": ( + self.config.google_workload_identity_audience + ), "DANDER_LAUNCHER": "fargate", "DANDER_PRINCIPAL": identity, "GCP_PROJECT_ID": project, diff --git a/src/dander/security/secret_manager.py b/src/dander/security/secret_manager.py index a05dfe84..88ea2aa6 100644 --- a/src/dander/security/secret_manager.py +++ b/src/dander/security/secret_manager.py @@ -38,7 +38,7 @@ def access_secret_version(self, *, request: Mapping[str, str]) -> object: """Access one Secret Manager version.""" -def _audit_access(reference: str, backend: str) -> None: +def audit_secret_access(reference: str, backend: str) -> None: """Emit a credential-access audit event without exposing secret material.""" _LOGGER.info( "credential_access", @@ -83,7 +83,7 @@ def get_secret(self, reference: str) -> str: self._client.access_secret_version(request={"name": reference}), ) value = response.payload.data.decode("utf-8") - _audit_access(reference, "gcp_secret_manager") + audit_secret_access(reference, "gcp_secret_manager") return value @@ -104,7 +104,7 @@ def get_secret(self, reference: str) -> str: raise SecretResolutionError( f"Secret environment reference {reference!r} is missing or empty" ) - _audit_access(reference, "environment") + audit_secret_access(reference, "environment") return value diff --git a/src/dander/state/failure.py b/src/dander/state/failure.py index 8663c2f4..58462295 100644 --- a/src/dander/state/failure.py +++ b/src/dander/state/failure.py @@ -31,7 +31,13 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu f"The runtime received a cancellation signal. A fresh run can retry safely; " f"inspect logs for run {run_id}.", ) - if 401 in statuses or names & {"Unauthenticated", "Unauthorized"}: + if 401 in statuses or names & { + "FargateIdentityError", + "NoCredentialsError", + "PartialCredentialsError", + "Unauthenticated", + "Unauthorized", + }: return _details( "authentication_failed", "Authentication failed. Verify the configured secret and provider credentials.", diff --git a/tests/cli/test_runtime_cli.py b/tests/cli/test_runtime_cli.py index 858fa1b2..b087eea6 100644 --- a/tests/cli/test_runtime_cli.py +++ b/tests/cli/test_runtime_cli.py @@ -11,6 +11,7 @@ import dander.cli.runtime_command as runtime_module from dander.cli.main import app from dander.executor import PipelineExecutionResult +from dander.identity import FargateIdentityError from dander.runtime import EndpointRunResult, PipelineRunResult from dander.runtime_contract import RuntimeCancelledError, RuntimeExitCode @@ -188,3 +189,31 @@ def test_runtime_execute_rejects_unknown_contract_before_start() -> None: assert result.exit_code == RuntimeExitCode.INVALID_INVOCATION assert "unsupported runtime contract" in result.output assert "runtime.started" not in result.output + + +def test_runtime_execute_sanitizes_fargate_identity_failure( + monkeypatch: MonkeyPatch, +) -> None: + def identity_failure(_context: object) -> None: + raise FargateIdentityError("temporary credential detail") + + monkeypatch.setattr(runtime_module, "prepare_launcher_identity", identity_failure) + result = CliRunner().invoke( + app, + [ + "runtime", + "execute", + "--contract", + "io.dander.runtime/v1", + "--pipeline", + "greenhouse_jobs", + "--platform", + "gcp", + ], + env={"DANDER_RUN_ID": "fargate:task-42", "DANDER_LAUNCHER": "fargate"}, + ) + + assert result.exit_code == RuntimeExitCode.PERMANENT_FAILURE + terminal = json.loads(result.output.splitlines()[-1]) + assert terminal["failure_code"] == "authentication_failed" + assert "temporary credential detail" not in result.output diff --git a/tests/identity/test_aws_google.py b/tests/identity/test_aws_google.py new file mode 100644 index 00000000..05271c7d --- /dev/null +++ b/tests/identity/test_aws_google.py @@ -0,0 +1,116 @@ +"""Fargate task-role adaptation for Google workload federation.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +import pytest + +from dander.identity import ( + FargateIdentityError, + prepare_fargate_google_identity, + prepare_fargate_task_credentials, +) + +if TYPE_CHECKING: + from pathlib import Path + +_RELATIVE_URI = "/v2/credentials/12345678-1234-1234-1234-123456789abc" +_NOW = datetime(2026, 8, 8, 12, 0, tzinfo=UTC) + + +def _document() -> dict[str, str]: + return { + "AccessKeyId": "ASIA" + "A" * 16, + "SecretAccessKey": "temporary-" + "secret", + "Token": "temporary-" + "session-token", + "Expiration": "2026-08-08T13:00:00Z", + } + + +def test_fargate_identity_uses_only_the_fixed_ecs_credential_origin() -> None: + environment = {"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": _RELATIVE_URI} + urls: list[str] = [] + + def fetch(url: str) -> object: + urls.append(url) + return _document() + + assert prepare_fargate_task_credentials( + environ=environment, + fetch=fetch, + now=_NOW, + ) + + assert urls == [f"http://169.254.170.2{_RELATIVE_URI}"] + assert environment["AWS_ACCESS_KEY_ID"].startswith("ASIA") + assert environment["AWS_SESSION_TOKEN"] + + +def test_fargate_google_identity_writes_only_non_secret_external_config(tmp_path: Path) -> None: + credential_path = tmp_path / "wif.json" + environment = { + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": _RELATIVE_URI, + "DANDER_GCP_SERVICE_ACCOUNT": ("dander-runtime@unit-project.iam.gserviceaccount.com"), + "DANDER_GCP_WIF_AUDIENCE": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase1b-aws/providers/fargate" + ), + } + + prepare_fargate_google_identity( + environ=environment, + fetch=lambda _url: _document(), + credential_path=credential_path, + now=_NOW, + ) + + config = json.loads(credential_path.read_text(encoding="utf-8")) + assert config["type"] == "external_account" + assert config["credential_source"]["environment_id"] == "aws1" + assert config["service_account_impersonation"] == {"token_lifetime_seconds": 600} + assert environment["GOOGLE_APPLICATION_CREDENTIALS"] == str(credential_path) + assert credential_path.stat().st_mode & 0o777 == 0o600 + serialized = credential_path.read_text(encoding="utf-8") + assert "private_key" not in serialized + assert "client_secret" not in serialized + assert environment["AWS_SECRET_ACCESS_KEY"] not in serialized + + +@pytest.mark.parametrize( + ("environment", "message"), + [ + ({}, "endpoint"), + ({"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "http://example.invalid"}, "endpoint"), + ({"AWS_ACCESS_KEY_ID": "AKIA" + "A" * 16}, "preconfigured"), + ( + { + "AWS_ACCESS_KEY_ID": "AKIA" + "A" * 16, + "AWS_SECRET_ACCESS_KEY": "not-temporary", + "AWS_SESSION_TOKEN": "not-temporary", + }, + "preconfigured", + ), + ], +) +def test_fargate_identity_rejects_missing_or_long_lived_credentials( + environment: dict[str, str], + message: str, +) -> None: + with pytest.raises(FargateIdentityError, match=message): + prepare_fargate_task_credentials(environ=environment, now=_NOW) + + +def test_fargate_identity_rejects_credentials_near_expiry() -> None: + environment = {"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": _RELATIVE_URI} + document = _document() + document["Expiration"] = "2026-08-08T12:04:59Z" + + with pytest.raises(FargateIdentityError, match="invalid"): + prepare_fargate_task_credentials( + environ=environment, + fetch=lambda _url: document, + now=_NOW, + ) diff --git a/tests/portability/test_phase1b_tools.py b/tests/portability/test_phase1b_tools.py index 8f1b1727..6ed2c78f 100644 --- a/tests/portability/test_phase1b_tools.py +++ b/tests/portability/test_phase1b_tools.py @@ -12,10 +12,11 @@ from scripts.portability.scan_long_lived_credentials import scan from scripts.portability.wif_bigquery_probe import ( ProbeError, - prepare_fargate_task_credentials, run_probe, ) +from dander.identity import FargateIdentityError, prepare_fargate_task_credentials + if TYPE_CHECKING: from collections.abc import Sequence from pathlib import Path @@ -196,9 +197,17 @@ def fetch(url: str) -> object: "AccessKeyId": access_key, "SecretAccessKey": secret_key, "Token": session_token, + "Expiration": "2026-08-07T13:00:00Z", } - assert prepare_fargate_task_credentials(environ=environment, fetch=fetch) is True + assert ( + prepare_fargate_task_credentials( + environ=environment, + fetch=fetch, + now=datetime(2026, 8, 7, 12, 0, tzinfo=UTC), + ) + is True + ) assert requested_urls == [f"http://169.254.170.2{relative_uri}"] assert environment["AWS_ACCESS_KEY_ID"] == access_key assert environment["AWS_SECRET_ACCESS_KEY"] == secret_key @@ -210,7 +219,7 @@ def fetch(url: str) -> object: ["https://example.invalid/credentials", "/latest/meta-data/iam/security-credentials"], ) def test_fargate_task_credentials_reject_an_untrusted_endpoint(relative_uri: str) -> None: - with pytest.raises(ProbeError, match="endpoint is invalid"): + with pytest.raises(FargateIdentityError, match="endpoint"): prepare_fargate_task_credentials( environ={"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": relative_uri}, fetch=lambda _url: pytest.fail("untrusted endpoint must not be fetched"), diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py index 0251c688..da34013a 100644 --- a/tests/providers/test_launcher_runtime.py +++ b/tests/providers/test_launcher_runtime.py @@ -88,6 +88,10 @@ def _fargate_runtime() -> LauncherRuntime: "provider": "fargate", "region": "us-east-1", "aws_account_id": "184463061564", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase1b-aws/providers/fargate" + ), "subnet_ids": ["subnet-0123456789abcdef0"], "security_group_ids": ["sg-0123456789abcdef0"], "architecture": "ARM64", @@ -109,6 +113,10 @@ def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> "provider": "fargate", "region": "us-east-1", "aws_account_id": "184463061564", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase1b-aws/providers/fargate" + ), "subnet_ids": ["subnet-0123456789abcdef0"], "security_group_ids": ["sg-0123456789abcdef0"], }, @@ -143,6 +151,9 @@ def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> "arn:aws:iam::184463061564:role/dander-runtime-salesforce" ) assert template.resources.ephemeral_storage_mib == 20_480 + assert dict(template.environment)["DANDER_GCP_SERVICE_ACCOUNT"] == ( + "dander-runtime-salesforce@unit-project.iam.gserviceaccount.com" + ) assert template.network.placement == "awsvpc" assert dict(template.network.extensions) == { "fargate_security_group_ids": "sg-0123456789abcdef0", @@ -184,3 +195,19 @@ def test_fargate_rejects_unhonored_runtime_intent( require_guarded_free_tier=guarded, alert_target=None, ) + + +def test_fargate_rejects_a_run_longer_than_the_task_role_session() -> None: + with pytest.raises(ExecutionProjectionError, match="deadline"): + _fargate_runtime().templates.build( + _PIPELINES, + image=_FARGATE_IMAGE, + project="unit-project", + cpu=1, + memory="2Gi", + deadline_seconds=3_601, + launcher_retry_count=1, + batch_rows=1_000, + require_guarded_free_tier=False, + alert_target=None, + ) diff --git a/tests/providers/test_secret_runtime.py b/tests/providers/test_secret_runtime.py index d688a22e..511ad2a2 100644 --- a/tests/providers/test_secret_runtime.py +++ b/tests/providers/test_secret_runtime.py @@ -2,13 +2,16 @@ from __future__ import annotations +import logging import sys from dataclasses import dataclass from typing import TYPE_CHECKING +import pytest + from dander.cli.provider_runtime import build_secret_store from dander.providers import ProviderKind, default_provider_registry -from dander.security import SecretRuntime +from dander.security import SecretResolutionError, SecretRuntime if TYPE_CHECKING: from collections.abc import Mapping @@ -33,6 +36,16 @@ def access_secret_version(self, *, request: Mapping[str, str]) -> _Response: return _Response(payload=_Payload(data=b"resolved-value")) +class _AwsClient: + def __init__(self, value: object = "resolved-value") -> None: + self.value = value + self.secret_ids: list[str] = [] + + def get_secret_value(self, *, SecretId: str) -> dict[str, object]: # noqa: N803 + self.secret_ids.append(SecretId) + return {"SecretString": self.value} + + def test_environment_provider_loads_without_gcp_runtime_or_sdk() -> None: gcp_runtime = "dander.providers.gcp_secret_manager.runtime" google_sdk = "google.cloud.secretmanager" @@ -81,3 +94,71 @@ def test_gcp_provider_preserves_environment_indirection_without_sdk_eagerness() def test_cli_builds_environment_store_through_provider_registry() -> None: store = build_secret_store("environment") assert type(store).__name__ == "EnvironmentSecretStore" + + +def test_aws_provider_is_lazy_and_audits_environment_indirection( + caplog: pytest.LogCaptureFixture, +) -> None: + runtime_module = "dander.providers.aws_secrets_manager.runtime" + sys.modules.pop(runtime_module, None) + sys.modules.pop("boto3", None) + client = _AwsClient() + reference = "aws-sm://arn:aws:secretsmanager:us-east-1:123456789012:secret:dander/source-token" + registry = default_provider_registry() + config = registry.parse( + ProviderKind.SECRETS, + {"provider": "aws_secret_manager", "region": "us-east-1"}, + ) + + assert runtime_module not in sys.modules + runtime = registry.build( + ProviderKind.SECRETS, + config, + context={"environment": {"DANDER_TEST_TOKEN": reference}, "client": client}, + ) + assert isinstance(runtime, SecretRuntime) + assert "boto3" not in sys.modules + with caplog.at_level(logging.INFO): + assert runtime.store.get_secret("DANDER_TEST_TOKEN") == "resolved-value" + + assert client.secret_ids == [reference.removeprefix("aws-sm://")] + assert caplog.records[-1].__dict__["secret_backend"] == "aws_secret_manager" + assert "resolved-value" not in caplog.text + assert runtime.capabilities.reference_forms == frozenset({"aws_secret_arn", "environment_name"}) + + +@pytest.mark.parametrize( + ("reference", "client", "message"), + [ + ("aws-sm://short-name", _AwsClient(), "full"), + ( + "aws-sm://arn:aws:secretsmanager:us-west-2:123456789012:secret:dander/token", + _AwsClient(), + "region", + ), + ( + "aws-sm://arn:aws:secretsmanager:us-east-1:123456789012:secret:dander/token", + _AwsClient(value=b"binary"), + "not text", + ), + ], +) +def test_aws_provider_rejects_ambiguous_cross_region_or_binary_secrets( + reference: str, + client: _AwsClient, + message: str, +) -> None: + registry = default_provider_registry() + config = registry.parse( + ProviderKind.SECRETS, + {"provider": "aws_secret_manager", "region": "us-east-1"}, + ) + runtime = registry.build( + ProviderKind.SECRETS, + config, + context={"client": client, "environment": {}}, + ) + assert isinstance(runtime, SecretRuntime) + + with pytest.raises(SecretResolutionError, match=message): + runtime.store.get_secret(reference) diff --git a/tickets/DANDER-90-fargate-launcher-projection.md b/tickets/DANDER-90-fargate-launcher-projection.md index 3a4d0f09..0f2fe86a 100644 --- a/tickets/DANDER-90-fargate-launcher-projection.md +++ b/tickets/DANDER-90-fargate-launcher-projection.md @@ -1,7 +1,7 @@ --- id: DANDER-90 title: Project the BigQuery runtime onto Fargate -status: in-review +status: completed component: deployment epic: cloud-portability depends_on: [DANDER-89] @@ -26,7 +26,7 @@ execution contract without implying that infrastructure or lifecycle support alr - [x] Fargate remains absent from the supported runtime-capability manifest. - [x] Focused projection and registry tests pass. - [x] Full local validation and isolated GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -36,5 +36,5 @@ Terraform, scheduler/controller behavior, credential assembly, and live executio ## Review Log -Full local validation and the isolated retained-project no-drift proof passed. Protected review -remains pending. +Full local validation, isolated retained-project no-drift, and protected CI passed. PR #121 merged +as `cff311ce23d6b7ef9dbad8b743bb9a8ae8321ac4`. diff --git a/tickets/DANDER-91-aws-secrets-keyless-runtime.md b/tickets/DANDER-91-aws-secrets-keyless-runtime.md new file mode 100644 index 00000000..dc11f72a --- /dev/null +++ b/tickets/DANDER-91-aws-secrets-keyless-runtime.md @@ -0,0 +1,43 @@ +--- +id: DANDER-91 +title: Prepare AWS secrets and keyless Fargate runtime identity +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-90] +created: 2026-08-08 +--- + +## Context + +The Fargate projection names task-role and Google workload-federation intent, but runtime startup +must prepare keyless Google credentials and AWS secret resolution before any provider client is +constructed. This slice keeps those adapters lazy and does not provision or advertise Fargate. + +## Acceptance Criteria + +- [x] AWS Secrets Manager registers lazily and does not load boto3 before real access. +- [x] Secret references require a full, region-matching ARN and text value. +- [x] Environment indirection and value-free access auditing remain available. +- [x] Fargate accepts only temporary ECS task-role credentials from the fixed link-local endpoint. +- [x] Static AWS credential environment variables and stale sessions fail closed. +- [x] Google external-account configuration contains no secret and limits impersonation to 600s. +- [x] Runtime identity preparation occurs before provider-client construction. +- [x] Identity failures produce sanitized authentication telemetry. +- [x] Fargate execution remains bounded while ECS credentials are not renewable in-process. +- [x] Focused tests, lint, formatting, and strict typing pass. +- [x] Full validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Reuse the Phase 1B ECS credential adapter in core, expose only one launcher identity hook, and keep +AWS-specific code behind lazy provider selection. Do not add AWS Terraform, scheduler lifecycle, +profile support, or a public support claim in this PR. + +## Review Log + +All 933 tests, lint, formatting, typing, distribution, dependency, container, security, and +Terraform checks passed. The isolated GCP plan reported exactly `No changes`, both schedules +remain paused, and read-only AWS identity confirmed account `184463061564` in `us-east-1`. +Protected review remains pending. From df83edb27b62c67c715422da106bb7237368eb54 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 15:17:11 -0400 Subject: [PATCH 041/163] feat: provision bounded Fargate runtime --- .github/workflows/ci.yml | 6 + CHANGELOG.md | 4 + HANDOFF.md | 41 +- docs/decisions.md | 10 + docs/known-limitations.md | 10 +- infra/aws/.terraform.lock.hcl | 27 + infra/aws/README.md | 18 + infra/aws/main.tf | 12 + infra/aws/modules/fargate/README.md | 12 + infra/aws/modules/fargate/main.tf | 894 ++++++++++++++++++ infra/aws/modules/fargate/outputs.tf | 33 + .../modules/fargate/tests/fargate.tftest.hcl | 117 +++ infra/aws/modules/fargate/variables.tf | 144 +++ infra/aws/outputs.tf | 24 + infra/aws/terraform.tfvars.example | 61 ++ infra/aws/variables.tf | 60 ++ infra/aws/versions.tf | 24 + pyproject.toml | 12 + src/dander/deployment/projection.py | 1 + src/dander/providers/fargate/config.py | 1 + src/dander/providers/fargate/runtime.py | 20 +- tests/infra/test_fargate_runtime.py | 90 ++ tests/providers/test_launcher_runtime.py | 46 + ...ER-92-fargate-infrastructure-controller.md | 44 + 24 files changed, 1685 insertions(+), 26 deletions(-) create mode 100644 infra/aws/.terraform.lock.hcl create mode 100644 infra/aws/README.md create mode 100644 infra/aws/main.tf create mode 100644 infra/aws/modules/fargate/README.md create mode 100644 infra/aws/modules/fargate/main.tf create mode 100644 infra/aws/modules/fargate/outputs.tf create mode 100644 infra/aws/modules/fargate/tests/fargate.tftest.hcl create mode 100644 infra/aws/modules/fargate/variables.tf create mode 100644 infra/aws/outputs.tf create mode 100644 infra/aws/terraform.tfvars.example create mode 100644 infra/aws/variables.tf create mode 100644 infra/aws/versions.tf create mode 100644 tests/infra/test_fargate_runtime.py create mode 100644 tickets/DANDER-92-fargate-infrastructure-controller.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f89873b..b786a524 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,12 @@ jobs: run: | terraform -chdir=bootstrap-admin init -backend=false -input=false terraform -chdir=bootstrap-admin validate + - name: Validate AWS Fargate stack + run: | + terraform -chdir=aws init -backend=false -input=false + terraform -chdir=aws validate + terraform -chdir=aws/modules/fargate init -backend=false -input=false + terraform -chdir=aws/modules/fargate test - name: Validate cross-cloud feasibility roots run: | terraform fmt -check -recursive ../acceptance/cloud-portability/phase1b diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da748da..a01cb6b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ capabilities enter through the next minor release. ### Added +- Add a separately packaged AWS Terraform stack for immutable ECR images, non-root Fargate task + definitions, distinct task/execution roles, paused-aware EventBridge schedules, and encrypted + failure routing. A Standard Step Functions controller enforces one absolute deadline, retries + only runtime exit code 75, and keeps scheduler delivery retries separate from launcher attempts. - Add a lazy AWS Secrets Manager runtime that accepts only full, region-matching secret ARNs, retains audited environment indirection, and never loads the AWS SDK until a secret is read. - Prepare Fargate's keyless Google identity before runtime construction by adapting only temporary diff --git a/HANDOFF.md b/HANDOFF.md index 59efe8e3..53c2271e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,39 @@ ## Finished -- Added lazy, audited AWS Secrets Manager resolution for full region-matching ARNs. -- Prepared keyless Fargate-to-Google identity before provider clients are constructed. -- Reused the hardened Phase 1B ECS credential adapter instead of maintaining a second copy. -- Sanitized launcher identity failures and bounded Fargate projections to one hour. -- Preserved the unsupported Fargate capability boundary and all existing GCP behavior. +- Added a separate packaged AWS Terraform root and product Fargate module. +- Projected immutable ECR images into non-root, read-only Fargate task definitions. +- Separated execution, per-pipeline task, controller, and scheduler IAM roles. +- Added an absolute-deadline Step Functions controller with exit-75-only retries. +- Added paused-aware scheduling, controller logs, encrypted failure queue, and a customer-key-encrypted notification topic. ## Try It -Build the `aws_secret_manager` provider with a synthetic client or build a Fargate execution -template with a valid WIF audience. Neither operation contacts or changes cloud infrastructure. +Run `terraform -chdir=infra/aws/modules/fargate test` for a provider-mocked plan. The example stack +is paused and no Terraform apply has been performed. ## Checks -- All 933 tests, Ruff, formatting, and strict mypy across 243 source files passed. -- Wheel, sdist, source-free installs, runtime-all assembly, and dependency audit passed. -- Container conformance, non-root/read-only checks, and bundled assets passed. -- Trivy found no high/critical findings; all Terraform roots validated. -- Isolated GCP reported `No changes`; AWS identity was confirmed read-only in `us-east-1`. +- Ruff, formatting, strict typing, and all 941 Python tests passed. +- Every Terraform root validated; the provider-mocked Fargate plan passed. +- Wheel/sdist installs, source-free generation, container conformance, dependency audit, and Trivy scans passed. +- Read-only AWS plan: 23 add, 0 change, 0 destroy; schedule disabled and nothing applied. +- Existing isolated GCP platform plan reported exactly `No changes.` ## Decisions -- Accept only temporary ECS task-role credentials from the fixed link-local endpoint. -- Keep the generated Google external-account file non-secret and its impersonated token at 600s. -- Cap Fargate at one hour until the task-role session can be renewed in-process. +- Use Standard Step Functions and its optimized ECS `.sync` integration. +- Retry only Dander exit code 75; keep scheduler delivery retries separate. +- Keep Fargate outside the support manifest until CLI lifecycle and live parity pass. ## Remaining -- Open the focused PR and let protected CI repeat validation. -- Add Fargate infrastructure and controller lifecycle in a separate slice. +- Let protected Linux CI repeat validation and merge the focused PR if clean. +- Add manifest-aware AWS planning and CLI lifecycle in a separate slice. +- Apply only after a separately reviewed plan; then prove live Fargate parity. ## Review First -- `src/dander/identity/aws_google.py` -- `src/dander/providers/aws_secrets_manager/runtime.py` -- `src/dander/cli/runtime_command.py` +- `infra/aws/modules/fargate/main.tf` +- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` +- `src/dander/providers/fargate/runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 4a4cc9e7..55d9432a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-08 — Fargate lifecycle is provider-native and bounded + +- **Controller:** One Standard Step Functions state machine per pipeline uses the optimized ECS + `.sync` integration. Its workflow timeout bounds all whole-runtime attempts and AWS owns + best-effort `StopTask` behavior on timeout or cancellation. +- **Retries:** Exit code 75 is the only runtime result eligible for a launcher retry. EventBridge + delivery retries remain a distinct counter and both exhausted paths reach an encrypted queue. +- **Boundary:** The AWS stack is packaged separately with native S3 backend configuration and + remains outside the public support manifest until CLI lifecycle and live parity are proven. + ## 2026-08-08 — Fargate identity remains keyless and bounded - **Identity:** Fargate accepts only temporary `ASIA` task-role credentials from the fixed ECS diff --git a/docs/known-limitations.md b/docs/known-limitations.md index f137ca5d..13cec975 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -3,11 +3,11 @@ Dander `0.7.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. -- AWS ECS/Fargate is not yet a supported Dander launcher. Phase 1B contains an isolated artifact - and keyless-identity feasibility stack only. The current runtime bridge copies one temporary ECS - task-role session into process memory and therefore caps a Fargate projection at one hour; - renewable credentials, launcher lifecycle, scheduling, alerts, rollback, and parity remain gated - on the later portable BigQuery vertical slice. +- AWS ECS/Fargate is not yet a supported Dander launcher. Its packaged, separately initialized + Terraform stack now plans ECR, ECS, Standard Step Functions control, scheduling, logs, and + failure routing, but public CLI plan/apply/status/cancel/replay integration and live parity remain + unproven. The runtime bridge copies one temporary ECS task-role session into process memory and + therefore caps a Fargate projection at one hour; renewable credentials remain a later gate. - Provider package extras and the full runtime image contain planned provider SDK dependencies, but do not by themselves implement or qualify Snowflake, Redshift, PostgreSQL, AWS, Azure, or OCI profiles. The packaged capability manifest is the support boundary. diff --git a/infra/aws/.terraform.lock.hcl b/infra/aws/.terraform.lock.hcl new file mode 100644 index 00000000..997bcb54 --- /dev/null +++ b/infra/aws/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = "~> 6.0" + hashes = [ + "h1:2kpake4zZKRX5437QVIRU3qFYH6Bjw/QE1fgVCPOrUg=", + "h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} diff --git a/infra/aws/README.md b/infra/aws/README.md new file mode 100644 index 00000000..af453449 --- /dev/null +++ b/infra/aws/README.md @@ -0,0 +1,18 @@ +# Dander AWS platform stack + +This separate Terraform root projects validated `io.dander.execution/v1` templates onto ECS +Fargate. It creates an immutable ECR repository, ECS task definitions, distinct execution/task +roles, a Standard Step Functions controller, paused-aware EventBridge schedules, CloudWatch logs, +and an encrypted failure queue plus notification topic. + +This root is construction-ready but is not yet selected by the public Dander plan/apply commands. +Initialize it with an operator-owned encrypted S3 backend and review a saved plan before any apply. +The stack accepts existing VPC subnet and security-group IDs; it does not create a network. + +The controller uses `ecs:runTask.sync`. One absolute Step Functions deadline bounds all attempts; +AWS performs a best-effort `StopTask` when the integration is cancelled or times out. Exit code 75 +is the only runtime outcome eligible for a bounded launcher retry. Scheduler delivery retries use a +separate counter and queue. + +No AWS deployment is supported until source-free live acceptance proves manual and scheduled runs, +deadline cancellation, replay, identity refresh, alerts, rollback, and no-change reconciliation. diff --git a/infra/aws/main.tf b/infra/aws/main.tf new file mode 100644 index 00000000..bd3d1b5c --- /dev/null +++ b/infra/aws/main.tf @@ -0,0 +1,12 @@ +module "fargate" { + source = "./modules/fargate" + + name = var.name + aws_account_id = var.aws_account_id + region = var.region + ecr_repository_name = var.ecr_repository_name + execution_projections = var.execution_projections + scheduler_delivery_retry_count = var.scheduler_delivery_retry_count + scheduler_delivery_max_age_seconds = var.scheduler_delivery_max_age_seconds + tags = var.tags +} diff --git a/infra/aws/modules/fargate/README.md b/infra/aws/modules/fargate/README.md new file mode 100644 index 00000000..676f9c2a --- /dev/null +++ b/infra/aws/modules/fargate/README.md @@ -0,0 +1,12 @@ +# Fargate launcher module + +This module consumes only validated execution projections. It deliberately keeps runtime task +roles separate from the shared image-pull/log execution role. AWS Secrets Manager access, when +declared, is attached only to the matching task role and exact secret ARNs. GCP secret references +remain resource names resolved through keyless Google federation inside the task. + +Schedules use EventBridge Scheduler's universal Step Functions `StartExecution` target so delivery +and launcher retry counters remain separate. The state machine is Standard, applies one absolute +deadline to every whole-runtime attempt, retries only Dander exit code 75, and emits safe exhausted +failure records to SQS. The AWS-managed ECS `.sync` integration owns task observation and performs +a best-effort `StopTask` on timeout or cancellation. diff --git a/infra/aws/modules/fargate/main.tf b/infra/aws/modules/fargate/main.tf new file mode 100644 index 00000000..cf5f4a28 --- /dev/null +++ b/infra/aws/modules/fargate/main.tf @@ -0,0 +1,894 @@ +locals { + partition = startswith(var.region, "us-gov-") ? "aws-us-gov" : "aws" + cpu_units = { + 1000 = 1024 + 2000 = 2048 + 4000 = 4096 + 8000 = 8192 + 16000 = 16384 + } + resource_names = { + for id in keys(var.execution_projections) : id => join("-", compact([ + substr(var.name, 0, 20), + substr(replace(id, "_", "-"), 0, 20), + substr(sha1(id), 0, 8), + ])) + } + task_role_names = { + for id, projection in var.execution_projections : id => element( + reverse(split("/", projection.workload_identity)), + 0, + ) + } + gcp_secret_environment = { + for id, projection in var.execution_projections : id => { + for name, binding in projection.secret_bindings : + name => trimprefix(binding.reference, "gcp-sm://") + if binding.provider == "gcp_secret_manager" + } + } + aws_secret_environment = { + for id, projection in var.execution_projections : id => { + for name, binding in projection.secret_bindings : name => binding.reference + if binding.provider == "aws_secret_manager" + } + } + aws_secret_arns = { + for id, bindings in local.aws_secret_environment : id => toset([ + for reference in values(bindings) : trimprefix(reference, "aws-sm://") + ]) + } + container_environment = { + for id, projection in var.execution_projections : id => merge( + projection.environment, + local.gcp_secret_environment[id], + local.aws_secret_environment[id], + ) + } + all_schedule_arns = [ + for id in keys(var.execution_projections) : + "arn:${local.partition}:scheduler:${var.region}:${var.aws_account_id}:schedule/default/${local.resource_names[id]}" + ] + tags = merge(var.tags, { + component = "fargate-launcher" + managed-by = "dander" + }) +} + +data "aws_caller_identity" "current" {} + +check "authenticated_account_matches_projection" { + assert { + condition = data.aws_caller_identity.current.account_id == var.aws_account_id + error_message = "Authenticated AWS account does not match aws_account_id." + } +} + +check "secret_references_match_deployment" { + assert { + condition = alltrue(flatten([ + for projection in values(var.execution_projections) : [ + for binding in values(projection.secret_bindings) : + binding.provider == "gcp_secret_manager" || startswith( + binding.reference, + "aws-sm://arn:${local.partition}:secretsmanager:${var.region}:${var.aws_account_id}:secret:", + ) + ] + ])) + error_message = "AWS secret references must be full ARNs in this account and region." + } +} + +resource "aws_ecr_repository" "runtime" { + name = var.ecr_repository_name + image_tag_mutability = "IMMUTABLE" + force_delete = false + + encryption_configuration { + encryption_type = "AES256" + } + + image_scanning_configuration { + scan_on_push = true + } + + tags = local.tags +} + +resource "aws_ecs_cluster" "runtime" { + name = var.name + + setting { + name = "containerInsights" + value = "enabled" + } + + tags = local.tags +} + +resource "aws_cloudwatch_log_group" "task" { + for_each = var.execution_projections + + name = "/dander/${var.name}/${each.key}" + retention_in_days = each.value.observability.retention_days + skip_destroy = false + tags = local.tags +} + +resource "aws_cloudwatch_log_group" "controller" { + for_each = var.execution_projections + + name = "/aws/vendedlogs/states/${local.resource_names[each.key]}" + retention_in_days = each.value.observability.retention_days + skip_destroy = false + tags = local.tags +} + +data "aws_iam_policy_document" "ecs_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "execution" { + name = "${var.name}-execution" + assume_role_policy = data.aws_iam_policy_document.ecs_assume.json + tags = local.tags +} + +data "aws_iam_policy_document" "execution" { + statement { + sid = "EcrAuthorization" + effect = "Allow" + actions = ["ecr:GetAuthorizationToken"] + resources = ["*"] + } + + statement { + sid = "PullRuntimeImage" + effect = "Allow" + actions = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ] + resources = [aws_ecr_repository.runtime.arn] + } + + statement { + sid = "WriteTaskLogs" + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [for group in aws_cloudwatch_log_group.task : "${group.arn}:*"] + } +} + +resource "aws_iam_role_policy" "execution" { + name = "dander-runtime-pull-and-logs" + role = aws_iam_role.execution.id + policy = data.aws_iam_policy_document.execution.json +} + +resource "aws_iam_role" "task" { + for_each = var.execution_projections + + name = local.task_role_names[each.key] + max_session_duration = 3600 + assume_role_policy = data.aws_iam_policy_document.ecs_assume.json + tags = merge(local.tags, { pipeline = each.key }) + + lifecycle { + precondition { + condition = each.value.workload_identity == ( + "arn:${local.partition}:iam::${var.aws_account_id}:role/${local.task_role_names[each.key]}" + ) + error_message = "Fargate workload_identity must be one path-free task role in this AWS account." + } + } +} + +data "aws_iam_policy_document" "task_secrets" { + for_each = { + for id, arns in local.aws_secret_arns : id => arns if length(arns) > 0 + } + + statement { + sid = "ReadDeclaredSecrets" + effect = "Allow" + actions = ["secretsmanager:GetSecretValue"] + resources = each.value + } +} + +resource "aws_iam_role_policy" "task_secrets" { + for_each = data.aws_iam_policy_document.task_secrets + + name = "dander-declared-secrets" + role = aws_iam_role.task[each.key].id + policy = each.value.json +} + +resource "aws_ecs_task_definition" "pipeline" { + for_each = var.execution_projections + + family = local.resource_names[each.key] + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = tostring(local.cpu_units[each.value.resources.cpu_millis]) + memory = tostring(each.value.resources.memory_mib) + execution_role_arn = aws_iam_role.execution.arn + task_role_arn = aws_iam_role.task[each.key].arn + + runtime_platform { + cpu_architecture = each.value.extensions.fargate_architecture + operating_system_family = "LINUX" + } + + dynamic "ephemeral_storage" { + for_each = each.value.resources.ephemeral_storage_mib > 20480 ? [1] : [] + content { + size_in_gib = each.value.resources.ephemeral_storage_mib / 1024 + } + } + + volume { + name = "dander-tmp" + } + + container_definitions = jsonencode([ + { + name = "dander" + image = each.value.image + essential = true + readonlyRootFilesystem = true + user = "65532:65532" + command = each.value.command + stopTimeout = tonumber(each.value.extensions.fargate_stop_timeout_seconds) + environment = [ + for name in sort(keys(local.container_environment[each.key])) : { + name = name + value = local.container_environment[each.key][name] + } + ] + linuxParameters = { + initProcessEnabled = true + } + mountPoints = [ + { + sourceVolume = "dander-tmp" + containerPath = "/tmp" + readOnly = false + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.task[each.key].name + awslogs-region = var.region + awslogs-stream-prefix = "runtime" + } + } + } + ]) + + tags = merge(local.tags, { pipeline = each.key }) + + lifecycle { + precondition { + condition = startswith( + each.value.image, + "${aws_ecr_repository.runtime.repository_url}@sha256:", + ) + error_message = "Fargate images must use an immutable digest in this stack's ECR repository." + } + precondition { + condition = ( + each.value.resources.ephemeral_storage_mib == 20480 || + ( + each.value.resources.ephemeral_storage_mib % 1024 == 0 && + each.value.resources.ephemeral_storage_mib >= 21504 + ) + ) + error_message = "Fargate ephemeral storage must be the 20 GiB default or a whole GiB from 21-200." + } + } +} + +resource "aws_sqs_queue" "failures" { + name = "${var.name}-failures" + message_retention_seconds = 1209600 + sqs_managed_sse_enabled = true + tags = local.tags +} + +data "aws_iam_policy_document" "failure_topic_key" { + statement { + sid = "AccountAdministration" + effect = "Allow" + actions = ["kms:*"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = ["arn:${local.partition}:iam::${var.aws_account_id}:root"] + } + } + + statement { + sid = "EventBridgeFailurePublication" + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + resources = ["*"] + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [var.aws_account_id] + } + } +} + +resource "aws_kms_key" "failure_topic" { + description = "Encrypt Dander Fargate controller failure notifications" + deletion_window_in_days = 30 + enable_key_rotation = true + policy = data.aws_iam_policy_document.failure_topic_key.json + tags = local.tags +} + +resource "aws_kms_alias" "failure_topic" { + name = "alias/${var.name}-failures" + target_key_id = aws_kms_key.failure_topic.key_id +} + +resource "aws_sns_topic" "failures" { + name = "${var.name}-failures" + kms_master_key_id = aws_kms_key.failure_topic.arn + tags = local.tags +} + +data "aws_iam_policy_document" "states_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["states.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "controller" { + for_each = var.execution_projections + + name = "${local.resource_names[each.key]}-controller" + assume_role_policy = data.aws_iam_policy_document.states_assume.json + tags = merge(local.tags, { pipeline = each.key }) +} + +data "aws_iam_policy_document" "controller" { + for_each = var.execution_projections + + statement { + sid = "RunPinnedTaskDefinition" + effect = "Allow" + actions = ["ecs:RunTask"] + resources = [aws_ecs_task_definition.pipeline[each.key].arn] + } + + statement { + sid = "ObserveAndStopOwnedTasks" + effect = "Allow" + actions = [ + "ecs:DescribeTasks", + "ecs:StopTask", + ] + resources = ["*"] + } + + statement { + sid = "PassExactTaskRoles" + effect = "Allow" + actions = ["iam:PassRole"] + resources = [ + aws_iam_role.execution.arn, + aws_iam_role.task[each.key].arn, + ] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ecs-tasks.amazonaws.com"] + } + } + + statement { + sid = "ObserveSynchronousTask" + effect = "Allow" + actions = [ + "events:DescribeRule", + "events:PutRule", + "events:PutTargets", + ] + resources = [ + "arn:${local.partition}:events:${var.region}:${var.aws_account_id}:rule/StepFunctionsGetEventsForECSTaskRule" + ] + } + + statement { + sid = "RecordExhaustedFailure" + effect = "Allow" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.failures.arn] + } + + # Step Functions log-delivery APIs do not support resource-level permissions. + statement { + sid = "DeliverControllerLogs" + effect = "Allow" + actions = [ + "logs:CreateLogDelivery", + "logs:DeleteLogDelivery", + "logs:DescribeLogGroups", + "logs:DescribeResourcePolicies", + "logs:GetLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:UpdateLogDelivery", + ] + resources = ["*"] + } +} + +resource "aws_iam_role_policy" "controller" { + for_each = var.execution_projections + + name = "dander-fargate-controller" + role = aws_iam_role.controller[each.key].id + policy = data.aws_iam_policy_document.controller[each.key].json +} + +resource "aws_sfn_state_machine" "pipeline" { + for_each = var.execution_projections + + name = local.resource_names[each.key] + role_arn = aws_iam_role.controller[each.key].arn + type = "STANDARD" + + logging_configuration { + include_execution_data = false + level = "ERROR" + log_destination = "${aws_cloudwatch_log_group.controller[each.key].arn}:*" + } + + definition = jsonencode({ + Comment = "Dander bounded whole-task Fargate controller" + StartAt = "Initialize" + TimeoutSeconds = each.value.resources.deadline_seconds + States = { + Initialize = { + Type = "Pass" + Parameters = { + "run_id.$" = "States.Format('${each.key}:{}', $.scheduled_time)" + "scheduled_time.$" = "$.scheduled_time" + "scheduler_attempt.$" = "$.scheduler_attempt" + "scheduler_execution_id.$" = "$.scheduler_execution_id" + launcher_attempt = 1 + } + Next = "Run task" + } + "Run task" = { + Type = "Task" + Resource = "arn:${local.partition}:states:::ecs:runTask.sync" + TimeoutSeconds = each.value.resources.deadline_seconds + Parameters = { + Cluster = aws_ecs_cluster.runtime.arn + TaskDefinition = aws_ecs_task_definition.pipeline[each.key].arn + LaunchType = "FARGATE" + PlatformVersion = "LATEST" + EnableExecuteCommand = false + PropagateTags = "TASK_DEFINITION" + NetworkConfiguration = { + AwsvpcConfiguration = { + AssignPublicIp = upper(each.value.extensions.fargate_assign_public_ip) + SecurityGroups = split(",", each.value.network.extensions.fargate_security_group_ids) + Subnets = split(",", each.value.network.extensions.fargate_subnet_ids) + } + } + Overrides = { + ContainerOverrides = [ + { + Name = "dander" + Environment = [ + { + Name = "DANDER_RUN_ID" + "Value.$" = "$.run_id" + }, + { + Name = "DANDER_LAUNCHER_EXECUTION_ID" + "Value.$" = "$$.Execution.Id" + }, + { + Name = "DANDER_ATTEMPT" + "Value.$" = "States.Format('{}', $.launcher_attempt)" + }, + ] + } + ] + } + } + ResultSelector = { + "task_arn.$" = "$.Tasks[0].TaskArn" + "exit_code.$" = "$.Tasks[0].Containers[0].ExitCode" + } + ResultPath = "$.task" + Retry = [ + { + ErrorEquals = [ + "ECS.ServerException", + "ECS.ThrottlingException", + ] + IntervalSeconds = 2 + BackoffRate = 2 + MaxAttempts = 3 + } + ] + Catch = [ + { + ErrorEquals = ["States.Timeout"] + ResultPath = "$.controller_failure" + Next = "Deadline failure" + }, + { + ErrorEquals = ["States.ALL"] + ResultPath = "$.controller_failure" + Next = "Controller failure" + }, + ] + Next = "Classify task" + } + "Classify task" = { + Type = "Choice" + Choices = [ + { + Variable = "$.task.exit_code" + NumericEquals = 0 + Next = "Success" + }, + { + Variable = "$.task.exit_code" + NumericEquals = 75 + Next = "Can retry" + }, + ] + Default = "Permanent failure" + } + "Can retry" = { + Type = "Choice" + Choices = [ + { + Variable = "$.launcher_attempt" + NumericLessThanEquals = each.value.resources.launcher_retry_count + Next = "Increment attempt" + } + ] + Default = "Retry exhausted" + } + "Increment attempt" = { + Type = "Pass" + Parameters = { + "run_id.$" = "$.run_id" + "scheduled_time.$" = "$.scheduled_time" + "scheduler_attempt.$" = "$.scheduler_attempt" + "scheduler_execution_id.$" = "$.scheduler_execution_id" + "launcher_attempt.$" = "States.MathAdd($.launcher_attempt, 1)" + } + Next = "Run task" + } + Success = { + Type = "Pass" + Parameters = { + contract = "io.dander.runtime/v1" + status = "succeeded" + pipeline_id = each.key + "run_id.$" = "$.run_id" + "task_arn.$" = "$.task.task_arn" + "container_exit_code.$" = "$.task.exit_code" + "launcher_attempt.$" = "$.launcher_attempt" + "scheduler_attempt.$" = "$.scheduler_attempt" + } + End = true + } + "Permanent failure" = { + Type = "Pass" + Parameters = { + failure = { + contract = "io.dander.runtime/v1" + status = "failed" + failure_code = "runtime_permanent_failure" + pipeline_id = each.key + "run_id.$" = "$.run_id" + "task_arn.$" = "$.task.task_arn" + "container_exit_code.$" = "$.task.exit_code" + "launcher_attempt.$" = "$.launcher_attempt" + } + } + Next = "Record failure" + } + "Retry exhausted" = { + Type = "Pass" + Parameters = { + failure = { + contract = "io.dander.runtime/v1" + status = "failed" + failure_code = "runtime_retry_exhausted" + pipeline_id = each.key + "run_id.$" = "$.run_id" + "task_arn.$" = "$.task.task_arn" + "container_exit_code.$" = "$.task.exit_code" + "launcher_attempt.$" = "$.launcher_attempt" + } + } + Next = "Record failure" + } + "Deadline failure" = { + Type = "Pass" + Parameters = { + failure = { + contract = "io.dander.runtime/v1" + status = "failed" + failure_code = "launcher_deadline_exceeded" + pipeline_id = each.key + "run_id.$" = "$.run_id" + "launcher_attempt.$" = "$.launcher_attempt" + } + } + Next = "Record failure" + } + "Controller failure" = { + Type = "Pass" + Parameters = { + failure = { + contract = "io.dander.runtime/v1" + status = "failed" + failure_code = "launcher_control_plane_failed" + pipeline_id = each.key + "run_id.$" = "$.run_id" + "launcher_attempt.$" = "$.launcher_attempt" + "error_code.$" = "$.controller_failure.Error" + } + } + Next = "Record failure" + } + "Record failure" = { + Type = "Task" + Resource = "arn:${local.partition}:states:::sqs:sendMessage" + Parameters = { + QueueUrl = aws_sqs_queue.failures.url + "MessageBody.$" = "States.JsonToString($.failure)" + } + ResultPath = null + Next = "Fail execution" + } + "Fail execution" = { + Type = "Fail" + Error = "Dander.RuntimeFailure" + Cause = "The Dander Fargate runtime did not succeed." + } + } + }) + + tags = merge(local.tags, { pipeline = each.key }) + + depends_on = [aws_iam_role_policy.controller] +} + +data "aws_iam_policy_document" "scheduler_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["scheduler.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "scheduler" { + for_each = var.execution_projections + + name = "${local.resource_names[each.key]}-schedule" + assume_role_policy = data.aws_iam_policy_document.scheduler_assume.json + tags = merge(local.tags, { pipeline = each.key }) +} + +data "aws_iam_policy_document" "scheduler" { + for_each = var.execution_projections + + statement { + sid = "StartExactStateMachine" + effect = "Allow" + actions = ["states:StartExecution"] + resources = [aws_sfn_state_machine.pipeline[each.key].arn] + } +} + +resource "aws_iam_role_policy" "scheduler" { + for_each = var.execution_projections + + name = "dander-start-controller" + role = aws_iam_role.scheduler[each.key].id + policy = data.aws_iam_policy_document.scheduler[each.key].json +} + +resource "aws_scheduler_schedule" "pipeline" { + for_each = var.execution_projections + + name = local.resource_names[each.key] + state = each.value.schedule.paused ? "DISABLED" : "ENABLED" + schedule_expression = each.value.schedule.expression + schedule_expression_timezone = each.value.schedule.time_zone + + flexible_time_window { + mode = "OFF" + } + + target { + arn = "arn:${local.partition}:scheduler:::aws-sdk:sfn:startExecution" + role_arn = aws_iam_role.scheduler[each.key].arn + input = jsonencode({ + StateMachineArn = aws_sfn_state_machine.pipeline[each.key].arn + Name = "" + Input = jsonencode({ + deployment_revision = each.value.labels.image_digest + scheduled_time = "" + scheduler_attempt = "" + scheduler_execution_id = "" + }) + }) + + dead_letter_config { + arn = aws_sqs_queue.failures.arn + } + + retry_policy { + maximum_event_age_in_seconds = var.scheduler_delivery_max_age_seconds + maximum_retry_attempts = var.scheduler_delivery_retry_count + } + } + + lifecycle { + precondition { + condition = ( + each.value.observability.alert_target == null || + each.value.observability.alert_target == aws_sns_topic.failures.arn + ) + error_message = "A declared Fargate alert target must equal this stack's failure topic ARN." + } + } +} + +resource "aws_cloudwatch_event_rule" "controller_failure" { + name = "${var.name}-controller-failures" + description = "Dander Fargate controllers that failed, timed out, or were aborted." + event_pattern = jsonencode({ + source = ["aws.states"] + detail-type = ["Step Functions Execution Status Change"] + detail = { + status = ["FAILED", "TIMED_OUT", "ABORTED"] + stateMachineArn = [ + for machine in aws_sfn_state_machine.pipeline : machine.arn + ] + } + }) + tags = local.tags +} + +resource "aws_cloudwatch_event_target" "failure_queue" { + rule = aws_cloudwatch_event_rule.controller_failure.name + arn = aws_sqs_queue.failures.arn +} + +resource "aws_cloudwatch_event_target" "failure_topic" { + rule = aws_cloudwatch_event_rule.controller_failure.name + arn = aws_sns_topic.failures.arn +} + +data "aws_iam_policy_document" "failure_queue" { + statement { + sid = "SchedulerDeadLetters" + effect = "Allow" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.failures.arn] + + principals { + type = "Service" + identifiers = ["scheduler.amazonaws.com"] + } + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = local.all_schedule_arns + } + } + + statement { + sid = "ControllerFailureEvents" + effect = "Allow" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.failures.arn] + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.controller_failure.arn] + } + } +} + +resource "aws_sqs_queue_policy" "failures" { + queue_url = aws_sqs_queue.failures.url + policy = data.aws_iam_policy_document.failure_queue.json +} + +data "aws_iam_policy_document" "failure_topic" { + statement { + sid = "AccountOwner" + effect = "Allow" + actions = ["SNS:*"] + resources = [aws_sns_topic.failures.arn] + + principals { + type = "AWS" + identifiers = ["arn:${local.partition}:iam::${var.aws_account_id}:root"] + } + } + + statement { + sid = "ControllerFailureEvents" + effect = "Allow" + actions = ["sns:Publish"] + resources = [aws_sns_topic.failures.arn] + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.controller_failure.arn] + } + } +} + +resource "aws_sns_topic_policy" "failures" { + arn = aws_sns_topic.failures.arn + policy = data.aws_iam_policy_document.failure_topic.json +} diff --git a/infra/aws/modules/fargate/outputs.tf b/infra/aws/modules/fargate/outputs.tf new file mode 100644 index 00000000..9f55387e --- /dev/null +++ b/infra/aws/modules/fargate/outputs.tf @@ -0,0 +1,33 @@ +output "runtime_repository_url" { + description = "ECR repository URL for immutable runtime manifests." + value = aws_ecr_repository.runtime.repository_url +} + +output "cluster_arn" { + description = "ECS cluster ARN." + value = aws_ecs_cluster.runtime.arn +} + +output "pipelines" { + description = "Per-pipeline runtime and controller resources." + value = { + for id in keys(var.execution_projections) : id => { + task_definition_arn = aws_ecs_task_definition.pipeline[id].arn + task_role_arn = aws_iam_role.task[id].arn + state_machine_arn = aws_sfn_state_machine.pipeline[id].arn + schedule_arn = aws_scheduler_schedule.pipeline[id].arn + schedule_state = aws_scheduler_schedule.pipeline[id].state + log_group_name = aws_cloudwatch_log_group.task[id].name + } + } +} + +output "failure_queue_url" { + description = "Encrypted queue for undelivered schedules and exhausted controller failures." + value = aws_sqs_queue.failures.url +} + +output "failure_topic_arn" { + description = "Notification topic for failed, timed-out, and aborted controller executions." + value = aws_sns_topic.failures.arn +} diff --git a/infra/aws/modules/fargate/tests/fargate.tftest.hcl b/infra/aws/modules/fargate/tests/fargate.tftest.hcl new file mode 100644 index 00000000..dfbfcf0f --- /dev/null +++ b/infra/aws/modules/fargate/tests/fargate.tftest.hcl @@ -0,0 +1,117 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ecs-tasks.amazonaws.com\"}}]}" + } + } + + mock_data "aws_caller_identity" { + defaults = { + account_id = "184463061564" + arn = "arn:aws:iam::184463061564:root" + user_id = "184463061564" + } + } +} + +variables { + name = "dander-portability" + aws_account_id = "184463061564" + region = "us-east-1" + ecr_repository_name = "dander" + scheduler_delivery_retry_count = 2 + scheduler_delivery_max_age_seconds = 3600 + tags = {} + + execution_projections = { + greenhouse_jobs = { + schema = "io.dander.execution/v1" + contract = "io.dander.runtime/v1" + pipeline_id = "greenhouse_jobs" + profile_id = "gcp" + launcher = "fargate" + image = "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command = ["runtime", "execute", "--contract", "io.dander.runtime/v1", "--pipeline", "greenhouse_jobs", "--platform", "gcp", "--config", "/app/dander.yaml"] + configuration_reference = "/app/dander.yaml" + environment = { + DANDER_LAUNCHER = "fargate" + GCP_PROJECT_ID = "unit-project" + } + secret_bindings = {} + workload_identity = "arn:aws:iam::184463061564:role/dander-runtime" + resources = { + cpu_millis = 1000 + memory_mib = 2048 + ephemeral_storage_mib = 20480 + deadline_seconds = 900 + runtime_retry_count = 0 + launcher_retry_count = 1 + } + schedule = { + task_count = 1 + maximum_parallelism = 1 + expression = "cron(0 9 * * ? *)" + time_zone = "America/New_York" + paused = true + } + network = { + placement = "awsvpc" + extensions = { + fargate_security_group_ids = "sg-0123456789abcdef0" + fargate_subnet_ids = "subnet-0123456789abcdef0" + } + } + labels = { + dander_version = "0.7.0" + image_digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + pipeline = "greenhouse_jobs" + profile = "gcp" + } + observability = { + log_destination = "cloudwatch_logs" + metric_namespace = "Dander" + alert_target = null + retention_days = 30 + } + extensions = { + fargate_architecture = "ARM64" + fargate_assign_public_ip = "disabled" + fargate_stop_timeout_seconds = "120" + } + } + } +} + +run "paused_bounded_controller" { + command = plan + + assert { + condition = aws_scheduler_schedule.pipeline["greenhouse_jobs"].state == "DISABLED" + error_message = "A paused Dander projection must create a disabled AWS schedule." + } + + assert { + condition = ( + jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].readonlyRootFilesystem == true && + jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].user == "65532:65532" && + jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].stopTimeout == 120 + ) + error_message = "The Fargate task must remain non-root, read-only, and explicitly stoppable." + } + + assert { + condition = ( + aws_iam_role.execution.name != + aws_iam_role.task["greenhouse_jobs"].name + ) + error_message = "ECS image-pull/log identity and runtime identity must be separate." + } + + assert { + condition = ( + aws_ecr_repository.runtime.image_tag_mutability == "IMMUTABLE" && + aws_ecr_repository.runtime.image_scanning_configuration[0].scan_on_push == true + ) + error_message = "The runtime repository must reject mutable tags and scan pushed images." + } +} diff --git a/infra/aws/modules/fargate/variables.tf b/infra/aws/modules/fargate/variables.tf new file mode 100644 index 00000000..9b19ea37 --- /dev/null +++ b/infra/aws/modules/fargate/variables.tf @@ -0,0 +1,144 @@ +variable "name" { + type = string + description = "Stable name prefix for this Dander Fargate deployment." +} + +variable "aws_account_id" { + type = string + description = "AWS account that owns the Fargate deployment." +} + +variable "region" { + type = string + description = "AWS region for all module resources." +} + +variable "ecr_repository_name" { + type = string + description = "ECR repository for immutable Dander runtime manifests." + default = "dander" +} + +variable "execution_projections" { + description = "Validated Fargate execution templates keyed by pipeline id." + type = map(object({ + schema = string + contract = string + pipeline_id = string + profile_id = string + launcher = string + image = string + command = list(string) + configuration_reference = string + environment = map(string) + secret_bindings = map(object({ + provider = string + reference = string + })) + workload_identity = string + resources = object({ + cpu_millis = number + memory_mib = number + ephemeral_storage_mib = number + deadline_seconds = number + runtime_retry_count = number + launcher_retry_count = number + }) + schedule = object({ + task_count = number + maximum_parallelism = number + expression = string + time_zone = string + paused = bool + }) + network = object({ + placement = string + extensions = map(string) + }) + labels = map(string) + observability = object({ + log_destination = string + metric_namespace = string + alert_target = optional(string) + retention_days = number + }) + extensions = map(string) + })) + + validation { + condition = length(var.execution_projections) > 0 && alltrue([ + for id, projection in var.execution_projections : + can(regex("^[a-z][a-z0-9_-]{1,62}$", id)) && + projection.pipeline_id == id && + projection.schema == "io.dander.execution/v1" && + projection.contract == "io.dander.runtime/v1" && + projection.launcher == "fargate" && + can(regex("^[a-z][a-z0-9_-]{0,62}$", projection.profile_id)) && + can(regex("@sha256:[0-9a-f]{64}$", projection.image)) && + projection.configuration_reference == "/app/dander.yaml" && + length(projection.command) >= 8 && + projection.command[0] == "runtime" && + projection.command[1] == "execute" && + projection.resources.runtime_retry_count == 0 && + contains([1000, 2000, 4000, 8000, 16000], projection.resources.cpu_millis) && + projection.resources.deadline_seconds >= 1 && + projection.resources.deadline_seconds <= 3600 && + projection.resources.launcher_retry_count >= 0 && + projection.resources.launcher_retry_count <= 10 && + projection.resources.ephemeral_storage_mib >= 20480 && + projection.resources.ephemeral_storage_mib <= 204800 && + projection.schedule.task_count == 1 && + projection.schedule.maximum_parallelism == 1 && + can(regex("^cron\\(.+\\)$", projection.schedule.expression)) && + length(trimspace(projection.schedule.time_zone)) > 0 && + projection.network.placement == "awsvpc" && + contains(keys(projection.network.extensions), "fargate_subnet_ids") && + contains(keys(projection.network.extensions), "fargate_security_group_ids") && + length(setsubtract(toset(keys(projection.network.extensions)), toset(["fargate_subnet_ids", "fargate_security_group_ids"]))) == 0 && + contains(["ARM64", "X86_64"], lookup(projection.extensions, "fargate_architecture", "")) && + contains(["enabled", "disabled"], lookup(projection.extensions, "fargate_assign_public_ip", "")) && + can(tonumber(lookup(projection.extensions, "fargate_stop_timeout_seconds", ""))) && + tonumber(lookup(projection.extensions, "fargate_stop_timeout_seconds", "0")) >= 2 && + tonumber(lookup(projection.extensions, "fargate_stop_timeout_seconds", "0")) <= 120 && + length(setsubtract(toset(keys(projection.extensions)), toset(["fargate_architecture", "fargate_assign_public_ip", "fargate_stop_timeout_seconds"]))) == 0 && + projection.observability.log_destination == "cloudwatch_logs" && + projection.observability.metric_namespace == "Dander" && + projection.observability.retention_days >= 1 && + length(setintersection(toset(keys(projection.environment)), toset(keys(projection.secret_bindings)))) == 0 && + projection.labels.image_digest == element(reverse(split("@", projection.image)), 0) && + alltrue([ + for binding in values(projection.secret_bindings) : + contains(["gcp_secret_manager", "aws_secret_manager"], binding.provider) + ]) + ]) + error_message = "Every entry must be one complete, immutable, bounded Fargate execution projection." + } +} + +variable "scheduler_delivery_retry_count" { + type = number + description = "Scheduler delivery retries, separate from launcher retries." + default = 2 + + validation { + condition = var.scheduler_delivery_retry_count >= 0 && var.scheduler_delivery_retry_count <= 185 && floor(var.scheduler_delivery_retry_count) == var.scheduler_delivery_retry_count + error_message = "scheduler_delivery_retry_count must be an integer from 0 through 185." + } +} + +variable "scheduler_delivery_max_age_seconds" { + type = number + description = "Maximum age for an undelivered scheduled invocation." + default = 3600 + + validation { + condition = var.scheduler_delivery_max_age_seconds >= 60 && var.scheduler_delivery_max_age_seconds <= 86400 && floor(var.scheduler_delivery_max_age_seconds) == var.scheduler_delivery_max_age_seconds + error_message = "scheduler_delivery_max_age_seconds must be an integer from 60 through 86400." + } +} + +variable "tags" { + type = map(string) + description = "Additional non-secret AWS tags." + default = {} +} diff --git a/infra/aws/outputs.tf b/infra/aws/outputs.tf new file mode 100644 index 00000000..5ae8e50f --- /dev/null +++ b/infra/aws/outputs.tf @@ -0,0 +1,24 @@ +output "runtime_repository_url" { + description = "ECR repository URL used for immutable Dander runtime manifests." + value = module.fargate.runtime_repository_url +} + +output "cluster_arn" { + description = "ECS cluster hosting the Dander Fargate tasks." + value = module.fargate.cluster_arn +} + +output "pipelines" { + description = "Controller, task-definition, schedule, and task-role identifiers by pipeline." + value = module.fargate.pipelines +} + +output "failure_queue_url" { + description = "SQS queue receiving exhausted controller and delivery failures." + value = module.fargate.failure_queue_url +} + +output "failure_topic_arn" { + description = "SNS topic receiving failed, timed-out, or aborted controller events." + value = module.fargate.failure_topic_arn +} diff --git a/infra/aws/terraform.tfvars.example b/infra/aws/terraform.tfvars.example new file mode 100644 index 00000000..974e75f7 --- /dev/null +++ b/infra/aws/terraform.tfvars.example @@ -0,0 +1,61 @@ +name = "dander-portability" +aws_account_id = "184463061564" +region = "us-east-1" + +execution_projections = { + greenhouse_jobs = { + schema = "io.dander.execution/v1" + contract = "io.dander.runtime/v1" + pipeline_id = "greenhouse_jobs" + profile_id = "gcp" + launcher = "fargate" + image = "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command = ["runtime", "execute", "--contract", "io.dander.runtime/v1", "--pipeline", "greenhouse_jobs", "--platform", "gcp", "--config", "/app/dander.yaml"] + configuration_reference = "/app/dander.yaml" + environment = { + DANDER_LAUNCHER = "fargate" + GCP_PROJECT_ID = "unit-project" + } + secret_bindings = {} + workload_identity = "arn:aws:iam::184463061564:role/dander-runtime" + resources = { + cpu_millis = 1000 + memory_mib = 2048 + ephemeral_storage_mib = 20480 + deadline_seconds = 900 + runtime_retry_count = 0 + launcher_retry_count = 1 + } + schedule = { + task_count = 1 + maximum_parallelism = 1 + expression = "cron(0 9 * * ? *)" + time_zone = "America/New_York" + paused = true + } + network = { + placement = "awsvpc" + extensions = { + fargate_security_group_ids = "sg-0123456789abcdef0" + fargate_subnet_ids = "subnet-0123456789abcdef0" + } + } + labels = { + dander_version = "0.7.0" + image_digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + pipeline = "greenhouse_jobs" + profile = "gcp" + } + observability = { + log_destination = "cloudwatch_logs" + metric_namespace = "Dander" + alert_target = null + retention_days = 30 + } + extensions = { + fargate_architecture = "ARM64" + fargate_assign_public_ip = "disabled" + fargate_stop_timeout_seconds = "120" + } + } +} diff --git a/infra/aws/variables.tf b/infra/aws/variables.tf new file mode 100644 index 00000000..9ee25794 --- /dev/null +++ b/infra/aws/variables.tf @@ -0,0 +1,60 @@ +variable "name" { + type = string + description = "Stable name prefix for this Dander Fargate deployment." + default = "dander" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,23}$", var.name)) + error_message = "name must be 2-24 lowercase letters, numbers, or hyphens." + } +} + +variable "aws_account_id" { + type = string + description = "AWS account that owns the Fargate deployment." + + validation { + condition = can(regex("^[0-9]{12}$", var.aws_account_id)) + error_message = "aws_account_id must contain exactly 12 digits." + } +} + +variable "region" { + type = string + description = "AWS region for ECR, ECS, Scheduler, Step Functions, logs, and failure routing." + default = "us-east-1" + + validation { + condition = can(regex("^[a-z]{2}(-gov)?-[a-z]+-[0-9]+$", var.region)) + error_message = "region must be a valid AWS region name." + } +} + +variable "ecr_repository_name" { + type = string + description = "ECR repository that receives immutable Dander runtime manifests." + default = "dander" +} + +variable "execution_projections" { + description = "Validated io.dander.execution/v1 Fargate templates keyed by pipeline id." + type = any +} + +variable "scheduler_delivery_retry_count" { + type = number + description = "EventBridge delivery retries; distinct from whole-runtime launcher attempts." + default = 2 +} + +variable "scheduler_delivery_max_age_seconds" { + type = number + description = "Maximum age for an undelivered scheduled invocation." + default = 3600 +} + +variable "tags" { + type = map(string) + description = "Additional non-secret AWS tags." + default = {} +} diff --git a/infra/aws/versions.tf b/infra/aws/versions.tf new file mode 100644 index 00000000..1a658dc2 --- /dev/null +++ b/infra/aws/versions.tf @@ -0,0 +1,24 @@ +terraform { + required_version = ">= 1.9" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } + + # Supply bucket, key, region, encryption, and native lockfile settings through + # `terraform init -backend-config`. Provider credentials never belong in this file. + backend "s3" {} +} + +provider "aws" { + region = var.region + + default_tags { + tags = merge(var.tags, { + managed-by = "dander" + }) + } +} diff --git a/pyproject.toml b/pyproject.toml index cda69949..207d26ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,18 @@ packages = ["src/dander"] "infra/bootstrap-admin/outputs.tf" = "dander/templates/project/infra/bootstrap-admin/outputs.tf" "infra/bootstrap-admin/variables.tf" = "dander/templates/project/infra/bootstrap-admin/variables.tf" "infra/bootstrap-admin/versions.tf" = "dander/templates/project/infra/bootstrap-admin/versions.tf" +"infra/aws/.terraform.lock.hcl" = "dander/templates/project/infra/aws/.terraform.lock.hcl" +"infra/aws/README.md" = "dander/templates/project/infra/aws/README.md" +"infra/aws/main.tf" = "dander/templates/project/infra/aws/main.tf" +"infra/aws/modules/fargate/README.md" = "dander/templates/project/infra/aws/modules/fargate/README.md" +"infra/aws/modules/fargate/main.tf" = "dander/templates/project/infra/aws/modules/fargate/main.tf" +"infra/aws/modules/fargate/outputs.tf" = "dander/templates/project/infra/aws/modules/fargate/outputs.tf" +"infra/aws/modules/fargate/tests/fargate.tftest.hcl" = "dander/templates/project/infra/aws/modules/fargate/tests/fargate.tftest.hcl" +"infra/aws/modules/fargate/variables.tf" = "dander/templates/project/infra/aws/modules/fargate/variables.tf" +"infra/aws/outputs.tf" = "dander/templates/project/infra/aws/outputs.tf" +"infra/aws/terraform.tfvars.example" = "dander/templates/project/infra/aws/terraform.tfvars.example" +"infra/aws/variables.tf" = "dander/templates/project/infra/aws/variables.tf" +"infra/aws/versions.tf" = "dander/templates/project/infra/aws/versions.tf" "infra/functions/__init__.py" = "dander/templates/project/infra/functions/__init__.py" "infra/functions/stop_billing/__init__.py" = "dander/templates/project/infra/functions/stop_billing/__init__.py" "infra/functions/stop_billing/handler.py" = "dander/templates/project/infra/functions/stop_billing/handler.py" diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index cb4df525..06828643 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -357,6 +357,7 @@ class LauncherCapabilities: "fargate_architecture", "fargate_assign_public_ip", "fargate_security_group_ids", + "fargate_stop_timeout_seconds", "fargate_subnet_ids", } ), diff --git a/src/dander/providers/fargate/config.py b/src/dander/providers/fargate/config.py index 2647df3d..921dfdad 100644 --- a/src/dander/providers/fargate/config.py +++ b/src/dander/providers/fargate/config.py @@ -28,6 +28,7 @@ class FargateLauncherConfig(BaseModel): architecture: Literal["ARM64", "X86_64"] = "ARM64" assign_public_ip: bool = False ephemeral_storage_mib: int = Field(default=20_480, ge=20_480, le=204_800) + stop_timeout_seconds: int = Field(default=120, ge=2, le=120) @field_validator("subnet_ids") @classmethod diff --git a/src/dander/providers/fargate/runtime.py b/src/dander/providers/fargate/runtime.py index a2212652..1a2ce7ab 100644 --- a/src/dander/providers/fargate/runtime.py +++ b/src/dander/providers/fargate/runtime.py @@ -140,7 +140,7 @@ def build( schedule=ScheduleProjection( task_count=1, maximum_parallelism=1, - expression=str(pipeline["schedule"]), + expression=_aws_schedule(str(pipeline["schedule"])), time_zone=str(pipeline["time_zone"]), paused=bool(pipeline["paused"]), ), @@ -169,6 +169,7 @@ def build( "fargate_assign_public_ip", "enabled" if self.config.assign_public_ip else "disabled", ), + ("fargate_stop_timeout_seconds", str(self.config.stop_timeout_seconds)), ), ) validate_launcher_projection(template, FARGATE_CAPABILITIES) @@ -212,6 +213,23 @@ def _validate_fargate_size(*, cpu_millis: int, memory_mib: int) -> None: raise ExecutionProjectionError("Fargate does not support the requested CPU/memory pair") +def _aws_schedule(value: str) -> str: + """Translate the compatible five-field cron subset to EventBridge Scheduler.""" + parts = value.split() + if len(parts) != 5 or any(re.fullmatch(r"[A-Za-z0-9*/,\-]+", part) is None for part in parts): + raise ExecutionProjectionError("Fargate requires a valid five-field cron schedule") + minute, hour, day_of_month, month, day_of_week = parts + if day_of_month != "*" and day_of_week != "*": + raise ExecutionProjectionError( + "Fargate cannot preserve a cron schedule that constrains both day fields" + ) + if day_of_week == "*": + day_of_week = "?" + else: + day_of_month = "?" + return f"cron({minute} {hour} {day_of_month} {month} {day_of_week} *)" + + FARGATE_LAUNCHER_FACTORY: ProviderFactory[LauncherRuntime] = ProviderFactory( kind=ProviderKind.LAUNCHER, provider_id="fargate", diff --git a/tests/infra/test_fargate_runtime.py b/tests/infra/test_fargate_runtime.py new file mode 100644 index 00000000..6d891275 --- /dev/null +++ b/tests/infra/test_fargate_runtime.py @@ -0,0 +1,90 @@ +"""Static safety and lifecycle contracts for the construction-ready AWS stack.""" + +from pathlib import Path + +ROOT = Path(__file__).parents[2] +AWS_ROOT = ROOT / "infra/aws" +MODULE = AWS_ROOT / "modules/fargate/main.tf" + + +def _source(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_aws_root_is_separate_and_uses_native_remote_state() -> None: + versions = _source(AWS_ROOT / "versions.tf") + root = _source(AWS_ROOT / "main.tf") + + assert 'backend "s3" {}' in versions + assert 'source = "./modules/fargate"' in root + assert "google" not in versions + assert "azurerm" not in versions + + +def test_fargate_task_is_immutable_nonroot_and_uses_separate_roles() -> None: + module = _source(MODULE) + + assert 'image_tag_mutability = "IMMUTABLE"' in module + assert "scan_on_push = true" in module + assert "readonlyRootFilesystem = true" in module + assert 'user = "65532:65532"' in module + assert 'containerPath = "/tmp"' in module + assert "execution_role_arn = aws_iam_role.execution.arn" in module + assert "task_role_arn = aws_iam_role.task[each.key].arn" in module + assert "secretsmanager:GetSecretValue" in module + assert "resources = each.value" in module + + +def test_controller_owns_deadline_and_only_runtime_exit_75_retries() -> None: + module = _source(MODULE) + + assert module.count('Resource = "arn:${local.partition}:states:::ecs:runTask.sync"') == 1 + assert "TimeoutSeconds = each.value.resources.deadline_seconds" in module + assert "NumericEquals = 75" in module + assert 'failure_code = "runtime_retry_exhausted"' in module + assert 'failure_code = "launcher_deadline_exceeded"' in module + assert '"ecs:StopTask"' in module + assert '"ecs:DescribeTasks"' in module + assert '"iam:PassRole"' in module + assert 'variable = "iam:PassedToService"' in module + assert 'values = ["ecs-tasks.amazonaws.com"]' in module + + +def test_scheduler_delivery_is_separate_paused_aware_and_dead_lettered() -> None: + module = _source(MODULE) + + assert ( + 'state = each.value.schedule.paused ? "DISABLED" : "ENABLED"' + in module + ) + assert 'arn = "arn:${local.partition}:scheduler:::aws-sdk:sfn:startExecution"' in module + assert 'Name = ""' in module + assert 'scheduled_time = ""' in module + assert 'scheduler_attempt = ""' in module + assert "maximum_retry_attempts = var.scheduler_delivery_retry_count" in module + assert "launcher_retry_count" in module + assert "dead_letter_config" in module + assert 'status = ["FAILED", "TIMED_OUT", "ABORTED"]' in module + assert 'resource "aws_kms_key" "failure_topic"' in module + assert "enable_key_rotation = true" in module + assert "kms_master_key_id = aws_kms_key.failure_topic.arn" in module + + +def test_aws_stack_contains_no_static_cloud_credentials_or_apply_path() -> None: + stack = "\n".join( + _source(path) + for path in AWS_ROOT.rglob("*") + if path.is_file() and ".terraform" not in path.parts and path.name != ".terraform.lock.hcl" + ) + + assert "AWS_ACCESS_KEY_ID" not in stack + assert "AWS_SECRET_ACCESS_KEY" not in stack + assert "terraform apply" not in stack + assert "private_key" not in stack.lower() + + +def test_fargate_remains_outside_the_public_supported_launcher_manifest() -> None: + capabilities = _source(ROOT / "src/dander/runtime-capabilities.json") + + assert '"launchers": ["cloud_run", "local"]' in capabilities + assert '"fargate"' not in capabilities diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py index da34013a..76dfaced 100644 --- a/tests/providers/test_launcher_runtime.py +++ b/tests/providers/test_launcher_runtime.py @@ -162,7 +162,9 @@ def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> assert dict(template.extensions) == { "fargate_architecture": "ARM64", "fargate_assign_public_ip": "disabled", + "fargate_stop_timeout_seconds": "120", } + assert template.schedule.expression == "cron(0 9 * * ? *)" secret = dict(template.secret_bindings)["SALESFORCE_KEY"] assert secret.reference.endswith("/secrets/salesforce-private-key/versions/latest") serialized = repr(template.as_dict()) @@ -211,3 +213,47 @@ def test_fargate_rejects_a_run_longer_than_the_task_role_session() -> None: require_guarded_free_tier=False, alert_target=None, ) + + +def test_fargate_rejects_cron_semantics_eventbridge_cannot_preserve() -> None: + pipelines = { + "greenhouse_jobs": { + **_PIPELINES["greenhouse_jobs"], + "schedule": "0 9 1 * MON", + } + } + with pytest.raises(ExecutionProjectionError, match="both day fields"): + _fargate_runtime().templates.build( + pipelines, + image=_FARGATE_IMAGE, + project="unit-project", + cpu=1, + memory="2Gi", + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=1_000, + require_guarded_free_tier=False, + alert_target=None, + ) + + +def test_fargate_rejects_provider_specific_cron_syntax() -> None: + pipelines = { + "greenhouse_jobs": { + **_PIPELINES["greenhouse_jobs"], + "schedule": "0 9 * * ?", + } + } + with pytest.raises(ExecutionProjectionError, match="valid five-field"): + _fargate_runtime().templates.build( + pipelines, + image=_FARGATE_IMAGE, + project="unit-project", + cpu=1, + memory="2Gi", + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=1_000, + require_guarded_free_tier=False, + alert_target=None, + ) diff --git a/tickets/DANDER-92-fargate-infrastructure-controller.md b/tickets/DANDER-92-fargate-infrastructure-controller.md new file mode 100644 index 00000000..67c2ef8c --- /dev/null +++ b/tickets/DANDER-92-fargate-infrastructure-controller.md @@ -0,0 +1,44 @@ +--- +id: DANDER-92 +title: Provision a bounded Fargate execution controller +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-90, DANDER-91] +created: 2026-08-08 +--- + +## Context + +The Fargate provider can build a safe execution projection and prepare keyless runtime identity, +but no product Terraform stack yet turns that projection into an executable, observable lifecycle. +This slice provisions the AWS resources without exposing them through public CLI lifecycle commands +or claiming support. + +## Acceptance Criteria + +- [x] Use a separate AWS Terraform root with an operator-configured encrypted S3 backend. +- [x] Create an immutable, scan-on-push ECR repository and one ECS cluster. +- [x] Create non-root, read-only task definitions from exact execution projections. +- [x] Keep the image-pull/log execution role separate from each runtime task role. +- [x] Grant AWS secret access only for exact declared ARNs. +- [x] Enforce one absolute deadline through a Standard Step Functions ECS `.sync` controller. +- [x] Retry only runtime exit code 75 within the declared launcher-attempt bound. +- [x] Keep Scheduler delivery retry state separate and honor paused projections. +- [x] Route exhausted, failed, timed-out, and aborted outcomes to encrypted failure targets. +- [x] Package every clean AWS Terraform asset and keep Fargate outside the support manifest. +- [x] Focused tests, Terraform validation, provider-mocked plan, and read-only AWS plan pass. +- [x] Full validation and isolated GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Use one state machine and task definition per pipeline. EventBridge Scheduler calls the universal +Step Functions `StartExecution` target with only non-secret schedule context. The optimized ECS +integration observes and best-effort stops its own tasks. Direct CLI lifecycle and live parity stay +in later, separately reviewed tickets. + +## Review Log + +The read-only plan proposed 23 creates, zero changes, and zero destroys against AWS account +`184463061564` in `us-east-1`. No Terraform apply or AWS resource mutation occurred. From 990395f58b5fbeac17c4241af6c4c9f817a31033 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 16:37:28 -0400 Subject: [PATCH 042/163] feat: plan manifest-defined Fargate deployments --- HANDOFF.md | 42 +-- infra/aws/README.md | 23 +- infra/aws/versions.tf | 2 +- src/dander/bootstrap/__init__.py | 3 + src/dander/bootstrap/aws_terraform.py | 270 ++++++++++++++++++ src/dander/bootstrap/terraform.py | 24 +- src/dander/cli/main.py | 105 +++++++ src/dander/project/config.py | 16 +- src/dander/project/portable_config.py | 11 +- tests/bootstrap/test_aws_terraform.py | 192 +++++++++++++ tests/cli/test_init_cli.py | 84 +++++- tests/project/test_portable_config.py | 36 +++ ...NDER-93-fargate-manifest-plan-lifecycle.md | 48 ++++ 13 files changed, 822 insertions(+), 34 deletions(-) create mode 100644 src/dander/bootstrap/aws_terraform.py create mode 100644 tests/bootstrap/test_aws_terraform.py create mode 100644 tickets/DANDER-93-fargate-manifest-plan-lifecycle.md diff --git a/HANDOFF.md b/HANDOFF.md index 53c2271e..213673a7 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,41 @@ ## Finished -- Added a separate packaged AWS Terraform root and product Fargate module. -- Projected immutable ECR images into non-root, read-only Fargate task definitions. -- Separated execution, per-pipeline task, controller, and scheduler IAM roles. -- Added an absolute-deadline Step Functions controller with exit-75-only retries. -- Added paused-aware scheduling, controller logs, encrypted failure queue, and a customer-key-encrypted notification topic. +- Added version-2 manifest selection for complete Fargate launcher configuration. +- Added `dander init-aws-plan` for saved, manifest-rendered plans against existing S3/DynamoDB state. +- Added `dander init-aws-apply` to apply only the previously reviewed saved plan after confirmation. +- Preserved version-1 and Cloud Run resolution behavior. +- Documented the intentionally unsupported boundary before AWS stage zero, image publication, and live proof. ## Try It -Run `terraform -chdir=infra/aws/modules/fargate test` for a provider-mocked plan. The example stack -is paused and no Terraform apply has been performed. +Define a version-2 Fargate deployment, then run `dander init-aws-plan --help`. Do not apply until +the AWS stage-zero and image-publication slices establish the prerequisites documented in +`infra/aws/README.md`. ## Checks -- Ruff, formatting, strict typing, and all 941 Python tests passed. -- Every Terraform root validated; the provider-mocked Fargate plan passed. -- Wheel/sdist installs, source-free generation, container conformance, dependency audit, and Trivy scans passed. -- Read-only AWS plan: 23 add, 0 change, 0 destroy; schedule disabled and nothing applied. -- Existing isolated GCP platform plan reported exactly `No changes.` +- Ruff, formatting, strict typing, and all 953 Python tests passed. +- Supported Terraform roots and feasibility roots validated; the provider-mocked Fargate test passed. +- Wheel/sdist installs, source-free generation, container conformance, dependency audit, Trivy, and Gitleaks passed. +- Credentialed read-only AWS plan: 23 add, 0 change, 0 destroy; nothing applied. +- Retained GCP platform plan reported exactly `No changes.` and left no state lock. ## Decisions -- Use Standard Step Functions and its optimized ECS `.sync` integration. -- Retry only Dander exit code 75; keep scheduler delivery retries separate. -- Keep Fargate outside the support manifest until CLI lifecycle and live parity pass. +- Keep GCP data-plane access explicit while Fargate supplies the launcher. +- Use DynamoDB state locking to preserve the Terraform 1.9+ compatibility contract. +- Keep Fargate unsupported until AWS stage zero, source-free ECR publication, and live parity pass. ## Remaining - Let protected Linux CI repeat validation and merge the focused PR if clean. -- Add manifest-aware AWS planning and CLI lifecycle in a separate slice. -- Apply only after a separately reviewed plan; then prove live Fargate parity. +- Add least-privilege AWS stage zero and source-free ECR image publication. +- Add AWS status, logs, cancellation, replay, and verification operations. +- Review a real saved plan before requesting separate approval for any AWS apply. ## Review First -- `infra/aws/modules/fargate/main.tf` -- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` -- `src/dander/providers/fargate/runtime.py` +- `src/dander/bootstrap/aws_terraform.py` +- `src/dander/cli/main.py` +- `tests/bootstrap/test_aws_terraform.py` diff --git a/infra/aws/README.md b/infra/aws/README.md index af453449..b1ad45f0 100644 --- a/infra/aws/README.md +++ b/infra/aws/README.md @@ -5,9 +5,26 @@ Fargate. It creates an immutable ECR repository, ECS task definitions, distinct roles, a Standard Step Functions controller, paused-aware EventBridge schedules, CloudWatch logs, and an encrypted failure queue plus notification topic. -This root is construction-ready but is not yet selected by the public Dander plan/apply commands. -Initialize it with an operator-owned encrypted S3 backend and review a saved plan before any apply. -The stack accepts existing VPC subnet and security-group IDs; it does not create a network. +`dander init-aws-plan` selects one version-2 Fargate deployment, renders its complete execution +projections, and saves a Terraform plan in this root. It requires an existing encrypted S3 state +bucket, a DynamoDB lock table, and an immutable image digest reference for the selected account and +region. The command never applies. After reviewing the saved plan, an operator may use +`dander init-aws-apply` to apply only that plan. + +```console +dander init-aws-plan \ + --project YOUR_GCP_DATA_PROJECT \ + --deployment aws_fargate \ + --state-bucket YOUR_AWS_STATE_BUCKET \ + --lock-table YOUR_TERRAFORM_LOCK_TABLE \ + --container-image 123456789012.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:DIGEST +``` + +The state bucket, lock table, and usable ECR image lifecycle are separate prerequisites until the +AWS stage-zero and image-publication commands ship. Do not apply this root before those commands +establish consistent repository ownership and publish the referenced digest. The stack accepts +existing VPC subnet and security-group IDs; it does not create a network. Use a least-privilege AWS +role for planning and application rather than the account root identity. The controller uses `ecs:runTask.sync`. One absolute Step Functions deadline bounds all attempts; AWS performs a best-effort `StopTask` when the integration is cancelled or times out. Exit code 75 diff --git a/infra/aws/versions.tf b/infra/aws/versions.tf index 1a658dc2..c590e2d5 100644 --- a/infra/aws/versions.tf +++ b/infra/aws/versions.tf @@ -8,7 +8,7 @@ terraform { } } - # Supply bucket, key, region, encryption, and native lockfile settings through + # Supply bucket, key, region, encryption, and DynamoDB locking through # `terraform init -backend-config`. Provider credentials never belong in this file. backend "s3" {} } diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index 1761f98e..b5d27ce5 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -1,6 +1,7 @@ """Infrastructure bootstrap adapters.""" from dander.bootstrap.admin import AdministrativeBootstrap, AdministrativeBootstrapError +from dander.bootstrap.aws_terraform import AwsTerraformBootstrap, AwsTerraformBootstrapError from dander.bootstrap.permissions import require_stage_zero_permissions from dander.bootstrap.project import ( ProjectBootstrapError, @@ -22,6 +23,8 @@ __all__ = [ "AdministrativeBootstrap", "AdministrativeBootstrapError", + "AwsTerraformBootstrap", + "AwsTerraformBootstrapError", "DeploymentSummary", "DeploymentVerificationError", "DeploymentVerifier", diff --git a/src/dander/bootstrap/aws_terraform.py b/src/dander/bootstrap/aws_terraform.py new file mode 100644 index 00000000..9d5eec38 --- /dev/null +++ b/src/dander/bootstrap/aws_terraform.py @@ -0,0 +1,270 @@ +"""Saved-plan Terraform lifecycle for one manifest-defined AWS Fargate deployment.""" + +from __future__ import annotations + +import os +import re +import subprocess +from json import dumps +from typing import TYPE_CHECKING + +from dander.bootstrap.terraform import ( + TerraformBootstrapError, + build_launcher_runtime, + validate_runtime_settings, + validate_terraform_pipelines, +) +from dander.deployment import ExecutionProjectionError +from dander.providers import ProviderFactoryError + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + +_AWS_ACCOUNT_ID = re.compile(r"^[0-9]{12}$") +_AWS_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_AWS_REGION = re.compile(r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") +_DEPLOYMENT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,23}$") +_DYNAMODB_TABLE = re.compile(r"^[A-Za-z0-9_.-]{3,255}$") +_GCP_PROJECT = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +_S3_BUCKET = re.compile(r"^(?![0-9]+(?:\.[0-9]+){3}$)[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$") +_STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") +_ECR_IMAGE = re.compile( + r"^(?P[0-9]{12})\.dkr\.ecr\.(?P[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+)" + r"\.amazonaws\.com/(?P[a-z0-9]+(?:[._/-][a-z0-9]+)*)" + r"@sha256:(?P[0-9a-f]{64})$" +) + + +class AwsTerraformBootstrapError(RuntimeError): + """Raised when the AWS saved-plan lifecycle cannot complete safely.""" + + +class AwsTerraformBootstrap: + """Plan or apply the packaged Fargate root using an existing S3 backend.""" + + def __init__(self, infra_dir: Path) -> None: + self._infra_dir = infra_dir.resolve() + self._plan_path = self._infra_dir / "dander-aws.tfplan" + + def execute( + self, + *, + project: str, + deployment_name: str, + state_bucket: str, + state_key: str, + state_region: str, + lock_table: str, + container_image: str, + launcher_config: Mapping[str, object], + runtime_cpu: int, + runtime_memory: str, + runtime_timeout_seconds: int, + runtime_max_retries: int, + runtime_batch_rows: int, + require_guarded_free_tier: bool, + pipelines: Mapping[str, Mapping[str, object]], + apply: bool, + aws_profile: str = "", + name: str = "dander", + ) -> Path: + """Produce one immutable AWS plan and optionally apply that exact plan.""" + self._validate_backend( + state_bucket=state_bucket, + state_key=state_key, + state_region=state_region, + lock_table=lock_table, + aws_profile=aws_profile, + ) + if not _GCP_PROJECT.fullmatch(project): + raise AwsTerraformBootstrapError(f"Invalid GCP data-plane project: {project!r}") + if not _DEPLOYMENT_NAME.fullmatch(name): + raise AwsTerraformBootstrapError( + "AWS deployment name must contain 2-24 lowercase letters, numbers, or hyphens" + ) + if not deployment_name.strip(): + raise AwsTerraformBootstrapError("Manifest deployment name must not be blank") + expanded_pipelines = dict(pipelines) + if not expanded_pipelines: + raise AwsTerraformBootstrapError("Fargate planning requires at least one pipeline") + try: + validate_runtime_settings( + cpu=runtime_cpu, + memory=runtime_memory, + timeout_seconds=runtime_timeout_seconds, + max_retries=runtime_max_retries, + batch_rows=runtime_batch_rows, + require_guarded_free_tier=require_guarded_free_tier, + ) + validate_terraform_pipelines(expanded_pipelines) + except TerraformBootstrapError as error: + raise AwsTerraformBootstrapError(str(error)) from error + + image_match = _ECR_IMAGE.fullmatch(container_image) + if image_match is None: + raise AwsTerraformBootstrapError( + "Fargate requires an immutable ECR image in account.dkr.ecr.region.amazonaws.com/" + "repository@sha256:digest form" + ) + raw_launcher = dict(launcher_config) + if raw_launcher.get("provider") != "fargate": + raise AwsTerraformBootstrapError("AWS planning requires launcher.provider='fargate'") + aws_account_id = raw_launcher.get("aws_account_id") + region = raw_launcher.get("region") + if not isinstance(aws_account_id, str) or not _AWS_ACCOUNT_ID.fullmatch(aws_account_id): + raise AwsTerraformBootstrapError("Fargate launcher requires a valid AWS account id") + if not isinstance(region, str) or not _AWS_REGION.fullmatch(region): + raise AwsTerraformBootstrapError("Fargate launcher requires a valid AWS region") + if image_match.group("account") != aws_account_id or image_match.group("region") != region: + raise AwsTerraformBootstrapError( + "ECR image account and region must match the selected Fargate launcher" + ) + + try: + launcher = build_launcher_runtime(launcher_config=raw_launcher) + projections = { + pipeline_id: template.as_dict() + for pipeline_id, template in launcher.templates.build( + expanded_pipelines, + image=container_image, + project=project, + cpu=runtime_cpu, + memory=runtime_memory, + deadline_seconds=runtime_timeout_seconds, + launcher_retry_count=runtime_max_retries, + batch_rows=runtime_batch_rows, + require_guarded_free_tier=require_guarded_free_tier, + alert_target=None, + ).items() + } + except (ExecutionProjectionError, ProviderFactoryError) as error: + raise AwsTerraformBootstrapError(str(error)) from error + + self._run( + "terraform", + "init", + "-reconfigure", + "-input=false", + f"-backend-config=bucket={state_bucket}", + f"-backend-config=key={state_key}", + f"-backend-config=region={state_region}", + "-backend-config=encrypt=true", + f"-backend-config=dynamodb_table={lock_table}", + aws_profile=aws_profile, + ) + self._run( + "terraform", + "plan", + "-input=false", + f"-var=name={name}", + f"-var=aws_account_id={aws_account_id}", + f"-var=region={region}", + f"-var=ecr_repository_name={image_match.group('repository')}", + "-var=execution_projections=" + f"{dumps(projections, sort_keys=True, separators=(',', ':'))}", + "-var=tags=" + + dumps( + { + "dander-deployment": deployment_name, + "dander-profile": "gcp", + }, + sort_keys=True, + separators=(",", ":"), + ), + f"-out={self._plan_path.name}", + aws_profile=aws_profile, + ) + if apply: + self._run( + "terraform", + "apply", + "-input=false", + self._plan_path.name, + aws_profile=aws_profile, + ) + return self._plan_path + + def apply_saved_plan( + self, + *, + state_bucket: str, + state_key: str, + state_region: str, + lock_table: str, + aws_profile: str = "", + ) -> Path: + """Apply only the saved plan produced by ``execute``.""" + self._validate_backend( + state_bucket=state_bucket, + state_key=state_key, + state_region=state_region, + lock_table=lock_table, + aws_profile=aws_profile, + ) + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AwsTerraformBootstrapError(f"Saved AWS plan is missing: {self._plan_path}") + self._run( + "terraform", + "init", + "-reconfigure", + "-input=false", + f"-backend-config=bucket={state_bucket}", + f"-backend-config=key={state_key}", + f"-backend-config=region={state_region}", + "-backend-config=encrypt=true", + f"-backend-config=dynamodb_table={lock_table}", + aws_profile=aws_profile, + ) + self._run( + "terraform", + "apply", + "-input=false", + self._plan_path.name, + aws_profile=aws_profile, + ) + return self._plan_path + + @staticmethod + def _validate_backend( + *, + state_bucket: str, + state_key: str, + state_region: str, + lock_table: str, + aws_profile: str, + ) -> None: + if not _S3_BUCKET.fullmatch(state_bucket): + raise AwsTerraformBootstrapError(f"Invalid S3 state bucket: {state_bucket!r}") + if not _STATE_KEY.fullmatch(state_key): + raise AwsTerraformBootstrapError(f"Invalid S3 state key: {state_key!r}") + if not _AWS_REGION.fullmatch(state_region): + raise AwsTerraformBootstrapError(f"Invalid S3 state region: {state_region!r}") + if not _DYNAMODB_TABLE.fullmatch(lock_table): + raise AwsTerraformBootstrapError(f"Invalid DynamoDB lock table: {lock_table!r}") + if aws_profile and not _AWS_PROFILE.fullmatch(aws_profile): + raise AwsTerraformBootstrapError(f"Invalid AWS profile: {aws_profile!r}") + + def _run(self, *args: str, aws_profile: str) -> None: + environment = os.environ.copy() + if aws_profile: + environment["AWS_PROFILE"] = aws_profile + try: + subprocess.run( + args, + cwd=self._infra_dir, + check=True, + env=environment, + ) + except FileNotFoundError as error: + raise AwsTerraformBootstrapError( + "Terraform is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + command = " ".join(args[:2]) + raise AwsTerraformBootstrapError( + f"{command} failed with exit code {error.returncode}" + ) from error + + +__all__ = ["AwsTerraformBootstrap", "AwsTerraformBootstrapError"] diff --git a/src/dander/bootstrap/terraform.py b/src/dander/bootstrap/terraform.py index 8ba6298b..ba97bd01 100644 --- a/src/dander/bootstrap/terraform.py +++ b/src/dander/bootstrap/terraform.py @@ -128,7 +128,7 @@ def execute( "Platform bootstrap requires --bootstrap-service-account with a valid " "service-account email" ) - _validate_runtime( + validate_runtime_settings( cpu=runtime_cpu, memory=runtime_memory, timeout_seconds=runtime_timeout_seconds, @@ -164,7 +164,7 @@ def execute( ) if not enable_runtime: raise TerraformBootstrapError("--druff-container-image requires --enable-runtime") - _validate_pipelines(expanded_pipelines) + validate_terraform_pipelines(expanded_pipelines) if failure_alert_email and not enable_runtime: raise TerraformBootstrapError("--failure-alert-email requires --enable-runtime") if failure_alert_email and ( @@ -174,7 +174,7 @@ def execute( execution_projections: dict[str, dict[str, object]] = {} if enable_runtime: try: - launcher = _build_launcher_runtime( + launcher = build_launcher_runtime( provider_id=launcher_provider, region=region, ) @@ -329,12 +329,22 @@ def _run(self, *args: str) -> None: ) from error -def _build_launcher_runtime(*, provider_id: str, region: str) -> LauncherRuntime: +def build_launcher_runtime( + *, + provider_id: str | None = None, + region: str | None = None, + launcher_config: Mapping[str, object] | None = None, +) -> LauncherRuntime: """Build one launcher through the shared lazy provider registry.""" + raw_config = dict(launcher_config or {}) + if not raw_config: + if provider_id is None or region is None: + raise ProviderFactoryError("Launcher provider and region are required") + raw_config = {"provider": provider_id, "region": region} registry = default_provider_registry() config = registry.parse( ProviderKind.LAUNCHER, - {"provider": provider_id, "region": region}, + raw_config, ) runtime = registry.build(ProviderKind.LAUNCHER, config) if not isinstance(runtime, LauncherRuntime): @@ -342,7 +352,7 @@ def _build_launcher_runtime(*, provider_id: str, region: str) -> LauncherRuntime return runtime -def _validate_pipelines(pipelines: Mapping[str, Mapping[str, object]]) -> None: +def validate_terraform_pipelines(pipelines: Mapping[str, Mapping[str, object]]) -> None: """Defensively validate the expanded project manifest before invoking Terraform.""" required = { "job_name", @@ -387,7 +397,7 @@ def _validate_pipelines(pipelines: Mapping[str, Mapping[str, object]]) -> None: raise TerraformBootstrapError(f"Pipeline {pipeline_id!r} has invalid secret bindings") -def _validate_runtime( +def validate_runtime_settings( *, cpu: int, memory: str, diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index c4f2c5f8..3eb6a237 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -21,6 +21,8 @@ from dander.bootstrap import ( AdministrativeBootstrap, AdministrativeBootstrapError, + AwsTerraformBootstrap, + AwsTerraformBootstrapError, DeploymentSummary, DeploymentVerifier, ProjectBootstrapError, @@ -135,6 +137,7 @@ _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") _DEFAULT_CONNECTORS_DIR = Path("connectors") _DEFAULT_INFRA_DIR = Path("infra") +_DEFAULT_AWS_INFRA_DIR = Path("infra/aws") _DEFAULT_MODELS_DIR = Path("models") _DEFAULT_CATALOG_PATH = Path(".dander/catalog.json") _DEFAULT_BOOTSTRAP_ADMIN_DIR = Path("infra/bootstrap-admin") @@ -1119,6 +1122,108 @@ def init_platform_apply( console.print(f"[green]Platform bootstrap applied.[/green] Saved plan: {plan_path}") +@app.command("init-aws-plan") +def init_aws_plan( + project: str = typer.Option(..., "--project", help="GCP BigQuery data-plane project id."), + state_bucket: str = typer.Option(..., "--state-bucket", help="Existing S3 state bucket."), + container_image: str = typer.Option( + ..., "--container-image", help="Immutable ECR image ending in @sha256 digest." + ), + deployment: str = typer.Option(..., "--deployment"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + state_key: str = typer.Option("dander/aws/state/terraform.tfstate", "--state-key"), + lock_table: str = typer.Option(..., "--lock-table", help="Existing DynamoDB lock table."), + state_region: str | None = typer.Option(None, "--state-region"), + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), + infra_dir: Path = typer.Option(_DEFAULT_AWS_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Plan one manifest-selected Fargate deployment without applying it.""" + try: + manifest = load_project_config( + config, + platforms_path=platforms_config, + deployment=deployment, + ) + if manifest.launcher_provider != "fargate": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select launcher.provider='fargate'" + ) + manifest.validate_references(config.resolve().parent) + runtime = manifest.platform.runtime + plan_path = AwsTerraformBootstrap(infra_dir).execute( + project=project, + deployment_name=manifest.deployment_name, + state_bucket=state_bucket, + state_key=state_key, + state_region=state_region or manifest.platform.region, + lock_table=lock_table, + container_image=container_image, + launcher_config=manifest.resolved_launcher_config(), + runtime_cpu=runtime.cpu, + runtime_memory=runtime.memory, + runtime_timeout_seconds=runtime.timeout_seconds, + runtime_max_retries=runtime.max_retries, + runtime_batch_rows=runtime.batch_rows, + require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, + pipelines=manifest.terraform_pipelines(), + apply=False, + aws_profile=aws_profile, + name=name, + ) + except (AwsTerraformBootstrapError, ProjectConfigError) as error: + raise ClickException(str(error)) from error + console.print(f"[green]AWS deployment planned.[/green] Saved plan: {plan_path}") + console.print( + "Review: " + + shlex.join(("terraform", f"-chdir={infra_dir}", "show", "-no-color", str(plan_path))) + ) + next_command = [ + "dander", + "init-aws-apply", + "--state-bucket", + state_bucket, + "--state-key", + state_key, + "--state-region", + state_region or manifest.platform.region, + "--lock-table", + lock_table, + ] + if aws_profile: + next_command.extend(("--aws-profile", aws_profile)) + console.print("Next after review: " + shlex.join(next_command)) + + +@app.command("init-aws-apply") +def init_aws_apply( + state_bucket: str = typer.Option(..., "--state-bucket", help="Existing S3 state bucket."), + state_key: str = typer.Option("dander/aws/state/terraform.tfstate", "--state-key"), + state_region: str = typer.Option(..., "--state-region"), + lock_table: str = typer.Option(..., "--lock-table", help="Existing DynamoDB lock table."), + aws_profile: str = typer.Option("", "--aws-profile"), + infra_dir: Path = typer.Option(_DEFAULT_AWS_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Apply only the previously reviewed AWS Terraform plan.""" + if not typer.confirm( + f"Apply the reviewed AWS deployment plan using state bucket {state_bucket!r}?", + default=False, + ): + raise typer.Abort() + try: + plan_path = AwsTerraformBootstrap(infra_dir).apply_saved_plan( + state_bucket=state_bucket, + state_key=state_key, + state_region=state_region, + lock_table=lock_table, + aws_profile=aws_profile, + ) + except AwsTerraformBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]AWS deployment applied.[/green] Saved plan: {plan_path}") + + @verify_app.command("deployment") def verify_deployment( project: str = typer.Option(..., "--project", help="GCP project id to inspect."), diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 90f760ee..54c92abb 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -210,7 +210,8 @@ class DanderProject(BaseModel): default="gcp_secret_manager", exclude=True, ) - launcher_provider: Literal["cloud_run"] = Field(default="cloud_run", exclude=True) + launcher_provider: Literal["cloud_run", "fargate"] = Field(default="cloud_run", exclude=True) + launcher_config: dict[str, object] = Field(default_factory=dict, exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) @field_validator("plugins") @@ -232,6 +233,10 @@ def validate_pipeline_ids(cls, values: dict[str, PipelineSpec]) -> dict[str, Pip @model_validator(mode="after") def validate_deployed_pipeline_ids(self) -> DanderProject: + if self.launcher_config: + configured_provider = self.launcher_config.get("provider") + if configured_provider != self.launcher_provider: + raise ValueError("launcher config provider must match launcher_provider") if self.deployed_pipeline_ids is None: return self if len(self.deployed_pipeline_ids) != len(set(self.deployed_pipeline_ids)): @@ -241,6 +246,15 @@ def validate_deployed_pipeline_ids(self) -> DanderProject: raise ValueError(f"deployed pipeline ids contain unknown pipeline {unknown[0]!r}") return self + def resolved_launcher_config(self) -> dict[str, object]: + """Return the complete selected launcher block for projection and planning.""" + if self.launcher_config: + return dict(self.launcher_config) + return { + "provider": self.launcher_provider, + "region": self.platform.region, + } + def terraform_pipelines(self) -> dict[str, dict[str, object]]: """Expand human pipeline definitions into the literal Terraform module contract.""" selected = ( diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 5d427750..4839c248 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -27,6 +27,7 @@ from dander.providers.cloud_run import CloudRunLauncherConfig # noqa: TC001 from dander.providers.dataplex import DataplexCatalogConfig # noqa: TC001 from dander.providers.environment_secrets import EnvironmentSecretConfig # noqa: TC001 +from dander.providers.fargate import FargateLauncherConfig # noqa: TC001 from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 @@ -124,6 +125,12 @@ def validate_pipeline_ids( ] +LauncherSpec = Annotated[ + CloudRunLauncherConfig | FargateLauncherConfig, + Field(discriminator="provider"), +] + + class PlatformProfileSpec(BaseModel): """One named combination of data-plane provider selections.""" @@ -172,7 +179,7 @@ class DeploymentSpec(BaseModel): model_config = ConfigDict(extra="forbid") platform: str = Field(pattern=_NAME.pattern) - launcher: CloudRunLauncherConfig + launcher: LauncherSpec runtime: PlatformRuntimeSpec = Field(default_factory=PlatformRuntimeSpec) safety: PlatformSafetySpec = Field(default_factory=PlatformSafetySpec) pipelines: dict[str, DeploymentPipelineSpec] = Field(min_length=1) @@ -347,6 +354,7 @@ def _resolve( catalog_provider=profile.catalog.provider, secret_provider=profile.secrets.provider, launcher_provider=selected.launcher.provider, + launcher_config=selected.launcher.model_dump(mode="json"), deployed_pipeline_ids=tuple(sorted(selected.pipelines)), ) @@ -438,6 +446,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: and legacy.catalog_provider == migrated.catalog_provider and legacy.secret_provider == migrated.secret_provider and legacy.launcher_provider == migrated.launcher_provider + and legacy.resolved_launcher_config() == migrated.resolved_launcher_config() and legacy.plugins == migrated.plugins and legacy.pipelines == migrated.pipelines and legacy.terraform_pipelines() == migrated.terraform_pipelines() diff --git a/tests/bootstrap/test_aws_terraform.py b/tests/bootstrap/test_aws_terraform.py new file mode 100644 index 00000000..19c9f10f --- /dev/null +++ b/tests/bootstrap/test_aws_terraform.py @@ -0,0 +1,192 @@ +"""Saved-plan lifecycle coverage for manifest-defined AWS deployments.""" + +from __future__ import annotations + +import json +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from dander.bootstrap import AwsTerraformBootstrap, AwsTerraformBootstrapError + +if TYPE_CHECKING: + from pathlib import Path + + +def _launcher() -> dict[str, object]: + return { + "provider": "fargate", + "region": "us-east-1", + "aws_account_id": "123456789012", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/123456789012/locations/global/" + "workloadIdentityPools/dander-aws/providers/dander-aws" + ), + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + "architecture": "X86_64", + "assign_public_ip": True, + } + + +def _pipelines() -> dict[str, dict[str, object]]: + return { + "greenhouse_jobs": { + "job_name": "dander-greenhouse-public", + "runtime_service_account_id": "dander-runtime", + "scheduler_service_account_id": "dander-scheduler", + "source": "greenhouse_job_board", + "models": ["stg_greenhouse__jobs"], + "build_models": True, + "publish_dataplex": True, + "schedule": "0 9 * * *", + "time_zone": "America/New_York", + "paused": True, + "secret_env": {"API_TOKEN": "greenhouse-token"}, + } + } + + +def _execute(bootstrap: AwsTerraformBootstrap, **overrides: object) -> Path: + arguments: dict[str, object] = { + "project": "unit-project", + "deployment_name": "aws_fargate", + "state_bucket": "unit-dander-state", + "state_key": "dander/aws/state/terraform.tfstate", + "state_region": "us-east-1", + "lock_table": "dander-terraform-locks", + "container_image": ( + "123456789012.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "a" * 64 + ), + "launcher_config": _launcher(), + "runtime_cpu": 1, + "runtime_memory": "2Gi", + "runtime_timeout_seconds": 900, + "runtime_max_retries": 1, + "runtime_batch_rows": 2_048, + "require_guarded_free_tier": False, + "pipelines": _pipelines(), + "apply": False, + "aws_profile": "dander-test", + "name": "dander-test", + } + arguments.update(overrides) + return bootstrap.execute(**arguments) # type: ignore[arg-type] + + +def test_aws_bootstrap_builds_manifest_projection_without_apply( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[tuple[tuple[str, ...], dict[str, str]]] = [] + + def fake_run( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + env: dict[str, str], + ) -> subprocess.CompletedProcess[str]: + assert cwd == tmp_path.resolve() + assert check + calls.append((args, env)) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + plan = _execute(AwsTerraformBootstrap(tmp_path)) + + assert plan == tmp_path.resolve() / "dander-aws.tfplan" + init, terraform_plan = (call[0] for call in calls) + assert init[:2] == ("terraform", "init") + assert "-backend-config=dynamodb_table=dander-terraform-locks" in init + assert terraform_plan[:2] == ("terraform", "plan") + assert all(call[0][:2] != ("terraform", "apply") for call in calls) + assert all(call[1]["AWS_PROFILE"] == "dander-test" for call in calls) + projection_argument = next( + item for item in terraform_plan if item.startswith("-var=execution_projections=") + ) + projection = json.loads(projection_argument.removeprefix("-var=execution_projections="))[ + "greenhouse_jobs" + ] + assert projection["launcher"] == "fargate" + assert projection["resources"]["memory_mib"] == 2_048 + assert projection["schedule"]["paused"] is True + assert projection["secret_bindings"]["API_TOKEN"] == { + "provider": "gcp_secret_manager", + "reference": "gcp-sm://projects/unit-project/secrets/greenhouse-token/versions/latest", + } + + +def test_aws_apply_uses_only_the_saved_plan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + plan_path = tmp_path / "dander-aws.tfplan" + plan_path.write_bytes(b"reviewed") + calls: list[tuple[str, ...]] = [] + + def fake_run( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + env: dict[str, str], + ) -> subprocess.CompletedProcess[str]: + del env + assert cwd == tmp_path.resolve() + assert check + calls.append(args) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = AwsTerraformBootstrap(tmp_path).apply_saved_plan( + state_bucket="unit-dander-state", + state_key="dander/aws/state/terraform.tfstate", + state_region="us-east-1", + lock_table="dander-terraform-locks", + aws_profile="dander-test", + ) + + assert result == plan_path + assert calls[0][:2] == ("terraform", "init") + assert calls[1] == ("terraform", "apply", "-input=false", "dander-aws.tfplan") + assert all(call[:2] != ("terraform", "plan") for call in calls) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"require_guarded_free_tier": True}, "guarded-free-tier"), + ({"runtime_memory": "512Mi"}, "CPU/memory"), + ({"state_key": "/absolute"}, "state key"), + ({"aws_profile": "bad profile"}, "AWS profile"), + ({"lock_table": "bad table"}, "lock table"), + ({"container_image": "example.invalid/dander@sha256:" + "a" * 64}, "ECR image"), + ( + { + "container_image": ( + "999999999999.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "a" * 64 + ) + }, + "account and region", + ), + ({"pipelines": {}}, "at least one pipeline"), + ], +) +def test_aws_bootstrap_rejects_unsafe_inputs_before_terraform( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + overrides: dict[str, object], + message: str, +) -> None: + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: pytest.fail("Terraform must not run"), + ) + + with pytest.raises(AwsTerraformBootstrapError, match=message): + _execute(AwsTerraformBootstrap(tmp_path), **overrides) diff --git a/tests/cli/test_init_cli.py b/tests/cli/test_init_cli.py index 5eb1581a..f0249cd7 100644 --- a/tests/cli/test_init_cli.py +++ b/tests/cli/test_init_cli.py @@ -8,7 +8,14 @@ from typer.testing import CliRunner from dander.cli.main import app -from dander.project import scaffold_project +from dander.project import ( + DanderProject, + PipelineSpec, + PlatformRuntimeSpec, + PlatformSafetySpec, + PlatformSpec, + scaffold_project, +) if TYPE_CHECKING: from pathlib import Path @@ -175,6 +182,81 @@ def fake_execute(self: object, **kwargs: object) -> Path: assert "init-platform-apply" in result.output +def test_aws_plan_resolves_selected_fargate_deployment_without_applying( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + manifest = DanderProject( + version=2, + platform=PlatformSpec( + region="us-east-1", + runtime=PlatformRuntimeSpec(memory="2Gi", timeout_seconds=900), + safety=PlatformSafetySpec(require_guarded_free_tier=False), + ), + pipelines={ + "greenhouse_jobs": PipelineSpec( + source="greenhouse_job_board", + models=["stg_greenhouse__jobs"], + ) + }, + deployment_name="aws_fargate", + launcher_provider="fargate", + launcher_config={ + "provider": "fargate", + "region": "us-east-1", + "aws_account_id": "123456789012", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/123456789012/locations/global/" + "workloadIdentityPools/dander-aws/providers/dander-aws" + ), + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + }, + ) + + def fake_load(*args: object, **kwargs: object) -> DanderProject: + assert kwargs["deployment"] == "aws_fargate" + return manifest + + def fake_validate(self: object, root: Path) -> None: + del self, root + + def fake_execute(self: object, **kwargs: object) -> Path: + captured.update(kwargs) + return tmp_path / "dander-aws.tfplan" + + monkeypatch.setattr("dander.cli.main.load_project_config", fake_load) + monkeypatch.setattr(DanderProject, "validate_references", fake_validate) + monkeypatch.setattr("dander.cli.main.AwsTerraformBootstrap.execute", fake_execute) + + result = CliRunner().invoke( + app, + [ + "init-aws-plan", + "--project", + "unit-project", + "--state-bucket", + "unit-dander-state", + "--container-image", + "123456789012.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "a" * 64, + "--deployment", + "aws_fargate", + "--lock-table", + "dander-terraform-locks", + "--aws-profile", + "dander-test", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["apply"] is False + assert captured["launcher_config"] == manifest.resolved_launcher_config() + assert captured["runtime_memory"] == "2Gi" + assert "AWS deployment planned" in result.output + assert "init-aws-apply" in result.output + + def test_admin_plan_runs_read_only_permission_preflight_before_terraform( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 8d6fbb70..4403efaf 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -164,6 +164,42 @@ def test_cloud_run_rejects_environment_only_secrets(tmp_path: Path) -> None: load_project_config(project_path) +def test_version_two_resolves_complete_fargate_launcher(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + deployment = platforms["deployments"].pop("gcp_cloud_run") + deployment["launcher"] = { + "provider": "fargate", + "region": "us-east-1", + "aws_account_id": "123456789012", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/123456789012/locations/global/" + "workloadIdentityPools/dander-aws/providers/dander-aws" + ), + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + "architecture": "X86_64", + "assign_public_ip": True, + } + deployment["runtime"]["memory"] = "2Gi" + deployment["safety"]["require_guarded_free_tier"] = False + platforms["deployments"]["aws_fargate"] = deployment + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path, deployment="aws_fargate") + + assert resolved.launcher_provider == "fargate" + assert resolved.platform.region == "us-east-1" + assert resolved.resolved_launcher_config() == deployment["launcher"] | { + "ephemeral_storage_mib": 20_480, + "stop_timeout_seconds": 120, + } + + def test_config_migrate_check_is_read_only_then_write_is_atomic(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tickets/DANDER-93-fargate-manifest-plan-lifecycle.md b/tickets/DANDER-93-fargate-manifest-plan-lifecycle.md new file mode 100644 index 00000000..be16d263 --- /dev/null +++ b/tickets/DANDER-93-fargate-manifest-plan-lifecycle.md @@ -0,0 +1,48 @@ +--- +id: DANDER-93 +title: Plan manifest-defined Fargate deployments +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-92] +created: 2026-08-08 +--- + +## Context + +The packaged Fargate Terraform root could be validated directly, but version-2 projects could not +select the launcher or drive a saved plan through Dander. This slice connects the existing +configuration, projection, and Terraform boundaries without creating AWS bootstrap resources or +claiming live support. + +## Acceptance Criteria + +- [x] Accept a validated Fargate launcher block in a version-2 deployment. +- [x] Preserve the complete selected launcher configuration after project resolution. +- [x] Select only an explicitly named deployment for AWS planning. +- [x] Render the existing Fargate execution projections from the resolved manifest. +- [x] Require an immutable ECR digest in the selected AWS account and region. +- [x] Initialize an existing encrypted S3 backend with a DynamoDB lock table. +- [x] Save a plan without applying and print exact review/apply commands. +- [x] Apply only a previously saved plan after explicit confirmation. +- [x] Preserve version-1 and Cloud Run behavior. +- [x] Full local validation and retained GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Keep the GCP bootstrap unchanged. Add one AWS-specific bootstrap adapter and two CLI commands that +consume the shared resolved project and launcher registry. AWS state/ECR creation, image +publication, and runtime operations remain separate follow-up slices. + +## Implementation Notes + +Terraform 1.9 does not support the newer S3 `use_lockfile` backend option, so this lifecycle uses +an explicitly named DynamoDB lock table and retains the existing Terraform 1.9+ contract. + +## Review Log + +No AWS or GCP apply occurred. The new AWS account was inspected read-only; it currently has a +default VPC but no S3 state bucket or DynamoDB lock table. A credentialed, local-backend Terraform +plan reported 23 additions, 0 changes, and 0 destructions. The retained GCP project reported exactly +`No changes.` From 038e89db9a54571848dfd1fb91c3ecc348c20dc7 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 17:43:08 -0400 Subject: [PATCH 043/163] feat: bootstrap AWS state and promote runtime images --- .github/workflows/ci.yml | 5 +- HANDOFF.md | 42 +- docs/decisions.md | 11 + docs/known-limitations.md | 11 +- infra/aws/README.md | 14 +- infra/aws/bootstrap-admin/.terraform.lock.hcl | 26 ++ infra/aws/bootstrap-admin/README.md | 13 + infra/aws/bootstrap-admin/main.tf | 351 +++++++++++++++ infra/aws/bootstrap-admin/outputs.tf | 19 + .../tests/bootstrap.tftest.hcl | 68 +++ infra/aws/bootstrap-admin/variables.tf | 80 ++++ infra/aws/bootstrap-admin/versions.tf | 20 + infra/aws/modules/fargate/main.tf | 20 +- infra/aws/modules/fargate/outputs.tf | 2 +- .../modules/fargate/tests/fargate.tftest.hcl | 16 +- pyproject.toml | 7 + src/dander/bootstrap/__init__.py | 8 + src/dander/bootstrap/aws_admin.py | 420 ++++++++++++++++++ src/dander/bootstrap/project.py | 233 ++++++++++ src/dander/cli/aws_command.py | 289 ++++++++++++ src/dander/cli/main.py | 107 +---- tests/bootstrap/test_aws_admin.py | 310 +++++++++++++ tests/bootstrap/test_aws_image_promotion.py | 189 ++++++++ tests/cli/test_init_cli.py | 78 +++- tests/infra/test_fargate_runtime.py | 17 +- ...ANDER-94-aws-stage-zero-image-promotion.md | 40 ++ 26 files changed, 2230 insertions(+), 166 deletions(-) create mode 100644 infra/aws/bootstrap-admin/.terraform.lock.hcl create mode 100644 infra/aws/bootstrap-admin/README.md create mode 100644 infra/aws/bootstrap-admin/main.tf create mode 100644 infra/aws/bootstrap-admin/outputs.tf create mode 100644 infra/aws/bootstrap-admin/tests/bootstrap.tftest.hcl create mode 100644 infra/aws/bootstrap-admin/variables.tf create mode 100644 infra/aws/bootstrap-admin/versions.tf create mode 100644 src/dander/bootstrap/aws_admin.py create mode 100644 src/dander/cli/aws_command.py create mode 100644 tests/bootstrap/test_aws_admin.py create mode 100644 tests/bootstrap/test_aws_image_promotion.py create mode 100644 tickets/DANDER-94-aws-stage-zero-image-promotion.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b786a524..9689cbdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,11 @@ jobs: run: | terraform -chdir=bootstrap-admin init -backend=false -input=false terraform -chdir=bootstrap-admin validate - - name: Validate AWS Fargate stack + - name: Validate AWS Terraform stacks run: | + terraform -chdir=aws/bootstrap-admin init -backend=false -input=false + terraform -chdir=aws/bootstrap-admin validate + terraform -chdir=aws/bootstrap-admin test terraform -chdir=aws init -backend=false -input=false terraform -chdir=aws validate terraform -chdir=aws/modules/fargate init -backend=false -input=false diff --git a/HANDOFF.md b/HANDOFF.md index 213673a7..4e279900 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,41 +2,41 @@ ## Finished -- Added version-2 manifest selection for complete Fargate launcher configuration. -- Added `dander init-aws-plan` for saved, manifest-rendered plans against existing S3/DynamoDB state. -- Added `dander init-aws-apply` to apply only the previously reviewed saved plan after confirmation. -- Preserved version-1 and Cloud Run resolution behavior. -- Documented the intentionally unsupported boundary before AWS stage zero, image publication, and live proof. +- Added saved-plan-only AWS stage-zero commands with post-apply local-to-S3 state migration. +- Added customer-key-encrypted S3 state, DynamoDB locking, immutable ECR, and a dedicated deployment role. +- Added idempotent source-free OCI promotion into ECR with exact index/platform digest verification. +- Moved ECR ownership out of the Fargate platform root and into stage zero. +- Packaged the new root, extended existing CI validation, and corrected AWS support documentation. ## Try It -Define a version-2 Fargate deployment, then run `dander init-aws-plan --help`. Do not apply until -the AWS stage-zero and image-publication slices establish the prerequisites documented in -`infra/aws/README.md`. +Run `dander init-aws-admin-plan --help` and `dander image-promote-aws --help`. A real reviewed plan +exists in `/tmp/dander-aws-bootstrap-plan`; do not apply it without separate approval. ## Checks -- Ruff, formatting, strict typing, and all 953 Python tests passed. -- Supported Terraform roots and feasibility roots validated; the provider-mocked Fargate test passed. -- Wheel/sdist installs, source-free generation, container conformance, dependency audit, Trivy, and Gitleaks passed. -- Credentialed read-only AWS plan: 23 add, 0 change, 0 destroy; nothing applied. -- Retained GCP platform plan reported exactly `No changes.` and left no state lock. +- Ruff, formatting, strict typing, and all 972 Python tests passed. +- Stage-zero and Fargate provider-mocked plans passed; protected Linux validation remains pending. +- Credentialed AWS stage-zero plan: 12 creates, 0 updates, 0 deletes; the bucket name is available. +- Wheel/sdist, source-free install, container conformance, dependency audit, focused Trivy scans, and Gitleaks passed. +- Retained GCP platform plan reported exactly `No changes.`; no cloud apply occurred. ## Decisions -- Keep GCP data-plane access explicit while Fargate supplies the launcher. -- Use DynamoDB state locking to preserve the Terraform 1.9+ compatibility contract. -- Keep Fargate unsupported until AWS stage zero, source-free ECR publication, and live parity pass. +- Stage zero exclusively owns AWS state, registry, encryption, and the deployment role. +- The first plan uses secured local state; only a successful apply migrates it to encrypted S3. +- Promote the accepted OCI artifact byte-for-byte; never rebuild separately for ECR. ## Remaining - Let protected Linux CI repeat validation and merge the focused PR if clean. -- Add least-privilege AWS stage zero and source-free ECR image publication. +- Obtain separate approval before applying the 12-resource AWS stage-zero plan. +- Configure a short-lived deployment-role profile and promote the accepted source-free image. - Add AWS status, logs, cancellation, replay, and verification operations. -- Review a real saved plan before requesting separate approval for any AWS apply. +- Keep Fargate unsupported until live keyless BigQuery parity passes. ## Review First -- `src/dander/bootstrap/aws_terraform.py` -- `src/dander/cli/main.py` -- `tests/bootstrap/test_aws_terraform.py` +- `src/dander/bootstrap/aws_admin.py` +- `src/dander/bootstrap/project.py` +- `src/dander/cli/aws_command.py` diff --git a/docs/decisions.md b/docs/decisions.md index 55d9432a..9ca7082e 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-08 — AWS stage zero owns state and artifact prerequisites + +- **Ownership:** A separate AWS stage-zero root owns the customer-key-encrypted S3 backend, + DynamoDB lock table, immutable ECR repository, and dedicated deployment role. The Fargate + platform root consumes the repository instead of attempting to create it after image promotion. +- **Lifecycle:** The first reviewed stage-zero plan uses secured local operator state. Applying + that exact plan creates the remote backend and immediately migrates state into S3; a failed + migration preserves the local recovery copy. +- **Artifact:** AWS publication copies the accepted source-free OCI index without rebuilding and + fails unless the index and every platform digest remain identical in ECR. + ## 2026-08-08 — Fargate lifecycle is provider-native and bounded - **Controller:** One Standard Step Functions state machine per pipeline uses the optimized ECS diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 13cec975..aff9ff98 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -3,11 +3,12 @@ Dander `0.7.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. -- AWS ECS/Fargate is not yet a supported Dander launcher. Its packaged, separately initialized - Terraform stack now plans ECR, ECS, Standard Step Functions control, scheduling, logs, and - failure routing, but public CLI plan/apply/status/cancel/replay integration and live parity remain - unproven. The runtime bridge copies one temporary ECS task-role session into process memory and - therefore caps a Fargate projection at one hour; renewable credentials remain a later gate. +- AWS ECS/Fargate is not yet a supported Dander launcher. Packaged stage-zero and platform roots + now have public saved-plan/apply commands, and accepted OCI artifacts can be copied into ECR + without rebuilding. Status, logs, cancellation, replay, deployment verification, and live parity + remain unproven. The runtime bridge copies one temporary ECS task-role session into process + memory and therefore caps a Fargate projection at one hour; renewable credentials remain a later + gate. - Provider package extras and the full runtime image contain planned provider SDK dependencies, but do not by themselves implement or qualify Snowflake, Redshift, PostgreSQL, AWS, Azure, or OCI profiles. The packaged capability manifest is the support boundary. diff --git a/infra/aws/README.md b/infra/aws/README.md index b1ad45f0..d9c264e0 100644 --- a/infra/aws/README.md +++ b/infra/aws/README.md @@ -1,9 +1,9 @@ # Dander AWS platform stack This separate Terraform root projects validated `io.dander.execution/v1` templates onto ECS -Fargate. It creates an immutable ECR repository, ECS task definitions, distinct execution/task -roles, a Standard Step Functions controller, paused-aware EventBridge schedules, CloudWatch logs, -and an encrypted failure queue plus notification topic. +Fargate. It consumes the immutable ECR repository created by AWS stage zero, then creates ECS task +definitions, distinct execution/task roles, a Standard Step Functions controller, paused-aware +EventBridge schedules, CloudWatch logs, and an encrypted failure queue plus notification topic. `dander init-aws-plan` selects one version-2 Fargate deployment, renders its complete execution projections, and saves a Terraform plan in this root. It requires an existing encrypted S3 state @@ -20,11 +20,9 @@ dander init-aws-plan \ --container-image 123456789012.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:DIGEST ``` -The state bucket, lock table, and usable ECR image lifecycle are separate prerequisites until the -AWS stage-zero and image-publication commands ship. Do not apply this root before those commands -establish consistent repository ownership and publish the referenced digest. The stack accepts -existing VPC subnet and security-group IDs; it does not create a network. Use a least-privilege AWS -role for planning and application rather than the account root identity. +Run AWS stage zero and promote the accepted source-free image before planning this root. The stack +accepts existing VPC subnet and security-group IDs; it does not create a network. Use the dedicated +stage-zero deployment role for planning and application rather than the account root identity. The controller uses `ecs:runTask.sync`. One absolute Step Functions deadline bounds all attempts; AWS performs a best-effort `StopTask` when the integration is cancelled or times out. Exit code 75 diff --git a/infra/aws/bootstrap-admin/.terraform.lock.hcl b/infra/aws/bootstrap-admin/.terraform.lock.hcl new file mode 100644 index 00000000..b0648221 --- /dev/null +++ b/infra/aws/bootstrap-admin/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = "~> 6.0" + hashes = [ + "h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} diff --git a/infra/aws/bootstrap-admin/README.md b/infra/aws/bootstrap-admin/README.md new file mode 100644 index 00000000..8dd16108 --- /dev/null +++ b/infra/aws/bootstrap-admin/README.md @@ -0,0 +1,13 @@ +# Dander AWS stage zero + +This Terraform root creates only the AWS prerequisites that the Fargate platform root cannot own: +an encrypted/versioned S3 state bucket, DynamoDB lock table, immutable ECR repository, and a +dedicated deployment role trusted by one exact operator principal. + +The public Dander lifecycle copies this root into a private operator-artifact directory and uses a +local backend for the first reviewed plan. After that exact plan is applied, Dander migrates the +state into the newly created S3 backend. The local state remains available as recovery evidence if +migration fails. Do not run this root directly from the repository checkout. + +Stage zero is the only AWS operation that may use an account administrator. Later platform plans, +image promotion, and applies use the returned deployment role through a short-lived AWS profile. diff --git a/infra/aws/bootstrap-admin/main.tf b/infra/aws/bootstrap-admin/main.tf new file mode 100644 index 00000000..a391969d --- /dev/null +++ b/infra/aws/bootstrap-admin/main.tf @@ -0,0 +1,351 @@ +locals { + partition = startswith(var.region, "us-gov-") ? "aws-us-gov" : "aws" + tags = merge(var.tags, { + component = "aws-stage-zero" + managed-by = "dander" + }) +} + +data "aws_caller_identity" "current" {} + +check "authenticated_account_matches_target" { + assert { + condition = data.aws_caller_identity.current.account_id == var.aws_account_id + error_message = "Authenticated AWS account does not match aws_account_id." + } +} + +resource "aws_kms_key" "stage_zero" { + description = "Encrypt Dander AWS state, lock, and registry artifacts" + deletion_window_in_days = 30 + enable_key_rotation = true + tags = local.tags + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "stage_zero" { + name = "alias/${var.name}-stage-zero" + target_key_id = aws_kms_key.stage_zero.key_id +} + +resource "aws_s3_bucket" "terraform_state" { + bucket = var.state_bucket + tags = local.tags + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_s3_bucket_ownership_controls" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_public_access_block" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.stage_zero.arn + sse_algorithm = "aws:kms" + } + bucket_key_enabled = true + } +} + +resource "aws_s3_bucket_versioning" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_dynamodb_table" "terraform_locks" { + name = var.lock_table + billing_mode = "PAY_PER_REQUEST" + hash_key = "LockID" + + attribute { + name = "LockID" + type = "S" + } + + point_in_time_recovery { + enabled = true + } + + server_side_encryption { + enabled = true + kms_key_arn = aws_kms_key.stage_zero.arn + } + + tags = local.tags + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_ecr_repository" "runtime" { + name = var.ecr_repository_name + image_tag_mutability = "IMMUTABLE" + force_delete = false + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.stage_zero.arn + } + + image_scanning_configuration { + scan_on_push = true + } + + tags = local.tags +} + +resource "aws_ecr_lifecycle_policy" "runtime" { + repository = aws_ecr_repository.runtime.name + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Remove untagged promotion residue after fourteen days" + selection = { + tagStatus = "untagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 14 + } + action = { + type = "expire" + } + } + ] + }) +} + +data "aws_iam_policy_document" "deployment_assume" { + statement { + sid = "ExactOperator" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "AWS" + identifiers = [var.admin_principal_arn] + } + + condition { + test = "ArnEquals" + variable = "aws:PrincipalArn" + values = [var.admin_principal_arn] + } + } +} + +resource "aws_iam_role" "deployment" { + name = "${var.name}-bootstrap" + max_session_duration = 3600 + assume_role_policy = data.aws_iam_policy_document.deployment_assume.json + tags = local.tags +} + +data "aws_iam_policy_document" "deployment" { + statement { + sid = "TerraformStateBucket" + effect = "Allow" + actions = [ + "s3:GetBucketLocation", + "s3:GetBucketVersioning", + "s3:ListBucket", + ] + resources = [aws_s3_bucket.terraform_state.arn] + } + + statement { + sid = "TerraformStateObjects" + effect = "Allow" + actions = [ + "s3:DeleteObject", + "s3:GetObject", + "s3:PutObject", + ] + resources = ["${aws_s3_bucket.terraform_state.arn}/*"] + } + + statement { + sid = "TerraformStateLocks" + effect = "Allow" + actions = [ + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + "dynamodb:GetItem", + "dynamodb:PutItem", + ] + resources = [aws_dynamodb_table.terraform_locks.arn] + } + + statement { + sid = "PublishRuntimeImage" + effect = "Allow" + actions = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:CompleteLayerUpload", + "ecr:DescribeImages", + "ecr:DescribeRepositories", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:ListImages", + "ecr:PutImage", + "ecr:UploadLayerPart", + ] + resources = [aws_ecr_repository.runtime.arn] + } + + statement { + sid = "EcrAuthorization" + effect = "Allow" + actions = ["ecr:GetAuthorizationToken"] + resources = ["*"] + } + + statement { + sid = "UseStageZeroEncryptionKey" + effect = "Allow" + actions = [ + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + ] + resources = [aws_kms_key.stage_zero.arn] + } + + statement { + sid = "DescribeDeploymentPrerequisites" + effect = "Allow" + actions = [ + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ecs:DescribeClusters", + "ecs:DescribeTaskDefinition", + "ecs:ListTagsForResource", + "events:DescribeRule", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListAttachedRolePolicies", + "iam:ListRolePolicies", + "kms:DescribeKey", + "kms:GetKeyPolicy", + "kms:ListAliases", + "kms:ListResourceTags", + "logs:DescribeLogGroups", + "scheduler:GetSchedule", + "sns:GetTopicAttributes", + "sqs:GetQueueAttributes", + "states:DescribeStateMachine", + "states:ListTagsForResource", + ] + resources = ["*"] + } + + statement { + sid = "ManageDanderRoles" + effect = "Allow" + actions = [ + "iam:CreateRole", + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:PassRole", + "iam:PutRolePolicy", + "iam:TagRole", + "iam:UntagRole", + "iam:UpdateAssumeRolePolicy", + ] + resources = [ + "arn:${local.partition}:iam::${var.aws_account_id}:role/${var.name}-*", + "arn:${local.partition}:iam::${var.aws_account_id}:role/dander-*", + ] + } + + statement { + sid = "ManageDanderDeployment" + effect = "Allow" + actions = [ + "ecs:CreateCluster", + "ecs:DeleteCluster", + "ecs:DeregisterTaskDefinition", + "ecs:RegisterTaskDefinition", + "ecs:TagResource", + "ecs:UntagResource", + "events:DeleteRule", + "events:PutRule", + "events:PutTargets", + "events:RemoveTargets", + "events:TagResource", + "events:UntagResource", + "kms:CreateAlias", + "kms:CreateKey", + "kms:DeleteAlias", + "kms:EnableKeyRotation", + "kms:PutKeyPolicy", + "kms:ScheduleKeyDeletion", + "kms:TagResource", + "kms:UntagResource", + "logs:CreateLogGroup", + "logs:DeleteLogGroup", + "logs:PutRetentionPolicy", + "logs:TagLogGroup", + "logs:UntagLogGroup", + "scheduler:CreateSchedule", + "scheduler:DeleteSchedule", + "scheduler:TagResource", + "scheduler:UntagResource", + "scheduler:UpdateSchedule", + "sns:CreateTopic", + "sns:DeleteTopic", + "sns:SetTopicAttributes", + "sns:TagResource", + "sns:UntagResource", + "sqs:CreateQueue", + "sqs:DeleteQueue", + "sqs:SetQueueAttributes", + "sqs:TagQueue", + "sqs:UntagQueue", + "states:CreateStateMachine", + "states:DeleteStateMachine", + "states:TagResource", + "states:UntagResource", + "states:UpdateStateMachine", + ] + resources = ["*"] + } +} + +resource "aws_iam_role_policy" "deployment" { + name = "dander-platform-administration" + role = aws_iam_role.deployment.id + policy = data.aws_iam_policy_document.deployment.json +} diff --git a/infra/aws/bootstrap-admin/outputs.tf b/infra/aws/bootstrap-admin/outputs.tf new file mode 100644 index 00000000..9c481a08 --- /dev/null +++ b/infra/aws/bootstrap-admin/outputs.tf @@ -0,0 +1,19 @@ +output "state_bucket" { + description = "Encrypted S3 bucket for Dander Terraform state." + value = aws_s3_bucket.terraform_state.bucket +} + +output "lock_table" { + description = "DynamoDB table used for Dander Terraform locking." + value = aws_dynamodb_table.terraform_locks.name +} + +output "runtime_repository_url" { + description = "Private ECR repository receiving promoted Dander artifacts." + value = aws_ecr_repository.runtime.repository_url +} + +output "deployment_role_arn" { + description = "Dedicated role for later Dander platform plans and applies." + value = aws_iam_role.deployment.arn +} diff --git a/infra/aws/bootstrap-admin/tests/bootstrap.tftest.hcl b/infra/aws/bootstrap-admin/tests/bootstrap.tftest.hcl new file mode 100644 index 00000000..131a5bf1 --- /dev/null +++ b/infra/aws/bootstrap-admin/tests/bootstrap.tftest.hcl @@ -0,0 +1,68 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "184463061564" + arn = "arn:aws:iam::184463061564:root" + user_id = "184463061564" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} + +variables { + aws_account_id = "184463061564" + region = "us-east-1" + name = "dander" + state_bucket = "dander-184463061564-state" + lock_table = "dander-terraform-locks" + ecr_repository_name = "dander" + admin_principal_arn = "arn:aws:iam::184463061564:root" +} + +run "hardened_stage_zero" { + command = plan + + assert { + condition = ( + aws_s3_bucket_public_access_block.terraform_state.block_public_acls && + aws_s3_bucket_public_access_block.terraform_state.block_public_policy && + aws_s3_bucket_public_access_block.terraform_state.ignore_public_acls && + aws_s3_bucket_public_access_block.terraform_state.restrict_public_buckets + ) + error_message = "Terraform state must block every form of public access." + } + + assert { + condition = ( + aws_kms_key.stage_zero.enable_key_rotation && + one(one(aws_s3_bucket_server_side_encryption_configuration.terraform_state.rule).apply_server_side_encryption_by_default).sse_algorithm == "aws:kms" + ) + error_message = "Stage-zero artifacts must use a rotating customer-managed encryption key." + } + + assert { + condition = aws_s3_bucket_versioning.terraform_state.versioning_configuration[0].status == "Enabled" + error_message = "Terraform state versioning must remain enabled." + } + + assert { + condition = ( + aws_dynamodb_table.terraform_locks.billing_mode == "PAY_PER_REQUEST" && + aws_dynamodb_table.terraform_locks.point_in_time_recovery[0].enabled + ) + error_message = "The lock table must be on-demand and recoverable." + } + + assert { + condition = ( + aws_ecr_repository.runtime.image_tag_mutability == "IMMUTABLE" && + aws_ecr_repository.runtime.image_scanning_configuration[0].scan_on_push + ) + error_message = "The runtime repository must reject mutable tags and scan pushes." + } +} diff --git a/infra/aws/bootstrap-admin/variables.tf b/infra/aws/bootstrap-admin/variables.tf new file mode 100644 index 00000000..e8b06416 --- /dev/null +++ b/infra/aws/bootstrap-admin/variables.tf @@ -0,0 +1,80 @@ +variable "aws_account_id" { + description = "Twelve-digit AWS account receiving Dander stage-zero resources." + type = string + + validation { + condition = can(regex("^[0-9]{12}$", var.aws_account_id)) + error_message = "aws_account_id must contain exactly twelve digits." + } +} + +variable "region" { + description = "AWS region for state, registry, and the initial Dander deployment." + type = string + + validation { + condition = can(regex("^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$", var.region)) + error_message = "region must be a valid AWS region name." + } +} + +variable "name" { + description = "Stable prefix for Dander administrative resources." + type = string + default = "dander" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,23}$", var.name)) + error_message = "name must contain 2-24 lowercase letters, numbers, or hyphens." + } +} + +variable "state_bucket" { + description = "Globally unique S3 bucket for encrypted Terraform state." + type = string + + validation { + condition = ( + can(regex("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", var.state_bucket)) && + !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$", var.state_bucket)) + ) + error_message = "state_bucket must be a valid S3 bucket name." + } +} + +variable "lock_table" { + description = "DynamoDB table used for Terraform state locking." + type = string + + validation { + condition = can(regex("^[A-Za-z0-9_.-]{3,255}$", var.lock_table)) + error_message = "lock_table must be a valid DynamoDB table name." + } +} + +variable "ecr_repository_name" { + description = "Private ECR repository receiving verified Dander OCI artifacts." + type = string + default = "dander" + + validation { + condition = can(regex("^[a-z0-9]+(?:[._/-][a-z0-9]+)*$", var.ecr_repository_name)) + error_message = "ecr_repository_name must be a valid private ECR repository name." + } +} + +variable "admin_principal_arn" { + description = "Exact AWS principal permitted to assume the Dander deployment role." + type = string + + validation { + condition = can(regex("^arn:(?:aws|aws-us-gov):iam::[0-9]{12}:(?:root|user/[A-Za-z0-9+=,.@_/-]+|role/[A-Za-z0-9+=,.@_/-]+)$", var.admin_principal_arn)) + error_message = "admin_principal_arn must be an IAM root, user, or role ARN." + } +} + +variable "tags" { + description = "Additional tags applied to all stage-zero resources." + type = map(string) + default = {} +} diff --git a/infra/aws/bootstrap-admin/versions.tf b/infra/aws/bootstrap-admin/versions.tf new file mode 100644 index 00000000..6c63e0f7 --- /dev/null +++ b/infra/aws/bootstrap-admin/versions.tf @@ -0,0 +1,20 @@ +terraform { + required_version = ">= 1.9" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} + +provider "aws" { + region = var.region + + default_tags { + tags = merge(var.tags, { + managed-by = "dander" + }) + } +} diff --git a/infra/aws/modules/fargate/main.tf b/infra/aws/modules/fargate/main.tf index cf5f4a28..1a138720 100644 --- a/infra/aws/modules/fargate/main.tf +++ b/infra/aws/modules/fargate/main.tf @@ -79,20 +79,8 @@ check "secret_references_match_deployment" { } } -resource "aws_ecr_repository" "runtime" { - name = var.ecr_repository_name - image_tag_mutability = "IMMUTABLE" - force_delete = false - - encryption_configuration { - encryption_type = "AES256" - } - - image_scanning_configuration { - scan_on_push = true - } - - tags = local.tags +data "aws_ecr_repository" "runtime" { + name = var.ecr_repository_name } resource "aws_ecs_cluster" "runtime" { @@ -158,7 +146,7 @@ data "aws_iam_policy_document" "execution" { "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer", ] - resources = [aws_ecr_repository.runtime.arn] + resources = [data.aws_ecr_repository.runtime.arn] } statement { @@ -286,7 +274,7 @@ resource "aws_ecs_task_definition" "pipeline" { precondition { condition = startswith( each.value.image, - "${aws_ecr_repository.runtime.repository_url}@sha256:", + "${data.aws_ecr_repository.runtime.repository_url}@sha256:", ) error_message = "Fargate images must use an immutable digest in this stack's ECR repository." } diff --git a/infra/aws/modules/fargate/outputs.tf b/infra/aws/modules/fargate/outputs.tf index 9f55387e..2781337d 100644 --- a/infra/aws/modules/fargate/outputs.tf +++ b/infra/aws/modules/fargate/outputs.tf @@ -1,6 +1,6 @@ output "runtime_repository_url" { description = "ECR repository URL for immutable runtime manifests." - value = aws_ecr_repository.runtime.repository_url + value = data.aws_ecr_repository.runtime.repository_url } output "cluster_arn" { diff --git a/infra/aws/modules/fargate/tests/fargate.tftest.hcl b/infra/aws/modules/fargate/tests/fargate.tftest.hcl index dfbfcf0f..d72e9160 100644 --- a/infra/aws/modules/fargate/tests/fargate.tftest.hcl +++ b/infra/aws/modules/fargate/tests/fargate.tftest.hcl @@ -12,6 +12,15 @@ mock_provider "aws" { user_id = "184463061564" } } + + mock_data "aws_ecr_repository" { + defaults = { + arn = "arn:aws:ecr:us-east-1:184463061564:repository/dander" + name = "dander" + registry_id = "184463061564" + repository_url = "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander" + } + } } variables { @@ -108,10 +117,7 @@ run "paused_bounded_controller" { } assert { - condition = ( - aws_ecr_repository.runtime.image_tag_mutability == "IMMUTABLE" && - aws_ecr_repository.runtime.image_scanning_configuration[0].scan_on_push == true - ) - error_message = "The runtime repository must reject mutable tags and scan pushed images." + condition = data.aws_ecr_repository.runtime.name == "dander" + error_message = "The Fargate stack must consume the stage-zero ECR repository." } } diff --git a/pyproject.toml b/pyproject.toml index 207d26ca..da0bd911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,13 @@ packages = ["src/dander"] "infra/bootstrap-admin/versions.tf" = "dander/templates/project/infra/bootstrap-admin/versions.tf" "infra/aws/.terraform.lock.hcl" = "dander/templates/project/infra/aws/.terraform.lock.hcl" "infra/aws/README.md" = "dander/templates/project/infra/aws/README.md" +"infra/aws/bootstrap-admin/.terraform.lock.hcl" = "dander/templates/project/infra/aws/bootstrap-admin/.terraform.lock.hcl" +"infra/aws/bootstrap-admin/README.md" = "dander/templates/project/infra/aws/bootstrap-admin/README.md" +"infra/aws/bootstrap-admin/main.tf" = "dander/templates/project/infra/aws/bootstrap-admin/main.tf" +"infra/aws/bootstrap-admin/outputs.tf" = "dander/templates/project/infra/aws/bootstrap-admin/outputs.tf" +"infra/aws/bootstrap-admin/tests/bootstrap.tftest.hcl" = "dander/templates/project/infra/aws/bootstrap-admin/tests/bootstrap.tftest.hcl" +"infra/aws/bootstrap-admin/variables.tf" = "dander/templates/project/infra/aws/bootstrap-admin/variables.tf" +"infra/aws/bootstrap-admin/versions.tf" = "dander/templates/project/infra/aws/bootstrap-admin/versions.tf" "infra/aws/main.tf" = "dander/templates/project/infra/aws/main.tf" "infra/aws/modules/fargate/README.md" = "dander/templates/project/infra/aws/modules/fargate/README.md" "infra/aws/modules/fargate/main.tf" = "dander/templates/project/infra/aws/modules/fargate/main.tf" diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index b5d27ce5..e646c4ce 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -1,10 +1,15 @@ """Infrastructure bootstrap adapters.""" from dander.bootstrap.admin import AdministrativeBootstrap, AdministrativeBootstrapError +from dander.bootstrap.aws_admin import ( + AwsAdministrativeBootstrap, + AwsAdministrativeBootstrapError, +) from dander.bootstrap.aws_terraform import AwsTerraformBootstrap, AwsTerraformBootstrapError from dander.bootstrap.permissions import require_stage_zero_permissions from dander.bootstrap.project import ( ProjectBootstrapError, + RuntimeImagePromoter, RuntimeImagePublisher, StateBucketBootstrap, active_admin_member, @@ -23,12 +28,15 @@ __all__ = [ "AdministrativeBootstrap", "AdministrativeBootstrapError", + "AwsAdministrativeBootstrap", + "AwsAdministrativeBootstrapError", "AwsTerraformBootstrap", "AwsTerraformBootstrapError", "DeploymentSummary", "DeploymentVerificationError", "DeploymentVerifier", "ProjectBootstrapError", + "RuntimeImagePromoter", "RuntimeImagePublisher", "StateBucketBootstrap", "TerraformBootstrap", diff --git a/src/dander/bootstrap/aws_admin.py b/src/dander/bootstrap/aws_admin.py new file mode 100644 index 00000000..d3df7e19 --- /dev/null +++ b/src/dander/bootstrap/aws_admin.py @@ -0,0 +1,420 @@ +"""Plan-first AWS stage-zero bootstrap with post-apply remote-state migration.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +_AWS_ACCOUNT_ID = re.compile(r"^[0-9]{12}$") +_AWS_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_AWS_REGION = re.compile(r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") +_DEPLOYMENT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,23}$") +_DYNAMODB_TABLE = re.compile(r"^[A-Za-z0-9_.-]{3,255}$") +_ECR_REPOSITORY = re.compile(r"^[a-z0-9]+(?:[._/-][a-z0-9]+)*$") +_PRINCIPAL_ARN = re.compile( + r"^arn:(?:aws|aws-us-gov):iam::(?P[0-9]{12}):" + r"(?:root|user/[A-Za-z0-9+=,.@_/-]+|role/[A-Za-z0-9+=,.@_/-]+)$" +) +_S3_BUCKET = re.compile(r"^(?![0-9]+(?:\.[0-9]+){3}$)[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$") +_STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") +_BACKEND_SCHEMA = "io.dander.aws-bootstrap-backend/v1" + + +class AwsAdministrativeBootstrapError(RuntimeError): + """Raised when AWS stage zero cannot be planned or applied safely.""" + + +class AwsAdministrativeBootstrap: + """Create AWS state, registry, and deployment-role prerequisites through a saved plan.""" + + def __init__(self, infra_dir: Path, operator_artifact_dir: Path) -> None: + self._infra_dir = infra_dir.resolve() + self._repository_dir = self._infra_dir.parent.parent.parent + self._operator_artifact_dir = operator_artifact_dir.expanduser().resolve() + if self._is_within(self._operator_artifact_dir, self._repository_dir): + raise AwsAdministrativeBootstrapError( + "AWS operator artifact directory must be outside the repository checkout" + ) + self._workspace = self._operator_artifact_dir / "terraform-workspace" + self._tf_data_dir = self._operator_artifact_dir / "terraform-data" + self._plan_path = self._operator_artifact_dir / "dander-aws-admin-bootstrap.tfplan" + self._backend_path = self._workspace / "backend.tf.json" + self._backend_record_path = self._operator_artifact_dir / "backend.json" + self._local_state_path = self._operator_artifact_dir / "terraform.tfstate" + + def execute( + self, + *, + aws_account_id: str, + region: str, + state_bucket: str, + state_key: str, + lock_table: str, + ecr_repository_name: str, + admin_principal_arn: str, + aws_profile: str = "", + name: str = "dander", + ) -> Path: + """Save a stage-zero plan without applying or creating cloud resources.""" + self._validate( + aws_account_id=aws_account_id, + region=region, + state_bucket=state_bucket, + state_key=state_key, + lock_table=lock_table, + ecr_repository_name=ecr_repository_name, + admin_principal_arn=admin_principal_arn, + aws_profile=aws_profile, + name=name, + ) + self._prepare_operator_directories() + backend = self._load_backend_record() + if backend is None: + self._write_backend( + "local", + {"path": str(self._local_state_path)}, + ) + else: + self._require_matching_backend( + backend, + state_bucket=state_bucket, + state_key=state_key, + region=region, + lock_table=lock_table, + ) + self._write_s3_backend( + state_bucket=state_bucket, + state_key=state_key, + region=region, + lock_table=lock_table, + ) + self._run("terraform", "init", "-reconfigure", "-input=false", aws_profile=aws_profile) + self._run( + "terraform", + "plan", + "-input=false", + f"-var=aws_account_id={aws_account_id}", + f"-var=region={region}", + f"-var=name={name}", + f"-var=state_bucket={state_bucket}", + f"-var=lock_table={lock_table}", + f"-var=ecr_repository_name={ecr_repository_name}", + f"-var=admin_principal_arn={admin_principal_arn}", + f"-out={self._plan_path}", + aws_profile=aws_profile, + ) + self._secure_plan() + return self._plan_path + + def apply_saved_plan( + self, + *, + aws_account_id: str, + region: str, + state_bucket: str, + state_key: str, + lock_table: str, + ecr_repository_name: str, + admin_principal_arn: str, + aws_profile: str = "", + name: str = "dander", + ) -> Path: + """Apply only the saved plan, then migrate initial local state into S3.""" + self._validate( + aws_account_id=aws_account_id, + region=region, + state_bucket=state_bucket, + state_key=state_key, + lock_table=lock_table, + ecr_repository_name=ecr_repository_name, + admin_principal_arn=admin_principal_arn, + aws_profile=aws_profile, + name=name, + ) + self._prepare_operator_directories(preserve_plan=True) + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AwsAdministrativeBootstrapError( + f"No reviewed AWS stage-zero plan exists at {self._plan_path}" + ) + backend = self._load_backend_record() + if backend is None: + self._write_backend("local", {"path": str(self._local_state_path)}) + else: + self._require_matching_backend( + backend, + state_bucket=state_bucket, + state_key=state_key, + region=region, + lock_table=lock_table, + ) + self._write_s3_backend( + state_bucket=state_bucket, + state_key=state_key, + region=region, + lock_table=lock_table, + ) + self._run("terraform", "init", "-reconfigure", "-input=false", aws_profile=aws_profile) + self._run( + "terraform", "apply", "-input=false", str(self._plan_path), aws_profile=aws_profile + ) + if backend is None: + self._migrate_state( + state_bucket=state_bucket, + state_key=state_key, + region=region, + lock_table=lock_table, + aws_profile=aws_profile, + ) + return self._plan_path + + @staticmethod + def deployment_role_arn(*, aws_account_id: str, region: str, name: str) -> str: + """Return the exact stage-zero deployment-role ARN.""" + partition = "aws-us-gov" if region.startswith("us-gov-") else "aws" + return f"arn:{partition}:iam::{aws_account_id}:role/{name}-bootstrap" + + def _migrate_state( + self, + *, + state_bucket: str, + state_key: str, + region: str, + lock_table: str, + aws_profile: str, + ) -> None: + self._write_s3_backend( + state_bucket=state_bucket, + state_key=state_key, + region=region, + lock_table=lock_table, + ) + try: + self._run( + "terraform", + "init", + "-migrate-state", + "-force-copy", + "-input=false", + aws_profile=aws_profile, + ) + except AwsAdministrativeBootstrapError: + self._write_backend("local", {"path": str(self._local_state_path)}) + raise + self._write_json( + self._backend_record_path, + { + "schema": _BACKEND_SCHEMA, + "bucket": state_bucket, + "key": state_key, + "region": region, + "dynamodb_table": lock_table, + }, + ) + + def _prepare_operator_directories(self, *, preserve_plan: bool = False) -> None: + try: + for path, label in ( + (self._operator_artifact_dir, "operator artifact directory"), + (self._workspace, "Terraform workspace"), + (self._tf_data_dir, "terraform-data"), + ): + if path.is_symlink(): + raise AwsAdministrativeBootstrapError(f"{label} must not be a symlink") + if path.exists() and not path.is_dir(): + raise AwsAdministrativeBootstrapError(f"{label} must be a directory") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + path.chmod(0o700) + if self._plan_path.is_symlink(): + raise AwsAdministrativeBootstrapError("Saved AWS plan must not be a symlink") + if self._plan_path.exists() and not self._plan_path.is_file(): + raise AwsAdministrativeBootstrapError("Saved AWS plan must be a regular file") + if self._plan_path.exists() and not preserve_plan: + self._plan_path.unlink() + for path, label in ( + (self._local_state_path, "Local Terraform state"), + (self._backend_record_path, "AWS backend record"), + (self._backend_path, "Terraform backend configuration"), + ): + if path.is_symlink(): + raise AwsAdministrativeBootstrapError(f"{label} must not be a symlink") + if path.exists() and not path.is_file(): + raise AwsAdministrativeBootstrapError(f"{label} must be a regular file") + for existing in self._workspace.iterdir(): + is_terraform_source = existing.suffix == ".tf" or ( + existing.name.endswith(".tf.json") and existing != self._backend_path + ) + if not is_terraform_source and existing.name != ".terraform.lock.hcl": + continue + if existing.is_symlink() or not existing.is_file(): + raise AwsAdministrativeBootstrapError( + "Terraform workspace files must be regular files, not symlinks" + ) + existing.unlink() + for source in self._infra_dir.iterdir(): + if source.is_file() and ( + source.suffix == ".tf" or source.name == ".terraform.lock.hcl" + ): + destination = self._workspace / source.name + if destination.is_symlink(): + raise AwsAdministrativeBootstrapError( + "Terraform workspace files must not be symlinks" + ) + shutil.copy2(source, destination) + except OSError as error: + raise AwsAdministrativeBootstrapError( + "Could not prepare the AWS operator artifact directory" + ) from error + + def _secure_plan(self) -> None: + try: + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AwsAdministrativeBootstrapError( + f"Terraform did not create a regular saved plan at {self._plan_path}" + ) + self._plan_path.chmod(0o600) + except OSError as error: + raise AwsAdministrativeBootstrapError( + "Could not secure the saved AWS Terraform plan" + ) from error + + def _load_backend_record(self) -> dict[str, object] | None: + if not self._backend_record_path.exists(): + return None + if not self._backend_record_path.is_file() or self._backend_record_path.is_symlink(): + raise AwsAdministrativeBootstrapError("AWS backend record must be a regular file") + try: + document = json.loads(self._backend_record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AwsAdministrativeBootstrapError("AWS backend record is unreadable") from error + if not isinstance(document, dict) or document.get("schema") != _BACKEND_SCHEMA: + raise AwsAdministrativeBootstrapError("AWS backend record has an unsupported schema") + return document + + @staticmethod + def _require_matching_backend( + backend: dict[str, object], + *, + state_bucket: str, + state_key: str, + region: str, + lock_table: str, + ) -> None: + expected = { + "bucket": state_bucket, + "key": state_key, + "region": region, + "dynamodb_table": lock_table, + } + if any(backend.get(key) != value for key, value in expected.items()): + raise AwsAdministrativeBootstrapError( + "AWS backend inputs do not match the previously migrated stage-zero state" + ) + + def _write_s3_backend( + self, + *, + state_bucket: str, + state_key: str, + region: str, + lock_table: str, + ) -> None: + self._write_backend( + "s3", + { + "bucket": state_bucket, + "key": state_key, + "region": region, + "encrypt": True, + "dynamodb_table": lock_table, + }, + ) + + def _write_backend(self, backend_type: str, config: dict[str, object]) -> None: + self._write_json( + self._backend_path, + {"terraform": {"backend": {backend_type: config}}}, + ) + + @staticmethod + def _write_json(path: Path, document: dict[str, object]) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + try: + temporary.write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.chmod(0o600) + os.replace(temporary, path) + path.chmod(0o600) + except OSError as error: + raise AwsAdministrativeBootstrapError("Could not write AWS backend metadata") from error + finally: + temporary.unlink(missing_ok=True) + + @staticmethod + def _is_within(candidate: Path, parent: Path) -> bool: + return candidate == parent or parent in candidate.parents + + @staticmethod + def _validate( + *, + aws_account_id: str, + region: str, + state_bucket: str, + state_key: str, + lock_table: str, + ecr_repository_name: str, + admin_principal_arn: str, + aws_profile: str, + name: str, + ) -> None: + for label, value, pattern in ( + ("AWS account id", aws_account_id, _AWS_ACCOUNT_ID), + ("AWS region", region, _AWS_REGION), + ("S3 state bucket", state_bucket, _S3_BUCKET), + ("state key", state_key, _STATE_KEY), + ("DynamoDB lock table", lock_table, _DYNAMODB_TABLE), + ("ECR repository", ecr_repository_name, _ECR_REPOSITORY), + ("deployment name", name, _DEPLOYMENT_NAME), + ): + if not pattern.fullmatch(value): + raise AwsAdministrativeBootstrapError(f"Invalid {label}: {value!r}") + principal = _PRINCIPAL_ARN.fullmatch(admin_principal_arn) + if principal is None or principal.group("account") != aws_account_id: + raise AwsAdministrativeBootstrapError( + "AWS admin principal must be an IAM root, user, or role ARN in the target account" + ) + if aws_profile and not _AWS_PROFILE.fullmatch(aws_profile): + raise AwsAdministrativeBootstrapError(f"Invalid AWS profile: {aws_profile!r}") + + def _run(self, *args: str, aws_profile: str) -> None: + environment = os.environ.copy() + environment["TF_DATA_DIR"] = str(self._tf_data_dir) + if aws_profile: + environment["AWS_PROFILE"] = aws_profile + try: + subprocess.run( + args, + cwd=self._workspace, + check=True, + env=environment, + umask=0o077, + ) + except FileNotFoundError as error: + raise AwsAdministrativeBootstrapError( + "Terraform is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + raise AwsAdministrativeBootstrapError( + f"Terraform {args[1]} failed with exit code {error.returncode}" + ) from error + + +__all__ = ["AwsAdministrativeBootstrap", "AwsAdministrativeBootstrapError"] diff --git a/src/dander/bootstrap/project.py b/src/dander/bootstrap/project.py index 76bce959..a241ea0b 100644 --- a/src/dander/bootstrap/project.py +++ b/src/dander/bootstrap/project.py @@ -21,6 +21,13 @@ _ACCOUNT = re.compile(r"^[^\s@]+@[^\s@]+$") _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") _PLATFORM_PART = re.compile(r"^[a-z0-9][a-z0-9_.-]*$") +_AWS_ACCOUNT_ID = re.compile(r"^[0-9]{12}$") +_AWS_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_ECR_REPOSITORY = re.compile(r"^[a-z0-9]+(?:[._/-][a-z0-9]+)*$") +_OCI_IMAGE = re.compile( + r"^(?P[a-z0-9.-]+(?::[0-9]+)?)/" + r"(?P[A-Za-z0-9][A-Za-z0-9._/-]*)@(?Psha256:[0-9a-f]{64})$" +) _ARTIFACT_SCHEMA = "io.dander.runtime.artifact/v1" _SOURCE_URL = "https://github.com/harrisonoconnorhover/dander" _RUNTIME_PLATFORMS = ("linux/amd64", "linux/arm64") @@ -370,6 +377,232 @@ def _content_digest(self) -> str: return hasher.hexdigest() +class RuntimeImagePromoter: + """Copy one accepted OCI index into ECR without rebuilding it.""" + + def __init__(self, project_dir: Path, *, runner: _Runner | None = None) -> None: + self._project_dir = project_dir.resolve() + self._runner = runner or _subprocess_runner + self._artifact_record_path: Path | None = None + + @property + def artifact_record_path(self) -> Path | None: + """Return the AWS artifact record written by the latest successful promotion.""" + return self._artifact_record_path + + def promote( + self, + *, + source_image: str, + aws_account_id: str, + region: str, + repository_name: str, + aws_profile: str = "", + tag_prefix: str = "promoted", + ) -> str: + """Promote an accepted source-free image and require byte-identical OCI digests.""" + self._artifact_record_path = None + source_match = _OCI_IMAGE.fullmatch(source_image) + if source_match is None: + raise ProjectBootstrapError("Source image must be an immutable OCI digest reference") + if not _AWS_ACCOUNT_ID.fullmatch(aws_account_id) or not _REGION.fullmatch(region): + raise ProjectBootstrapError("Invalid AWS image-promotion account or region") + if not _ECR_REPOSITORY.fullmatch(repository_name): + raise ProjectBootstrapError("Invalid ECR repository name") + if aws_profile and not _AWS_PROFILE.fullmatch(aws_profile): + raise ProjectBootstrapError("Invalid AWS profile") + if not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", tag_prefix): + raise ProjectBootstrapError("Invalid ECR promotion tag prefix") + + source_record_path = self._project_dir / ".dander" / "runtime-artifact.json" + try: + source_record = json.loads(source_record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ProjectBootstrapError( + "A valid source-free runtime artifact record is required before AWS promotion" + ) from error + expected_digest = source_match.group("digest") + if not isinstance(source_record, dict): + raise ProjectBootstrapError( + "Source image does not match its accepted runtime artifact record" + ) + expected_platforms = source_record.get("platform_manifests") + if ( + source_record.get("schema") != _ARTIFACT_SCHEMA + or source_record.get("image") != source_image + or source_record.get("index_digest") != expected_digest + or not isinstance(expected_platforms, dict) + or not expected_platforms + or any( + not isinstance(key, str) + or not isinstance(value, str) + or not _DIGEST.fullmatch(value) + for key, value in expected_platforms.items() + ) + ): + raise ProjectBootstrapError( + "Source image does not match its accepted runtime artifact record" + ) + + aws_prefix = ("aws", "--profile", aws_profile) if aws_profile else ("aws",) + host = f"{aws_account_id}.dkr.ecr.{region}.amazonaws.com" + destination_repository = f"{host}/{repository_name}" + tag = f"{tag_prefix}-{expected_digest.removeprefix('sha256:')[:12]}" + tagged_destination = f"{destination_repository}:{tag}" + try: + repository = self._runner( + ( + *aws_prefix, + "ecr", + "describe-repositories", + "--region", + region, + "--repository-names", + repository_name, + "--query", + "repositories[0].repositoryUri", + "--output", + "text", + ), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if repository != destination_repository: + raise ProjectBootstrapError("ECR returned an unexpected repository URI") + password = self._runner( + (*aws_prefix, "ecr", "get-login-password", "--region", region), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if not password: + raise ProjectBootstrapError("AWS returned an empty ECR login password") + self._runner( + ("docker", "login", host, "--username", "AWS", "--password-stdin"), + cwd=self._project_dir, + check=True, + text=True, + input=password, + ) + source_raw = self._runner( + ("docker", "buildx", "imagetools", "inspect", "--raw", source_image), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout + source_platforms = _platform_manifests( + source_raw, + default_digest=expected_digest, + ) + if source_platforms != expected_platforms: + raise ProjectBootstrapError( + "Source registry content no longer matches the accepted artifact record" + ) + describe_destination = ( + *aws_prefix, + "ecr", + "describe-images", + "--region", + region, + "--repository-name", + repository_name, + "--image-ids", + f"imageTag={tag}", + "--query", + "imageDetails[0].imageDigest", + "--output", + "text", + ) + existing = self._runner( + describe_destination, + cwd=self._project_dir, + check=False, + capture_output=True, + text=True, + ) + created = existing.returncode != 0 + if existing.returncode == 0: + destination_digest = existing.stdout.strip() + else: + self._runner( + ( + "docker", + "buildx", + "imagetools", + "create", + "--prefer-index=false", + "--tag", + tagged_destination, + source_image, + ), + cwd=self._project_dir, + check=True, + ) + destination_digest = self._runner( + describe_destination, + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if destination_digest != expected_digest: + if created: + raise ProjectBootstrapError( + "ECR rewrote the OCI index; the promoted artifact is not byte-identical" + ) + raise ProjectBootstrapError( + "Existing immutable ECR tag does not match the accepted OCI index" + ) + immutable_destination = f"{destination_repository}@{destination_digest}" + destination_raw = self._runner( + ( + "docker", + "buildx", + "imagetools", + "inspect", + "--raw", + immutable_destination, + ), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout + destination_platforms = _platform_manifests( + destination_raw, + default_digest=destination_digest, + ) + if destination_platforms != expected_platforms: + raise ProjectBootstrapError( + "ECR platform manifests differ from the accepted source-free artifact" + ) + except (FileNotFoundError, subprocess.CalledProcessError) as error: + raise ProjectBootstrapError("Could not promote the runtime image into ECR") from error + + record_path = self._project_dir / ".dander" / "runtime-artifact-aws.json" + try: + _write_artifact_record( + record_path, + { + **source_record, + "image": immutable_destination, + "tagged_image": tagged_destination, + "source_image": source_image, + "promotion": "registry-copy", + }, + ) + except OSError as error: + raise ProjectBootstrapError( + "Runtime image was promoted but its AWS artifact record could not be written" + ) from error + self._artifact_record_path = record_path + return immutable_destination + + def _is_build_context_file(path: Path) -> bool: if any(part in {".terraform", "__pycache__"} for part in path.parts): return False diff --git a/src/dander/cli/aws_command.py b/src/dander/cli/aws_command.py new file mode 100644 index 00000000..73679ce1 --- /dev/null +++ b/src/dander/cli/aws_command.py @@ -0,0 +1,289 @@ +"""AWS stage-zero, artifact-promotion, and Fargate Terraform CLI commands.""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +import typer +from click import ClickException +from rich.console import Console + +from dander.bootstrap import ( + AwsAdministrativeBootstrap, + AwsAdministrativeBootstrapError, + AwsTerraformBootstrap, + AwsTerraformBootstrapError, + ProjectBootstrapError, + RuntimeImagePromoter, +) +from dander.project import ProjectConfigError, load_project_config + +_DEFAULT_AWS_INFRA_DIR = Path("infra/aws") +_DEFAULT_AWS_BOOTSTRAP_ADMIN_DIR = Path("infra/aws/bootstrap-admin") +_DEFAULT_PROJECT_CONFIG = Path("dander.yaml") +console = Console() + + +def init_aws_admin_plan( + aws_account_id: str = typer.Option(..., "--aws-account-id"), + region: str = typer.Option("us-east-1", "--region"), + state_bucket: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/aws/bootstrap-admin/terraform.tfstate", "--state-key"), + lock_table: str = typer.Option("dander-terraform-locks", "--lock-table"), + ecr_repository_name: str = typer.Option("dander", "--ecr-repository"), + admin_principal_arn: str = typer.Option(..., "--admin-principal-arn"), + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), + operator_artifact_dir: Path = typer.Option( # noqa: B008 + ..., + "--operator-artifact-dir", + help="Secured directory outside the repository for plans, state, and Terraform metadata.", + ), + infra_dir: Path = typer.Option( # noqa: B008 + _DEFAULT_AWS_BOOTSTRAP_ADMIN_DIR, hidden=True + ), +) -> None: + """Plan AWS state, registry, and deployment-role prerequisites without applying.""" + try: + plan_path = AwsAdministrativeBootstrap(infra_dir, operator_artifact_dir).execute( + aws_account_id=aws_account_id, + region=region, + state_bucket=state_bucket, + state_key=state_key, + lock_table=lock_table, + ecr_repository_name=ecr_repository_name, + admin_principal_arn=admin_principal_arn, + aws_profile=aws_profile, + name=name, + ) + except AwsAdministrativeBootstrapError as error: + raise ClickException(str(error)) from error + workspace = operator_artifact_dir.expanduser().resolve() / "terraform-workspace" + console.print(f"[green]AWS administrative bootstrap planned.[/green] Saved plan: {plan_path}") + console.print( + "Review: " + + shlex.join(("terraform", f"-chdir={workspace}", "show", "-no-color", str(plan_path))) + ) + next_command = [ + "dander", + "init-aws-admin-apply", + "--aws-account-id", + aws_account_id, + "--region", + region, + "--state-bucket", + state_bucket, + "--state-key", + state_key, + "--lock-table", + lock_table, + "--ecr-repository", + ecr_repository_name, + "--admin-principal-arn", + admin_principal_arn, + "--name", + name, + "--operator-artifact-dir", + str(operator_artifact_dir), + ] + if aws_profile: + next_command.extend(("--aws-profile", aws_profile)) + console.print("Next after review: " + shlex.join(next_command)) + + +def init_aws_admin_apply( + aws_account_id: str = typer.Option(..., "--aws-account-id"), + region: str = typer.Option("us-east-1", "--region"), + state_bucket: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/aws/bootstrap-admin/terraform.tfstate", "--state-key"), + lock_table: str = typer.Option("dander-terraform-locks", "--lock-table"), + ecr_repository_name: str = typer.Option("dander", "--ecr-repository"), + admin_principal_arn: str = typer.Option(..., "--admin-principal-arn"), + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), + operator_artifact_dir: Path = typer.Option(..., "--operator-artifact-dir"), # noqa: B008 + infra_dir: Path = typer.Option( # noqa: B008 + _DEFAULT_AWS_BOOTSTRAP_ADMIN_DIR, hidden=True + ), +) -> None: + """Apply the reviewed AWS stage-zero plan and migrate its state into S3.""" + if not typer.confirm( + f"Apply the reviewed AWS stage-zero plan to account {aws_account_id!r}?", + default=False, + ): + raise typer.Abort() + try: + bootstrap = AwsAdministrativeBootstrap(infra_dir, operator_artifact_dir) + plan_path = bootstrap.apply_saved_plan( + aws_account_id=aws_account_id, + region=region, + state_bucket=state_bucket, + state_key=state_key, + lock_table=lock_table, + ecr_repository_name=ecr_repository_name, + admin_principal_arn=admin_principal_arn, + aws_profile=aws_profile, + name=name, + ) + except AwsAdministrativeBootstrapError as error: + raise ClickException(str(error)) from error + role_arn = AwsAdministrativeBootstrap.deployment_role_arn( + aws_account_id=aws_account_id, + region=region, + name=name, + ) + console.print(f"[green]AWS administrative bootstrap applied.[/green] Saved plan: {plan_path}") + console.print(f"Deployment role: {role_arn}") + console.print( + "Next: configure a short-lived AWS profile that assumes this role, then promote the " + "accepted source-free image with dander image-promote-aws." + ) + + +def image_promote_aws( + source_image: str = typer.Option( + ..., "--source-image", help="Accepted source-free OCI image ending in @sha256 digest." + ), + aws_account_id: str = typer.Option(..., "--aws-account-id"), + region: str = typer.Option("us-east-1", "--region"), + ecr_repository_name: str = typer.Option("dander", "--ecr-repository"), + aws_profile: str = typer.Option("", "--aws-profile"), + tag_prefix: str = typer.Option("promoted", "--tag-prefix"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 +) -> None: + """Copy an accepted source-free OCI index into ECR without rebuilding it.""" + project_dir = config.resolve().parent + if not typer.confirm( + f"Copy the accepted runtime image into AWS account {aws_account_id!r}?", + default=False, + ): + raise typer.Abort() + try: + promoter = RuntimeImagePromoter(project_dir) + image = promoter.promote( + source_image=source_image, + aws_account_id=aws_account_id, + region=region, + repository_name=ecr_repository_name, + aws_profile=aws_profile, + tag_prefix=tag_prefix, + ) + except ProjectBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]Promoted byte-identical runtime image:[/green] {image}") + if promoter.artifact_record_path is not None: + console.print(f"AWS artifact record: {promoter.artifact_record_path}") + console.print("Next: use this immutable digest with dander init-aws-plan.") + + +def init_aws_plan( + project: str = typer.Option(..., "--project", help="GCP BigQuery data-plane project id."), + state_bucket: str = typer.Option(..., "--state-bucket", help="Existing S3 state bucket."), + container_image: str = typer.Option( + ..., "--container-image", help="Immutable ECR image ending in @sha256 digest." + ), + deployment: str = typer.Option(..., "--deployment"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + state_key: str = typer.Option("dander/aws/state/terraform.tfstate", "--state-key"), + lock_table: str = typer.Option(..., "--lock-table", help="Existing DynamoDB lock table."), + state_region: str | None = typer.Option(None, "--state-region"), + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), + infra_dir: Path = typer.Option(_DEFAULT_AWS_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Plan one manifest-selected Fargate deployment without applying it.""" + try: + manifest = load_project_config( + config, + platforms_path=platforms_config, + deployment=deployment, + ) + if manifest.launcher_provider != "fargate": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select launcher.provider='fargate'" + ) + manifest.validate_references(config.resolve().parent) + runtime = manifest.platform.runtime + plan_path = AwsTerraformBootstrap(infra_dir).execute( + project=project, + deployment_name=manifest.deployment_name, + state_bucket=state_bucket, + state_key=state_key, + state_region=state_region or manifest.platform.region, + lock_table=lock_table, + container_image=container_image, + launcher_config=manifest.resolved_launcher_config(), + runtime_cpu=runtime.cpu, + runtime_memory=runtime.memory, + runtime_timeout_seconds=runtime.timeout_seconds, + runtime_max_retries=runtime.max_retries, + runtime_batch_rows=runtime.batch_rows, + require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, + pipelines=manifest.terraform_pipelines(), + apply=False, + aws_profile=aws_profile, + name=name, + ) + except (AwsTerraformBootstrapError, ProjectConfigError) as error: + raise ClickException(str(error)) from error + console.print(f"[green]AWS deployment planned.[/green] Saved plan: {plan_path}") + console.print( + "Review: " + + shlex.join(("terraform", f"-chdir={infra_dir}", "show", "-no-color", str(plan_path))) + ) + next_command = [ + "dander", + "init-aws-apply", + "--state-bucket", + state_bucket, + "--state-key", + state_key, + "--state-region", + state_region or manifest.platform.region, + "--lock-table", + lock_table, + ] + if aws_profile: + next_command.extend(("--aws-profile", aws_profile)) + console.print("Next after review: " + shlex.join(next_command)) + + +def init_aws_apply( + state_bucket: str = typer.Option(..., "--state-bucket", help="Existing S3 state bucket."), + state_key: str = typer.Option("dander/aws/state/terraform.tfstate", "--state-key"), + state_region: str = typer.Option(..., "--state-region"), + lock_table: str = typer.Option(..., "--lock-table", help="Existing DynamoDB lock table."), + aws_profile: str = typer.Option("", "--aws-profile"), + infra_dir: Path = typer.Option(_DEFAULT_AWS_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Apply only the previously reviewed AWS Terraform plan.""" + if not typer.confirm( + f"Apply the reviewed AWS deployment plan using state bucket {state_bucket!r}?", + default=False, + ): + raise typer.Abort() + try: + plan_path = AwsTerraformBootstrap(infra_dir).apply_saved_plan( + state_bucket=state_bucket, + state_key=state_key, + state_region=state_region, + lock_table=lock_table, + aws_profile=aws_profile, + ) + except AwsTerraformBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]AWS deployment applied.[/green] Saved plan: {plan_path}") + + +def register_aws_commands(app: typer.Typer) -> None: + """Register flat, compatibility-preserving AWS command names on the root CLI.""" + app.command("init-aws-admin-plan")(init_aws_admin_plan) + app.command("init-aws-admin-apply")(init_aws_admin_apply) + app.command("image-promote-aws")(image_promote_aws) + app.command("init-aws-plan")(init_aws_plan) + app.command("init-aws-apply")(init_aws_apply) + + +__all__ = ["register_aws_commands"] diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 3eb6a237..844b17f7 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -21,8 +21,6 @@ from dander.bootstrap import ( AdministrativeBootstrap, AdministrativeBootstrapError, - AwsTerraformBootstrap, - AwsTerraformBootstrapError, DeploymentSummary, DeploymentVerifier, ProjectBootstrapError, @@ -45,6 +43,7 @@ SemanticRegistryPublisher, SqliteMetadataStore, ) +from dander.cli.aws_command import register_aws_commands from dander.cli.config_command import config_app from dander.cli.init_command import ( InitOptions, @@ -133,11 +132,11 @@ app.add_typer(plugins_app, name="plugins") app.add_typer(connector_app, name="connector") app.add_typer(runtime_app, name="runtime") +register_aws_commands(app) console = Console() _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") _DEFAULT_CONNECTORS_DIR = Path("connectors") _DEFAULT_INFRA_DIR = Path("infra") -_DEFAULT_AWS_INFRA_DIR = Path("infra/aws") _DEFAULT_MODELS_DIR = Path("models") _DEFAULT_CATALOG_PATH = Path(".dander/catalog.json") _DEFAULT_BOOTSTRAP_ADMIN_DIR = Path("infra/bootstrap-admin") @@ -1122,108 +1121,6 @@ def init_platform_apply( console.print(f"[green]Platform bootstrap applied.[/green] Saved plan: {plan_path}") -@app.command("init-aws-plan") -def init_aws_plan( - project: str = typer.Option(..., "--project", help="GCP BigQuery data-plane project id."), - state_bucket: str = typer.Option(..., "--state-bucket", help="Existing S3 state bucket."), - container_image: str = typer.Option( - ..., "--container-image", help="Immutable ECR image ending in @sha256 digest." - ), - deployment: str = typer.Option(..., "--deployment"), - config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 - platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 - state_key: str = typer.Option("dander/aws/state/terraform.tfstate", "--state-key"), - lock_table: str = typer.Option(..., "--lock-table", help="Existing DynamoDB lock table."), - state_region: str | None = typer.Option(None, "--state-region"), - aws_profile: str = typer.Option("", "--aws-profile"), - name: str = typer.Option("dander", "--name"), - infra_dir: Path = typer.Option(_DEFAULT_AWS_INFRA_DIR, hidden=True), # noqa: B008 -) -> None: - """Plan one manifest-selected Fargate deployment without applying it.""" - try: - manifest = load_project_config( - config, - platforms_path=platforms_config, - deployment=deployment, - ) - if manifest.launcher_provider != "fargate": - raise ProjectConfigError( - f"Deployment {deployment!r} does not select launcher.provider='fargate'" - ) - manifest.validate_references(config.resolve().parent) - runtime = manifest.platform.runtime - plan_path = AwsTerraformBootstrap(infra_dir).execute( - project=project, - deployment_name=manifest.deployment_name, - state_bucket=state_bucket, - state_key=state_key, - state_region=state_region or manifest.platform.region, - lock_table=lock_table, - container_image=container_image, - launcher_config=manifest.resolved_launcher_config(), - runtime_cpu=runtime.cpu, - runtime_memory=runtime.memory, - runtime_timeout_seconds=runtime.timeout_seconds, - runtime_max_retries=runtime.max_retries, - runtime_batch_rows=runtime.batch_rows, - require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, - pipelines=manifest.terraform_pipelines(), - apply=False, - aws_profile=aws_profile, - name=name, - ) - except (AwsTerraformBootstrapError, ProjectConfigError) as error: - raise ClickException(str(error)) from error - console.print(f"[green]AWS deployment planned.[/green] Saved plan: {plan_path}") - console.print( - "Review: " - + shlex.join(("terraform", f"-chdir={infra_dir}", "show", "-no-color", str(plan_path))) - ) - next_command = [ - "dander", - "init-aws-apply", - "--state-bucket", - state_bucket, - "--state-key", - state_key, - "--state-region", - state_region or manifest.platform.region, - "--lock-table", - lock_table, - ] - if aws_profile: - next_command.extend(("--aws-profile", aws_profile)) - console.print("Next after review: " + shlex.join(next_command)) - - -@app.command("init-aws-apply") -def init_aws_apply( - state_bucket: str = typer.Option(..., "--state-bucket", help="Existing S3 state bucket."), - state_key: str = typer.Option("dander/aws/state/terraform.tfstate", "--state-key"), - state_region: str = typer.Option(..., "--state-region"), - lock_table: str = typer.Option(..., "--lock-table", help="Existing DynamoDB lock table."), - aws_profile: str = typer.Option("", "--aws-profile"), - infra_dir: Path = typer.Option(_DEFAULT_AWS_INFRA_DIR, hidden=True), # noqa: B008 -) -> None: - """Apply only the previously reviewed AWS Terraform plan.""" - if not typer.confirm( - f"Apply the reviewed AWS deployment plan using state bucket {state_bucket!r}?", - default=False, - ): - raise typer.Abort() - try: - plan_path = AwsTerraformBootstrap(infra_dir).apply_saved_plan( - state_bucket=state_bucket, - state_key=state_key, - state_region=state_region, - lock_table=lock_table, - aws_profile=aws_profile, - ) - except AwsTerraformBootstrapError as error: - raise ClickException(str(error)) from error - console.print(f"[green]AWS deployment applied.[/green] Saved plan: {plan_path}") - - @verify_app.command("deployment") def verify_deployment( project: str = typer.Option(..., "--project", help="GCP project id to inspect."), diff --git a/tests/bootstrap/test_aws_admin.py b/tests/bootstrap/test_aws_admin.py new file mode 100644 index 00000000..23eab488 --- /dev/null +++ b/tests/bootstrap/test_aws_admin.py @@ -0,0 +1,310 @@ +"""AWS stage-zero saved-plan and state-migration coverage.""" + +from __future__ import annotations + +import json +import stat +import subprocess +from pathlib import Path +from typing import TypedDict + +import pytest + +from dander.bootstrap import AwsAdministrativeBootstrap, AwsAdministrativeBootstrapError + + +class _Arguments(TypedDict): + aws_account_id: str + region: str + state_bucket: str + state_key: str + lock_table: str + ecr_repository_name: str + admin_principal_arn: str + aws_profile: str + name: str + + +def _arguments(tmp_path: Path) -> _Arguments: + del tmp_path + return { + "aws_account_id": "184463061564", + "region": "us-east-1", + "state_bucket": "dander-184463061564-state", + "state_key": "dander/aws/bootstrap-admin/terraform.tfstate", + "lock_table": "dander-terraform-locks", + "ecr_repository_name": "dander", + "admin_principal_arn": "arn:aws:iam::184463061564:root", + "aws_profile": "dander-phase1b", + "name": "dander", + } + + +def _layout(tmp_path: Path) -> tuple[Path, Path]: + infra_dir = tmp_path / "checkout" / "infra" / "aws" / "bootstrap-admin" + infra_dir.mkdir(parents=True) + (infra_dir / "main.tf").write_text('resource "terraform_data" "unit" {}\n') + return infra_dir, tmp_path / "operator" + + +def test_aws_admin_plan_uses_secured_local_state_without_applying( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + commands: list[tuple[str, ...]] = [] + backends: list[dict[str, object]] = [] + environments: list[dict[str, str]] = [] + + def fake_run( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + env: dict[str, str], + umask: int, + ) -> subprocess.CompletedProcess[str]: + assert check and umask == 0o077 + commands.append(args) + environments.append(env) + backends.append(json.loads((cwd / "backend.tf.json").read_text(encoding="utf-8"))) + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch(mode=0o644) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + plan = AwsAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments(tmp_path)) + + assert plan == operator_dir / "dander-aws-admin-bootstrap.tfplan" + assert stat.S_IMODE(plan.stat().st_mode) == 0o600 + assert commands[0][:2] == ("terraform", "init") + assert commands[1][:2] == ("terraform", "plan") + assert not any(command[:2] == ("terraform", "apply") for command in commands) + backend = backends[0]["terraform"]["backend"] # type: ignore[index] + assert backend == {"local": {"path": str(operator_dir / "terraform.tfstate")}} + assert all(environment["AWS_PROFILE"] == "dander-phase1b" for environment in environments) + assert all( + environment["TF_DATA_DIR"] == str(operator_dir / "terraform-data") + for environment in environments + ) + + +def test_aws_admin_plan_removes_stale_terraform_source_from_private_workspace( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + workspace = operator_dir / "terraform-workspace" + workspace.mkdir(parents=True) + stale = workspace / "removed-resource.tf" + stale.write_text('resource "terraform_data" "stale" {}\n', encoding="utf-8") + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del kwargs + assert not stale.exists() + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + AwsAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments(tmp_path)) + + assert not stale.exists() + + +def test_aws_admin_apply_uses_saved_plan_then_migrates_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + commands: list[tuple[str, ...]] = [] + backend_types: list[str] = [] + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del kwargs + commands.append(args) + backend = json.loads((cwd / "backend.tf.json").read_text(encoding="utf-8")) + backend_types.append(next(iter(backend["terraform"]["backend"]))) + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + bootstrap = AwsAdministrativeBootstrap(infra_dir, operator_dir) + plan = bootstrap.execute(**_arguments(tmp_path)) + commands.clear() + backend_types.clear() + + applied = bootstrap.apply_saved_plan(**_arguments(tmp_path)) + + assert applied == plan + assert commands[0][:2] == ("terraform", "init") + assert commands[1] == ("terraform", "apply", "-input=false", str(plan)) + assert commands[2] == ( + "terraform", + "init", + "-migrate-state", + "-force-copy", + "-input=false", + ) + assert backend_types == ["local", "local", "s3"] + record = json.loads((operator_dir / "backend.json").read_text(encoding="utf-8")) + assert record == { + "schema": "io.dander.aws-bootstrap-backend/v1", + "bucket": "dander-184463061564-state", + "key": "dander/aws/bootstrap-admin/terraform.tfstate", + "region": "us-east-1", + "dynamodb_table": "dander-terraform-locks", + } + + +def test_aws_admin_subsequent_plan_uses_migrated_s3_backend( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + operator_dir.mkdir() + (operator_dir / "backend.json").write_text( + json.dumps( + { + "schema": "io.dander.aws-bootstrap-backend/v1", + "bucket": "dander-184463061564-state", + "key": "dander/aws/bootstrap-admin/terraform.tfstate", + "region": "us-east-1", + "dynamodb_table": "dander-terraform-locks", + } + ), + encoding="utf-8", + ) + backends: list[dict[str, object]] = [] + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del kwargs + backends.append(json.loads((cwd / "backend.tf.json").read_text(encoding="utf-8"))) + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + AwsAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments(tmp_path)) + + assert backends[0]["terraform"]["backend"] == { # type: ignore[index] + "s3": { + "bucket": "dander-184463061564-state", + "dynamodb_table": "dander-terraform-locks", + "encrypt": True, + "key": "dander/aws/bootstrap-admin/terraform.tfstate", + "region": "us-east-1", + } + } + + +def test_aws_admin_failed_state_migration_preserves_local_recovery_backend( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del cwd, kwargs + if "-migrate-state" in args: + raise subprocess.CalledProcessError(1, args) + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + bootstrap = AwsAdministrativeBootstrap(infra_dir, operator_dir) + bootstrap.execute(**_arguments(tmp_path)) + + with pytest.raises(AwsAdministrativeBootstrapError, match="init failed"): + bootstrap.apply_saved_plan(**_arguments(tmp_path)) + + backend = json.loads( + (operator_dir / "terraform-workspace" / "backend.tf.json").read_text(encoding="utf-8") + ) + assert backend["terraform"]["backend"] == { + "local": {"path": str(operator_dir / "terraform.tfstate")} + } + assert not (operator_dir / "backend.json").exists() + + +def test_aws_admin_rejects_mismatched_backend_inputs_before_terraform( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + operator_dir.mkdir() + (operator_dir / "backend.json").write_text( + json.dumps( + { + "schema": "io.dander.aws-bootstrap-backend/v1", + "bucket": "different-state-bucket", + "key": "dander/aws/bootstrap-admin/terraform.tfstate", + "region": "us-east-1", + "dynamodb_table": "dander-terraform-locks", + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: pytest.fail("Terraform must not run"), + ) + + with pytest.raises(AwsAdministrativeBootstrapError, match="do not match"): + AwsAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments(tmp_path)) + + +def test_aws_admin_rejects_a_symlinked_local_state_before_terraform( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + operator_dir.mkdir() + (operator_dir / "terraform.tfstate").symlink_to(tmp_path / "unrelated") + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: pytest.fail("Terraform must not run"), + ) + + with pytest.raises(AwsAdministrativeBootstrapError, match="must not be a symlink"): + AwsAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments(tmp_path)) + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"aws_account_id": "123"}, "account"), + ({"state_bucket": "bad bucket"}, "bucket"), + ({"state_key": "/absolute"}, "state key"), + ({"admin_principal_arn": "arn:aws:iam::999999999999:root"}, "principal"), + ({"aws_profile": "bad profile"}, "profile"), + ], +) +def test_aws_admin_rejects_invalid_inputs( + tmp_path: Path, + override: dict[str, object], + message: str, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + arguments = {**_arguments(tmp_path), **override} + + with pytest.raises(AwsAdministrativeBootstrapError, match=message): + AwsAdministrativeBootstrap(infra_dir, operator_dir).execute(**arguments) # type: ignore[arg-type] diff --git a/tests/bootstrap/test_aws_image_promotion.py b/tests/bootstrap/test_aws_image_promotion.py new file mode 100644 index 00000000..e1ff95fa --- /dev/null +++ b/tests/bootstrap/test_aws_image_promotion.py @@ -0,0 +1,189 @@ +"""Registry-copy promotion coverage for accepted source-free runtime images.""" + +from __future__ import annotations + +import json +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from dander.bootstrap import ProjectBootstrapError, RuntimeImagePromoter + +if TYPE_CHECKING: + from pathlib import Path + +_INDEX_DIGEST = "sha256:" + "a" * 64 +_AMD64_DIGEST = "sha256:" + "b" * 64 +_ARM64_DIGEST = "sha256:" + "c" * 64 +_SOURCE_IMAGE = f"us-central1-docker.pkg.dev/unit-project/dander/dander@{_INDEX_DIGEST}" + + +def _index() -> str: + return json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "digest": _AMD64_DIGEST, + "platform": {"os": "linux", "architecture": "amd64"}, + }, + { + "digest": _ARM64_DIGEST, + "platform": {"os": "linux", "architecture": "arm64"}, + }, + ], + } + ) + + +def _write_source_record(project_dir: Path, *, image: str = _SOURCE_IMAGE) -> None: + artifact_dir = project_dir / ".dander" + artifact_dir.mkdir() + (artifact_dir / "runtime-artifact.json").write_text( + json.dumps( + { + "schema": "io.dander.runtime.artifact/v1", + "image": image, + "tagged_image": "source:accepted", + "index_digest": _INDEX_DIGEST, + "platform_manifests": { + "linux/amd64": _AMD64_DIGEST, + "linux/arm64": _ARM64_DIGEST, + }, + "source": "https://github.com/harrisonoconnorhover/dander", + "revision": "unit", + "created": "2026-08-08T00:00:00Z", + "sbom_attached": True, + "provenance_attached": True, + } + ), + encoding="utf-8", + ) + + +class _Runner: + def __init__(self, *, destination_digest: str = _INDEX_DIGEST) -> None: + self.destination_digest = destination_digest + self.commands: list[tuple[str, ...]] = [] + self.inputs: list[str | None] = [] + self.describe_image_calls = 0 + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: + del capture_output, text + assert cwd.is_absolute() + assert check or "describe-images" in args + self.commands.append(args) + self.inputs.append(input) + if "describe-repositories" in args: + return subprocess.CompletedProcess( + args, + 0, + stdout="184463061564.dkr.ecr.us-east-1.amazonaws.com/dander\n", + ) + if "get-login-password" in args: + return subprocess.CompletedProcess(args, 0, stdout="temporary-password\n") + if "describe-images" in args: + self.describe_image_calls += 1 + if self.describe_image_calls == 1: + return subprocess.CompletedProcess(args, 254, stdout="") + return subprocess.CompletedProcess(args, 0, stdout=self.destination_digest + "\n") + if args[:4] == ("docker", "buildx", "imagetools", "inspect"): + return subprocess.CompletedProcess(args, 0, stdout=_index()) + return subprocess.CompletedProcess(args, 0, stdout="") + + +def test_runtime_image_promoter_copies_and_verifies_the_accepted_index(tmp_path: Path) -> None: + _write_source_record(tmp_path) + runner = _Runner() + promoter = RuntimeImagePromoter(tmp_path, runner=runner) + + image = promoter.promote( + source_image=_SOURCE_IMAGE, + aws_account_id="184463061564", + region="us-east-1", + repository_name="dander", + aws_profile="dander-deploy", + ) + + assert image == f"184463061564.dkr.ecr.us-east-1.amazonaws.com/dander@{_INDEX_DIGEST}" + create = next( + command + for command in runner.commands + if command[:4] == ("docker", "buildx", "imagetools", "create") + ) + assert "--prefer-index=false" in create + assert create[-1] == _SOURCE_IMAGE + login_index = next( + index for index, command in enumerate(runner.commands) if command[:2] == ("docker", "login") + ) + assert runner.inputs[login_index] == "temporary-password" + assert all("temporary-password" not in part for command in runner.commands for part in command) + assert all(command[:3] != ("docker", "buildx", "build") for command in runner.commands) + assert promoter.artifact_record_path == tmp_path / ".dander" / "runtime-artifact-aws.json" + record = json.loads(promoter.artifact_record_path.read_text(encoding="utf-8")) + assert record["source_image"] == _SOURCE_IMAGE + assert record["image"] == image + assert record["promotion"] == "registry-copy" + + +def test_runtime_image_promoter_rejects_registry_digest_rewrites(tmp_path: Path) -> None: + _write_source_record(tmp_path) + + with pytest.raises(ProjectBootstrapError, match="rewrote the OCI index"): + RuntimeImagePromoter( + tmp_path, + runner=_Runner(destination_digest="sha256:" + "d" * 64), + ).promote( + source_image=_SOURCE_IMAGE, + aws_account_id="184463061564", + region="us-east-1", + repository_name="dander", + ) + + +def test_runtime_image_promoter_is_idempotent_for_an_existing_identical_tag( + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + runner = _Runner() + runner.describe_image_calls = 1 + + image = RuntimeImagePromoter(tmp_path, runner=runner).promote( + source_image=_SOURCE_IMAGE, + aws_account_id="184463061564", + region="us-east-1", + repository_name="dander", + ) + + assert image.endswith("@" + _INDEX_DIGEST) + assert not any( + command[:4] == ("docker", "buildx", "imagetools", "create") for command in runner.commands + ) + + +def test_runtime_image_promoter_rejects_a_source_record_mismatch_before_copy( + tmp_path: Path, +) -> None: + _write_source_record(tmp_path, image="registry.example.invalid/dander@" + _INDEX_DIGEST) + runner = _Runner() + + with pytest.raises(ProjectBootstrapError, match="does not match"): + RuntimeImagePromoter(tmp_path, runner=runner).promote( + source_image=_SOURCE_IMAGE, + aws_account_id="184463061564", + region="us-east-1", + repository_name="dander", + ) + + assert runner.commands == [] diff --git a/tests/cli/test_init_cli.py b/tests/cli/test_init_cli.py index f0249cd7..5371b613 100644 --- a/tests/cli/test_init_cli.py +++ b/tests/cli/test_init_cli.py @@ -226,9 +226,9 @@ def fake_execute(self: object, **kwargs: object) -> Path: captured.update(kwargs) return tmp_path / "dander-aws.tfplan" - monkeypatch.setattr("dander.cli.main.load_project_config", fake_load) + monkeypatch.setattr("dander.cli.aws_command.load_project_config", fake_load) monkeypatch.setattr(DanderProject, "validate_references", fake_validate) - monkeypatch.setattr("dander.cli.main.AwsTerraformBootstrap.execute", fake_execute) + monkeypatch.setattr("dander.cli.aws_command.AwsTerraformBootstrap.execute", fake_execute) result = CliRunner().invoke( app, @@ -257,6 +257,80 @@ def fake_execute(self: object, **kwargs: object) -> Path: assert "init-aws-apply" in result.output +def test_aws_admin_plan_saves_without_applying_and_prints_exact_next_command( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + + def fake_execute(self: object, **kwargs: object) -> Path: + captured.update(kwargs) + return tmp_path / "operator" / "dander-aws-admin-bootstrap.tfplan" + + monkeypatch.setattr("dander.cli.aws_command.AwsAdministrativeBootstrap.execute", fake_execute) + operator_dir = tmp_path / "operator" + result = CliRunner().invoke( + app, + [ + "init-aws-admin-plan", + "--aws-account-id", + "184463061564", + "--state-bucket", + "dander-184463061564-state", + "--admin-principal-arn", + "arn:aws:iam::184463061564:root", + "--operator-artifact-dir", + str(operator_dir), + "--aws-profile", + "dander-phase1b", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["aws_account_id"] == "184463061564" + assert captured["lock_table"] == "dander-terraform-locks" + assert "AWS administrative bootstrap planned" in result.output + assert "init-aws-admin-apply" in result.output + assert "terraform-workspace" in result.output + + +def test_aws_image_promotion_requires_confirmation_and_does_not_rebuild( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + + def fake_promote(self: object, **kwargs: object) -> str: + captured.update(kwargs) + return "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "a" * 64 + + monkeypatch.setattr("dander.cli.aws_command.RuntimeImagePromoter.promote", fake_promote) + config = tmp_path / "dander.yaml" + config.write_text("version: 1\n", encoding="utf-8") + source = "us-central1-docker.pkg.dev/unit-project/dander/dander@sha256:" + "a" * 64 + result = CliRunner().invoke( + app, + [ + "image-promote-aws", + "--source-image", + source, + "--aws-account-id", + "184463061564", + "--aws-profile", + "dander-deploy", + "--config", + str(config), + ], + input="y\n", + ) + + assert result.exit_code == 0, result.output + assert captured["source_image"] == source + assert captured["aws_profile"] == "dander-deploy" + assert "Promoted byte-identical runtime image" in result.output + assert "init-aws-plan" in result.output + + def test_admin_plan_runs_read_only_permission_preflight_before_terraform( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/infra/test_fargate_runtime.py b/tests/infra/test_fargate_runtime.py index 6d891275..1170406d 100644 --- a/tests/infra/test_fargate_runtime.py +++ b/tests/infra/test_fargate_runtime.py @@ -21,11 +21,24 @@ def test_aws_root_is_separate_and_uses_native_remote_state() -> None: assert "azurerm" not in versions +def test_aws_stage_zero_uses_customer_encryption_and_scoped_role_passing() -> None: + stage_zero = _source(AWS_ROOT / "bootstrap-admin/main.tf") + + assert 'resource "aws_kms_key" "stage_zero"' in stage_zero + assert "enable_key_rotation = true" in stage_zero + assert 'sse_algorithm = "aws:kms"' in stage_zero + assert 'sid = "ManageDanderRoles"' in stage_zero + assert '"arn:${local.partition}:iam::${var.aws_account_id}:role/dander-*"' in stage_zero + assert 'variable = "aws:PrincipalArn"' in stage_zero + + def test_fargate_task_is_immutable_nonroot_and_uses_separate_roles() -> None: module = _source(MODULE) + stage_zero = _source(AWS_ROOT / "bootstrap-admin/main.tf") - assert 'image_tag_mutability = "IMMUTABLE"' in module - assert "scan_on_push = true" in module + assert 'data "aws_ecr_repository" "runtime"' in module + assert 'image_tag_mutability = "IMMUTABLE"' in stage_zero + assert "scan_on_push = true" in stage_zero assert "readonlyRootFilesystem = true" in module assert 'user = "65532:65532"' in module assert 'containerPath = "/tmp"' in module diff --git a/tickets/DANDER-94-aws-stage-zero-image-promotion.md b/tickets/DANDER-94-aws-stage-zero-image-promotion.md new file mode 100644 index 00000000..beaf3aa7 --- /dev/null +++ b/tickets/DANDER-94-aws-stage-zero-image-promotion.md @@ -0,0 +1,40 @@ +--- +id: DANDER-94 +title: Bootstrap AWS state and promote source-free artifacts +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-93] +created: 2026-08-08 +--- + +## Context + +Manifest-aware Fargate planning required an existing S3 backend, DynamoDB lock table, and ECR +image, while the platform root still attempted to own ECR. This slice resolves that bootstrap +cycle without applying infrastructure or claiming Fargate support. + +## Acceptance Criteria + +- [x] Plan stage zero locally before any AWS resource exists. +- [x] Create only encrypted/versioned state, locking, immutable registry, and deployment-role prerequisites. +- [x] Apply only a reviewed saved plan after confirmation. +- [x] Migrate initial local state into the created encrypted S3 backend after apply. +- [x] Preserve a local recovery copy if state migration fails. +- [x] Make stage zero, not the Fargate platform root, own ECR. +- [x] Copy an accepted source-free OCI index into ECR without rebuilding it. +- [x] Fail promotion unless the index and platform digests remain identical. +- [x] Focused Python tests, Terraform tests, real read-only AWS plan, and security scan pass. +- [x] Full local validation and retained GCP no-drift pass. +- [ ] Protected CI passes. + +## Design + +Copy the packaged stage-zero root into an operator-owned directory outside the checkout. Use local +state for the first saved plan, then migrate state after that exact plan creates S3 and DynamoDB. +Use the resulting short-lived deployment role for later platform plans and registry promotion. + +## Review Log + +The credentialed read-only AWS plan proposed 12 creates, zero updates, and zero deletes. No AWS or +GCP apply occurred. From 27e3c8ff6ff7852b2f3a0e8fb95db5e0b7df0a5f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 17:46:56 -0400 Subject: [PATCH 044/163] fix: keep AWS review commands copyable --- src/dander/cli/aws_command.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/dander/cli/aws_command.py b/src/dander/cli/aws_command.py index 73679ce1..efa6d0f8 100644 --- a/src/dander/cli/aws_command.py +++ b/src/dander/cli/aws_command.py @@ -63,7 +63,8 @@ def init_aws_admin_plan( console.print(f"[green]AWS administrative bootstrap planned.[/green] Saved plan: {plan_path}") console.print( "Review: " - + shlex.join(("terraform", f"-chdir={workspace}", "show", "-no-color", str(plan_path))) + + shlex.join(("terraform", f"-chdir={workspace}", "show", "-no-color", str(plan_path))), + soft_wrap=True, ) next_command = [ "dander", @@ -89,7 +90,7 @@ def init_aws_admin_plan( ] if aws_profile: next_command.extend(("--aws-profile", aws_profile)) - console.print("Next after review: " + shlex.join(next_command)) + console.print("Next after review: " + shlex.join(next_command), soft_wrap=True) def init_aws_admin_apply( @@ -231,7 +232,8 @@ def init_aws_plan( console.print(f"[green]AWS deployment planned.[/green] Saved plan: {plan_path}") console.print( "Review: " - + shlex.join(("terraform", f"-chdir={infra_dir}", "show", "-no-color", str(plan_path))) + + shlex.join(("terraform", f"-chdir={infra_dir}", "show", "-no-color", str(plan_path))), + soft_wrap=True, ) next_command = [ "dander", @@ -247,7 +249,7 @@ def init_aws_plan( ] if aws_profile: next_command.extend(("--aws-profile", aws_profile)) - console.print("Next after review: " + shlex.join(next_command)) + console.print("Next after review: " + shlex.join(next_command), soft_wrap=True) def init_aws_apply( From f71241da5c770c0496638a80dea92e1b2aa0aa52 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 18:17:21 -0400 Subject: [PATCH 045/163] feat: add manifest-bound Fargate operations --- HANDOFF.md | 42 +- docs/decisions.md | 9 + docs/known-limitations.md | 8 +- infra/aws/README.md | 19 + infra/aws/bootstrap-admin/main.tf | 38 ++ src/dander/cli/aws_command.py | 169 ++++++ src/dander/providers/fargate/__init__.py | 18 +- src/dander/providers/fargate/operations.py | 629 +++++++++++++++++++++ tests/bootstrap/test_aws_admin.py | 21 + tests/cli/test_aws_operations_cli.py | 186 ++++++ tests/providers/test_fargate_operations.py | 439 ++++++++++++++ tickets/DANDER-95-fargate-operations.md | 39 ++ 12 files changed, 1591 insertions(+), 26 deletions(-) create mode 100644 src/dander/providers/fargate/operations.py create mode 100644 tests/cli/test_aws_operations_cli.py create mode 100644 tests/providers/test_fargate_operations.py create mode 100644 tickets/DANDER-95-fargate-operations.md diff --git a/HANDOFF.md b/HANDOFF.md index 4e279900..5523b62b 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,41 +2,41 @@ ## Finished -- Added saved-plan-only AWS stage-zero commands with post-apply local-to-S3 state migration. -- Added customer-key-encrypted S3 state, DynamoDB locking, immutable ECR, and a dedicated deployment role. -- Added idempotent source-free OCI promotion into ECR with exact index/platform digest verification. -- Moved ECR ownership out of the Fargate platform root and into stage zero. -- Packaged the new root, extended existing CI validation, and corrected AWS support documentation. +- Added manifest-bound Fargate run, status, logs, cancel, replay, and verify commands. +- Correlated exact Step Functions executions, ECS tasks, and `runtime/dander/...` log streams. +- Normalized successful and failed controller results without returning unrestricted AWS payloads. +- Scoped the stage-zero deployment role to Dander controllers, executions, and task logs. +- Updated AWS operator documentation, decisions, limitations, tests, and DANDER-95. ## Try It -Run `dander init-aws-admin-plan --help` and `dander image-promote-aws --help`. A real reviewed plan -exists in `/tmp/dander-aws-bootstrap-plan`; do not apply it without separate approval. +Run `dander aws --help`. The commands require a validated Fargate deployment and the short-lived +AWS deployment-role profile; mutating commands require confirmation. ## Checks -- Ruff, formatting, strict typing, and all 972 Python tests passed. -- Stage-zero and Fargate provider-mocked plans passed; protected Linux validation remains pending. -- Credentialed AWS stage-zero plan: 12 creates, 0 updates, 0 deletes; the bucket name is available. -- Wheel/sdist, source-free install, container conformance, dependency audit, focused Trivy scans, and Gitleaks passed. -- Retained GCP platform plan reported exactly `No changes.`; no cloud apply occurred. +- Ruff, formatting, strict typing, and all 989 Python tests passed. +- All Terraform validation and both provider-mocked AWS plans passed. +- Wheel/sdist inspection, source-free installs, full runtime dependencies, and container contract passed. +- Dependency audit and Terraform HIGH/CRITICAL scan passed; protected image and Git secret scans remain pending. +- AWS stage-zero plan: 12 creates, 0 updates, 0 deletes. Retained GCP plan: exactly `No changes.` ## Decisions -- Stage zero exclusively owns AWS state, registry, encryption, and the deployment role. -- The first plan uses secured local state; only a successful apply migrates it to encrypted S3. -- Promote the accepted OCI artifact byte-for-byte; never rebuild separately for ECR. +- The manifest, not operator-supplied AWS identifiers, owns every Fargate resource binding. +- Step Functions remains the lifecycle authority; failed status reads history only for allow-listed fields. +- Fargate remains unsupported until the source-free keyless live proof passes. ## Remaining -- Let protected Linux CI repeat validation and merge the focused PR if clean. +- Let protected CI repeat Linux, image, and Git-history security checks. +- Merge the focused PR if protected CI remains clean. - Obtain separate approval before applying the 12-resource AWS stage-zero plan. -- Configure a short-lived deployment-role profile and promote the accepted source-free image. -- Add AWS status, logs, cancellation, replay, and verification operations. -- Keep Fargate unsupported until live keyless BigQuery parity passes. +- Complete the Fargate live comparison before claiming AWS support. +- Continue the portability roadmap with PostgreSQL state and warehouse. ## Review First -- `src/dander/bootstrap/aws_admin.py` -- `src/dander/bootstrap/project.py` +- `src/dander/providers/fargate/operations.py` - `src/dander/cli/aws_command.py` +- `infra/aws/bootstrap-admin/main.tf` diff --git a/docs/decisions.md b/docs/decisions.md index 9ca7082e..99e19ecf 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-08 — Fargate operations bind to the manifest and controller + +- **Ownership:** The validated deployment and pipeline determine the exact state machine, schedule, + task definition, log group, and ECR image checks; operators do not supply arbitrary AWS resources. +- **Lifecycle:** Step Functions remains authoritative for start, status, cancellation, and replay; + CloudWatch events are correlated through the execution's exact ECS task identifier. +- **Output:** Commands expose small normalized records and scoped errors, not unrestricted AWS + responses. This completes the operator surface but does not claim support before live acceptance. + ## 2026-08-08 — AWS stage zero owns state and artifact prerequisites - **Ownership:** A separate AWS stage-zero root owns the customer-key-encrypted S3 backend, diff --git a/docs/known-limitations.md b/docs/known-limitations.md index aff9ff98..02fffbd9 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -5,10 +5,10 @@ evaluate these limits before using it for an unattended system containing busine - AWS ECS/Fargate is not yet a supported Dander launcher. Packaged stage-zero and platform roots now have public saved-plan/apply commands, and accepted OCI artifacts can be copied into ECR - without rebuilding. Status, logs, cancellation, replay, deployment verification, and live parity - remain unproven. The runtime bridge copies one temporary ECS task-role session into process - memory and therefore caps a Fargate projection at one hour; renewable credentials remain a later - gate. + without rebuilding. Manifest-bound status, logs, cancellation, replay, and deployment + verification are implemented but not live-qualified. The runtime bridge copies one temporary ECS + task-role session into process memory and therefore caps a Fargate projection at one hour; + renewable credentials and live parity remain later gates. - Provider package extras and the full runtime image contain planned provider SDK dependencies, but do not by themselves implement or qualify Snowflake, Redshift, PostgreSQL, AWS, Azure, or OCI profiles. The packaged capability manifest is the support boundary. diff --git a/infra/aws/README.md b/infra/aws/README.md index d9c264e0..6207eb56 100644 --- a/infra/aws/README.md +++ b/infra/aws/README.md @@ -29,5 +29,24 @@ AWS performs a best-effort `StopTask` when the integration is cancelled or times is the only runtime outcome eligible for a bounded launcher retry. Scheduler delivery retries use a separate counter and queue. +After a reviewed platform apply, operate one exact manifest-bound pipeline with the dedicated +short-lived deployment-role profile: + +```console +dander aws run --deployment aws_fargate --pipeline greenhouse_jobs --aws-profile dander-deploy +dander aws status --deployment aws_fargate --pipeline greenhouse_jobs --aws-profile dander-deploy +dander aws logs --deployment aws_fargate --pipeline greenhouse_jobs \ + --execution-arn EXECUTION_ARN --aws-profile dander-deploy +dander aws replay --deployment aws_fargate --pipeline greenhouse_jobs \ + --execution-arn TERMINAL_EXECUTION_ARN --aws-profile dander-deploy +dander aws verify --deployment aws_fargate --pipeline greenhouse_jobs \ + --expected-image 123456789012.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:DIGEST \ + --aws-profile dander-deploy +``` + +`run`, `cancel`, and `replay` require confirmation because they mutate paid AWS execution state. +Status, logs, and verification are read-only. Commands validate that supplied execution ARNs belong +to the selected pipeline and print only Dander's normalized operation records. + No AWS deployment is supported until source-free live acceptance proves manual and scheduled runs, deadline cancellation, replay, identity refresh, alerts, rollback, and no-change reconciliation. diff --git a/infra/aws/bootstrap-admin/main.tf b/infra/aws/bootstrap-admin/main.tf index a391969d..569c1921 100644 --- a/infra/aws/bootstrap-admin/main.tf +++ b/infra/aws/bootstrap-admin/main.tf @@ -227,6 +227,44 @@ data "aws_iam_policy_document" "deployment" { resources = ["*"] } + statement { + sid = "OperateDanderControllers" + effect = "Allow" + actions = [ + "states:ListExecutions", + "states:StartExecution", + ] + resources = [ + "arn:${local.partition}:states:${var.region}:${var.aws_account_id}:stateMachine:${var.name}-*" + ] + } + + statement { + sid = "ObserveAndCancelDanderExecutions" + effect = "Allow" + actions = [ + "states:DescribeExecution", + "states:GetExecutionHistory", + "states:StopExecution", + ] + resources = [ + "arn:${local.partition}:states:${var.region}:${var.aws_account_id}:execution:${var.name}-*:*" + ] + } + + statement { + sid = "ReadDanderTaskLogs" + effect = "Allow" + actions = [ + "logs:DescribeLogStreams", + "logs:FilterLogEvents", + "logs:GetLogEvents", + ] + resources = [ + "arn:${local.partition}:logs:${var.region}:${var.aws_account_id}:log-group:/dander/${var.name}/*:*" + ] + } + statement { sid = "UseStageZeroEncryptionKey" effect = "Allow" diff --git a/src/dander/cli/aws_command.py b/src/dander/cli/aws_command.py index efa6d0f8..05f3fa33 100644 --- a/src/dander/cli/aws_command.py +++ b/src/dander/cli/aws_command.py @@ -18,11 +18,179 @@ RuntimeImagePromoter, ) from dander.project import ProjectConfigError, load_project_config +from dander.providers.fargate import FargateBinding, FargateOperationError, FargateOperations _DEFAULT_AWS_INFRA_DIR = Path("infra/aws") _DEFAULT_AWS_BOOTSTRAP_ADMIN_DIR = Path("infra/aws/bootstrap-admin") _DEFAULT_PROJECT_CONFIG = Path("dander.yaml") console = Console() +aws_app = typer.Typer(help="Operate and verify manifest-bound AWS Fargate pipelines.") + + +def _fargate_operations( + *, + config: Path, + deployment: str, + pipeline: str, + name: str, + aws_profile: str, +) -> FargateOperations: + binding = FargateBinding.from_project( + config=config, + deployment=deployment, + pipeline_id=pipeline, + name=name, + ) + return FargateOperations(binding, aws_profile=aws_profile) + + +@aws_app.command("run") +def aws_run( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), + execution_name: str | None = typer.Option(None, "--execution-name"), +) -> None: + """Start one paid manual Fargate execution after confirmation.""" + if not typer.confirm(f"Start Fargate pipeline {pipeline!r}?", default=False): + raise typer.Abort() + try: + execution = _fargate_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + aws_profile=aws_profile, + ).start(execution_name=execution_name) + except FargateOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + +@aws_app.command("status") +def aws_status( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_arn: str | None = typer.Option(None, "--execution-arn"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Show sanitized status for one execution, or the latest execution.""" + try: + operations = _fargate_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + aws_profile=aws_profile, + ) + execution = ( + operations.describe(execution_arn) if execution_arn is not None else operations.latest() + ) + except FargateOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data={"execution": execution.as_dict() if execution is not None else None}) + + +@aws_app.command("logs") +def aws_logs( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_arn: str = typer.Option(..., "--execution-arn"), + limit: int = typer.Option(100, "--limit", min=1, max=10_000), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Read CloudWatch events correlated to one controller execution.""" + try: + events = _fargate_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + aws_profile=aws_profile, + ).logs(execution_arn, limit=limit) + except FargateOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data={"events": [event.as_dict() for event in events]}) + + +@aws_app.command("cancel") +def aws_cancel( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_arn: str = typer.Option(..., "--execution-arn"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Cancel one running controller execution after confirmation.""" + if not typer.confirm(f"Cancel Fargate execution {execution_arn!r}?", default=False): + raise typer.Abort() + try: + execution = _fargate_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + aws_profile=aws_profile, + ).cancel(execution_arn) + except FargateOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + +@aws_app.command("replay") +def aws_replay( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_arn: str = typer.Option(..., "--execution-arn"), + execution_name: str | None = typer.Option(None, "--execution-name"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Replay one terminal pipeline at its inclusive cursor boundary.""" + if not typer.confirm(f"Replay Fargate pipeline {pipeline!r}?", default=False): + raise typer.Abort() + try: + execution = _fargate_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + aws_profile=aws_profile, + ).replay(execution_arn, execution_name=execution_name) + except FargateOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + +@aws_app.command("verify") +def aws_verify( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + expected_image: str = typer.Option(..., "--expected-image"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + aws_profile: str = typer.Option("", "--aws-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Verify one deployed Fargate pipeline through read-only provider checks.""" + try: + verification = _fargate_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + aws_profile=aws_profile, + ).verify(expected_image=expected_image) + except FargateOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data=verification.as_dict()) def init_aws_admin_plan( @@ -281,6 +449,7 @@ def init_aws_apply( def register_aws_commands(app: typer.Typer) -> None: """Register flat, compatibility-preserving AWS command names on the root CLI.""" + app.add_typer(aws_app, name="aws") app.command("init-aws-admin-plan")(init_aws_admin_plan) app.command("init-aws-admin-apply")(init_aws_admin_apply) app.command("image-promote-aws")(image_promote_aws) diff --git a/src/dander/providers/fargate/__init__.py b/src/dander/providers/fargate/__init__.py index f41b498d..2cf5cc86 100644 --- a/src/dander/providers/fargate/__init__.py +++ b/src/dander/providers/fargate/__init__.py @@ -1,5 +1,21 @@ """Dependency-light ECS/Fargate launcher configuration.""" from dander.providers.fargate.config import FargateLauncherConfig +from dander.providers.fargate.operations import ( + FargateBinding, + FargateDeploymentVerification, + FargateExecution, + FargateLogEvent, + FargateOperationError, + FargateOperations, +) -__all__ = ["FargateLauncherConfig"] +__all__ = [ + "FargateBinding", + "FargateDeploymentVerification", + "FargateExecution", + "FargateLauncherConfig", + "FargateLogEvent", + "FargateOperationError", + "FargateOperations", +] diff --git a/src/dander/providers/fargate/operations.py b/src/dander/providers/fargate/operations.py new file mode 100644 index 00000000..891af98d --- /dev/null +++ b/src/dander/providers/fargate/operations.py @@ -0,0 +1,629 @@ +"""Provider-native ECS/Fargate execution controls through the AWS CLI.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Protocol, cast +from uuid import uuid4 + +from dander.project import ProjectConfigError, load_project_config + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + +_AWS_ACCOUNT_ID = re.compile(r"^[0-9]{12}$") +_AWS_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_AWS_REGION = re.compile(r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") +_DEPLOYMENT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,23}$") +_EXECUTION_NAME = re.compile(r"^[A-Za-z0-9_-]{1,80}$") +_PIPELINE_ID = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,127}$") +_TASK_ID = re.compile(r"^[0-9a-f]{32}$") +_ECR_IMAGE = re.compile( + r"^(?P[0-9]{12})\.dkr\.ecr\.(?P[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+)" + r"\.amazonaws\.com/(?P[a-z0-9]+(?:[._/-][a-z0-9]+)*)" + r"@sha256:(?P[0-9a-f]{64})$" +) +_TERMINAL_STATUSES = frozenset({"SUCCEEDED", "FAILED", "TIMED_OUT", "ABORTED"}) + + +class FargateOperationError(RuntimeError): + """Raised when an AWS execution operation cannot complete or returns invalid data.""" + + +class _Runner(Protocol): + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _subprocess_runner( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + ) + + +@dataclass(frozen=True, slots=True) +class FargateBinding: + """Exact manifest pipeline to AWS controller-resource binding.""" + + account_id: str + region: str + deployment_name: str + pipeline_id: str + resource_name: str + state_machine_arn: str + cluster_name: str + log_group_name: str + schedule_paused: bool + project_dir: Path + + @classmethod + def from_project( + cls, + *, + config: Path, + deployment: str, + pipeline_id: str, + name: str = "dander", + ) -> FargateBinding: + """Resolve one Fargate pipeline from the validated project manifest.""" + resolved_config = config.expanduser().resolve() + if not _DEPLOYMENT_NAME.fullmatch(name): + raise FargateOperationError("Invalid AWS deployment name") + if not _PIPELINE_ID.fullmatch(pipeline_id): + raise FargateOperationError("Invalid pipeline identifier") + try: + manifest = load_project_config(resolved_config, deployment=deployment) + if manifest.launcher_provider != "fargate": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select launcher.provider='fargate'" + ) + pipeline = manifest.pipelines[pipeline_id] + manifest.validate_references(resolved_config.parent) + launcher = manifest.resolved_launcher_config() + except KeyError as error: + raise FargateOperationError( + f"Pipeline {pipeline_id!r} is not declared in the project manifest" + ) from error + except ProjectConfigError as error: + raise FargateOperationError(str(error)) from error + account_id = launcher.get("aws_account_id") + region = launcher.get("region") + if not isinstance(account_id, str) or _AWS_ACCOUNT_ID.fullmatch(account_id) is None: + raise FargateOperationError("Fargate deployment has an invalid AWS account") + if not isinstance(region, str) or _AWS_REGION.fullmatch(region) is None: + raise FargateOperationError("Fargate deployment has an invalid AWS region") + resource_name = _resource_name(name=name, pipeline_id=pipeline_id) + partition = "aws-us-gov" if region.startswith("us-gov-") else "aws" + return cls( + account_id=account_id, + region=region, + deployment_name=deployment, + pipeline_id=pipeline_id, + resource_name=resource_name, + state_machine_arn=( + f"arn:{partition}:states:{region}:{account_id}:stateMachine:{resource_name}" + ), + cluster_name=name, + log_group_name=f"/dander/{name}/{pipeline_id}", + schedule_paused=pipeline.paused, + project_dir=resolved_config.parent, + ) + + def validate_execution_arn(self, execution_arn: str) -> str: + """Require an execution owned by this exact state machine.""" + state_machine_prefix = self.state_machine_arn.replace(":stateMachine:", ":execution:") + prefix = state_machine_prefix + ":" + if not execution_arn.startswith(prefix): + raise FargateOperationError("Execution does not belong to the selected pipeline") + name = execution_arn.removeprefix(prefix) + if _EXECUTION_NAME.fullmatch(name) is None: + raise FargateOperationError("Execution ARN has an invalid name") + return name + + +@dataclass(frozen=True, slots=True) +class FargateExecution: + """Small sanitized view of one Step Functions controller execution.""" + + execution_arn: str + name: str + state: str + started_at: str | None = None + stopped_at: str | None = None + run_id: str | None = None + task_arn: str | None = None + container_exit_code: int | None = None + failure_code: str | None = None + + @property + def terminal(self) -> bool: + return self.state in {"succeeded", "failed", "cancelled"} + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class FargateLogEvent: + """One CloudWatch event correlated to the selected ECS task.""" + + timestamp: int + message: str + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class FargateDeploymentVerification: + """Read-only verification result for one deployed pipeline.""" + + state_machine: str + cluster: str + schedule: str + schedule_state: str + task_definition: str + image: str + log_group: str + repository: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +class FargateOperations: + """Start, observe, cancel, replay, and verify one bound Fargate pipeline.""" + + def __init__( + self, + binding: FargateBinding, + *, + aws_profile: str = "", + runner: _Runner | None = None, + clock: Callable[[], datetime] | None = None, + nonce: Callable[[], str] | None = None, + ) -> None: + if aws_profile and _AWS_PROFILE.fullmatch(aws_profile) is None: + raise FargateOperationError("Invalid AWS profile") + self.binding = binding + self._aws_prefix = ("aws", "--profile", aws_profile) if aws_profile else ("aws",) + self._runner = runner or _subprocess_runner + self._clock = clock or (lambda: datetime.now(UTC)) + self._nonce = nonce or (lambda: uuid4().hex[:8]) + + def start(self, *, execution_name: str | None = None) -> FargateExecution: + """Start one manual controller execution.""" + now = self._clock().astimezone(UTC) + name = execution_name or _new_execution_name("manual", now, self._nonce()) + return self._start(name=name, now=now, correlation=f"manual:{name}") + + def replay( + self, + execution_arn: str, + *, + execution_name: str | None = None, + ) -> FargateExecution: + """Start a fresh inclusive-boundary run after one terminal execution.""" + previous = self.describe(execution_arn) + if not previous.terminal: + raise FargateOperationError("Only a terminal execution can be replayed") + now = self._clock().astimezone(UTC) + name = execution_name or _new_execution_name("replay", now, self._nonce()) + return self._start(name=name, now=now, correlation=f"replay:{previous.name}") + + def latest(self) -> FargateExecution | None: + """Return the latest controller execution, if one exists.""" + payload = self._json( + "states", + "list-executions", + "--state-machine-arn", + self.binding.state_machine_arn, + "--max-results", + "1", + ) + executions = payload.get("executions") + if not isinstance(executions, list): + raise FargateOperationError("AWS returned an invalid execution list") + if not executions: + return None + summary = executions[0] + if not isinstance(summary, dict) or not isinstance(summary.get("executionArn"), str): + raise FargateOperationError("AWS returned an invalid execution summary") + return self.describe(cast("str", summary["executionArn"])) + + def describe(self, execution_arn: str) -> FargateExecution: + """Return normalized status for one owned execution.""" + name = self.binding.validate_execution_arn(execution_arn) + payload = self._json("states", "describe-execution", "--execution-arn", execution_arn) + if payload.get("executionArn") != execution_arn: + raise FargateOperationError("AWS returned an unexpected execution") + status = payload.get("status") + if not isinstance(status, str): + raise FargateOperationError("AWS returned an invalid execution status") + output = _json_document(payload.get("output")) + state = _normalized_state(status) + details: object = output + if state == "failed": + details = [output, self._execution_history(execution_arn)] + failure_code = _deep_string(details, "failure_code") + if failure_code is None and status == "TIMED_OUT": + failure_code = "launcher_deadline_exceeded" + elif failure_code is None and state == "failed": + failure_code = "launcher_execution_failed" + return FargateExecution( + execution_arn=execution_arn, + name=name, + state=state, + started_at=_optional_string(payload.get("startDate")), + stopped_at=_optional_string(payload.get("stopDate")), + run_id=_deep_string(details, "run_id"), + task_arn=_deep_string(details, "task_arn", "taskArn", "TaskArn"), + container_exit_code=_deep_integer(details, "container_exit_code", "exit_code"), + failure_code="operator_cancelled" if state == "cancelled" else failure_code, + ) + + def logs(self, execution_arn: str, *, limit: int = 100) -> tuple[FargateLogEvent, ...]: + """Read task logs correlated through the controller execution history.""" + if not 1 <= limit <= 10_000: + raise FargateOperationError("Log limit must be between 1 and 10000") + execution = self.describe(execution_arn) + task_arn = execution.task_arn or self._task_arn_from_history(execution_arn) + if task_arn is None: + raise FargateOperationError("The execution has not exposed an ECS task yet") + task_id = task_arn.rsplit("/", maxsplit=1)[-1] + if _TASK_ID.fullmatch(task_id) is None: + raise FargateOperationError("AWS returned an invalid ECS task ARN") + payload = self._json( + "logs", + "filter-log-events", + "--log-group-name", + self.binding.log_group_name, + "--log-stream-name-prefix", + f"runtime/dander/{task_id}", + "--limit", + str(limit), + ) + events = payload.get("events") + if not isinstance(events, list): + raise FargateOperationError("AWS returned an invalid CloudWatch log response") + result: list[FargateLogEvent] = [] + for event in events: + if not isinstance(event, dict): + raise FargateOperationError("AWS returned an invalid CloudWatch log event") + timestamp = event.get("timestamp") + message = event.get("message") + if ( + isinstance(timestamp, bool) + or not isinstance(timestamp, int) + or not isinstance(message, str) + ): + raise FargateOperationError("AWS returned an invalid CloudWatch log event") + result.append(FargateLogEvent(timestamp=timestamp, message=message)) + return tuple(result) + + def cancel(self, execution_arn: str) -> FargateExecution: + """Stop one running controller; the synchronous ECS integration stops its task.""" + execution = self.describe(execution_arn) + if execution.state != "running": + raise FargateOperationError("Only a running execution can be cancelled") + payload = self._json( + "states", + "stop-execution", + "--execution-arn", + execution_arn, + "--error", + "Dander.OperatorCancelled", + "--cause", + "Cancelled by the Dander operator", + ) + return FargateExecution( + execution_arn=execution_arn, + name=execution.name, + state="cancelled", + started_at=execution.started_at, + stopped_at=_optional_string(payload.get("stopDate")), + run_id=execution.run_id, + task_arn=execution.task_arn, + failure_code="operator_cancelled", + ) + + def verify(self, *, expected_image: str) -> FargateDeploymentVerification: + """Verify immutable image, controller, scheduler, logs, and cluster read-only.""" + image = _ECR_IMAGE.fullmatch(expected_image) + if ( + image is None + or image.group("account") != self.binding.account_id + or image.group("region") != self.binding.region + ): + raise FargateOperationError( + "Expected image is not an immutable ECR image in this deployment" + ) + machine = self._json( + "states", + "describe-state-machine", + "--state-machine-arn", + self.binding.state_machine_arn, + ) + if machine.get("status") != "ACTIVE": + raise FargateOperationError("Fargate controller is not active") + clusters = self._json("ecs", "describe-clusters", "--clusters", self.binding.cluster_name) + cluster_items = clusters.get("clusters") + if ( + not isinstance(cluster_items, list) + or len(cluster_items) != 1 + or not isinstance(cluster_items[0], dict) + or cluster_items[0].get("status") != "ACTIVE" + or clusters.get("failures") not in ([], None) + ): + raise FargateOperationError("Fargate cluster is unavailable") + schedule = self._json("scheduler", "get-schedule", "--name", self.binding.resource_name) + expected_schedule = "DISABLED" if self.binding.schedule_paused else "ENABLED" + if schedule.get("State") != expected_schedule: + raise FargateOperationError("Fargate schedule does not match the manifest") + log_groups = self._json( + "logs", + "describe-log-groups", + "--log-group-name-prefix", + self.binding.log_group_name, + ).get("logGroups") + if not isinstance(log_groups, list) or not any( + isinstance(group, dict) and group.get("logGroupName") == self.binding.log_group_name + for group in log_groups + ): + raise FargateOperationError("Fargate log group is unavailable") + task = self._json( + "ecs", + "describe-task-definition", + "--task-definition", + self.binding.resource_name, + ).get("taskDefinition") + if not isinstance(task, dict) or task.get("status") != "ACTIVE": + raise FargateOperationError("Fargate task definition is unavailable") + containers = task.get("containerDefinitions") + if ( + not isinstance(containers, list) + or len(containers) != 1 + or not isinstance(containers[0], dict) + ): + raise FargateOperationError("Fargate task definition has invalid containers") + container = containers[0] + if ( + container.get("image") != expected_image + or container.get("readonlyRootFilesystem") is not True + or container.get("user") != "65532:65532" + ): + raise FargateOperationError( + "Fargate task does not match the immutable runtime contract" + ) + repository = self._json( + "ecr", + "describe-repositories", + "--repository-names", + image.group("repository"), + ).get("repositories") + if ( + not isinstance(repository, list) + or len(repository) != 1 + or not isinstance(repository[0], dict) + ): + raise FargateOperationError("ECR repository is unavailable") + repository_item = repository[0] + encryption = repository_item.get("encryptionConfiguration") + scanning = repository_item.get("imageScanningConfiguration") + if ( + repository_item.get("imageTagMutability") != "IMMUTABLE" + or not isinstance(encryption, dict) + or encryption.get("encryptionType") != "KMS" + or not isinstance(scanning, dict) + or scanning.get("scanOnPush") is not True + ): + raise FargateOperationError( + "ECR repository does not meet the runtime artifact contract" + ) + task_arn = task.get("taskDefinitionArn") + cluster_arn = cluster_items[0].get("clusterArn") + schedule_arn = schedule.get("Arn") + repository_arn = repository_item.get("repositoryArn") + if not all( + isinstance(value, str) + for value in (task_arn, cluster_arn, schedule_arn, repository_arn) + ): + raise FargateOperationError("AWS verification response is incomplete") + return FargateDeploymentVerification( + state_machine=self.binding.state_machine_arn, + cluster=cast("str", cluster_arn), + schedule=cast("str", schedule_arn), + schedule_state=expected_schedule, + task_definition=cast("str", task_arn), + image=expected_image, + log_group=self.binding.log_group_name, + repository=cast("str", repository_arn), + ) + + def _start(self, *, name: str, now: datetime, correlation: str) -> FargateExecution: + if _EXECUTION_NAME.fullmatch(name) is None: + raise FargateOperationError( + "Execution name must use 1-80 letters, numbers, hyphens, or underscores" + ) + scheduled_time = now.isoformat(timespec="seconds").replace("+00:00", "Z") + request = { + "deployment_revision": "manual", + "scheduled_time": scheduled_time, + "scheduler_attempt": 1, + "scheduler_execution_id": correlation, + } + payload = self._json( + "states", + "start-execution", + "--state-machine-arn", + self.binding.state_machine_arn, + "--name", + name, + "--input", + json.dumps(request, separators=(",", ":"), sort_keys=True), + ) + execution_arn = payload.get("executionArn") + if not isinstance(execution_arn, str): + raise FargateOperationError("AWS did not return an execution ARN") + self.binding.validate_execution_arn(execution_arn) + return FargateExecution( + execution_arn=execution_arn, + name=name, + state="running", + started_at=_optional_string(payload.get("startDate")) or scheduled_time, + ) + + def _task_arn_from_history(self, execution_arn: str) -> str | None: + self.binding.validate_execution_arn(execution_arn) + return _deep_string( + self._execution_history(execution_arn), "task_arn", "taskArn", "TaskArn" + ) + + def _execution_history(self, execution_arn: str) -> list[object]: + """Return one owned execution's bounded history for allow-listed field extraction.""" + self.binding.validate_execution_arn(execution_arn) + payload = self._json( + "states", + "get-execution-history", + "--execution-arn", + execution_arn, + "--reverse-order", + "--max-results", + "100", + ) + events = payload.get("events") + if not isinstance(events, list): + raise FargateOperationError("AWS returned invalid execution history") + return events + + def _json(self, service: str, operation: str, *arguments: str) -> dict[str, object]: + command = ( + *self._aws_prefix, + service, + operation, + "--region", + self.binding.region, + *arguments, + "--output", + "json", + ) + try: + completed = self._runner( + command, + cwd=self.binding.project_dir, + check=True, + capture_output=True, + text=True, + ) + payload: object = json.loads(completed.stdout) + except FileNotFoundError as error: + raise FargateOperationError("The AWS CLI is not available") from error + except (subprocess.CalledProcessError, json.JSONDecodeError) as error: + raise FargateOperationError( + f"AWS {service} {operation} did not complete successfully" + ) from error + if not isinstance(payload, dict): + raise FargateOperationError(f"AWS {service} returned an invalid response") + return cast("dict[str, object]", payload) + + +def _resource_name(*, name: str, pipeline_id: str) -> str: + normalized = pipeline_id.replace("_", "-")[:20] + digest = hashlib.sha1(pipeline_id.encode(), usedforsecurity=False).hexdigest()[:8] + return "-".join(part for part in (name[:20], normalized, digest) if part) + + +def _new_execution_name(prefix: str, now: datetime, nonce: str) -> str: + if re.fullmatch(r"[0-9a-f]{8}", nonce) is None: + raise FargateOperationError("Execution nonce must contain eight lowercase hex characters") + timestamp = now.strftime("%Y%m%dT%H%M%S") + return f"{prefix}-{timestamp}-{nonce}" + + +def _normalized_state(status: str) -> str: + if status == "RUNNING": + return "running" + if status == "SUCCEEDED": + return "succeeded" + if status == "ABORTED": + return "cancelled" + if status in _TERMINAL_STATUSES: + return "failed" + raise FargateOperationError("AWS returned an unknown execution status") + + +def _json_document(value: object) -> object: + if not isinstance(value, str): + return None + try: + return json.loads(value) + except json.JSONDecodeError: + return None + + +def _deep_string(value: object, *keys: str) -> str | None: + found = _deep_value(value, frozenset(keys), expected=str) + return cast("str | None", found) + + +def _deep_integer(value: object, *keys: str) -> int | None: + found = _deep_value(value, frozenset(keys), expected=int) + return cast("int | None", found) + + +def _deep_value(value: object, keys: frozenset[str], *, expected: type[object]) -> object: + if isinstance(value, dict): + for key, child in value.items(): + if key in keys and isinstance(child, expected) and not isinstance(child, bool): + return child + found = _deep_value(child, keys, expected=expected) + if found is not None: + return found + elif isinstance(value, list): + for child in value: + found = _deep_value(child, keys, expected=expected) + if found is not None: + return found + elif isinstance(value, str) and value[:1] in {"{", "["}: + nested = _json_document(value) + if nested is not None: + return _deep_value(nested, keys, expected=expected) + return None + + +def _optional_string(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +__all__ = [ + "FargateBinding", + "FargateDeploymentVerification", + "FargateExecution", + "FargateLogEvent", + "FargateOperationError", + "FargateOperations", +] diff --git a/tests/bootstrap/test_aws_admin.py b/tests/bootstrap/test_aws_admin.py index 23eab488..cfb270a0 100644 --- a/tests/bootstrap/test_aws_admin.py +++ b/tests/bootstrap/test_aws_admin.py @@ -12,6 +12,8 @@ from dander.bootstrap import AwsAdministrativeBootstrap, AwsAdministrativeBootstrapError +_REPO_ROOT = Path(__file__).parents[2] + class _Arguments(TypedDict): aws_account_id: str @@ -47,6 +49,25 @@ def _layout(tmp_path: Path) -> tuple[Path, Path]: return infra_dir, tmp_path / "operator" +def test_deployment_role_scopes_fargate_operations_to_dander_resources() -> None: + terraform = (_REPO_ROOT / "infra/aws/bootstrap-admin/main.tf").read_text(encoding="utf-8") + + for action in ( + "states:StartExecution", + "states:ListExecutions", + "states:DescribeExecution", + "states:GetExecutionHistory", + "states:StopExecution", + "logs:FilterLogEvents", + ): + assert f'"{action}"' in terraform + assert "stateMachine:${var.name}-*" in terraform + assert "execution:${var.name}-*:*" in terraform + assert "log-group:/dander/${var.name}/*:*" in terraform + assert '"states:*"' not in terraform + assert '"logs:*"' not in terraform + + def test_aws_admin_plan_uses_secured_local_state_without_applying( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/cli/test_aws_operations_cli.py b/tests/cli/test_aws_operations_cli.py new file mode 100644 index 00000000..bfa8fff1 --- /dev/null +++ b/tests/cli/test_aws_operations_cli.py @@ -0,0 +1,186 @@ +"""Thin CLI coverage for manifest-bound Fargate operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from typer.testing import CliRunner + +import dander.cli.aws_command as aws_command +from dander.cli.main import app +from dander.providers.fargate import ( + FargateDeploymentVerification, + FargateExecution, + FargateLogEvent, +) + +if TYPE_CHECKING: + from pathlib import Path + + from pytest import MonkeyPatch + +EXECUTION = ( + "arn:aws:states:us-east-1:123456789012:execution:dander-salesforce-crm-deadbeef:manual-unit" +) +IMAGE = "123456789012.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "a" * 64 + + +class _Operations: + def __init__(self) -> None: + self.calls: list[tuple[str, object]] = [] + self.execution = FargateExecution( + execution_arn=EXECUTION, + name="manual-unit", + state="succeeded", + run_id="salesforce_crm:unit", + ) + + def start(self, *, execution_name: str | None = None) -> FargateExecution: + self.calls.append(("start", execution_name)) + return self.execution + + def latest(self) -> FargateExecution: + self.calls.append(("latest", None)) + return self.execution + + def describe(self, execution_arn: str) -> FargateExecution: + self.calls.append(("describe", execution_arn)) + return self.execution + + def logs(self, execution_arn: str, *, limit: int) -> tuple[FargateLogEvent, ...]: + self.calls.append(("logs", (execution_arn, limit))) + return (FargateLogEvent(timestamp=1, message="runtime complete"),) + + def cancel(self, execution_arn: str) -> FargateExecution: + self.calls.append(("cancel", execution_arn)) + return self.execution + + def replay( + self, + execution_arn: str, + *, + execution_name: str | None = None, + ) -> FargateExecution: + self.calls.append(("replay", (execution_arn, execution_name))) + return self.execution + + def verify(self, *, expected_image: str) -> FargateDeploymentVerification: + self.calls.append(("verify", expected_image)) + return FargateDeploymentVerification( + state_machine="machine", + cluster="cluster", + schedule="schedule", + schedule_state="DISABLED", + task_definition="task", + image=expected_image, + log_group="logs", + repository="repository", + ) + + +def _install_fake(monkeypatch: MonkeyPatch) -> tuple[_Operations, dict[str, object]]: + operations = _Operations() + binding: dict[str, object] = {} + + def factory(**kwargs: object) -> _Operations: + binding.update(kwargs) + return operations + + monkeypatch.setattr(aws_command, "_fargate_operations", factory) + return operations, binding + + +def _base_args(tmp_path: Path) -> list[str]: + return [ + "--deployment", + "aws_fargate", + "--pipeline", + "salesforce_crm", + "--config", + str(tmp_path / "dander.yaml"), + "--aws-profile", + "dander-deploy", + ] + + +def test_status_and_logs_are_bound_to_the_selected_manifest_pipeline( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + operations, binding = _install_fake(monkeypatch) + + status = CliRunner().invoke( + app, + ["aws", "status", *_base_args(tmp_path), "--execution-arn", EXECUTION], + ) + logs = CliRunner().invoke( + app, + ["aws", "logs", *_base_args(tmp_path), "--execution-arn", EXECUTION, "--limit", "7"], + ) + + assert status.exit_code == 0, status.output + assert logs.exit_code == 0, logs.output + assert '"state": "succeeded"' in status.output + assert '"message": "runtime complete"' in logs.output + assert operations.calls == [("describe", EXECUTION), ("logs", (EXECUTION, 7))] + assert binding == { + "config": tmp_path / "dander.yaml", + "deployment": "aws_fargate", + "pipeline": "salesforce_crm", + "name": "dander", + "aws_profile": "dander-deploy", + } + + +@pytest.mark.parametrize( + ("command", "extra", "expected"), + [ + ("run", ["--execution-name", "manual-unit"], ("start", "manual-unit")), + ("cancel", ["--execution-arn", EXECUTION], ("cancel", EXECUTION)), + ( + "replay", + ["--execution-arn", EXECUTION, "--execution-name", "replay-unit"], + ("replay", (EXECUTION, "replay-unit")), + ), + ], +) +def test_mutating_operations_require_confirmation_and_invoke_one_action( + monkeypatch: MonkeyPatch, + tmp_path: Path, + command: str, + extra: list[str], + expected: tuple[str, object], +) -> None: + operations, _binding = _install_fake(monkeypatch) + + refused = CliRunner().invoke( + app, + ["aws", command, *_base_args(tmp_path), *extra], + input="n\n", + ) + accepted = CliRunner().invoke( + app, + ["aws", command, *_base_args(tmp_path), *extra], + input="y\n", + ) + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert operations.calls == [expected] + + +def test_verify_reports_the_exact_expected_image( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + operations, _binding = _install_fake(monkeypatch) + + result = CliRunner().invoke( + app, + ["aws", "verify", *_base_args(tmp_path), "--expected-image", IMAGE], + ) + + assert result.exit_code == 0, result.output + assert IMAGE in result.output + assert operations.calls == [("verify", IMAGE)] diff --git a/tests/providers/test_fargate_operations.py b/tests/providers/test_fargate_operations.py new file mode 100644 index 00000000..35ac6fef --- /dev/null +++ b/tests/providers/test_fargate_operations.py @@ -0,0 +1,439 @@ +"""Provider-native Fargate execution, logs, replay, and verification tests.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from collections import defaultdict, deque +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +import pytest +import yaml + +from dander.project import prepare_version_one_migration +from dander.providers.fargate import ( + FargateBinding, + FargateOperationError, + FargateOperations, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + +ACCOUNT = "123456789012" +REGION = "us-east-1" +PIPELINE = "salesforce_crm" +RESOURCE = ( + "dander-salesforce-crm-" + + hashlib.sha1(PIPELINE.encode(), usedforsecurity=False).hexdigest()[:8] +) +MACHINE = f"arn:aws:states:{REGION}:{ACCOUNT}:stateMachine:{RESOURCE}" +EXECUTION = MACHINE.replace(":stateMachine:", ":execution:") + ":manual-unit" +TASK_ID = "a" * 32 +TASK_ARN = f"arn:aws:ecs:{REGION}:{ACCOUNT}:task/dander/{TASK_ID}" +IMAGE = f"{ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com/dander@sha256:" + "b" * 64 + + +def _binding(tmp_path: Path, *, paused: bool = True) -> FargateBinding: + return FargateBinding( + account_id=ACCOUNT, + region=REGION, + deployment_name="aws_fargate", + pipeline_id=PIPELINE, + resource_name=RESOURCE, + state_machine_arn=MACHINE, + cluster_name="dander", + log_group_name=f"/dander/dander/{PIPELINE}", + schedule_paused=paused, + project_dir=tmp_path, + ) + + +class _Runner: + def __init__(self, responses: Mapping[tuple[str, str], list[dict[str, object]]]) -> None: + self.responses: defaultdict[tuple[str, str], deque[dict[str, object]]] = defaultdict(deque) + for key, values in responses.items(): + self.responses[key].extend(values) + self.commands: list[tuple[str, ...]] = [] + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert cwd.is_absolute() and check and capture_output and text + self.commands.append(args) + service_index = 3 if args[1:2] == ("--profile",) else 1 + key = (args[service_index], args[service_index + 1]) + if not self.responses[key]: + raise AssertionError(f"No fake AWS response for {key}") + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps(self.responses[key].popleft()), + ) + + +def _operations( + tmp_path: Path, + responses: Mapping[tuple[str, str], list[dict[str, object]]], + *, + paused: bool = True, +) -> tuple[FargateOperations, _Runner]: + runner = _Runner(responses) + operations = FargateOperations( + _binding(tmp_path, paused=paused), + aws_profile="dander-deploy", + runner=runner, + clock=lambda: datetime(2026, 8, 8, 21, 0, tzinfo=UTC), + nonce=lambda: "deadbeef", + ) + return operations, runner + + +def _describe( + *, + status: str = "SUCCEEDED", + execution_arn: str = EXECUTION, + output: dict[str, object] | None = None, +) -> dict[str, object]: + payload: dict[str, object] = { + "executionArn": execution_arn, + "status": status, + "startDate": "2026-08-08T21:00:00Z", + } + if status != "RUNNING": + payload["stopDate"] = "2026-08-08T21:01:00Z" + if output is not None: + payload["output"] = json.dumps(output) + return payload + + +def test_binding_resolves_the_exact_manifest_pipeline_and_terraform_name(tmp_path: Path) -> None: + project = tmp_path / "dander.yaml" + project.write_text( + """ +version: 1 +platform: + region: us-central1 + safety: + require_guarded_free_tier: false +pipelines: + salesforce_crm: + source: salesforce + models: [] + build_models: false + paused: true + resources: + job: dander-salesforce-crm + runtime_service_account: dander-salesforce-run + scheduler_service_account: dander-salesforce-sched +""".strip(), + encoding="utf-8", + ) + connectors = tmp_path / "connectors" + connectors.mkdir() + (connectors / "salesforce.yaml").write_text( + """ +name: salesforce +base_url: https://example.com +auth_strategy: none +endpoints: + - name: records + path: /records + primary_key: [id] + raw_schema: + - {name: id, type: STRING, mode: REQUIRED} +""".strip(), + encoding="utf-8", + ) + migration = prepare_version_one_migration(project) + project.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + deployment = platforms["deployments"].pop("gcp_cloud_run") + deployment["launcher"] = { + "provider": "fargate", + "region": REGION, + "aws_account_id": ACCOUNT, + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/123456789012/locations/global/" + "workloadIdentityPools/dander-aws/providers/dander-aws" + ), + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + } + deployment["runtime"]["memory"] = "2Gi" + deployment["safety"]["require_guarded_free_tier"] = False + platforms["deployments"]["aws_fargate"] = deployment + (tmp_path / "dander.platforms.yaml").write_text(yaml.safe_dump(platforms), encoding="utf-8") + + binding = FargateBinding.from_project( + config=project, + deployment="aws_fargate", + pipeline_id=PIPELINE, + ) + + assert binding.resource_name == RESOURCE + assert binding.state_machine_arn == MACHINE + assert binding.log_group_name == f"/dander/dander/{PIPELINE}" + assert binding.schedule_paused is True + + +def test_start_uses_exact_state_machine_and_operator_correlation(tmp_path: Path) -> None: + started_arn = MACHINE.replace(":stateMachine:", ":execution:") + ":manual-unit" + operations, runner = _operations( + tmp_path, + { + ("states", "start-execution"): [ + {"executionArn": started_arn, "startDate": "2026-08-08T21:00:00Z"} + ] + }, + ) + + execution = operations.start(execution_name="manual-unit") + + assert execution.execution_arn == started_arn + assert execution.state == "running" + command = runner.commands[0] + assert command[:5] == ("aws", "--profile", "dander-deploy", "states", "start-execution") + assert command[command.index("--state-machine-arn") + 1] == MACHINE + request = json.loads(command[command.index("--input") + 1]) + assert request == { + "deployment_revision": "manual", + "scheduled_time": "2026-08-08T21:00:00Z", + "scheduler_attempt": 1, + "scheduler_execution_id": "manual:manual-unit", + } + + +def test_latest_status_normalizes_runtime_output_without_provider_noise(tmp_path: Path) -> None: + operations, _runner = _operations( + tmp_path, + { + ("states", "list-executions"): [{"executions": [{"executionArn": EXECUTION}]}], + ("states", "describe-execution"): [ + _describe( + output={ + "status": "succeeded", + "run_id": f"{PIPELINE}:2026-08-08T21:00:00Z", + "task_arn": TASK_ARN, + "container_exit_code": 0, + } + ) + ], + }, + ) + + execution = operations.latest() + + assert execution is not None + assert execution.state == "succeeded" + assert execution.task_arn == TASK_ARN + assert execution.container_exit_code == 0 + assert execution.run_id == f"{PIPELINE}:2026-08-08T21:00:00Z" + + +def test_failed_status_recovers_only_normalized_fields_from_execution_history( + tmp_path: Path, +) -> None: + failure = { + "failure_code": "runtime_permanent_failure", + "run_id": f"{PIPELINE}:2026-08-08T21:00:00Z", + "task_arn": TASK_ARN, + "container_exit_code": 1, + "provider_error": "must not be returned", + } + operations, _runner = _operations( + tmp_path, + { + ("states", "describe-execution"): [_describe(status="FAILED")], + ("states", "get-execution-history"): [ + {"events": [{"stateExitedEventDetails": {"output": json.dumps(failure)}}]} + ], + }, + ) + + execution = operations.describe(EXECUTION) + + assert execution.state == "failed" + assert execution.failure_code == "runtime_permanent_failure" + assert execution.run_id == f"{PIPELINE}:2026-08-08T21:00:00Z" + assert execution.task_arn == TASK_ARN + assert execution.container_exit_code == 1 + assert "provider_error" not in execution.as_dict() + + +def test_logs_find_the_task_in_execution_history_and_use_its_exact_stream(tmp_path: Path) -> None: + operations, runner = _operations( + tmp_path, + { + ("states", "describe-execution"): [_describe(status="RUNNING")], + ("states", "get-execution-history"): [ + { + "events": [ + { + "taskSubmittedEventDetails": { + "output": json.dumps({"Tasks": [{"TaskArn": TASK_ARN}]}) + } + } + ] + } + ], + ("logs", "filter-log-events"): [ + {"events": [{"timestamp": 1234, "message": "runtime started"}]} + ], + }, + ) + + events = operations.logs(EXECUTION, limit=25) + + assert events[0].message == "runtime started" + logs = runner.commands[-1] + assert logs[logs.index("--log-group-name") + 1] == f"/dander/dander/{PIPELINE}" + assert logs[logs.index("--log-stream-name-prefix") + 1] == f"runtime/dander/{TASK_ID}" + assert logs[logs.index("--limit") + 1] == "25" + + +def test_cancel_only_stops_an_owned_running_execution(tmp_path: Path) -> None: + operations, runner = _operations( + tmp_path, + { + ("states", "describe-execution"): [_describe(status="RUNNING")], + ("states", "stop-execution"): [{"stopDate": "2026-08-08T21:00:30Z"}], + }, + ) + + cancelled = operations.cancel(EXECUTION) + + assert cancelled.state == "cancelled" + assert cancelled.failure_code == "operator_cancelled" + assert (runner.commands[-1][3], runner.commands[-1][4]) == ("states", "stop-execution") + + +def test_replay_requires_terminal_execution_and_starts_a_fresh_name(tmp_path: Path) -> None: + replay_arn = MACHINE.replace(":stateMachine:", ":execution:") + ":replay-unit" + operations, runner = _operations( + tmp_path, + { + ("states", "describe-execution"): [_describe()], + ("states", "start-execution"): [ + {"executionArn": replay_arn, "startDate": "2026-08-08T21:02:00Z"} + ], + }, + ) + + replay = operations.replay(EXECUTION, execution_name="replay-unit") + + assert replay.name == "replay-unit" + request = json.loads(runner.commands[-1][runner.commands[-1].index("--input") + 1]) + assert request["scheduler_execution_id"] == "replay:manual-unit" + + +def test_replay_rejects_a_running_execution_without_starting_another(tmp_path: Path) -> None: + operations, runner = _operations( + tmp_path, + {("states", "describe-execution"): [_describe(status="RUNNING")]}, + ) + + with pytest.raises(FargateOperationError, match="terminal"): + operations.replay(EXECUTION) + + assert len(runner.commands) == 1 + + +def test_execution_arn_must_belong_to_the_bound_pipeline(tmp_path: Path) -> None: + operations, runner = _operations(tmp_path, {}) + + with pytest.raises(FargateOperationError, match="selected pipeline"): + operations.describe( + f"arn:aws:states:{REGION}:{ACCOUNT}:execution:other-pipeline:manual-unit" + ) + + assert runner.commands == [] + + +def test_verify_checks_controller_schedule_image_logs_cluster_and_registry(tmp_path: Path) -> None: + operations, _runner = _operations( + tmp_path, + { + ("states", "describe-state-machine"): [{"status": "ACTIVE"}], + ("ecs", "describe-clusters"): [ + { + "clusters": [ + { + "clusterArn": f"arn:aws:ecs:{REGION}:{ACCOUNT}:cluster/dander", + "status": "ACTIVE", + } + ], + "failures": [], + } + ], + ("scheduler", "get-schedule"): [ + { + "Arn": (f"arn:aws:scheduler:{REGION}:{ACCOUNT}:schedule/default/{RESOURCE}"), + "State": "DISABLED", + } + ], + ("logs", "describe-log-groups"): [ + {"logGroups": [{"logGroupName": f"/dander/dander/{PIPELINE}"}]} + ], + ("ecs", "describe-task-definition"): [ + { + "taskDefinition": { + "taskDefinitionArn": ( + f"arn:aws:ecs:{REGION}:{ACCOUNT}:task-definition/{RESOURCE}:1" + ), + "status": "ACTIVE", + "containerDefinitions": [ + { + "image": IMAGE, + "readonlyRootFilesystem": True, + "user": "65532:65532", + } + ], + } + } + ], + ("ecr", "describe-repositories"): [ + { + "repositories": [ + { + "repositoryArn": (f"arn:aws:ecr:{REGION}:{ACCOUNT}:repository/dander"), + "imageTagMutability": "IMMUTABLE", + "encryptionConfiguration": {"encryptionType": "KMS"}, + "imageScanningConfiguration": {"scanOnPush": True}, + } + ] + } + ], + }, + ) + + result = operations.verify(expected_image=IMAGE) + + assert result.image == IMAGE + assert result.schedule_state == "DISABLED" + assert result.state_machine == MACHINE + + +def test_verify_fails_when_schedule_state_differs_from_manifest(tmp_path: Path) -> None: + operations, _runner = _operations( + tmp_path, + { + ("states", "describe-state-machine"): [{"status": "ACTIVE"}], + ("ecs", "describe-clusters"): [ + {"clusters": [{"clusterArn": "cluster", "status": "ACTIVE"}], "failures": []} + ], + ("scheduler", "get-schedule"): [{"Arn": "schedule", "State": "ENABLED"}], + }, + ) + + with pytest.raises(FargateOperationError, match="schedule"): + operations.verify(expected_image=IMAGE) diff --git a/tickets/DANDER-95-fargate-operations.md b/tickets/DANDER-95-fargate-operations.md new file mode 100644 index 00000000..3c0e90df --- /dev/null +++ b/tickets/DANDER-95-fargate-operations.md @@ -0,0 +1,39 @@ +--- +id: DANDER-95 +title: Operate and verify manifest-bound Fargate pipelines +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-94] +created: 2026-08-08 +--- + +## Context + +The packaged Fargate controller had plan/apply commands but no provider-native operator surface. +Operators need exact, manifest-bound start, status, logs, cancellation, replay, and verification +without copying AWS resource identifiers by hand or exposing provider response details. + +## Acceptance Criteria + +- [x] Resolve one exact Fargate deployment and pipeline from the project manifest. +- [x] Start and replay only after explicit CLI confirmation. +- [x] Normalize execution state and correlate CloudWatch logs to the exact ECS task. +- [x] Reject execution ARNs that do not belong to the selected pipeline. +- [x] Cancel only running controller executions. +- [x] Verify the controller, cluster, schedule, task image, log group, and immutable ECR policy. +- [x] Scope deployment-role operation permissions to Dander state machines, executions, and logs. +- [x] Keep Fargate unsupported until source-free live acceptance passes. +- [ ] Protected CI passes. + +## Design + +Use the AWS CLI under the existing short-lived deployment profile. Derive every resource name from +the validated manifest and the same deterministic naming rule as Terraform. Return small sanitized +records rather than unrestricted AWS responses. Step Functions remains the lifecycle authority; +replay starts a fresh execution at Dander's inclusive watermark boundary. + +## Review Log + +No AWS or GCP apply is part of this slice. A credentialed AWS stage-zero plan and retained GCP +no-drift plan are required before merge. From 08ce0a0e60752b743c1071a8c7b063154b16f76c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 18:21:30 -0400 Subject: [PATCH 046/163] docs: record Fargate operations validation --- HANDOFF.md | 3 +-- tickets/DANDER-95-fargate-operations.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 5523b62b..b4585098 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -18,7 +18,7 @@ AWS deployment-role profile; mutating commands require confirmation. - Ruff, formatting, strict typing, and all 989 Python tests passed. - All Terraform validation and both provider-mocked AWS plans passed. - Wheel/sdist inspection, source-free installs, full runtime dependencies, and container contract passed. -- Dependency audit and Terraform HIGH/CRITICAL scan passed; protected image and Git secret scans remain pending. +- Protected Python, Terraform, distribution, container, image, and Git secret checks all passed. - AWS stage-zero plan: 12 creates, 0 updates, 0 deletes. Retained GCP plan: exactly `No changes.` ## Decisions @@ -29,7 +29,6 @@ AWS deployment-role profile; mutating commands require confirmation. ## Remaining -- Let protected CI repeat Linux, image, and Git-history security checks. - Merge the focused PR if protected CI remains clean. - Obtain separate approval before applying the 12-resource AWS stage-zero plan. - Complete the Fargate live comparison before claiming AWS support. diff --git a/tickets/DANDER-95-fargate-operations.md b/tickets/DANDER-95-fargate-operations.md index 3c0e90df..cc0a9c6e 100644 --- a/tickets/DANDER-95-fargate-operations.md +++ b/tickets/DANDER-95-fargate-operations.md @@ -24,7 +24,7 @@ without copying AWS resource identifiers by hand or exposing provider response d - [x] Verify the controller, cluster, schedule, task image, log group, and immutable ECR policy. - [x] Scope deployment-role operation permissions to Dander state machines, executions, and logs. - [x] Keep Fargate unsupported until source-free live acceptance passes. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design From 92f7873c71874f67b56b7d210fb54523c4085f52 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 18:50:06 -0400 Subject: [PATCH 047/163] feat: add PostgreSQL durable state --- .github/workflows/ci.yml | 17 +- HANDOFF.md | 44 +- docs/decisions.md | 15 +- docs/known-limitations.md | 7 +- docs/platform-profiles.md | 46 +- pyproject.toml | 6 +- src/dander/cli/run_command.py | 18 +- src/dander/project/config.py | 12 +- src/dander/project/portable_config.py | 11 +- src/dander/providers/defaults.py | 9 + src/dander/providers/postgresql/__init__.py | 5 + src/dander/providers/postgresql/config.py | 43 ++ src/dander/providers/postgresql/state.py | 633 ++++++++++++++++++ tests/cli/test_run_command.py | 15 + tests/project/test_portable_config.py | 33 + .../test_postgresql_state_runtime.py | 384 +++++++++++ tickets/DANDER-96-postgresql-state.md | 44 ++ uv.lock | 53 +- 18 files changed, 1350 insertions(+), 45 deletions(-) create mode 100644 src/dander/providers/postgresql/__init__.py create mode 100644 src/dander/providers/postgresql/config.py create mode 100644 src/dander/providers/postgresql/state.py create mode 100644 tests/providers/test_postgresql_state_runtime.py create mode 100644 tickets/DANDER-96-postgresql-state.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9689cbdf..4c1ea061 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,13 +17,28 @@ jobs: python-quality: name: Python quality runs-on: ubuntu-latest + env: + DANDER_TEST_POSTGRES_DSN: postgresql://postgres:dander_test@127.0.0.1:5432/dander_test + services: + postgres: + image: postgres:15-alpine@sha256:3d0f7584ed7d04e27fa050d6683a74746608faf21f202be78460d679cc56461f + env: + POSTGRES_PASSWORD: dander_test + POSTGRES_DB: dander_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d dander_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2 - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 with: enable-cache: true - name: Install locked environment - run: uv sync --frozen --extra dev + run: uv sync --frozen --extra dev --extra postgres - name: Ruff lint run: uv run ruff check . - name: Ruff format diff --git a/HANDOFF.md b/HANDOFF.md index b4585098..c8ea2e6e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,40 @@ ## Finished -- Added manifest-bound Fargate run, status, logs, cancel, replay, and verify commands. -- Correlated exact Step Functions executions, ECS tasks, and `runtime/dander/...` log streams. -- Normalized successful and failed controller results without returning unrestricted AWS payloads. -- Scoped the stage-zero deployment role to Dander controllers, executions, and task logs. -- Updated AWS operator documentation, decisions, limitations, tests, and DANDER-95. +- Added a lazy PostgreSQL 15+ durable-state provider selected from version 2 platform profiles. +- Implemented versioned migrations, server-time leases, monotonic fences, watermark CAS, history, and JSONB metadata. +- Added bounded pooling, terminal-history retention, interruption reconciliation, and fail-closed BigQuery pairing. +- Added live PostgreSQL CI conformance for contention, migration rollback, connection loss, and pool exhaustion. +- Kept version 1 and BigQuery state behavior unchanged. ## Try It -Run `dander aws --help`. The commands require a validated Fargate deployment and the short-lived -AWS deployment-role profile; mutating commands require confirmation. +Install `dander-platform[postgres]`, set the manifest's `state.provider: postgresql`, and inject the +DSN through its configured environment variable. See `docs/platform-profiles.md`. Runtime execution +with BigQuery intentionally stops until the next destination-fence ticket is merged. ## Checks -- Ruff, formatting, strict typing, and all 989 Python tests passed. -- All Terraform validation and both provider-mocked AWS plans passed. -- Wheel/sdist inspection, source-free installs, full runtime dependencies, and container contract passed. -- Protected Python, Terraform, distribution, container, image, and Git secret checks all passed. -- AWS stage-zero plan: 12 creates, 0 updates, 0 deletes. Retained GCP plan: exactly `No changes.` +- Ruff, formatting, strict mypy, and all 998 Python tests passed. +- Seven live PostgreSQL 15 provider tests passed; manual container restart recovery passed. +- Terraform formatting, GCP/AWS validation, and provider-mocked AWS tests passed. +- Wheel/sdist inspection, source-free validation, PostgreSQL-extra installation, and OCI build passed. +- Linux OCI uses system `libpq`; non-Linux PostgreSQL installs use Psycopg's binary runtime. ## Decisions -- The manifest, not operator-supplied AWS identifiers, owns every Fargate resource binding. -- Step Functions remains the lifecycle authority; failed status reads history only for allow-listed fields. -- Fargate remains unsupported until the source-free keyless live proof passes. +- Manifests retain only the DSN environment-variable name; credentials are runtime-injected. +- Interrupted history is exempt from terminal retention. +- PostgreSQL state does not weaken publication fencing while the cross-backend protocol is pending. ## Remaining -- Merge the focused PR if protected CI remains clean. -- Obtain separate approval before applying the 12-resource AWS stage-zero plan. -- Complete the Fargate live comparison before claiming AWS support. -- Continue the portability roadmap with PostgreSQL state and warehouse. +- Run protected CI and merge the focused PostgreSQL-state PR if clean. +- Implement the generic destination target-fence protocol as a separate PR. +- Implement PostgreSQL warehouse capabilities, then the Kubernetes launcher and live profile. ## Review First -- `src/dander/providers/fargate/operations.py` -- `src/dander/cli/aws_command.py` -- `infra/aws/bootstrap-admin/main.tf` +- `src/dander/providers/postgresql/state.py` +- `tests/providers/test_postgresql_state_runtime.py` +- `src/dander/project/portable_config.py` diff --git a/docs/decisions.md b/docs/decisions.md index 99e19ecf..9ec6cc3f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -87,6 +87,16 @@ - **Compatibility:** Existing table identities, per-pipeline lease tables, server-time leases, fencing, watermark CAS, and interrupted-run reconciliation remain unchanged. +## 2026-08-08 — PostgreSQL is the second durable-state implementation + +- **Connection boundary:** Version 2 profiles store only an environment-variable name; the + PostgreSQL connection string is runtime-injected and never serialized into project manifests. +- **Correctness:** PostgreSQL 15+ uses one bounded pool, short transactions, server-time leases, + monotonic fencing tokens, atomic cursor CAS, sanitized history, deterministic JSONB snapshots, + and an advisory-locked version ledger that rejects newer schemas. +- **Qualification:** The adapter passes live local contention and exhaustion tests, but it does + not make a PostgreSQL/Kubernetes profile supported before warehouse and live-profile gates pass. + ## 2026-08-08 — BigQuery enters portability through one composed runtime - **Selection:** Version 1 projects retain an implicit BigQuery warehouse; version 2 resolution @@ -103,8 +113,9 @@ empty while Oracle's SDK requires a cryptography version with a known fixed-in-50 advisory. - **Release image:** `runtime-all` is their checked union; repository and generated Dockerfiles install and validate it so one immutable multi-platform image has deterministic dependencies. - PostgreSQL uses pure-Python Psycopg with Debian's maintained `libpq5`, avoiding a bundled wheel - whose embedded native-library SBOM fails the release image scan. + Linux uses pure-Python Psycopg with Debian's maintained `libpq5`, avoiding a bundled wheel whose + embedded native-library SBOM fails the release image scan. Non-Linux extras use Psycopg's binary + package so local provider installation does not require a separately linked `libpq`. - **Support boundary:** `runtime-capabilities.json`, concrete adapter conformance, and live profile gates remain authoritative. A present SDK or package extra never makes a provider supported. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 02fffbd9..48c522ce 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,8 +10,11 @@ evaluate these limits before using it for an unattended system containing busine task-role session into process memory and therefore caps a Fargate projection at one hour; renewable credentials and live parity remain later gates. - Provider package extras and the full runtime image contain planned provider SDK dependencies, - but do not by themselves implement or qualify Snowflake, Redshift, PostgreSQL, AWS, Azure, or - OCI profiles. The packaged capability manifest is the support boundary. + but do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. PostgreSQL + durable state is implemented and locally conformance-tested, but no PostgreSQL warehouse or + Kubernetes live profile is qualified yet. BigQuery execution with PostgreSQL state fails closed + until cross-backend destination fencing is enabled. The packaged capability manifest is the + support boundary. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 5e91b1d1..ec920b3d 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -7,8 +7,10 @@ Dander version 2 separates reusable pipeline intent from environment-specific de plus runtime limits, schedules, secret references, and provider resource names. The first supported hosted profile remains BigQuery, BigQuery state, Dataplex (or no cloud -catalog), GCP Secret Manager, and Cloud Run. A provider name outside the installed and -supported contract fails validation; configuration does not imply provider support. +catalog), GCP Secret Manager, and Cloud Run. PostgreSQL 15 or newer is implemented as an +alternative durable-state backend for version 2 projects, but the complete PostgreSQL/Kubernetes +profile is not yet live-qualified. A provider name outside the installed and supported contract +fails validation; configuration alone does not imply profile support. One logical project may have multiple named platforms and deployments. `dander validate` accepts `--deployment`; the Python configuration loader accepts the same explicit selection. Other current @@ -62,3 +64,43 @@ Provider SDKs can be installed independently with extras such as `dander-platform[runtime-all]` so one immutable image can later serve any first-class adapter. Installing an extra does not make its provider selectable; configuration remains gated by the registered adapter and capability manifest. + +## PostgreSQL durable state + +Select PostgreSQL only in `dander.platforms.yaml`; keep the connection string out of Git: + +```yaml +platforms: + portable: + warehouse: + provider: bigquery + location: US + state: + provider: postgresql + dsn_env: DANDER_POSTGRES_DSN + schema_name: dander_meta + pool_min_size: 1 + pool_max_size: 5 + pool_timeout_seconds: 10 + lease_seconds: 120 + terminal_history_retention_days: 90 + catalog: + provider: none + secrets: + provider: environment +``` + +Install `dander-platform[postgres]` for a provider-specific environment, or use the official +`runtime-all` image. Inject `DANDER_POSTGRES_DSN` through the launcher's existing secret binding; +the value is never part of either manifest. Dander creates and migrates only the configured schema. +It uses a bounded pool, PostgreSQL server time for leases, atomic watermark comparison, sanitized +run history, and deterministic JSONB metadata snapshots. + +The current BigQuery warehouse/PostgreSQL state combination fails closed at execution because its +cross-backend destination fence is the next portability ticket. The state adapter is available for +conformance and composition work now; it is not a shortcut around that publication boundary. + +Terminal `succeeded`, `failed`, and `skipped` history older than the configured retention is +removed when migrations run. Active runs and `interrupted_run` records are retained. Dander +refuses a state ledger newer than the running package. Changing state backends is not an online +migration: keep schedules paused until the destination-fence and cutover workflow is complete. diff --git a/pyproject.toml b/pyproject.toml index da0bd911..192e658f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,8 @@ redshift = [ "redshift-connector>=2.1,<3", ] postgres = [ - "psycopg[pool]>=3.2,<4", + "psycopg[pool]>=3.2,<4; sys_platform == 'linux'", + "psycopg[binary,pool]>=3.2,<4; sys_platform != 'linux'", ] gcp = [ "google-auth>=2.35", @@ -87,7 +88,8 @@ runtime-all = [ "google-cloud-bigquery-storage>=2.30", "google-cloud-dataplex>=2.20,<3", "google-cloud-secret-manager>=2.20", - "psycopg[pool]>=3.2,<4", + "psycopg[pool]>=3.2,<4; sys_platform == 'linux'", + "psycopg[binary,pool]>=3.2,<4; sys_platform != 'linux'", "redshift-connector>=2.1,<3", "snowflake-connector-python>=3.12,<5", ] diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 5c7d2f96..47d5fb12 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -113,6 +113,7 @@ class _ResolvedRun: warehouse_provider: str warehouse_location: str state_provider: str + state_config: dict[str, object] catalog_provider: str secret_provider: str @@ -169,6 +170,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_provider = "bigquery" warehouse_location = "US" state_provider = "bigquery" + state_config: dict[str, object] = {"provider": "bigquery"} catalog_provider = "dataplex" secret_provider = "gcp_secret_manager" plugin_registry: ConnectorPluginRegistry | None = None @@ -196,6 +198,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_provider = manifest.warehouse_provider warehouse_location = manifest.platform.bigquery_location state_provider = manifest.state_provider + state_config = manifest.state_config catalog_provider = ( "dataplex" if options.publish_dataplex else manifest.catalog_provider ) @@ -245,6 +248,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: warehouse_provider=warehouse_provider, warehouse_location=warehouse_location, state_provider=state_provider, + state_config=state_config, catalog_provider=catalog_provider, secret_provider=secret_provider, ) @@ -422,11 +426,15 @@ def _build_warehouse_runtime(resolved: _ResolvedRun) -> WarehouseRuntime: def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: + _require_supported_state_pair( + state_provider=resolved.state_provider, + warehouse_provider=resolved.warehouse_provider, + ) registry = default_provider_registry() try: config = registry.parse( ProviderKind.STATE, - {"provider": resolved.state_provider}, + resolved.state_config, ) runtime = registry.build( ProviderKind.STATE, @@ -446,6 +454,14 @@ def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: return runtime +def _require_supported_state_pair(*, state_provider: str, warehouse_provider: str) -> None: + if state_provider == "postgresql" and warehouse_provider == "bigquery": + raise ClickException( + "PostgreSQL state with BigQuery execution is not available until the " + "cross-backend destination fence is enabled" + ) + + def build_source_adapter( config: SourceConfig, auth: AuthStrategy, diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 54c92abb..71b1c95e 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -20,6 +20,12 @@ _SERVICE_ACCOUNT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") _RUNTIME_MEMORY = re.compile(r"^[1-9][0-9]*(?:Mi|Gi)$") _PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") + + +def _default_state_config() -> dict[str, object]: + return {"provider": "bigquery"} + + _DISTRIBUTION = re.compile(r"^[A-Za-z0-9]+(?:[-_.][A-Za-z0-9]+)*$") if TYPE_CHECKING: @@ -204,7 +210,11 @@ class DanderProject(BaseModel): platform_name: str = Field(default="gcp", exclude=True) deployment_name: str = Field(default="gcp_cloud_run", exclude=True) warehouse_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) - state_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) + state_provider: Literal["bigquery", "postgresql"] = Field(default="bigquery", exclude=True) + state_config: dict[str, object] = Field( + default_factory=_default_state_config, + exclude=True, + ) catalog_provider: Literal["dataplex", "none"] = Field(default="dataplex", exclude=True) secret_provider: Literal["gcp_secret_manager", "environment"] = Field( default="gcp_secret_manager", diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 4839c248..10249240 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -30,6 +30,7 @@ from dander.providers.fargate import FargateLauncherConfig # noqa: TC001 from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 +from dander.providers.postgresql import PostgreSQLStateConfig # noqa: TC001 _NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") _PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") @@ -131,13 +132,19 @@ def validate_pipeline_ids( ] +StateSpec = Annotated[ + BigQueryStateConfig | PostgreSQLStateConfig, + Field(discriminator="provider"), +] + + class PlatformProfileSpec(BaseModel): """One named combination of data-plane provider selections.""" model_config = ConfigDict(extra="forbid") warehouse: BigQueryWarehouseConfig - state: BigQueryStateConfig + state: StateSpec catalog: CatalogSpec secrets: SecretProviderSpec @@ -351,6 +358,7 @@ def _resolve( deployment_name=selected_name, warehouse_provider=profile.warehouse.provider, state_provider=profile.state.provider, + state_config=profile.state.model_dump(mode="json"), catalog_provider=profile.catalog.provider, secret_provider=profile.secrets.provider, launcher_provider=selected.launcher.provider, @@ -443,6 +451,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: legacy.platform == migrated.platform and legacy.warehouse_provider == migrated.warehouse_provider and legacy.state_provider == migrated.state_provider + and legacy.state_config == migrated.state_config and legacy.catalog_provider == migrated.catalog_provider and legacy.secret_provider == migrated.secret_provider and legacy.launcher_provider == migrated.launcher_provider diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 2d1b1c4a..51cc044d 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -8,6 +8,7 @@ from dander.providers.fargate import FargateLauncherConfig from dander.providers.gcp_secret_manager import GcpSecretManagerConfig from dander.providers.no_catalog import NoCatalogConfig +from dander.providers.postgresql import PostgreSQLStateConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -30,6 +31,14 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.bigquery.state:BIGQUERY_STATE_FACTORY" ), ) + registry.register( + kind=ProviderKind.STATE, + provider_id="postgresql", + config_model=PostgreSQLStateConfig, + load_factory=lazy_provider_factory( + "dander.providers.postgresql.state:POSTGRESQL_STATE_FACTORY" + ), + ) registry.register( kind=ProviderKind.CATALOG, provider_id="dataplex", diff --git a/src/dander/providers/postgresql/__init__.py b/src/dander/providers/postgresql/__init__.py new file mode 100644 index 00000000..91de5e2d --- /dev/null +++ b/src/dander/providers/postgresql/__init__.py @@ -0,0 +1,5 @@ +"""PostgreSQL provider configuration exports.""" + +from dander.providers.postgresql.config import PostgreSQLStateConfig + +__all__ = ["PostgreSQLStateConfig"] diff --git a/src/dander/providers/postgresql/config.py b/src/dander/providers/postgresql/config.py new file mode 100644 index 00000000..74e9a142 --- /dev/null +++ b/src/dander/providers/postgresql/config.py @@ -0,0 +1,43 @@ +"""Dependency-light PostgreSQL provider configuration.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +_ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]{0,126}$") +_SCHEMA_NAME = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") + + +class PostgreSQLStateConfig(BaseModel): + """PostgreSQL durable-state connection and bounded-pool settings. + + The manifest stores only the name of an environment variable. The connection + string itself remains runtime-injected secret material. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["postgresql"] + dsn_env: str = "DANDER_POSTGRES_DSN" + schema_name: str = "dander_meta" + pool_min_size: int = Field(default=1, ge=1, le=20) + pool_max_size: int = Field(default=5, ge=1, le=50) + pool_timeout_seconds: float = Field(default=10.0, gt=0, le=120) + lease_seconds: int = Field(default=120, ge=10, le=3600) + terminal_history_retention_days: int = Field(default=90, ge=1, le=3650) + + @model_validator(mode="after") + def validate_settings(self) -> PostgreSQLStateConfig: + if not _ENVIRONMENT_NAME.fullmatch(self.dsn_env): + raise ValueError("dsn_env must be an uppercase environment-variable name") + if not _SCHEMA_NAME.fullmatch(self.schema_name): + raise ValueError("schema_name must be a safe lowercase PostgreSQL identifier") + if self.pool_min_size > self.pool_max_size: + raise ValueError("pool_min_size must not exceed pool_max_size") + return self + + +__all__ = ["PostgreSQLStateConfig"] diff --git a/src/dander/providers/postgresql/state.py b/src/dander/providers/postgresql/state.py new file mode 100644 index 00000000..37f67ced --- /dev/null +++ b/src/dander/providers/postgresql/state.py @@ -0,0 +1,633 @@ +"""PostgreSQL durable-state runtime and versioned schema migrations.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime +from typing import TYPE_CHECKING, Any, cast + +from psycopg import Connection, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.catalog import MetadataSnapshot, MetadataStore +from dander.providers.postgresql.config import PostgreSQLStateConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.state import ( + LeaseHandle, + LeaseStore, + RunHistoryStore, + RunRecord, + RunStage, + RunStatus, + StateCapabilities, + StateMigration, + StateRuntime, + WatermarkStore, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from pydantic import BaseModel + + from dander.concurrency import FencingToken + +_STATE_SCHEMA_VERSION = 1 +_MIGRATIONS = (StateMigration(version=1, name="initial_state_schema"),) +_INTERRUPTED_SUMMARY = "A later execution acquired the expired pipeline lease." + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +class PostgreSQLStateDatabase: + """Shared bounded connection pool and safe state-schema identifiers.""" + + def __init__(self, *, pool: PostgreSQLPool, schema_name: str) -> None: + self.pool = pool + self.schema_name = schema_name + + def relation(self, name: str) -> sql.Identifier: + return sql.Identifier(self.schema_name, name) + + +class PostgreSQLStateMigrator: + """Apply the PostgreSQL state schema atomically and reject newer ledgers.""" + + def __init__( + self, + database: PostgreSQLStateDatabase, + *, + terminal_history_retention_days: int, + ) -> None: + self._database = database + self._retention_days = terminal_history_retention_days + + @property + def migrations(self) -> tuple[StateMigration, ...]: + return _MIGRATIONS + + def current_version(self) -> int: + with self._database.pool.connection() as connection, connection.transaction(): + self._ensure_ledger(connection) + version = self._read_version(connection) + return version + + def migrate(self) -> int: + with self._database.pool.connection() as connection, connection.transaction(): + self._ensure_ledger(connection) + connection.execute( + "SELECT pg_advisory_xact_lock(hashtext(%s))", + (f"dander-state:{self._database.schema_name}",), + ) + version = self._read_version(connection) + if version < _STATE_SCHEMA_VERSION: + self._apply_initial_schema(connection) + connection.execute( + sql.SQL( + "INSERT INTO {} (version, name, applied_at) " + "VALUES (%s, %s, clock_timestamp()) ON CONFLICT (version) DO NOTHING" + ).format(self._database.relation("dander_schema_migrations")), + (_MIGRATIONS[0].version, _MIGRATIONS[0].name), + ) + self._prune_terminal_history(connection) + return _STATE_SCHEMA_VERSION + + def _ensure_ledger(self, connection: Connection[PostgreSQLRow]) -> None: + connection.execute( + sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format( + sql.Identifier(self._database.schema_name) + ) + ) + connection.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS {} (" + "version INTEGER PRIMARY KEY CHECK (version > 0), " + "name TEXT NOT NULL UNIQUE, applied_at TIMESTAMPTZ NOT NULL)" + ).format(self._database.relation("dander_schema_migrations")) + ) + + def _read_version(self, connection: Connection[PostgreSQLRow]) -> int: + row = connection.execute( + sql.SQL("SELECT COALESCE(MAX(version), 0) AS version FROM {}").format( + self._database.relation("dander_schema_migrations") + ) + ).fetchone() + if row is None: + raise RuntimeError("PostgreSQL state migration ledger returned no result") + value = row["version"] + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError("PostgreSQL state migration ledger contains an invalid version") + if value > _STATE_SCHEMA_VERSION: + raise RuntimeError( + f"PostgreSQL state schema version {value} is newer than this Dander runtime" + ) + return value + + def _apply_initial_schema(self, connection: Connection[PostgreSQLRow]) -> None: + relation = self._database.relation + statements = ( + sql.SQL( + "CREATE TABLE IF NOT EXISTS {} (" + "pipeline_id TEXT PRIMARY KEY, run_id TEXT, " + "fencing_token BIGINT NOT NULL DEFAULT 0 CHECK (fencing_token >= 0), " + "heartbeat_at TIMESTAMPTZ, " + "lease_expires_at TIMESTAMPTZ NOT NULL DEFAULT to_timestamp(0))" + ).format(relation("dander_pipeline_leases")), + sql.SQL( + "CREATE TABLE IF NOT EXISTS {} (" + "source_name TEXT NOT NULL, entity_name TEXT NOT NULL, " + "last_cursor TEXT NOT NULL, updated_at TIMESTAMPTZ NOT NULL, " + "PRIMARY KEY (source_name, entity_name))" + ).format(relation("dander_watermarks")), + sql.SQL( + "CREATE TABLE IF NOT EXISTS {} (" + "run_id TEXT PRIMARY KEY, pipeline_id TEXT NOT NULL, source_name TEXT NOT NULL, " + "status TEXT NOT NULL, stage TEXT NOT NULL, started_at TIMESTAMPTZ NOT NULL, " + "finished_at TIMESTAMPTZ, endpoints BIGINT NOT NULL DEFAULT 0, " + "extracted BIGINT NOT NULL DEFAULT 0, affected BIGINT NOT NULL DEFAULT 0, " + "models BIGINT NOT NULL DEFAULT 0, assertions BIGINT NOT NULL DEFAULT 0, " + "assets BIGINT NOT NULL DEFAULT 0, failure_stage TEXT, failure_code TEXT, " + "failure_summary VARCHAR(512))" + ).format(relation("dander_runs")), + sql.SQL( + "CREATE INDEX IF NOT EXISTS {} ON {} (pipeline_id, status, started_at DESC)" + ).format( + sql.Identifier("dander_runs_pipeline_status_started_idx"), + relation("dander_runs"), + ), + sql.SQL( + "CREATE TABLE IF NOT EXISTS {} (" + "pipeline_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, " + "manifest_json JSONB NOT NULL, updated_at TIMESTAMPTZ NOT NULL)" + ).format(relation("dander_metadata_snapshots")), + ) + for statement in statements: + connection.execute(statement) + + def _prune_terminal_history(self, connection: Connection[PostgreSQLRow]) -> None: + connection.execute( + sql.SQL( + "DELETE FROM {} WHERE status IN ('succeeded', 'failed', 'skipped') " + "AND failure_code IS DISTINCT FROM 'interrupted_run' " + "AND finished_at < clock_timestamp() - (%s * INTERVAL '1 day')" + ).format(self._database.relation("dander_runs")), + (self._retention_days,), + ) + + +class PostgreSQLLeaseStore(LeaseStore): + """Coordinate pipeline ownership using PostgreSQL server time and row locks.""" + + def __init__(self, database: PostgreSQLStateDatabase, *, lease_seconds: int) -> None: + self._database = database + self._lease_seconds = lease_seconds + + @property + def lease_seconds(self) -> int: + return self._lease_seconds + + def acquire(self, pipeline_id: str, run_id: str) -> LeaseHandle | None: + table = self._database.relation("dander_pipeline_leases") + with self._database.pool.connection() as connection, connection.transaction(): + connection.execute( + sql.SQL( + "INSERT INTO {} (pipeline_id, run_id, fencing_token, " + "heartbeat_at, lease_expires_at) " + "VALUES (%s, NULL, 0, NULL, to_timestamp(0)) " + "ON CONFLICT (pipeline_id) DO NOTHING" + ).format(table), + (pipeline_id,), + ) + row = connection.execute( + sql.SQL( + "UPDATE {} SET run_id = %s, fencing_token = fencing_token + 1, " + "heartbeat_at = clock_timestamp(), " + "lease_expires_at = clock_timestamp() + (%s * INTERVAL '1 second') " + "WHERE pipeline_id = %s AND lease_expires_at <= clock_timestamp() " + "RETURNING fencing_token" + ).format(table), + (run_id, self._lease_seconds, pipeline_id), + ).fetchone() + if row is None: + return None + token = row["fencing_token"] + if isinstance(token, bool) or not isinstance(token, int) or token <= 0: + raise RuntimeError("PostgreSQL lease returned an invalid fencing token") + return LeaseHandle( + pipeline_id=pipeline_id, + run_id=run_id, + fencing_token=token, + lease_seconds=self._lease_seconds, + ) + + def heartbeat(self, lease: LeaseHandle) -> bool: + with self._database.pool.connection() as connection: + cursor = connection.execute( + sql.SQL( + "UPDATE {} SET heartbeat_at = clock_timestamp(), " + "lease_expires_at = clock_timestamp() + (%s * INTERVAL '1 second') " + "WHERE pipeline_id = %s AND run_id = %s AND fencing_token = %s " + "AND lease_expires_at > clock_timestamp()" + ).format(self._database.relation("dander_pipeline_leases")), + ( + self._lease_seconds, + lease.pipeline_id, + lease.run_id, + lease.fencing_token, + ), + ) + return cursor.rowcount == 1 + + def release(self, lease: LeaseHandle) -> bool: + with self._database.pool.connection() as connection: + cursor = connection.execute( + sql.SQL( + "UPDATE {} SET run_id = NULL, heartbeat_at = clock_timestamp(), " + "lease_expires_at = clock_timestamp() " + "WHERE pipeline_id = %s AND run_id = %s AND fencing_token = %s" + ).format(self._database.relation("dander_pipeline_leases")), + (lease.pipeline_id, lease.run_id, lease.fencing_token), + ) + return cursor.rowcount == 1 + + +class PostgreSQLWatermarkStore(WatermarkStore): + """Persist cursors with atomic expected-boundary comparison.""" + + def __init__(self, database: PostgreSQLStateDatabase) -> None: + self._database = database + + def get(self, source: str, entity: str) -> str | None: + with self._database.pool.connection() as connection: + row = connection.execute( + sql.SQL( + "SELECT last_cursor FROM {} WHERE source_name = %s AND entity_name = %s" + ).format(self._database.relation("dander_watermarks")), + (source, entity), + ).fetchone() + return None if row is None else str(row["last_cursor"]) + + def set(self, source: str, entity: str, cursor: str) -> None: + with self._database.pool.connection() as connection: + connection.execute( + sql.SQL( + "INSERT INTO {} (source_name, entity_name, last_cursor, updated_at) " + "VALUES (%s, %s, %s, clock_timestamp()) " + "ON CONFLICT (source_name, entity_name) DO UPDATE SET " + "last_cursor = EXCLUDED.last_cursor, updated_at = clock_timestamp()" + ).format(self._database.relation("dander_watermarks")), + (source, entity, cursor), + ) + + def compare_and_set( + self, + source: str, + entity: str, + *, + expected: str | None, + cursor: str, + fence: FencingToken | None = None, + ) -> bool: + del fence # Destination fencing is provider-neutral and handled at publication. + table = self._database.relation("dander_watermarks") + with self._database.pool.connection() as connection, connection.transaction(): + if expected is None: + result = connection.execute( + sql.SQL( + "INSERT INTO {} (source_name, entity_name, last_cursor, updated_at) " + "VALUES (%s, %s, %s, clock_timestamp()) " + "ON CONFLICT (source_name, entity_name) DO NOTHING" + ).format(table), + (source, entity, cursor), + ) + else: + result = connection.execute( + sql.SQL( + "UPDATE {} SET last_cursor = %s, updated_at = clock_timestamp() " + "WHERE source_name = %s AND entity_name = %s " + "AND last_cursor IS NOT DISTINCT FROM %s" + ).format(table), + (cursor, source, entity, expected), + ) + return result.rowcount == 1 + + +class PostgreSQLRunHistoryStore(RunHistoryStore): + """Persist sanitized pipeline lifecycle summaries in PostgreSQL.""" + + def __init__(self, database: PostgreSQLStateDatabase) -> None: + self._database = database + + def start(self, run_id: str, source: str, *, pipeline_id: str | None = None) -> None: + with self._database.pool.connection() as connection: + connection.execute( + sql.SQL( + "INSERT INTO {} (run_id, pipeline_id, source_name, status, stage, started_at) " + "VALUES (%s, %s, %s, 'running', 'ingest', clock_timestamp())" + ).format(self._database.relation("dander_runs")), + (run_id, pipeline_id or source, source), + ) + + def checkpoint( + self, + run_id: str, + stage: RunStage, + *, + endpoints: int = 0, + extracted: int = 0, + affected: int = 0, + models: int = 0, + assertions: int = 0, + assets: int = 0, + ) -> None: + self._update( + run_id, + status=None, + stage=stage, + endpoints=endpoints, + extracted=extracted, + affected=affected, + models=models, + assertions=assertions, + assets=assets, + failure_stage=None, + ) + + def finish( + self, + run_id: str, + status: RunStatus, + *, + endpoints: int, + extracted: int, + affected: int, + models: int = 0, + assertions: int = 0, + assets: int = 0, + failure_stage: RunStage | None = None, + failure_code: str | None = None, + failure_summary: str | None = None, + ) -> None: + self._update( + run_id, + status=status, + stage=( + RunStage.COMPLETE + if status in {RunStatus.SUCCEEDED, RunStatus.SKIPPED} + else failure_stage or RunStage.INGEST + ), + endpoints=endpoints, + extracted=extracted, + affected=affected, + models=models, + assertions=assertions, + assets=assets, + failure_stage=failure_stage, + failure_code=failure_code, + failure_summary=failure_summary, + ) + + def _update( + self, + run_id: str, + *, + status: RunStatus | None, + stage: RunStage, + endpoints: int, + extracted: int, + affected: int, + models: int, + assertions: int, + assets: int, + failure_stage: RunStage | None, + failure_code: str | None = None, + failure_summary: str | None = None, + ) -> None: + terminal = status is not None + with self._database.pool.connection() as connection: + connection.execute( + sql.SQL( + "UPDATE {} SET status = %s, stage = %s, " + "finished_at = CASE WHEN %s THEN clock_timestamp() ELSE finished_at END, " + "endpoints = %s, extracted = %s, affected = %s, models = %s, " + "assertions = %s, assets = %s, failure_stage = %s, " + "failure_code = %s, failure_summary = %s WHERE run_id = %s" + ).format(self._database.relation("dander_runs")), + ( + status.value if status is not None else RunStatus.RUNNING.value, + stage.value, + terminal, + endpoints, + extracted, + affected, + models, + assertions, + assets, + failure_stage.value if failure_stage is not None else None, + failure_code, + failure_summary, + run_id, + ), + ) + + def recent( + self, + *, + limit: int = 20, + pipeline_id: str | None = None, + ) -> tuple[RunRecord, ...]: + if not 1 <= limit <= 1000: + raise ValueError("limit must be between 1 and 1000") + where = sql.SQL("") + parameters: list[object] = [] + if pipeline_id is not None: + where = sql.SQL(" WHERE pipeline_id = %s") + parameters.append(pipeline_id) + parameters.append(limit) + with self._database.pool.connection() as connection: + rows = connection.execute( + sql.SQL( + "SELECT run_id, pipeline_id, source_name, status, stage, started_at, " + "finished_at, endpoints, extracted, affected, models, assertions, assets, " + "failure_stage, failure_code, failure_summary FROM {}{} " + "ORDER BY started_at DESC LIMIT %s" + ).format(self._database.relation("dander_runs"), where), + parameters, + ).fetchall() + return tuple(_run_record(row) for row in rows) + + def reconcile_interrupted(self, pipeline_id: str, *, current_run_id: str) -> None: + with self._database.pool.connection() as connection: + connection.execute( + sql.SQL( + "UPDATE {} SET status = 'failed', finished_at = clock_timestamp(), " + "failure_stage = stage, failure_code = 'interrupted_run', " + "failure_summary = %s WHERE status = 'running' AND pipeline_id = %s " + "AND run_id != %s" + ).format(self._database.relation("dander_runs")), + (_INTERRUPTED_SUMMARY, pipeline_id, current_run_id), + ) + + +class PostgreSQLMetadataStore(MetadataStore): + """Store one deterministic JSONB metadata snapshot per pipeline.""" + + def __init__(self, database: PostgreSQLStateDatabase) -> None: + self._database = database + + def publish( + self, + *, + pipeline_id: str, + run_id: str, + manifest: dict[str, object], + ) -> None: + payload = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + with self._database.pool.connection() as connection: + connection.execute( + sql.SQL( + "INSERT INTO {} (pipeline_id, run_id, manifest_json, updated_at) " + "VALUES (%s, %s, %s::jsonb, clock_timestamp()) " + "ON CONFLICT (pipeline_id) DO UPDATE SET run_id = EXCLUDED.run_id, " + "manifest_json = EXCLUDED.manifest_json, updated_at = clock_timestamp()" + ).format(self._database.relation("dander_metadata_snapshots")), + (pipeline_id, run_id, payload), + ) + + def snapshots(self, *, pipeline_id: str | None = None) -> tuple[MetadataSnapshot, ...]: + where = sql.SQL("") + parameters: list[object] = [] + if pipeline_id is not None: + where = sql.SQL(" WHERE pipeline_id = %s") + parameters.append(pipeline_id) + with self._database.pool.connection() as connection: + rows = connection.execute( + sql.SQL( + "SELECT pipeline_id, run_id, manifest_json, updated_at FROM {}{} " + "ORDER BY pipeline_id" + ).format(self._database.relation("dander_metadata_snapshots"), where), + parameters, + ).fetchall() + return tuple(_metadata_snapshot(row) for row in rows) + + +def _run_record(row: PostgreSQLRow) -> RunRecord: + failure_stage = row["failure_stage"] + return RunRecord( + run_id=str(row["run_id"]), + pipeline_id=str(row["pipeline_id"]), + source=str(row["source_name"]), + status=RunStatus(str(row["status"])), + stage=RunStage(str(row["stage"])), + started_at=_timestamp(row["started_at"]), + finished_at=_optional_timestamp(row["finished_at"]), + endpoints=int(row["endpoints"]), + extracted=int(row["extracted"]), + affected=int(row["affected"]), + models=int(row["models"]), + assertions=int(row["assertions"]), + assets=int(row["assets"]), + failure_stage=RunStage(str(failure_stage)) if failure_stage is not None else None, + failure_code=str(row["failure_code"]) if row["failure_code"] is not None else None, + failure_summary=( + str(row["failure_summary"]) if row["failure_summary"] is not None else None + ), + ) + + +def _metadata_snapshot(row: PostgreSQLRow) -> MetadataSnapshot: + manifest = row["manifest_json"] + if not isinstance(manifest, dict): + raise RuntimeError("Stored Dander metadata is invalid") + return MetadataSnapshot( + pipeline_id=str(row["pipeline_id"]), + run_id=str(row["run_id"]), + manifest=cast("dict[str, object]", manifest), + updated_at=_timestamp(row["updated_at"]), + ) + + +def _timestamp(value: object) -> str: + if isinstance(value, datetime): + return value.isoformat() + return str(value) + + +def _optional_timestamp(value: object) -> str | None: + return None if value is None else _timestamp(value) + + +def _build_postgresql_state( + config: BaseModel, + context: Mapping[str, object], +) -> StateRuntime: + if not isinstance(config, PostgreSQLStateConfig): + raise TypeError("PostgreSQL state factory received the wrong configuration") + supplied_pool = context.get("pool") + if supplied_pool is None: + dsn = os.environ.get(config.dsn_env) + if not dsn: + raise ValueError(f"PostgreSQL state requires a connection string in {config.dsn_env}") + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=config.pool_min_size, + max_size=config.pool_max_size, + timeout=config.pool_timeout_seconds, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=config.pool_timeout_seconds) + elif isinstance(supplied_pool, ConnectionPool): + pool = supplied_pool + else: + raise TypeError("PostgreSQL state context pool must be a psycopg ConnectionPool") + + database = PostgreSQLStateDatabase(pool=pool, schema_name=config.schema_name) + metadata_enabled = context.get("metadata_enabled", True) + if not isinstance(metadata_enabled, bool): + raise ValueError("PostgreSQL state runtime requires a boolean metadata_enabled") + migrator = PostgreSQLStateMigrator( + database, + terminal_history_retention_days=config.terminal_history_retention_days, + ) + return StateRuntime( + provider_id="postgresql", + leases=PostgreSQLLeaseStore(database, lease_seconds=config.lease_seconds), + watermarks=PostgreSQLWatermarkStore(database), + history=PostgreSQLRunHistoryStore(database), + metadata=PostgreSQLMetadataStore(database) if metadata_enabled else None, + migrator=migrator, + capabilities=StateCapabilities( + provider_id="postgresql", + schema_version=_STATE_SCHEMA_VERSION, + server_time=True, + atomic_leases=True, + monotonic_fencing=True, + atomic_watermark_cas=True, + interrupted_run_reconciliation=True, + ), + ) + + +POSTGRESQL_STATE_FACTORY = ProviderFactory[StateRuntime]( + kind=ProviderKind.STATE, + provider_id="postgresql", + api_version=PROVIDER_API_VERSION, + build=_build_postgresql_state, +) + +__all__ = [ + "POSTGRESQL_STATE_FACTORY", + "PostgreSQLLeaseStore", + "PostgreSQLMetadataStore", + "PostgreSQLRunHistoryStore", + "PostgreSQLStateDatabase", + "PostgreSQLStateMigrator", + "PostgreSQLWatermarkStore", +] diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 14326544..5b9d9365 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Protocol, cast +import pytest +from click import ClickException from typer.testing import CliRunner import dander.cli.run_command as run_module @@ -219,3 +221,16 @@ def build_secrets(provider_id: str) -> _Built: } migrator = cast("_Migrator", captured["migrator"]) assert migrator.calls == 1 + + +def test_postgresql_state_fails_closed_before_cross_backend_fencing() -> None: + with pytest.raises(ClickException, match="cross-backend destination fence"): + run_module._require_supported_state_pair( + state_provider="postgresql", + warehouse_provider="bigquery", + ) + + run_module._require_supported_state_pair( + state_provider="bigquery", + warehouse_provider="bigquery", + ) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 4403efaf..3aaa5d87 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -150,6 +150,39 @@ def test_version_two_none_catalog_disables_external_publication(tmp_path: Path) assert resolved.pipelines["example_records"].publish_dataplex is False +def test_version_two_preserves_postgresql_state_connection_reference(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["platforms"]["gcp"]["state"] = { + "provider": "postgresql", + "dsn_env": "DANDER_STATE_DATABASE_URL", + "schema_name": "dander_control", + "pool_min_size": 2, + "pool_max_size": 8, + "lease_seconds": 180, + "terminal_history_retention_days": 120, + } + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path) + + assert resolved.state_provider == "postgresql" + assert resolved.state_config == { + "provider": "postgresql", + "dsn_env": "DANDER_STATE_DATABASE_URL", + "schema_name": "dander_control", + "pool_min_size": 2, + "pool_max_size": 8, + "pool_timeout_seconds": 10.0, + "lease_seconds": 180, + "terminal_history_retention_days": 120, + } + + def test_cloud_run_rejects_environment_only_secrets(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_postgresql_state_runtime.py b/tests/providers/test_postgresql_state_runtime.py new file mode 100644 index 00000000..2a8ccc0d --- /dev/null +++ b/tests/providers/test_postgresql_state_runtime.py @@ -0,0 +1,384 @@ +"""PostgreSQL durable-state provider configuration and live conformance.""" + +from __future__ import annotations + +import os +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING, Any, cast + +import pytest +from psycopg import Connection, OperationalError, connect, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from dander.providers import ProviderKind, default_provider_registry +from dander.providers.postgresql.state import ( + PostgreSQLLeaseStore, + PostgreSQLMetadataStore, + PostgreSQLRunHistoryStore, + PostgreSQLStateMigrator, + PostgreSQLWatermarkStore, +) +from dander.state import LeaseHandle, RunStage, RunStatus, StateRuntime + +if TYPE_CHECKING: + from collections.abc import Iterator + + from dander.catalog import MetadataStore + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +@pytest.fixture +def postgresql_runtime() -> Iterator[tuple[StateRuntime, PostgreSQLPool, str]]: + dsn = os.environ.get("DANDER_TEST_POSTGRES_DSN") + if not dsn: + pytest.skip("DANDER_TEST_POSTGRES_DSN is not configured") + schema_name = f"dander_test_{uuid.uuid4().hex}" + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=5, + timeout=2, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.STATE, + { + "provider": "postgresql", + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "schema_name": schema_name, + "lease_seconds": 10, + "terminal_history_retention_days": 30, + }, + ) + runtime = registry.build( + ProviderKind.STATE, + config, + context={"pool": pool, "metadata_enabled": True}, + ) + assert isinstance(runtime, StateRuntime) + try: + yield runtime, pool, schema_name + finally: + with pool.connection() as connection: + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_name)) + ) + pool.close() + + +def test_postgresql_registration_requires_only_an_environment_reference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = default_provider_registry() + + config = registry.parse( + ProviderKind.STATE, + { + "provider": "postgresql", + "dsn_env": "DATABASE_URL", + "schema_name": "dander_control", + "pool_min_size": 2, + "pool_max_size": 4, + }, + ) + + assert config.model_dump(mode="json") == { + "provider": "postgresql", + "dsn_env": "DATABASE_URL", + "schema_name": "dander_control", + "pool_min_size": 2, + "pool_max_size": 4, + "pool_timeout_seconds": 10.0, + "lease_seconds": 120, + "terminal_history_retention_days": 90, + } + monkeypatch.delenv("DATABASE_URL", raising=False) + with pytest.raises(ValueError, match="connection string in DATABASE_URL"): + registry.build(ProviderKind.STATE, config) + + +def test_postgresql_state_schema_and_stores_conform( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], +) -> None: + runtime, pool, schema_name = postgresql_runtime + + assert runtime.migrator.current_version() == 0 + assert runtime.migrator.migrate() == 1 + assert runtime.migrator.migrate() == 1 + assert runtime.migrator.current_version() == 1 + assert isinstance(runtime.leases, PostgreSQLLeaseStore) + assert isinstance(runtime.watermarks, PostgreSQLWatermarkStore) + assert isinstance(runtime.history, PostgreSQLRunHistoryStore) + assert isinstance(runtime.metadata, PostgreSQLMetadataStore) + assert runtime.capabilities.server_time is True + assert runtime.capabilities.atomic_leases is True + assert runtime.capabilities.monotonic_fencing is True + assert runtime.capabilities.atomic_watermark_cas is True + assert runtime.capabilities.interrupted_run_reconciliation is True + + first = runtime.leases.acquire("salesforce", "run-1") + assert first is not None + assert first.fencing_token == 1 + assert runtime.leases.acquire("salesforce", "run-2") is None + assert runtime.leases.heartbeat(first) is True + stale = LeaseHandle("salesforce", "run-1", 999, 10) + assert runtime.leases.heartbeat(stale) is False + assert runtime.leases.release(stale) is False + assert runtime.leases.release(first) is True + second = runtime.leases.acquire("salesforce", "run-2") + assert second is not None + assert second.fencing_token == 2 + + assert runtime.watermarks.get("salesforce", "accounts") is None + assert runtime.watermarks.compare_and_set( + "salesforce", "accounts", expected=None, cursor="2026-08-08T12:00:00Z" + ) + assert not runtime.watermarks.compare_and_set( + "salesforce", "accounts", expected=None, cursor="stale" + ) + assert runtime.watermarks.compare_and_set( + "salesforce", + "accounts", + expected="2026-08-08T12:00:00Z", + cursor="2026-08-08T13:00:00Z", + ) + assert not runtime.watermarks.compare_and_set( + "salesforce", "accounts", expected="old", cursor="regressed" + ) + assert runtime.watermarks.get("salesforce", "accounts") == "2026-08-08T13:00:00Z" + + runtime.history.start("abandoned", "salesforce", pipeline_id="salesforce") + runtime.history.start("current", "salesforce", pipeline_id="salesforce") + runtime.history.reconcile_interrupted("salesforce", current_run_id="current") + runtime.history.checkpoint( + "current", + RunStage.TRANSFORM, + endpoints=4, + extracted=12, + affected=10, + models=2, + assertions=3, + ) + runtime.history.finish( + "current", + RunStatus.SUCCEEDED, + endpoints=4, + extracted=12, + affected=10, + models=5, + assertions=8, + assets=5, + ) + records = {record.run_id: record for record in runtime.history.recent(limit=10)} + assert records["abandoned"].status is RunStatus.FAILED + assert records["abandoned"].failure_code == "interrupted_run" + assert records["current"].status is RunStatus.SUCCEEDED + assert records["current"].stage is RunStage.COMPLETE + assert records["current"].models == 5 + + metadata = cast("MetadataStore", runtime.metadata) + metadata.publish( + pipeline_id="salesforce", + run_id="current", + manifest={"z": 2, "a": {"columns": ["id"]}}, + ) + metadata.publish( + pipeline_id="salesforce", + run_id="replacement", + manifest={"a": {"columns": ["id", "name"]}}, + ) + snapshots = metadata.snapshots(pipeline_id="salesforce") + assert len(snapshots) == 1 + assert snapshots[0].run_id == "replacement" + assert snapshots[0].manifest == {"a": {"columns": ["id", "name"]}} + + with pool.connection() as connection: + lease_times = connection.execute( + sql.SQL( + "SELECT heartbeat_at, lease_expires_at, clock_timestamp() AS server_now " + "FROM {} WHERE pipeline_id = 'salesforce'" + ).format(sql.Identifier(schema_name, "dander_pipeline_leases")) + ).fetchone() + assert lease_times is not None + assert lease_times["heartbeat_at"].tzinfo is not None + assert lease_times["lease_expires_at"] > lease_times["server_now"] + + +def test_postgresql_lease_contention_has_one_owner( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], +) -> None: + runtime, _pool, _schema_name = postgresql_runtime + runtime.migrator.migrate() + + with ThreadPoolExecutor(max_workers=5) as executor: + handles = list( + executor.map( + lambda run_id: runtime.leases.acquire("contended", run_id), + (f"run-{index}" for index in range(10)), + ) + ) + + owners = [handle for handle in handles if handle is not None] + assert len(owners) == 1 + assert owners[0].fencing_token == 1 + + +def test_postgresql_failed_migration_records_no_version( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime, _pool, _schema_name = postgresql_runtime + migrator = cast("PostgreSQLStateMigrator", runtime.migrator) + apply_initial_schema = migrator._apply_initial_schema + + def fail_after_ddl(connection: Connection[PostgreSQLRow]) -> None: + apply_initial_schema(connection) + raise RuntimeError("synthetic migration failure") + + monkeypatch.setattr(migrator, "_apply_initial_schema", fail_after_ddl) + + with pytest.raises(RuntimeError, match="synthetic migration failure"): + migrator.migrate() + + assert migrator.current_version() == 0 + + +def test_postgresql_retention_preserves_interrupted_runs_and_rejects_newer_schema( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], +) -> None: + runtime, pool, schema_name = postgresql_runtime + runtime.migrator.migrate() + runs = sql.Identifier(schema_name, "dander_runs") + ledger = sql.Identifier(schema_name, "dander_schema_migrations") + with pool.connection() as connection: + connection.execute( + sql.SQL( + "INSERT INTO {} (run_id, pipeline_id, source_name, status, stage, started_at, " + "finished_at, failure_code) VALUES " + "('old-success', 'p', 's', 'succeeded', 'complete', now() - interval '60 days', " + "now() - interval '60 days', NULL), " + "('old-interrupted', 'p', 's', 'failed', 'ingest', now() - interval '60 days', " + "now() - interval '60 days', 'interrupted_run')" + ).format(runs) + ) + runtime.migrator.migrate() + remaining = {record.run_id for record in runtime.history.recent(limit=10)} + assert "old-success" not in remaining + assert "old-interrupted" in remaining + + with pool.connection() as connection: + connection.execute( + sql.SQL( + "INSERT INTO {} (version, name, applied_at) VALUES (2, 'future', now())" + ).format(ledger) + ) + with pytest.raises(RuntimeError, match="newer than this Dander runtime"): + runtime.migrator.current_version() + + +def test_postgresql_pool_exhaustion_fails_boundedly( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], +) -> None: + _runtime, parent_pool, schema_name = postgresql_runtime + dsn = os.environ["DANDER_TEST_POSTGRES_DSN"] + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=1, + timeout=0.1, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.STATE, + { + "provider": "postgresql", + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "schema_name": schema_name, + "pool_min_size": 1, + "pool_max_size": 1, + "pool_timeout_seconds": 0.1, + }, + ) + runtime = registry.build( + ProviderKind.STATE, + config, + context={"pool": pool, "metadata_enabled": False}, + ) + assert isinstance(runtime, StateRuntime) + runtime.migrator.migrate() + try: + with pool.connection(), pytest.raises(PoolTimeout): + runtime.watermarks.get("source", "entity") + finally: + pool.close() + with parent_pool.connection() as connection: + assert connection.execute("SELECT 1 AS value").fetchone() == {"value": 1} + + +def test_postgresql_pool_replaces_a_lost_connection( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], +) -> None: + _runtime, _parent_pool, schema_name = postgresql_runtime + dsn = os.environ["DANDER_TEST_POSTGRES_DSN"] + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=1, + timeout=1, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.STATE, + { + "provider": "postgresql", + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "schema_name": schema_name, + "pool_min_size": 1, + "pool_max_size": 1, + }, + ) + runtime = registry.build( + ProviderKind.STATE, + config, + context={"pool": pool, "metadata_enabled": False}, + ) + assert isinstance(runtime, StateRuntime) + runtime.migrator.migrate() + with pool.connection() as connection: + row = connection.execute("SELECT pg_backend_pid() AS pid").fetchone() + assert row is not None + with connect(dsn, autocommit=True) as killer: + assert killer.execute("SELECT pg_terminate_backend(%s)", (row["pid"],)).fetchone() == ( + True, + ) + + try: + with pytest.raises(OperationalError): + runtime.watermarks.get("source", "lost-connection") + pool.wait(timeout=5) + runtime.watermarks.set("source", "lost-connection", "recovered") + assert runtime.watermarks.get("source", "lost-connection") == "recovered" + finally: + pool.close() diff --git a/tickets/DANDER-96-postgresql-state.md b/tickets/DANDER-96-postgresql-state.md new file mode 100644 index 00000000..4ff09cf6 --- /dev/null +++ b/tickets/DANDER-96-postgresql-state.md @@ -0,0 +1,44 @@ +--- +id: DANDER-96 +title: Implement PostgreSQL durable state +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-86] +created: 2026-08-08 +--- + +## Context + +Portable launchers and warehouses need a durable state backend that is not tied to BigQuery. +PostgreSQL is the roadmap's reference implementation for leases, watermarks, history, and metadata. + +## Acceptance Criteria + +- [x] Select PostgreSQL state from a version 2 platform profile without storing its connection. +- [x] Apply an idempotent, advisory-locked migration and reject newer schema versions. +- [x] Use server time, atomic lease ownership, and monotonically increasing fencing tokens. +- [x] Compare-and-set watermarks atomically and reject stale boundaries. +- [x] Persist sanitized run history, reconcile interrupted runs, and retain interrupted records. +- [x] Persist deterministic JSONB metadata snapshots. +- [x] Bound connection pooling and fail boundedly when the pool is exhausted. +- [x] Preserve the implicit version 1 BigQuery state path. +- [x] Fail closed for BigQuery execution until its cross-backend destination fence exists. +- [x] Run live PostgreSQL 15 contention and state-conformance tests in protected CI. + +## Design + +Register one lazy `postgresql` state provider. Its manifest block stores a safe schema name, bounded +pool settings, retention, and the name of the environment variable containing the DSN. Keep all +PostgreSQL implementation and Psycopg imports inside the selected provider module. Share one pool +across small store classes implementing the existing state contracts. + +## Implementation Notes + +Migration 1 creates the five shared control tables in one transaction. Terminal retention excludes +active and interrupted rows. Linux keeps the audited system-libpq package path; non-Linux installs +the binary Psycopg runtime so the public extra works without local linker setup. + +## Review Log + +No cloud resource or retained-project mutation is part of this ticket. diff --git a/uv.lock b/uv.lock index cdbd3305..92f59daf 100644 --- a/uv.lock +++ b/uv.lock @@ -2,22 +2,30 @@ version = 1 revision = 3 requires-python = ">=3.12" resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux'", + "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux'", + "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform == 'emscripten'", - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform == 'emscripten'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version < '3.13' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version < '3.13' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux'", + "python_full_version < '3.13' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version < '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", "python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version < '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version < '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", ] [[package]] @@ -663,6 +671,7 @@ gcp = [ { name = "google-cloud-secret-manager" }, ] postgres = [ + { name = "psycopg", extra = ["binary"], marker = "sys_platform != 'linux'" }, { name = "psycopg", extra = ["pool"] }, ] redshift = [ @@ -679,6 +688,7 @@ runtime-all = [ { name = "google-cloud-bigquery-storage" }, { name = "google-cloud-dataplex" }, { name = "google-cloud-secret-manager" }, + { name = "psycopg", extra = ["binary"], marker = "sys_platform != 'linux'" }, { name = "psycopg", extra = ["pool"] }, { name = "redshift-connector" }, { name = "snowflake-connector-python" }, @@ -719,8 +729,10 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11" }, { name = "packaging", specifier = ">=24.0" }, - { name = "psycopg", extras = ["pool"], marker = "extra == 'postgres'", specifier = ">=3.2,<4" }, - { name = "psycopg", extras = ["pool"], marker = "extra == 'runtime-all'", specifier = ">=3.2,<4" }, + { name = "psycopg", extras = ["binary", "pool"], marker = "sys_platform != 'linux' and extra == 'postgres'", specifier = ">=3.2,<4" }, + { name = "psycopg", extras = ["binary", "pool"], marker = "sys_platform != 'linux' and extra == 'runtime-all'", specifier = ">=3.2,<4" }, + { name = "psycopg", extras = ["pool"], marker = "sys_platform == 'linux' and extra == 'postgres'", specifier = ">=3.2,<4" }, + { name = "psycopg", extras = ["pool"], marker = "sys_platform == 'linux' and extra == 'runtime-all'", specifier = ">=3.2,<4" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pydantic-settings", specifier = ">=2.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3" }, @@ -795,7 +807,7 @@ dependencies = [ { name = "tomlkit" }, { name = "typing-extensions" }, { name = "tzdata" }, - { name = "win-precise-time", marker = "python_full_version < '3.13' and os_name == 'nt'" }, + { name = "win-precise-time", marker = "python_full_version < '3.13' and os_name == 'nt' and sys_platform != 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/91/d1ba83149b894628fffbfab47509627e42625e562f01c892d238d02d9501/dlt-1.28.1.tar.gz", hash = "sha256:152adddcda24b57d6321100170c0aa11dfb45dd3cd71723b616d5c9bc46d4326", size = 1085998, upload-time = "2026-06-19T19:50:34.869Z" } wheels = [ @@ -2183,10 +2195,29 @@ wheels = [ ] [package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] pool = [ { name = "psycopg-pool" }, ] +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, +] + [[package]] name = "psycopg-pool" version = "3.3.1" From 530a915a01ea2e1949528c77e530f181c97b0b17 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 19:20:31 -0400 Subject: [PATCH 048/163] feat: add destination target fencing --- HANDOFF.md | 44 ++-- docs/decisions.md | 10 + docs/platform-profiles.md | 5 + src/dander/concurrency.py | 56 ++++- src/dander/providers/bigquery/fence.py | 192 ++++++++++++++++++ src/dander/providers/bigquery/runtime.py | 19 +- src/dander/providers/postgresql/config.py | 5 + src/dander/providers/postgresql/fence.py | 170 ++++++++++++++++ src/dander/providers/postgresql/state.py | 29 ++- src/dander/state/lease.py | 6 + src/dander/warehouse/runtime.py | 12 +- tests/project/test_portable_config.py | 3 + tests/providers/test_bigquery_runtime.py | 93 ++++++++- .../test_postgresql_state_runtime.py | 101 +++++++++ tests/state/test_lease.py | 3 + tickets/DANDER-97-destination-target-fence.md | 37 ++++ 16 files changed, 730 insertions(+), 55 deletions(-) create mode 100644 src/dander/providers/bigquery/fence.py create mode 100644 src/dander/providers/postgresql/fence.py create mode 100644 tickets/DANDER-97-destination-target-fence.md diff --git a/HANDOFF.md b/HANDOFF.md index c8ea2e6e..30e9433f 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,40 @@ ## Finished -- Added a lazy PostgreSQL 15+ durable-state provider selected from version 2 platform profiles. -- Implemented versioned migrations, server-time leases, monotonic fences, watermark CAS, history, and JSONB metadata. -- Added bounded pooling, terminal-history retention, interruption reconciliation, and fail-closed BigQuery pairing. -- Added live PostgreSQL CI conformance for contention, migration rollback, connection loss, and pool exhaustion. -- Kept version 1 and BigQuery state behavior unchanged. +- Added provider-neutral state authority/epoch and destination `TargetFence` contracts. +- Implemented atomic target claims and transactionally fenced DML for BigQuery destinations. +- Implemented the same claim/publication boundary for PostgreSQL destinations. +- Made PostgreSQL state issue usable cross-backend tokens with an explicit stable authority ID. +- Kept PostgreSQL-state/BigQuery execution fail-closed until every runtime caller is wired. ## Try It -Install `dander-platform[postgres]`, set the manifest's `state.provider: postgresql`, and inject the -DSN through its configured environment variable. See `docs/platform-profiles.md`. Runtime execution -with BigQuery intentionally stops until the next destination-fence ticket is merged. +Run the focused provider tests. PostgreSQL live coverage needs `DANDER_TEST_POSTGRES_DSN` pointing +to a disposable PostgreSQL 15+ database; the test creates and drops its own schemas. ## Checks -- Ruff, formatting, strict mypy, and all 998 Python tests passed. -- Seven live PostgreSQL 15 provider tests passed; manual container restart recovery passed. -- Terraform formatting, GCP/AWS validation, and provider-mocked AWS tests passed. -- Wheel/sdist inspection, source-free validation, PostgreSQL-extra installation, and OCI build passed. -- Linux OCI uses system `libpq`; non-Linux PostgreSQL installs use Psycopg's binary runtime. +- All 1,001 Python tests passed locally; PostgreSQL tests skip without a configured DSN. +- Eight PostgreSQL 15 live tests passed against a disposable local container. +- Repository-wide Ruff, formatting, and strict mypy checks passed. +- The stale PostgreSQL claimant test proved target data stayed unchanged until the newer owner ran. +- Distribution inspection/install, dependency audit, and all Terraform validations passed. ## Decisions -- Manifests retain only the DSN environment-variable name; credentials are runtime-injected. -- Interrupted history is exempt from terminal retention. -- PostgreSQL state does not weaken publication fencing while the cross-backend protocol is pending. +- State authority identity is explicit and non-secret; epoch changes belong to a future cutover. +- Target claims accept only a newer token or the exact same run/token retry. +- Destination fencing is not advertised as usable until ingestion and materialization callers bind it. ## Remaining -- Run protected CI and merge the focused PostgreSQL-state PR if clean. -- Implement the generic destination target-fence protocol as a separate PR. -- Implement PostgreSQL warehouse capabilities, then the Kubernetes launcher and live profile. +- Run protected CI on the focused pull request. +- Wire the target fence into every supported writer and materialization finalizer. +- Implement PostgreSQL warehouse schema, loading, transforms, assertions, and telemetry. +- Then add the Kubernetes launcher and cross-backend matrix proofs. ## Review First -- `src/dander/providers/postgresql/state.py` -- `tests/providers/test_postgresql_state_runtime.py` -- `src/dander/project/portable_config.py` +- `src/dander/concurrency.py` +- `src/dander/providers/bigquery/fence.py` +- `src/dander/providers/postgresql/fence.py` diff --git a/docs/decisions.md b/docs/decisions.md index 9ec6cc3f..476747fe 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -779,3 +779,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc its short-lived task-role values only in the current process. The generated external-account config uses the library-supported `service_account_impersonation` lifetime field. This does not make Fargate a supported launcher; that remains Phase 3. + +## 2026-08-08 — Destination publication has its own fence ledger + +- A state lease identifies its immutable authority and epoch, but never assumes that its control + table is transactionally reachable from a different destination warehouse. +- Each destination claims `(pipeline, target)` in `dander_target_commits` before staging. Only a + newer token or an exact retry from the current authority/epoch may replace that claim. +- Final publication touches the exact authority/epoch/pipeline/target/run/token tuple and records + completion in the same destination transaction. Cross-backend execution remains fail-closed + until every writer and materialization caller uses this boundary. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index ec920b3d..a6f680f6 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -77,6 +77,8 @@ platforms: location: US state: provider: postgresql + authority_id: postgresql:portable-state + authority_epoch: 1 dsn_env: DANDER_POSTGRES_DSN schema_name: dander_meta pool_min_size: 1 @@ -96,6 +98,9 @@ the value is never part of either manifest. Dander creates and migrates only the It uses a bounded pool, PostgreSQL server time for leases, atomic watermark comparison, sanitized run history, and deterministic JSONB metadata snapshots. +`authority_id` is a stable, non-secret identifier for this state deployment. Do not reuse it for a +different database or change `authority_epoch` outside a reviewed state-backend cutover. + The current BigQuery warehouse/PostgreSQL state combination fails closed at execution because its cross-backend destination fence is the next portability ticket. The state adapter is available for conformance and composition work now; it is not a shortcut around that publication boundary. diff --git a/src/dander/concurrency.py b/src/dander/concurrency.py index 9e3355e9..dbd08188 100644 --- a/src/dander/concurrency.py +++ b/src/dander/concurrency.py @@ -1,4 +1,4 @@ -"""Shared lease-fencing contract for BigQuery mutations.""" +"""Shared state-lease and destination-publication fencing contracts.""" from __future__ import annotations @@ -9,24 +9,70 @@ from google.cloud import bigquery _TABLE_ID = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*\.[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*$") +_AUTHORITY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{2,255}$") @dataclass(frozen=True) class FencingToken: - """One run's ownership token for a BigQuery pipeline lease row.""" + """One run's ownership token issued by a durable-state authority.""" - lease_table: str + lease_table: str | None pipeline_id: str run_id: str token: int + authority_id: str | None = None + authority_epoch: int = 1 def __post_init__(self) -> None: - if not _TABLE_ID.fullmatch(self.lease_table): + if self.lease_table is not None and not _TABLE_ID.fullmatch(self.lease_table): raise ValueError("Invalid BigQuery lease table") if not self.pipeline_id or not self.run_id: raise ValueError("Fencing identifiers must be non-empty") if isinstance(self.token, bool) or self.token <= 0: raise ValueError("Fencing token must be a positive integer") + if self.authority_id is None and self.lease_table is None: + raise ValueError("Fencing token requires an authority id") + if self.authority_id is not None and not _AUTHORITY_ID.fullmatch(self.authority_id): + raise ValueError("Invalid fencing authority id") + if isinstance(self.authority_epoch, bool) or self.authority_epoch <= 0: + raise ValueError("Fencing authority epoch must be a positive integer") + + @property + def resolved_authority_id(self) -> str: + """Return the explicit authority or the legacy BigQuery deployment identity.""" + if self.authority_id is not None: + return self.authority_id + assert self.lease_table is not None + return f"bigquery:{self.lease_table.rsplit('.', 1)[0]}" + + +@dataclass(frozen=True) +class TargetFence: + """One destination target claimed by an exact state lease owner.""" + + fence_table: str + target_id: str + authority_id: str + authority_epoch: int + pipeline_id: str + run_id: str + token: int + + def __post_init__(self) -> None: + if not self.fence_table or not self.target_id: + raise ValueError("Target fence identifiers must be non-empty") + if not _AUTHORITY_ID.fullmatch(self.authority_id): + raise ValueError("Invalid target-fence authority id") + if isinstance(self.authority_epoch, bool) or self.authority_epoch <= 0: + raise ValueError("Target-fence authority epoch must be a positive integer") + if not self.pipeline_id or not self.run_id: + raise ValueError("Target-fence run identifiers must be non-empty") + if isinstance(self.token, bool) or self.token <= 0: + raise ValueError("Target-fence token must be a positive integer") + + +class TargetFenceLostError(RuntimeError): + """Raised when a destination rejects a stale or foreign publication owner.""" class OwnershipGuard(Protocol): @@ -48,6 +94,8 @@ def fenced_dml(statement: str, fence: FencingToken) -> str: def fencing_touch_sql(fence: FencingToken) -> str: """Return the conditional lease-row DML and assertion for an existing transaction.""" + if fence.lease_table is None: + raise ValueError("State-side BigQuery fencing requires a BigQuery lease table") return ( f"UPDATE `{fence.lease_table}`\n" "SET heartbeat_at = heartbeat_at\n" diff --git a/src/dander/providers/bigquery/fence.py b/src/dander/providers/bigquery/fence.py new file mode 100644 index 00000000..4d755110 --- /dev/null +++ b/src/dander/providers/bigquery/fence.py @@ -0,0 +1,192 @@ +"""BigQuery destination-side target fencing.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, cast + +from google.cloud import bigquery + +from dander._bigquery_retry import run_mutation_with_retry +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.warehouse.runtime import PreparedWarehouseStatement + +if TYPE_CHECKING: + from collections.abc import Iterable + + from dander.warehouse.contracts import RelationRef + +_TABLE_ID = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*\.[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*$") + + +class _FenceJob(Protocol): + num_dml_affected_rows: int | None + + def result(self) -> Iterable[object]: + """Wait for the destination-fence mutation.""" + ... + + +class _FenceClient(Protocol): + def query( + self, + query: str, + *, + job_config: bigquery.QueryJobConfig | None = None, + ) -> _FenceJob: + """Submit one BigQuery fence statement.""" + ... + + +@dataclass(slots=True) +class BigQueryTargetFence: + """Claim and verify a target using a BigQuery destination-side ledger.""" + + project: str + client: _FenceClient | None = None + + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + """Atomically accept a newer token or an exact idempotent retry.""" + if target.catalog != self.project: + raise ValueError("BigQuery target fence must use the runtime project") + table = f"{target.catalog}.{target.namespace}._dander_target_commits" + target_id = ".".join(target.coordinates) + client = self._required_client() + run_mutation_with_retry(lambda: client.query(_target_fence_table_sql(table))) + claim = run_mutation_with_retry( + lambda: client.query( + _target_claim_sql(table), + job_config=_target_fence_config(target_id, fence), + ) + ) + if claim.num_dml_affected_rows != 1: + raise TargetFenceLostError( + f"Destination target {target_id!r} rejected stale publication ownership" + ) + return TargetFence( + fence_table=table, + target_id=target_id, + authority_id=fence.resolved_authority_id, + authority_epoch=fence.authority_epoch, + pipeline_id=fence.pipeline_id, + run_id=fence.run_id, + token=fence.token, + ) + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: + """Verify, publish, and record completion in one BigQuery transaction.""" + if not _TABLE_ID.fullmatch(fence.fence_table) or not fence.fence_table.startswith( + f"{self.project}." + ): + raise ValueError("BigQuery target fence belongs to another runtime project") + finalizer = statement.strip().removesuffix(";") + return PreparedWarehouseStatement( + sql=( + "BEGIN TRANSACTION;\n" + f"{_target_touch_sql(fence)}\n" + f"{finalizer};\n" + f"{_target_commit_sql(fence)}\n" + "COMMIT TRANSACTION;" + ), + options=_claim_config(fence), + ) + + def _required_client(self) -> _FenceClient: + if self.client is None: + self.client = cast("_FenceClient", bigquery.Client(project=self.project)) + return self.client + + +def _target_fence_table_sql(table: str) -> str: + return ( + f"CREATE TABLE IF NOT EXISTS `{table}` (" + "target_id STRING NOT NULL, pipeline_id STRING NOT NULL, " + "authority_id STRING NOT NULL, authority_epoch INT64 NOT NULL, " + "run_id STRING NOT NULL, fencing_token INT64 NOT NULL, " + "status STRING NOT NULL, claimed_at TIMESTAMP NOT NULL, committed_at TIMESTAMP) " + "CLUSTER BY target_id, pipeline_id" + ) + + +def _target_claim_sql(table: str) -> str: + return ( + f"MERGE `{table}` AS current\n" + "USING (SELECT @dander_target_id AS target_id, " + "@dander_pipeline_id AS pipeline_id, @dander_authority_id AS authority_id, " + "@dander_authority_epoch AS authority_epoch, @dander_run_id AS run_id, " + "@dander_fencing_token AS fencing_token) AS incoming\n" + "ON current.target_id = incoming.target_id " + "AND current.pipeline_id = incoming.pipeline_id\n" + "WHEN MATCHED AND current.authority_id = incoming.authority_id " + "AND current.authority_epoch = incoming.authority_epoch " + "AND (incoming.fencing_token > current.fencing_token OR " + "(incoming.fencing_token = current.fencing_token " + "AND incoming.run_id = current.run_id)) THEN\n" + " UPDATE SET run_id = incoming.run_id, fencing_token = incoming.fencing_token, " + "status = 'claimed', claimed_at = CURRENT_TIMESTAMP(), committed_at = NULL\n" + "WHEN NOT MATCHED THEN\n" + " INSERT (target_id, pipeline_id, authority_id, authority_epoch, run_id, " + "fencing_token, status, claimed_at, committed_at)\n" + " VALUES (incoming.target_id, incoming.pipeline_id, incoming.authority_id, " + "incoming.authority_epoch, incoming.run_id, incoming.fencing_token, 'claimed', " + "CURRENT_TIMESTAMP(), NULL)" + ) + + +def _target_touch_sql(fence: TargetFence) -> str: + return ( + f"UPDATE `{fence.fence_table}` SET claimed_at = claimed_at\n" + f"WHERE {_target_match_sql()};\n" + "ASSERT @@row_count = 1 AS 'Dander destination fence lost';" + ) + + +def _target_commit_sql(fence: TargetFence) -> str: + return ( + f"UPDATE `{fence.fence_table}`\n" + "SET status = 'committed', committed_at = CURRENT_TIMESTAMP()\n" + f"WHERE {_target_match_sql()};\n" + "ASSERT @@row_count = 1 AS 'Dander destination fence lost';" + ) + + +def _target_match_sql() -> str: + return ( + "target_id = @dander_target_id AND pipeline_id = @dander_pipeline_id " + "AND authority_id = @dander_authority_id " + "AND authority_epoch = @dander_authority_epoch " + "AND run_id = @dander_run_id AND fencing_token = @dander_fencing_token " + "AND status = 'claimed'" + ) + + +def _target_fence_config(target_id: str, fence: FencingToken) -> bigquery.QueryJobConfig: + return bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter("dander_target_id", "STRING", target_id), + bigquery.ScalarQueryParameter("dander_pipeline_id", "STRING", fence.pipeline_id), + bigquery.ScalarQueryParameter( + "dander_authority_id", "STRING", fence.resolved_authority_id + ), + bigquery.ScalarQueryParameter("dander_authority_epoch", "INT64", fence.authority_epoch), + bigquery.ScalarQueryParameter("dander_run_id", "STRING", fence.run_id), + bigquery.ScalarQueryParameter("dander_fencing_token", "INT64", fence.token), + ] + ) + + +def _claim_config(fence: TargetFence) -> bigquery.QueryJobConfig: + return bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter("dander_target_id", "STRING", fence.target_id), + bigquery.ScalarQueryParameter("dander_pipeline_id", "STRING", fence.pipeline_id), + bigquery.ScalarQueryParameter("dander_authority_id", "STRING", fence.authority_id), + bigquery.ScalarQueryParameter("dander_authority_epoch", "INT64", fence.authority_epoch), + bigquery.ScalarQueryParameter("dander_run_id", "STRING", fence.run_id), + bigquery.ScalarQueryParameter("dander_fencing_token", "INT64", fence.token), + ] + ) + + +__all__ = ["BigQueryTargetFence"] diff --git a/src/dander/providers/bigquery/runtime.py b/src/dander/providers/bigquery/runtime.py index f3ef005c..6767edce 100644 --- a/src/dander/providers/bigquery/runtime.py +++ b/src/dander/providers/bigquery/runtime.py @@ -5,15 +5,14 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, cast -from dander.concurrency import FencingToken, fenced_dml, fencing_job_config from dander.pipeline.runtime import BigQueryGraphRunner, GraphExecutionPlan from dander.providers.bigquery.config import BigQueryWarehouseConfig +from dander.providers.bigquery.fence import BigQueryTargetFence from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import BigQueryTransformRunner from dander.warehouse.bigquery_compat import canonical_schema_from_bigquery from dander.warehouse.runtime import ( - PreparedWarehouseStatement, WarehouseCapabilities, WarehouseRuntime, WarehouseTransformRunner, @@ -101,20 +100,6 @@ def build_transform_runner( return None -@dataclass(frozen=True, slots=True) -class BigQueryTargetFence: - """Bind BigQuery lease ownership inside the destination transaction.""" - - def prepare_dml(self, statement: str, fence: object) -> PreparedWarehouseStatement: - """Return the existing conditional DML touch and parameter bindings.""" - if not isinstance(fence, FencingToken): - raise TypeError("BigQuery target fencing requires a FencingToken") - return PreparedWarehouseStatement( - sql=fenced_dml(statement, fence), - options=fencing_job_config(fence), - ) - - @dataclass(frozen=True, slots=True) class BigQueryTelemetry: """Normalize stable BigQuery job counters and identifiers.""" @@ -168,7 +153,7 @@ def build_bigquery_warehouse( schema_mapper=BigQuerySchemaMapper(), writers=BigQueryWriterFactory(project), transforms=BigQueryTransformFactory(project), - target_fence=BigQueryTargetFence(), + target_fence=BigQueryTargetFence(project), telemetry=BigQueryTelemetry(), capabilities=BIGQUERY_CAPABILITIES, ) diff --git a/src/dander/providers/postgresql/config.py b/src/dander/providers/postgresql/config.py index 74e9a142..f0149c49 100644 --- a/src/dander/providers/postgresql/config.py +++ b/src/dander/providers/postgresql/config.py @@ -9,6 +9,7 @@ _ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]{0,126}$") _SCHEMA_NAME = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") +_AUTHORITY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{2,255}$") class PostgreSQLStateConfig(BaseModel): @@ -21,6 +22,8 @@ class PostgreSQLStateConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) provider: Literal["postgresql"] + authority_id: str + authority_epoch: int = Field(default=1, ge=1) dsn_env: str = "DANDER_POSTGRES_DSN" schema_name: str = "dander_meta" pool_min_size: int = Field(default=1, ge=1, le=20) @@ -35,6 +38,8 @@ def validate_settings(self) -> PostgreSQLStateConfig: raise ValueError("dsn_env must be an uppercase environment-variable name") if not _SCHEMA_NAME.fullmatch(self.schema_name): raise ValueError("schema_name must be a safe lowercase PostgreSQL identifier") + if not _AUTHORITY_ID.fullmatch(self.authority_id): + raise ValueError("authority_id must be a stable non-secret deployment identifier") if self.pool_min_size > self.pool_max_size: raise ValueError("pool_min_size must not exceed pool_max_size") return self diff --git a/src/dander/providers/postgresql/fence.py b/src/dander/providers/postgresql/fence.py new file mode 100644 index 00000000..1e1877a6 --- /dev/null +++ b/src/dander/providers/postgresql/fence.py @@ -0,0 +1,170 @@ +"""PostgreSQL destination-side target fencing.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from psycopg import Connection, Cursor, sql +from psycopg_pool import ConnectionPool + +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.warehouse.runtime import PreparedWarehouseStatement + +if TYPE_CHECKING: + from dander.warehouse.contracts import RelationRef + +PostgreSQLPool = ConnectionPool[Connection[dict[str, object]]] + + +@dataclass(frozen=True, slots=True) +class PostgreSQLFencePlan: + """Provider-native statements surrounding one publication transaction.""" + + touch_sql: sql.Composed + commit_sql: sql.Composed + parameters: tuple[object, ...] + + +@dataclass(frozen=True, slots=True) +class PostgreSQLTargetFence: + """Claim and verify publication ownership in the destination database.""" + + pool: PostgreSQLPool + catalog: str + + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + """Atomically accept a newer token or an exact idempotent retry.""" + if target.catalog != self.catalog: + raise ValueError("PostgreSQL target fence must use the runtime database") + table = sql.Identifier(target.namespace, "dander_target_commits") + target_id = ".".join(target.coordinates) + with self.pool.connection() as connection, connection.transaction(): + connection.execute(_postgresql_target_fence_table_sql(table)) + row = connection.execute( + _postgresql_target_claim_sql(table), + ( + target_id, + fence.pipeline_id, + fence.resolved_authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + ), + ).fetchone() + if row is None: + raise TargetFenceLostError( + f"Destination target {target_id!r} rejected stale publication ownership" + ) + return TargetFence( + fence_table=f"{target.namespace}.dander_target_commits", + target_id=target_id, + authority_id=fence.resolved_authority_id, + authority_epoch=fence.authority_epoch, + pipeline_id=fence.pipeline_id, + run_id=fence.run_id, + token=fence.token, + ) + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: + """Return DML plus the exact transactional fence operations it requires.""" + finalizer = statement.strip().removesuffix(";") + table = _postgresql_fence_identifier(fence.fence_table) + return PreparedWarehouseStatement( + sql=finalizer, + options=PostgreSQLFencePlan( + touch_sql=_postgresql_target_touch_sql(table), + commit_sql=_postgresql_target_commit_sql(table), + parameters=_postgresql_claim_parameters(fence), + ), + ) + + def execute_dml( + self, + connection: Connection[dict[str, object]], + statement: str, + fence: TargetFence, + parameters: tuple[object, ...] = (), + ) -> Cursor[dict[str, object]]: + """Execute exact ownership checks and publication in one transaction.""" + prepared = self.prepare_dml(statement, fence) + plan = prepared.options + assert isinstance(plan, PostgreSQLFencePlan) + with connection.transaction(): + if connection.execute(plan.touch_sql, plan.parameters).fetchone() is None: + raise TargetFenceLostError("Dander destination fence lost before publication") + result = connection.execute(prepared.sql, parameters) + if connection.execute(plan.commit_sql, plan.parameters).fetchone() is None: + raise TargetFenceLostError("Dander destination fence lost during publication") + return result + + +def _postgresql_target_fence_table_sql(table: sql.Identifier) -> sql.Composed: + return sql.SQL( + "CREATE TABLE IF NOT EXISTS {} (" + "target_id TEXT NOT NULL, pipeline_id TEXT NOT NULL, " + "authority_id TEXT NOT NULL, authority_epoch BIGINT NOT NULL CHECK (authority_epoch > 0), " + "run_id TEXT NOT NULL, fencing_token BIGINT NOT NULL CHECK (fencing_token > 0), " + "status TEXT NOT NULL CHECK (status IN ('claimed', 'committed')), " + "claimed_at TIMESTAMPTZ NOT NULL, committed_at TIMESTAMPTZ, " + "PRIMARY KEY (target_id, pipeline_id))" + ).format(table) + + +def _postgresql_target_claim_sql(table: sql.Identifier) -> sql.Composed: + return sql.SQL( + "INSERT INTO {} AS current " + "(target_id, pipeline_id, authority_id, authority_epoch, run_id, fencing_token, " + "status, claimed_at, committed_at) " + "VALUES (%s, %s, %s, %s, %s, %s, 'claimed', clock_timestamp(), NULL) " + "ON CONFLICT (target_id, pipeline_id) DO UPDATE SET " + "run_id = EXCLUDED.run_id, fencing_token = EXCLUDED.fencing_token, " + "status = 'claimed', claimed_at = clock_timestamp(), committed_at = NULL " + "WHERE current.authority_id = EXCLUDED.authority_id " + "AND current.authority_epoch = EXCLUDED.authority_epoch " + "AND (EXCLUDED.fencing_token > current.fencing_token OR " + "(EXCLUDED.fencing_token = current.fencing_token " + "AND EXCLUDED.run_id = current.run_id)) RETURNING fencing_token" + ).format(table) + + +def _postgresql_target_touch_sql(table: sql.Identifier) -> sql.Composed: + return sql.SQL("UPDATE {} SET claimed_at = claimed_at WHERE {} RETURNING fencing_token").format( + table, sql.SQL(_postgresql_target_match_sql()) + ) + + +def _postgresql_target_commit_sql(table: sql.Identifier) -> sql.Composed: + return sql.SQL( + "UPDATE {} SET status = 'committed', committed_at = clock_timestamp() " + "WHERE {} RETURNING fencing_token" + ).format(table, sql.SQL(_postgresql_target_match_sql())) + + +def _postgresql_target_match_sql() -> str: + return ( + "target_id = %s AND pipeline_id = %s AND authority_id = %s " + "AND authority_epoch = %s AND run_id = %s AND fencing_token = %s " + "AND status = 'claimed'" + ) + + +def _postgresql_claim_parameters(fence: TargetFence) -> tuple[object, ...]: + return ( + fence.target_id, + fence.pipeline_id, + fence.authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + ) + + +def _postgresql_fence_identifier(value: str) -> sql.Identifier: + coordinates = value.split(".") + if len(coordinates) != 2 or any(not coordinate for coordinate in coordinates): + raise ValueError("Invalid PostgreSQL target-fence table") + return sql.Identifier(*coordinates) + + +__all__ = ["PostgreSQLFencePlan", "PostgreSQLTargetFence"] diff --git a/src/dander/providers/postgresql/state.py b/src/dander/providers/postgresql/state.py index 37f67ced..c033e0b1 100644 --- a/src/dander/providers/postgresql/state.py +++ b/src/dander/providers/postgresql/state.py @@ -12,6 +12,7 @@ from psycopg_pool import ConnectionPool from dander.catalog import MetadataSnapshot, MetadataStore +from dander.concurrency import FencingToken from dander.providers.postgresql.config import PostgreSQLStateConfig from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.state import ( @@ -32,8 +33,6 @@ from pydantic import BaseModel - from dander.concurrency import FencingToken - _STATE_SCHEMA_VERSION = 1 _MIGRATIONS = (StateMigration(version=1, name="initial_state_schema"),) _INTERRUPTED_SUMMARY = "A later execution acquired the expired pipeline lease." @@ -181,9 +180,18 @@ def _prune_terminal_history(self, connection: Connection[PostgreSQLRow]) -> None class PostgreSQLLeaseStore(LeaseStore): """Coordinate pipeline ownership using PostgreSQL server time and row locks.""" - def __init__(self, database: PostgreSQLStateDatabase, *, lease_seconds: int) -> None: + def __init__( + self, + database: PostgreSQLStateDatabase, + *, + lease_seconds: int, + authority_id: str, + authority_epoch: int, + ) -> None: self._database = database self._lease_seconds = lease_seconds + self._authority_id = authority_id + self._authority_epoch = authority_epoch @property def lease_seconds(self) -> int: @@ -221,6 +229,14 @@ def acquire(self, pipeline_id: str, run_id: str) -> LeaseHandle | None: run_id=run_id, fencing_token=token, lease_seconds=self._lease_seconds, + fence=FencingToken( + lease_table=None, + pipeline_id=pipeline_id, + run_id=run_id, + token=token, + authority_id=self._authority_id, + authority_epoch=self._authority_epoch, + ), ) def heartbeat(self, lease: LeaseHandle) -> bool: @@ -598,7 +614,12 @@ def _build_postgresql_state( ) return StateRuntime( provider_id="postgresql", - leases=PostgreSQLLeaseStore(database, lease_seconds=config.lease_seconds), + leases=PostgreSQLLeaseStore( + database, + lease_seconds=config.lease_seconds, + authority_id=config.authority_id, + authority_epoch=config.authority_epoch, + ), watermarks=PostgreSQLWatermarkStore(database), history=PostgreSQLRunHistoryStore(database), metadata=PostgreSQLMetadataStore(database) if metadata_enabled else None, diff --git a/src/dander/state/lease.py b/src/dander/state/lease.py index 504e1c59..16bb8bc2 100644 --- a/src/dander/state/lease.py +++ b/src/dander/state/lease.py @@ -87,6 +87,8 @@ def __init__( dataset: str, client: _BigQueryClient | None = None, lease_seconds: int = 120, + authority_id: str | None = None, + authority_epoch: int = 1, ) -> None: if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", project): raise ValueError(f"Invalid BigQuery project: {project!r}") @@ -94,6 +96,8 @@ def __init__( raise ValueError(f"Invalid BigQuery dataset: {dataset!r}") self._lease_seconds = _validated_lease_seconds(lease_seconds) self._table_prefix = f"{project}.{dataset}._dander_lease_" + self._authority_id = authority_id or f"bigquery:{project}:{dataset}" + self._authority_epoch = authority_epoch self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) self._ready: set[str] = set() @@ -154,6 +158,8 @@ def acquire(self, pipeline_id: str, run_id: str) -> LeaseHandle | None: pipeline_id=pipeline_id, run_id=run_id, token=token, + authority_id=self._authority_id, + authority_epoch=self._authority_epoch, ), ) diff --git a/src/dander/warehouse/runtime.py b/src/dander/warehouse/runtime.py index 3416e6d3..6fb33013 100644 --- a/src/dander/warehouse/runtime.py +++ b/src/dander/warehouse/runtime.py @@ -9,10 +9,10 @@ from collections.abc import Iterable, Sequence from pathlib import Path - from dander.concurrency import OwnershipGuard + from dander.concurrency import FencingToken, OwnershipGuard, TargetFence from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import TransformRunResult - from dander.warehouse.contracts import RelationCodec, RelationSchema + from dander.warehouse.contracts import RelationCodec, RelationRef, RelationSchema from dander.writer.base import SchemaEvolution, WriteMode, WritePattern, WriteTransport @@ -104,9 +104,13 @@ def build_transform_runner( @runtime_checkable class WarehouseTargetFence(Protocol): - """Prepare one destination-side fenced DML finalizer.""" + """Claim and transactionally verify destination publication ownership.""" - def prepare_dml(self, statement: str, fence: object) -> PreparedWarehouseStatement: + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + """Claim ``target`` before creating run-scoped staging.""" + ... + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: """Bind the provider fencing statement and its execution options.""" ... diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 3aaa5d87..e2feeb7b 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -159,6 +159,7 @@ def test_version_two_preserves_postgresql_state_connection_reference(tmp_path: P platforms = yaml.safe_load(migration.platforms_yaml) platforms["platforms"]["gcp"]["state"] = { "provider": "postgresql", + "authority_id": "postgresql:portable-state", "dsn_env": "DANDER_STATE_DATABASE_URL", "schema_name": "dander_control", "pool_min_size": 2, @@ -173,6 +174,8 @@ def test_version_two_preserves_postgresql_state_connection_reference(tmp_path: P assert resolved.state_provider == "postgresql" assert resolved.state_config == { "provider": "postgresql", + "authority_id": "postgresql:portable-state", + "authority_epoch": 1, "dsn_env": "DANDER_STATE_DATABASE_URL", "schema_name": "dander_control", "pool_min_size": 2, diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index 41652e9f..bb883698 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -6,10 +6,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +import pytest from google.cloud import bigquery -from dander.concurrency import FencingToken +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError from dander.providers import ProviderKind, default_provider_registry +from dander.providers.bigquery.fence import BigQueryTargetFence from dander.telemetry import TelemetryOperation from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTransport @@ -62,8 +64,11 @@ def test_bigquery_runtime_exposes_codec_schema_fence_and_telemetry() -> None: schema = runtime.schema_mapper.canonical_schema( [WriteField(name="id", data_type="STRING", mode="REQUIRED")] ) - fence = FencingToken( - lease_table="unit-project.dander_meta._dander_leases", + fence = TargetFence( + fence_table="unit-project.raw._dander_target_commits", + target_id="unit-project.raw.records", + authority_id="bigquery:unit-project:dander_meta", + authority_epoch=1, pipeline_id="records", run_id="run-123", token=7, @@ -87,7 +92,8 @@ class _Job: assert runtime.relation_codec.render(relation) == "`unit-project`.`raw`.`records`" assert schema.fields[0].data_type.kind is LogicalTypeKind.STRING - assert "ASSERT @@row_count = 1" in prepared.sql + assert prepared.sql.count("ASSERT @@row_count = 1") == 2 + assert "status = 'committed'" in prepared.sql assert isinstance(prepared.options, bigquery.QueryJobConfig) assert telemetry.to_payload() == { "provider": "bigquery", @@ -105,6 +111,85 @@ class _Job: } +class _FenceJob: + def __init__(self, *, affected: int | None = None) -> None: + self.num_dml_affected_rows = affected + + def result(self) -> list[object]: + return [] + + +class _FenceClient: + def __init__(self, *, claim_affected: int = 1) -> None: + self.claim_affected = claim_affected + self.queries: list[tuple[str, bigquery.QueryJobConfig | None]] = [] + + def query( + self, + query: str, + *, + job_config: bigquery.QueryJobConfig | None = None, + ) -> _FenceJob: + self.queries.append((query, job_config)) + return _FenceJob(affected=self.claim_affected if query.startswith("MERGE") else None) + + +def test_bigquery_target_fence_claims_before_preparing_publication() -> None: + client = _FenceClient() + capability = BigQueryTargetFence(project="unit-project", client=client) + relation = RelationRef(catalog="unit-project", namespace="raw", name="records") + lease = FencingToken( + lease_table=None, + pipeline_id="records", + run_id="run-123", + token=7, + authority_id="postgresql:state-primary", + authority_epoch=3, + ) + + claim = capability.claim(relation, lease) + prepared = capability.prepare_dml("MERGE `target` USING `stage` ON FALSE", claim) + + assert client.queries[0][0].startswith( + "CREATE TABLE IF NOT EXISTS `unit-project.raw._dander_target_commits`" + ) + claim_sql, claim_config = client.queries[1] + assert claim_sql.startswith("MERGE `unit-project.raw._dander_target_commits`") + assert "incoming.fencing_token > current.fencing_token" in claim_sql + assert "incoming.run_id = current.run_id" in claim_sql + assert claim_config is not None + parameters = {parameter.name: parameter.value for parameter in claim_config.query_parameters} + assert parameters == { + "dander_target_id": "unit-project.raw.records", + "dander_pipeline_id": "records", + "dander_authority_id": "postgresql:state-primary", + "dander_authority_epoch": 3, + "dander_run_id": "run-123", + "dander_fencing_token": 7, + } + assert prepared.sql.startswith("BEGIN TRANSACTION;\nUPDATE") + assert "MERGE `target` USING `stage` ON FALSE;" in prepared.sql + assert prepared.sql.endswith("COMMIT TRANSACTION;") + + +def test_bigquery_target_fence_rejects_zero_row_claim() -> None: + capability = BigQueryTargetFence( + project="unit-project", + client=_FenceClient(claim_affected=0), + ) + relation = RelationRef(catalog="unit-project", namespace="raw", name="records") + lease = FencingToken( + lease_table=None, + pipeline_id="records", + run_id="stale-run", + token=2, + authority_id="postgresql:state-primary", + ) + + with pytest.raises(TargetFenceLostError, match="rejected stale"): + capability.claim(relation, lease) + + def test_bigquery_runtime_constructs_writers_through_capability( monkeypatch: MonkeyPatch, ) -> None: diff --git a/tests/providers/test_postgresql_state_runtime.py b/tests/providers/test_postgresql_state_runtime.py index 2a8ccc0d..77cec895 100644 --- a/tests/providers/test_postgresql_state_runtime.py +++ b/tests/providers/test_postgresql_state_runtime.py @@ -12,7 +12,9 @@ from psycopg.rows import dict_row from psycopg_pool import ConnectionPool, PoolTimeout +from dander.concurrency import FencingToken, TargetFenceLostError from dander.providers import ProviderKind, default_provider_registry +from dander.providers.postgresql.fence import PostgreSQLTargetFence from dander.providers.postgresql.state import ( PostgreSQLLeaseStore, PostgreSQLMetadataStore, @@ -21,6 +23,7 @@ PostgreSQLWatermarkStore, ) from dander.state import LeaseHandle, RunStage, RunStatus, StateRuntime +from dander.warehouse import RelationRef if TYPE_CHECKING: from collections.abc import Iterator @@ -54,6 +57,7 @@ def postgresql_runtime() -> Iterator[tuple[StateRuntime, PostgreSQLPool, str]]: ProviderKind.STATE, { "provider": "postgresql", + "authority_id": "postgresql:test-state", "dsn_env": "DANDER_TEST_POSTGRES_DSN", "schema_name": schema_name, "lease_seconds": 10, @@ -85,6 +89,7 @@ def test_postgresql_registration_requires_only_an_environment_reference( ProviderKind.STATE, { "provider": "postgresql", + "authority_id": "postgresql:test-state", "dsn_env": "DATABASE_URL", "schema_name": "dander_control", "pool_min_size": 2, @@ -94,6 +99,8 @@ def test_postgresql_registration_requires_only_an_environment_reference( assert config.model_dump(mode="json") == { "provider": "postgresql", + "authority_id": "postgresql:test-state", + "authority_epoch": 1, "dsn_env": "DATABASE_URL", "schema_name": "dander_control", "pool_min_size": 2, @@ -129,6 +136,10 @@ def test_postgresql_state_schema_and_stores_conform( first = runtime.leases.acquire("salesforce", "run-1") assert first is not None assert first.fencing_token == 1 + assert first.fence is not None + assert first.fence.lease_table is None + assert first.fence.resolved_authority_id == "postgresql:test-state" + assert first.fence.authority_epoch == 1 assert runtime.leases.acquire("salesforce", "run-2") is None assert runtime.leases.heartbeat(first) is True stale = LeaseHandle("salesforce", "run-1", 999, 10) @@ -233,6 +244,94 @@ def test_postgresql_lease_contention_has_one_owner( assert owners[0].fencing_token == 1 +def test_postgresql_destination_fence_rejects_stale_publication( + postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], +) -> None: + runtime, pool, schema_name = postgresql_runtime + runtime.migrator.migrate() + with pool.connection() as connection: + database_row = connection.execute("SELECT current_database() AS name").fetchone() + assert database_row is not None + catalog = cast("str", database_row["name"]) + connection.execute( + sql.SQL("CREATE TABLE {} (id TEXT PRIMARY KEY, label TEXT NOT NULL)").format( + sql.Identifier(schema_name, "destination_records") + ) + ) + connection.execute( + sql.SQL("INSERT INTO {} VALUES ('one', 'original')").format( + sql.Identifier(schema_name, "destination_records") + ) + ) + capability = PostgreSQLTargetFence(pool=pool, catalog=catalog) + relation = RelationRef( + catalog=catalog, + namespace=schema_name, + name="destination_records", + ) + first = FencingToken( + lease_table=None, + pipeline_id="fenced-pipeline", + run_id="run-one", + token=1, + authority_id="postgresql:test-state", + ) + first_claim = capability.claim(relation, first) + assert capability.claim(relation, first) == first_claim + + stale_retry = FencingToken( + lease_table=None, + pipeline_id="fenced-pipeline", + run_id="different-run", + token=1, + authority_id="postgresql:test-state", + ) + with pytest.raises(TargetFenceLostError, match="rejected stale"): + capability.claim(relation, stale_retry) + + second = FencingToken( + lease_table=None, + pipeline_id="fenced-pipeline", + run_id="run-two", + token=2, + authority_id="postgresql:test-state", + ) + second_claim = capability.claim(relation, second) + update = ( + sql.SQL("UPDATE {} SET label = %s WHERE id = %s") + .format(sql.Identifier(schema_name, "destination_records")) + .as_string() + ) + with pool.connection() as connection: + with pytest.raises(TargetFenceLostError, match="lost before publication"): + capability.execute_dml( + connection, + update, + first_claim, + ("stale", "one"), + ) + capability.execute_dml( + connection, + update, + second_claim, + ("published", "one"), + ) + row = connection.execute( + sql.SQL("SELECT label FROM {} WHERE id = 'one'").format( + sql.Identifier(schema_name, "destination_records") + ) + ).fetchone() + commit = connection.execute( + sql.SQL( + "SELECT status, run_id, fencing_token FROM {} " + "WHERE target_id = %s AND pipeline_id = %s" + ).format(sql.Identifier(schema_name, "dander_target_commits")), + (".".join(relation.coordinates), "fenced-pipeline"), + ).fetchone() + assert row == {"label": "published"} + assert commit == {"status": "committed", "run_id": "run-two", "fencing_token": 2} + + def test_postgresql_failed_migration_records_no_version( postgresql_runtime: tuple[StateRuntime, PostgreSQLPool, str], monkeypatch: pytest.MonkeyPatch, @@ -308,6 +407,7 @@ def test_postgresql_pool_exhaustion_fails_boundedly( ProviderKind.STATE, { "provider": "postgresql", + "authority_id": "postgresql:test-state", "dsn_env": "DANDER_TEST_POSTGRES_DSN", "schema_name": schema_name, "pool_min_size": 1, @@ -353,6 +453,7 @@ def test_postgresql_pool_replaces_a_lost_connection( ProviderKind.STATE, { "provider": "postgresql", + "authority_id": "postgresql:test-state", "dsn_env": "DANDER_TEST_POSTGRES_DSN", "schema_name": schema_name, "pool_min_size": 1, diff --git a/tests/state/test_lease.py b/tests/state/test_lease.py index f6e06a2c..8d5d06b5 100644 --- a/tests/state/test_lease.py +++ b/tests/state/test_lease.py @@ -77,6 +77,7 @@ def test_bigquery_lease_claims_with_conditional_dml_and_builds_fence() -> None: assert lease is not None assert lease.fencing_token == 7 assert lease.fence is not None + assert lease.fence.lease_table is not None assert lease.fence.lease_table.startswith("unit-project.meta._dander_lease_") claim = next(query for query in client.queries if "fencing_token + 1" in query) assert "lease_expires_at <= CURRENT_TIMESTAMP()" in claim @@ -96,6 +97,8 @@ def test_bigquery_leases_isolate_pipeline_mutations_in_separate_tables() -> None assert greenhouse is not None and greenhouse.fence is not None assert hubspot is not None and hubspot.fence is not None + assert greenhouse.fence.lease_table is not None + assert hubspot.fence.lease_table is not None assert greenhouse.fence.lease_table != hubspot.fence.lease_table assert greenhouse.fence.lease_table.startswith("unit-project.meta._dander_lease_") assert hubspot.fence.lease_table.startswith("unit-project.meta._dander_lease_") diff --git a/tickets/DANDER-97-destination-target-fence.md b/tickets/DANDER-97-destination-target-fence.md new file mode 100644 index 00000000..862ef0f8 --- /dev/null +++ b/tickets/DANDER-97-destination-target-fence.md @@ -0,0 +1,37 @@ +--- +id: DANDER-97 +title: Add destination-side target-fence protocol +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-96] +created: 2026-08-08 +--- + +## Context + +The durable-state backend and destination warehouse may be different systems. A state-side lease +check cannot therefore be transactionally coupled to target publication. + +## Acceptance Criteria + +- [x] Give every state deployment a stable authority ID and positive authority epoch. +- [x] Claim a target before staging using authority, epoch, pipeline, target, run, and token. +- [x] Accept only a newer token or an exact idempotent retry from the current authority and epoch. +- [x] Verify the complete ownership tuple in the same transaction as destination DML. +- [x] Mark a successful target publication committed inside that transaction. +- [x] Implement the protocol for BigQuery and PostgreSQL destinations. +- [x] Prove that an older PostgreSQL claimant cannot publish after a newer claim. +- [x] Keep unsupported cross-backend runtime combinations fail-closed until all callers are wired. + +## Design + +State providers issue provider-neutral `FencingToken` values. Warehouse adapters own +`dander_target_commits` and return a `TargetFence` only after an atomic destination claim. BigQuery +prepares a parameterized transaction script. PostgreSQL exposes the same claim contract and runs +the touch, publication, and completion statements in one Psycopg transaction. + +## Review Log + +This ticket adds no cloud resources or Terraform. The existing PostgreSQL-state/BigQuery runtime +guard remains in place; wiring every writer and materialization path is the next focused change. From f9f25c54f375fdb930f1975fe8e19496bae67faa Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 19:49:35 -0400 Subject: [PATCH 049/163] feat: add bounded PostgreSQL warehouse writes --- HANDOFF.md | 44 +- docs/decisions.md | 12 + docs/platform-profiles.md | 29 +- src/dander/providers/bigquery/fence.py | 2 +- src/dander/providers/bigquery/runtime.py | 2 +- src/dander/providers/defaults.py | 10 +- src/dander/providers/postgresql/__init__.py | 4 +- src/dander/providers/postgresql/config.py | 28 +- src/dander/providers/postgresql/fence.py | 2 +- src/dander/providers/postgresql/runtime.py | 223 ++++++++++ src/dander/providers/postgresql/writer.py | 416 ++++++++++++++++++ src/dander/writer/base.py | 4 +- tests/providers/test_bigquery_runtime.py | 4 +- .../test_postgresql_warehouse_runtime.py | 304 +++++++++++++ tickets/DANDER-98-postgresql-warehouse.md | 39 ++ 15 files changed, 1085 insertions(+), 38 deletions(-) create mode 100644 src/dander/providers/postgresql/runtime.py create mode 100644 src/dander/providers/postgresql/writer.py create mode 100644 tests/providers/test_postgresql_warehouse_runtime.py create mode 100644 tickets/DANDER-98-postgresql-warehouse.md diff --git a/HANDOFF.md b/HANDOFF.md index 30e9433f..37931388 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,40 @@ ## Finished -- Added provider-neutral state authority/epoch and destination `TargetFence` contracts. -- Implemented atomic target claims and transactionally fenced DML for BigQuery destinations. -- Implemented the same claim/publication boundary for PostgreSQL destinations. -- Made PostgreSQL state issue usable cross-backend tokens with an explicit stable authority ID. -- Kept PostgreSQL-state/BigQuery execution fail-closed until every runtime caller is wired. +- Added the lazy PostgreSQL warehouse runtime and explicit SCD1/COPY capability declaration. +- Implemented bounded `COPY` staging and deterministic last-record-wins upserts. +- Bound every PostgreSQL target mutation to the destination target-fence transaction. +- Added canonical PostgreSQL type mapping and narrow nullable-column evolution. +- Added PostgreSQL 15 live conformance coverage without enabling a hosted profile. ## Try It -Run the focused provider tests. PostgreSQL live coverage needs `DANDER_TEST_POSTGRES_DSN` pointing -to a disposable PostgreSQL 15+ database; the test creates and drops its own schemas. +Set `DANDER_TEST_POSTGRES_DSN` to a disposable PostgreSQL 15+ database, then run the two +PostgreSQL provider test modules. Tests create and remove isolated schemas. ## Checks -- All 1,001 Python tests passed locally; PostgreSQL tests skip without a configured DSN. -- Eight PostgreSQL 15 live tests passed against a disposable local container. -- Repository-wide Ruff, formatting, and strict mypy checks passed. -- The stale PostgreSQL claimant test proved target data stayed unchanged until the newer owner ran. -- Distribution inspection/install, dependency audit, and all Terraform validations passed. +- Eleven PostgreSQL 15 state and warehouse tests passed against a local disposable container. +- All 1,004 tests, repository-wide Ruff/formatting, and strict mypy passed locally. +- Dependency audit found no known vulnerabilities; wheel and sdist inspection/install passed. +- GCP, AWS/Fargate, and cross-cloud Terraform formatting, tests, and validation passed. +- The runtime-all container built successfully and passed runtime conformance. ## Decisions -- State authority identity is explicit and non-secret; epoch changes belong to a future cutover. -- Target claims accept only a newer token or the exact same run/token retry. -- Destination fencing is not advertised as usable until ingestion and materialization callers bind it. +- Dander owns endpoint batching; PostgreSQL streams each supplied batch with `COPY`. +- Temporary staging is transaction-scoped with `ON COMMIT DROP`. +- Profile selection, transforms, assertions, and Kubernetes remain separate slices. ## Remaining -- Run protected CI on the focused pull request. -- Wire the target fence into every supported writer and materialization finalizer. -- Implement PostgreSQL warehouse schema, loading, transforms, assertions, and telemetry. -- Then add the Kubernetes launcher and cross-backend matrix proofs. +- Run protected CI and merge the focused pull request. +- Add version 2 PostgreSQL profile selection and bind state fences in runtime assembly. +- Implement PostgreSQL transforms and assertions against the portable SQL contract. +- Add the Kubernetes launcher after the warehouse composition is complete. ## Review First -- `src/dander/concurrency.py` -- `src/dander/providers/bigquery/fence.py` -- `src/dander/providers/postgresql/fence.py` +- `src/dander/providers/postgresql/writer.py` +- `src/dander/providers/postgresql/runtime.py` +- `tests/providers/test_postgresql_warehouse_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 476747fe..b4828257 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -789,3 +789,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - Final publication touches the exact authority/epoch/pipeline/target/run/token tuple and records completion in the same destination transaction. Cross-backend execution remains fail-closed until every writer and materialization caller uses this boundary. + +## 2026-08-08 — PostgreSQL warehouse starts with bounded SCD1 publication + +- PostgreSQL's first warehouse slice supports only the existing SCD1 contract. Dander supplies + bounded batches; the adapter streams each batch through `COPY` into an `ON COMMIT DROP` temporary + relation and selects the final ordinal per business key before upsert. +- The destination target fence is touched and committed in the same transaction as each batch. + An exact run/token claim may therefore finalize multiple batches, while an older token remains + unable to publish after a newer claim. +- PostgreSQL types derive from canonical schema v1. Only declared nullable additions evolve + automatically; extra columns, type or nullability drift, required additions, and malformed + records fail before target DML. Profile selection and transforms remain separate milestones. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index a6f680f6..c4ddd3f0 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -7,10 +7,10 @@ Dander version 2 separates reusable pipeline intent from environment-specific de plus runtime limits, schedules, secret references, and provider resource names. The first supported hosted profile remains BigQuery, BigQuery state, Dataplex (or no cloud -catalog), GCP Secret Manager, and Cloud Run. PostgreSQL 15 or newer is implemented as an -alternative durable-state backend for version 2 projects, but the complete PostgreSQL/Kubernetes -profile is not yet live-qualified. A provider name outside the installed and supported contract -fails validation; configuration alone does not imply profile support. +catalog), GCP Secret Manager, and Cloud Run. PostgreSQL 15 or newer has conformance adapters for +durable state and bounded SCD1 warehouse writes, but the version 2 profile schema does not yet +select a complete PostgreSQL/Kubernetes composition. A provider name outside the installed and +supported contract fails validation; adapter registration alone does not imply profile support. One logical project may have multiple named platforms and deployments. `dander validate` accepts `--deployment`; the Python configuration loader accepts the same explicit selection. Other current @@ -101,11 +101,26 @@ run history, and deterministic JSONB metadata snapshots. `authority_id` is a stable, non-secret identifier for this state deployment. Do not reuse it for a different database or change `authority_epoch` outside a reviewed state-backend cutover. -The current BigQuery warehouse/PostgreSQL state combination fails closed at execution because its -cross-backend destination fence is the next portability ticket. The state adapter is available for -conformance and composition work now; it is not a shortcut around that publication boundary. +The current BigQuery warehouse/PostgreSQL state combination remains fail-closed at execution. +Destination-side target fencing now exists for both systems, but every execution caller must bind +the selected state's token to its warehouse target before any cross-backend combination can be +enabled. Terminal `succeeded`, `failed`, and `skipped` history older than the configured retention is removed when migrations run. Active runs and `interrupted_run` records are retained. Dander refuses a state ledger newer than the running package. Changing state backends is not an online migration: keep schedules paused until the destination-fence and cutover workflow is complete. + +## PostgreSQL warehouse adapter + +The PostgreSQL warehouse adapter is an implementation and conformance boundary, not yet a hosted +profile. It accepts declared schemas, creates database-local schemas and relations, streams each +bounded Dander batch through PostgreSQL `COPY`, and performs deterministic last-record-wins SCD1 +publication inside a transactionally verified destination fence. Temporary staging uses +`ON COMMIT DROP`; nullable top-level columns may be added when additive evolution is selected. + +The configured database must already exist. Runtime-created connections require TLS, and the DSN +is supplied only through the configured environment variable. The first slice supports PostgreSQL +15+, SCD1, and canonical scalar/array/JSON mappings. Transform execution, assertions, launcher +projection, and version 2 profile selection remain separate follow-up work; until then this adapter +must not be represented as an end-to-end supported PostgreSQL deployment. diff --git a/src/dander/providers/bigquery/fence.py b/src/dander/providers/bigquery/fence.py index 4d755110..673b6cd7 100644 --- a/src/dander/providers/bigquery/fence.py +++ b/src/dander/providers/bigquery/fence.py @@ -157,7 +157,7 @@ def _target_match_sql() -> str: "AND authority_id = @dander_authority_id " "AND authority_epoch = @dander_authority_epoch " "AND run_id = @dander_run_id AND fencing_token = @dander_fencing_token " - "AND status = 'claimed'" + "AND status IN ('claimed', 'committed')" ) diff --git a/src/dander/providers/bigquery/runtime.py b/src/dander/providers/bigquery/runtime.py index 6767edce..ea13528c 100644 --- a/src/dander/providers/bigquery/runtime.py +++ b/src/dander/providers/bigquery/runtime.py @@ -130,7 +130,7 @@ def operation( provider_id="bigquery", schema_contract_version=1, write_modes=frozenset(WriteMode), - transports=frozenset(WriteTransport), + transports=frozenset({WriteTransport.LOAD_JOB, WriteTransport.STORAGE_WRITE}), supports_transforms=True, supports_graphs=True, supports_target_fencing=True, diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 51cc044d..b74fd279 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -8,7 +8,7 @@ from dander.providers.fargate import FargateLauncherConfig from dander.providers.gcp_secret_manager import GcpSecretManagerConfig from dander.providers.no_catalog import NoCatalogConfig -from dander.providers.postgresql import PostgreSQLStateConfig +from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -31,6 +31,14 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.bigquery.state:BIGQUERY_STATE_FACTORY" ), ) + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="postgresql", + config_model=PostgreSQLWarehouseConfig, + load_factory=lazy_provider_factory( + "dander.providers.postgresql.runtime:POSTGRESQL_WAREHOUSE_FACTORY" + ), + ) registry.register( kind=ProviderKind.STATE, provider_id="postgresql", diff --git a/src/dander/providers/postgresql/__init__.py b/src/dander/providers/postgresql/__init__.py index 91de5e2d..9f64200b 100644 --- a/src/dander/providers/postgresql/__init__.py +++ b/src/dander/providers/postgresql/__init__.py @@ -1,5 +1,5 @@ """PostgreSQL provider configuration exports.""" -from dander.providers.postgresql.config import PostgreSQLStateConfig +from dander.providers.postgresql.config import PostgreSQLStateConfig, PostgreSQLWarehouseConfig -__all__ = ["PostgreSQLStateConfig"] +__all__ = ["PostgreSQLStateConfig", "PostgreSQLWarehouseConfig"] diff --git a/src/dander/providers/postgresql/config.py b/src/dander/providers/postgresql/config.py index f0149c49..b5b3bd02 100644 --- a/src/dander/providers/postgresql/config.py +++ b/src/dander/providers/postgresql/config.py @@ -45,4 +45,30 @@ def validate_settings(self) -> PostgreSQLStateConfig: return self -__all__ = ["PostgreSQLStateConfig"] +class PostgreSQLWarehouseConfig(BaseModel): + """PostgreSQL warehouse connection, pool, and transaction limits.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["postgresql"] + database: str + dsn_env: str = "DANDER_POSTGRES_DSN" + pool_min_size: int = Field(default=1, ge=1, le=20) + pool_max_size: int = Field(default=5, ge=1, le=50) + pool_timeout_seconds: float = Field(default=10.0, gt=0, le=120) + statement_timeout_ms: int = Field(default=300_000, ge=1_000, le=3_600_000) + lock_timeout_ms: int = Field(default=30_000, ge=100, le=300_000) + idle_transaction_timeout_ms: int = Field(default=60_000, ge=1_000, le=600_000) + + @model_validator(mode="after") + def validate_settings(self) -> PostgreSQLWarehouseConfig: + if not _SCHEMA_NAME.fullmatch(self.database): + raise ValueError("database must be a safe lowercase PostgreSQL identifier") + if not _ENVIRONMENT_NAME.fullmatch(self.dsn_env): + raise ValueError("dsn_env must be an uppercase environment-variable name") + if self.pool_min_size > self.pool_max_size: + raise ValueError("pool_min_size must not exceed pool_max_size") + return self + + +__all__ = ["PostgreSQLStateConfig", "PostgreSQLWarehouseConfig"] diff --git a/src/dander/providers/postgresql/fence.py b/src/dander/providers/postgresql/fence.py index 1e1877a6..8ef0902a 100644 --- a/src/dander/providers/postgresql/fence.py +++ b/src/dander/providers/postgresql/fence.py @@ -145,7 +145,7 @@ def _postgresql_target_match_sql() -> str: return ( "target_id = %s AND pipeline_id = %s AND authority_id = %s " "AND authority_epoch = %s AND run_id = %s AND fencing_token = %s " - "AND status = 'claimed'" + "AND status IN ('claimed', 'committed')" ) diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py new file mode 100644 index 00000000..33e82032 --- /dev/null +++ b/src/dander/providers/postgresql/runtime.py @@ -0,0 +1,223 @@ +"""PostgreSQL implementation of Dander's warehouse capability bundle.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING, cast + +from psycopg import Connection +from psycopg.conninfo import conninfo_to_dict +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.providers.postgresql.config import PostgreSQLWarehouseConfig +from dander.providers.postgresql.fence import PostgreSQLTargetFence +from dander.providers.postgresql.writer import PostgreSQLScd1Writer, PostgreSQLTimeouts +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.telemetry import OperationTelemetry, TelemetryOperation +from dander.warehouse import CanonicalField, RelationRef, RelationSchema +from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.writer import ( + SchemaEvolution, + WriteField, + WriteMode, + WritePattern, + WriteTransport, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from pydantic import BaseModel + +PostgreSQLRow = dict[str, object] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +@dataclass(frozen=True, slots=True) +class PostgreSQLRelationCodec: + """Render one database-local PostgreSQL schema/relation coordinate.""" + + database: str + provider_id: str = "postgresql" + + def render(self, relation: RelationRef) -> str: + if relation.catalog != self.database: + raise ValueError("PostgreSQL relation belongs to another database") + return f"{_quote(relation.namespace)}.{_quote(relation.name)}" + + +@dataclass(frozen=True, slots=True) +class PostgreSQLSchemaMapper: + """Map compatibility `WriteField` declarations into canonical schema v1.""" + + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + canonical: list[CanonicalField] = [] + for field in fields: + if isinstance(field, WriteField): + canonical.append(field.to_canonical()) + elif isinstance(field, CanonicalField): + canonical.append(field) + else: + raise TypeError("PostgreSQL schema mapper received an unsupported field") + return RelationSchema(fields=tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class PostgreSQLWriterFactory: + """Construct PostgreSQL's first bounded hosted ingestion writer.""" + + database: str + pool: PostgreSQLPool + target_fence: PostgreSQLTargetFence + timeouts: PostgreSQLTimeouts + + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + ) -> WritePattern: + del batch_rows + if sandbox: + raise ValueError("PostgreSQL warehouse does not use Dander's BigQuery sandbox mode") + return PostgreSQLScd1Writer( + database=self.database, + pool=self.pool, + target_fence=self.target_fence, + schema_evolution=schema_evolution, + timeouts=self.timeouts, + ) + + +@dataclass(frozen=True, slots=True) +class PostgreSQLTransformFactory: + """Fail clearly until the PostgreSQL transform runner lands.""" + + def build_transform_runner( + self, + *, + graph_plan: object | None, + build_models: bool, + ) -> None: + if graph_plan is not None or build_models: + raise ValueError("PostgreSQL transforms are not available in this adapter slice") + return None + + +@dataclass(frozen=True, slots=True) +class PostgreSQLTelemetry: + """Normalize stable Psycopg cursor counters without exposing statements or values.""" + + def operation( + self, + job: object, + *, + operation: TelemetryOperation, + duration_ms: int = 0, + retry_count: int = 0, + ) -> OperationTelemetry: + rowcount = getattr(job, "rowcount", 0) + affected = rowcount if isinstance(rowcount, int) and rowcount >= 0 else 0 + return OperationTelemetry( + provider="postgresql", + operation=operation, + duration_ms=duration_ms, + retry_count=retry_count, + rows_affected=affected, + ) + + +POSTGRESQL_CAPABILITIES = WarehouseCapabilities( + provider_id="postgresql", + schema_contract_version=1, + write_modes=frozenset({WriteMode.SCD1}), + transports=frozenset({WriteTransport.COPY}), + supports_transforms=False, + supports_graphs=False, + supports_target_fencing=True, +) + + +def build_postgresql_warehouse( + config: BaseModel, + context: Mapping[str, object], +) -> WarehouseRuntime: + """Build a TLS-required PostgreSQL runtime or use an injected conformance pool.""" + if not isinstance(config, PostgreSQLWarehouseConfig): + raise TypeError("PostgreSQL warehouse factory received the wrong configuration") + supplied_pool = context.get("pool") + if supplied_pool is None: + dsn = os.environ.get(config.dsn_env) + if not dsn: + raise ValueError( + f"PostgreSQL warehouse requires a connection string in {config.dsn_env}" + ) + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=config.pool_min_size, + max_size=config.pool_max_size, + timeout=config.pool_timeout_seconds, + kwargs={"row_factory": dict_row, "sslmode": _required_sslmode(dsn)}, + open=True, + ), + ) + pool.wait(timeout=config.pool_timeout_seconds) + elif isinstance(supplied_pool, ConnectionPool): + pool = supplied_pool + else: + raise TypeError("PostgreSQL warehouse context pool must be a psycopg ConnectionPool") + + with pool.connection() as connection: + row = connection.execute("SELECT current_database() AS database").fetchone() + if row is None or row["database"] != config.database: + raise ValueError("PostgreSQL warehouse database does not match the selected connection") + + target_fence = PostgreSQLTargetFence(pool=pool, catalog=config.database) + timeouts = PostgreSQLTimeouts( + statement_ms=config.statement_timeout_ms, + lock_ms=config.lock_timeout_ms, + idle_transaction_ms=config.idle_transaction_timeout_ms, + ) + return WarehouseRuntime( + provider_id="postgresql", + relation_codec=PostgreSQLRelationCodec(config.database), + schema_mapper=PostgreSQLSchemaMapper(), + writers=PostgreSQLWriterFactory(config.database, pool, target_fence, timeouts), + transforms=PostgreSQLTransformFactory(), + target_fence=target_fence, + telemetry=PostgreSQLTelemetry(), + capabilities=POSTGRESQL_CAPABILITIES, + ) + + +POSTGRESQL_WAREHOUSE_FACTORY = ProviderFactory[WarehouseRuntime]( + kind=ProviderKind.WAREHOUSE, + provider_id="postgresql", + api_version=PROVIDER_API_VERSION, + build=build_postgresql_warehouse, +) + + +def _quote(identifier: str) -> str: + return f'"{identifier.replace(chr(34), chr(34) * 2)}"' + + +def _required_sslmode(dsn: str) -> str: + """Require encrypted transport without weakening stricter caller verification.""" + configured = conninfo_to_dict(dsn).get("sslmode") + if configured in {"require", "verify-ca", "verify-full"}: + return configured + return "require" + + +__all__ = [ + "POSTGRESQL_CAPABILITIES", + "POSTGRESQL_WAREHOUSE_FACTORY", + "PostgreSQLRelationCodec", + "PostgreSQLSchemaMapper", +] diff --git a/src/dander/providers/postgresql/writer.py b/src/dander/providers/postgresql/writer.py new file mode 100644 index 00000000..8f3e5bc0 --- /dev/null +++ b/src/dander/providers/postgresql/writer.py @@ -0,0 +1,416 @@ +"""PostgreSQL schema management and bounded COPY-backed SCD1 writes.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from typing import TYPE_CHECKING +from uuid import uuid4 + +from psycopg import Connection, sql +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from dander.warehouse import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + RelationSchema, +) +from dander.writer import SchemaEvolution, WriteMode, WritePattern, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping, Sequence + + from dander.providers.postgresql.fence import PostgreSQLTargetFence + +PostgreSQLRow = dict[str, object] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +class PostgreSQLWriteError(ValueError): + """Raised when records or a deployed relation violate the PostgreSQL contract.""" + + +@dataclass(frozen=True, slots=True) +class PostgreSQLTimeouts: + """Transaction-local PostgreSQL timeout values in milliseconds.""" + + statement_ms: int + lock_ms: int + idle_transaction_ms: int + + +class PostgreSQLScd1Writer(WritePattern): + """Stream records through COPY and publish keyed rows transactionally.""" + + mode = WriteMode.SCD1 + supports_batched_writes = True + + def __init__( + self, + *, + database: str, + pool: PostgreSQLPool, + target_fence: PostgreSQLTargetFence, + schema_evolution: SchemaEvolution = SchemaEvolution.STRICT, + timeouts: PostgreSQLTimeouts, + ) -> None: + self._database = database + self._pool = pool + self._target_fence = target_fence + self._schema_evolution = schema_evolution + self._timeouts = timeouts + + def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: + """COPY one bounded batch and upsert its deterministic last record per key.""" + _validate_target(target, database=self._database) + publication_fence = target.publication_fence + assert publication_fence is not None + schema = target.canonical_schema + fields = {field.name: field for field in schema.fields} + missing_keys = sorted(set(target.business_key) - set(fields)) + if missing_keys: + raise PostgreSQLWriteError( + f"Business-key column {missing_keys[0]!r} is absent from the declared schema" + ) + + with self._pool.connection() as connection, connection.transaction(): + _set_timeouts(connection, self._timeouts) + _ensure_target( + connection, + target, + schema, + evolution=self._schema_evolution, + ) + staging = f"dander_stage_{uuid4().hex}" + _create_staging(connection, staging, schema, target.business_key) + written = _copy_records( + connection, + staging, + records, + schema.fields, + target.business_key, + ) + if written == 0: + no_op = sql.SQL("UPDATE {} SET {} = {} WHERE FALSE").format( + _target_identifier(target), + sql.Identifier(target.business_key[0]), + sql.Identifier(target.business_key[0]), + ) + self._target_fence.execute_dml( + connection, + no_op.as_string(connection), + publication_fence, + ) + return 0 + finalizer = _scd1_sql(target, staging, schema.fields) + cursor = self._target_fence.execute_dml( + connection, + finalizer.as_string(connection), + publication_fence, + ) + return max(cursor.rowcount, 0) + + +def _validate_target(target: WriteTarget, *, database: str) -> None: + if target.project != database: + raise PostgreSQLWriteError( + f"Writer database {database!r} does not match target catalog {target.project!r}" + ) + if not target.business_key: + raise PostgreSQLWriteError("PostgreSQL SCD1 writes require a business key") + if not target.schema: + raise PostgreSQLWriteError("PostgreSQL writes require a declared schema") + publication_fence = target.publication_fence + if publication_fence is None: + raise PostgreSQLWriteError("PostgreSQL hosted writes require a destination target fence") + expected_target = ".".join(target.relation_ref.coordinates) + expected_fence_table = f"{target.dataset}.dander_target_commits" + if ( + publication_fence.target_id != expected_target + or publication_fence.fence_table != expected_fence_table + ): + raise PostgreSQLWriteError("PostgreSQL destination target fence does not match the target") + + +def _set_timeouts( + connection: Connection[PostgreSQLRow], + timeouts: PostgreSQLTimeouts, +) -> None: + for name, value in ( + ("statement_timeout", timeouts.statement_ms), + ("lock_timeout", timeouts.lock_ms), + ("idle_in_transaction_session_timeout", timeouts.idle_transaction_ms), + ): + connection.execute("SELECT set_config(%s, %s, true)", (name, str(value))) + + +def _ensure_target( + connection: Connection[PostgreSQLRow], + target: WriteTarget, + schema: RelationSchema, + *, + evolution: SchemaEvolution, +) -> None: + connection.execute( + sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format(sql.Identifier(target.dataset)) + ) + definitions = [ + sql.SQL("{} {}{}").format( + sql.Identifier(field.name), + sql.SQL(_postgresql_type(field.data_type)), + sql.SQL(" NOT NULL" if _requires_value(field, target.business_key) else ""), + ) + for field in schema.fields + ] + connection.execute( + sql.SQL("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))").format( + _target_identifier(target), + sql.SQL(", ").join(definitions), + sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key), + ) + ) + deployed = _deployed_columns(connection, target) + expected = {field.name: field for field in schema.fields} + extras = sorted(set(deployed) - set(expected)) + if extras: + raise PostgreSQLWriteError(f"Deployed target has undeclared column {extras[0]!r}") + for field in schema.fields: + current = deployed.get(field.name) + if current is None: + if evolution is not SchemaEvolution.ADDITIVE: + raise PostgreSQLWriteError(f"Deployed target is missing column {field.name!r}") + if _requires_value(field, target.business_key): + raise PostgreSQLWriteError( + f"Cannot add required PostgreSQL column {field.name!r} automatically" + ) + connection.execute( + sql.SQL("ALTER TABLE {} ADD COLUMN {} {}").format( + _target_identifier(target), + sql.Identifier(field.name), + sql.SQL(_postgresql_type(field.data_type)), + ) + ) + continue + deployed_type, deployed_required = current + expected_type = _normalize_type(_postgresql_type(field.data_type)) + expected_required = _requires_value(field, target.business_key) + if _normalize_type(deployed_type) != expected_type: + raise PostgreSQLWriteError(f"PostgreSQL type drift for column {field.name!r}") + if deployed_required != expected_required: + raise PostgreSQLWriteError(f"PostgreSQL nullability drift for column {field.name!r}") + + index_name = _unique_index_name(target) + connection.execute( + sql.SQL("CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} ({})").format( + sql.Identifier(index_name), + _target_identifier(target), + sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key), + ) + ) + + +def _deployed_columns( + connection: Connection[PostgreSQLRow], + target: WriteTarget, +) -> dict[str, tuple[str, bool]]: + rows = connection.execute( + "SELECT attribute.attname AS name, " + "pg_catalog.format_type(attribute.atttypid, attribute.atttypmod) AS data_type, " + "attribute.attnotnull AS required " + "FROM pg_catalog.pg_attribute AS attribute " + "JOIN pg_catalog.pg_class AS relation ON relation.oid = attribute.attrelid " + "JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace " + "WHERE namespace.nspname = %s AND relation.relname = %s " + "AND attribute.attnum > 0 AND NOT attribute.attisdropped", + (target.dataset, target.table), + ).fetchall() + deployed: dict[str, tuple[str, bool]] = {} + for row in rows: + name = row["name"] + data_type = row["data_type"] + required = row["required"] + if ( + not isinstance(name, str) + or not isinstance(data_type, str) + or not isinstance(required, bool) + ): + raise PostgreSQLWriteError("PostgreSQL returned invalid relation metadata") + deployed[name] = (data_type, required) + return deployed + + +def _create_staging( + connection: Connection[PostgreSQLRow], + staging: str, + schema: RelationSchema, + business_key: Sequence[str], +) -> None: + definitions = [ + sql.SQL("{} {}{}").format( + sql.Identifier(field.name), + sql.SQL(_postgresql_type(field.data_type)), + sql.SQL(" NOT NULL" if _requires_value(field, business_key) else ""), + ) + for field in schema.fields + ] + definitions.append( + sql.SQL("{} BIGINT GENERATED ALWAYS AS IDENTITY").format(sql.Identifier("_dander_ordinal")) + ) + connection.execute( + sql.SQL("CREATE TEMP TABLE {} ({}) ON COMMIT DROP").format( + sql.Identifier(staging), + sql.SQL(", ").join(definitions), + ) + ) + + +def _copy_records( + connection: Connection[PostgreSQLRow], + staging: str, + records: Iterable[Mapping[str, object]], + fields: Sequence[CanonicalField], + business_key: Sequence[str], +) -> int: + names = tuple(field.name for field in fields) + expected = set(names) + copied = 0 + statement = sql.SQL("COPY {} ({}) FROM STDIN").format( + sql.Identifier(staging), + sql.SQL(", ").join(sql.Identifier(name) for name in names), + ) + with connection.cursor().copy(statement) as copy: + for index, record in enumerate(records): + if set(record) != expected: + raise PostgreSQLWriteError( + f"Record {index} does not match the declared PostgreSQL schema" + ) + if any(record[key] is None for key in business_key): + raise PostgreSQLWriteError(f"Record {index} has a null business-key value") + copy.write_row( + tuple(_postgresql_value(record[field.name], field.data_type) for field in fields) + ) + copied += 1 + return copied + + +def _scd1_sql( + target: WriteTarget, + staging: str, + fields: Sequence[CanonicalField], +) -> sql.Composed: + names = tuple(field.name for field in fields) + mutable = tuple(name for name in names if name not in target.business_key) + conflict = ( + sql.SQL("DO UPDATE SET {} ").format( + sql.SQL(", ").join( + sql.SQL("{} = EXCLUDED.{}").format( + sql.Identifier(name), + sql.Identifier(name), + ) + for name in mutable + ) + ) + if mutable + else sql.SQL("DO NOTHING ") + ) + selected = sql.SQL(", ").join(sql.Identifier(name) for name in names) + keys = sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key) + ordering = sql.SQL(", ").join( + [*(sql.Identifier(key) for key in target.business_key), sql.Identifier("_dander_ordinal")] + ) + return sql.SQL( + "INSERT INTO {} ({}) SELECT {} FROM (" + "SELECT DISTINCT ON ({}) {}, {} FROM {} ORDER BY {} DESC" + ") AS incoming ON CONFLICT ({}) {}RETURNING 1" + ).format( + _target_identifier(target), + selected, + selected, + keys, + selected, + sql.Identifier("_dander_ordinal"), + sql.Identifier(staging), + ordering, + keys, + conflict, + ) + + +def _target_identifier(target: WriteTarget) -> sql.Identifier: + return sql.Identifier(target.dataset, target.table) + + +def _unique_index_name(target: WriteTarget) -> str: + digest = hashlib.sha256( + f"{target.dataset}.{target.table}:{','.join(target.business_key)}".encode() + ).hexdigest()[:10] + prefix = f"dander_uq_{target.table}"[:51] + return f"{prefix}_{digest}" + + +def _requires_value(field: CanonicalField, business_key: Sequence[str]) -> bool: + return field.cardinality is FieldCardinality.REQUIRED or field.name in business_key + + +def _normalize_type(value: str) -> str: + return " ".join(value.lower().split()) + + +def _postgresql_type(data_type: CanonicalType) -> str: + match data_type.kind: + case LogicalTypeKind.BOOLEAN: + return "boolean" + case LogicalTypeKind.INTEGER: + if data_type.bit_width in {8, 16}: + return "smallint" + if data_type.bit_width == 32: + return "integer" + return "bigint" + case LogicalTypeKind.DECIMAL: + return f"numeric({data_type.precision},{data_type.scale})" + case LogicalTypeKind.FLOAT: + return "real" if data_type.bit_width == 32 else "double precision" + case LogicalTypeKind.STRING: + return "text" + case LogicalTypeKind.BINARY: + return "bytea" + case LogicalTypeKind.DATE: + return "date" + case LogicalTypeKind.TIME: + return f"time({data_type.fractional_second_precision}) without time zone" + case LogicalTypeKind.TIMESTAMP: + timezone = "with" if data_type.with_timezone else "without" + return f"timestamp({data_type.fractional_second_precision}) {timezone} time zone" + case LogicalTypeKind.JSON | LogicalTypeKind.RECORD: + return "jsonb" + case LogicalTypeKind.ARRAY: + assert data_type.element is not None + return f"{_postgresql_type(data_type.element)}[]" + raise AssertionError("Unhandled canonical PostgreSQL type") + + +def _postgresql_value(value: object, data_type: CanonicalType) -> object: + if value is None: + return None + if data_type.kind in {LogicalTypeKind.JSON, LogicalTypeKind.RECORD}: + return Jsonb(value) + if data_type.kind is LogicalTypeKind.ARRAY: + assert data_type.element is not None + if not isinstance(value, (list, tuple)): + raise PostgreSQLWriteError("PostgreSQL array values must be lists or tuples") + return [_postgresql_value(item, data_type.element) for item in value] + if isinstance(value, (str, int, float, bool, bytes, Decimal, date, time, datetime)): + return value + raise PostgreSQLWriteError("PostgreSQL record contains an unsupported scalar value") + + +__all__ = [ + "PostgreSQLScd1Writer", + "PostgreSQLTimeouts", + "PostgreSQLWriteError", +] diff --git a/src/dander/writer/base.py b/src/dander/writer/base.py index 685198c0..0db32726 100644 --- a/src/dander/writer/base.py +++ b/src/dander/writer/base.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Mapping - from dander.concurrency import FencingToken + from dander.concurrency import FencingToken, TargetFence from dander.warehouse import CanonicalField, RelationRef, RelationSchema @@ -41,6 +41,7 @@ class WriteTransport(StrEnum): LOAD_JOB = "load_job" STORAGE_WRITE = "storage_write" + COPY = "copy" @dataclass(frozen=True) @@ -69,6 +70,7 @@ class WriteTarget: business_key: tuple[str, ...] = field(default_factory=tuple) schema: tuple[WriteField, ...] = field(default_factory=tuple) fence: FencingToken | None = None + publication_fence: TargetFence | None = None @property def relation_ref(self) -> RelationRef: diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index bb883698..90ec5d4f 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -54,7 +54,9 @@ def test_default_registry_loads_bigquery_runtime_only_after_selection() -> None: assert isinstance(runtime, WarehouseRuntime) assert module_name in sys.modules assert runtime.capabilities.write_modes == frozenset(WriteMode) - assert runtime.capabilities.transports == frozenset(WriteTransport) + assert runtime.capabilities.transports == frozenset( + {WriteTransport.LOAD_JOB, WriteTransport.STORAGE_WRITE} + ) def test_bigquery_runtime_exposes_codec_schema_fence_and_telemetry() -> None: diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py new file mode 100644 index 00000000..ca4be22f --- /dev/null +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -0,0 +1,304 @@ +"""PostgreSQL warehouse registration and live SCD1 conformance.""" + +from __future__ import annotations + +import os +import sys +import uuid +from datetime import UTC, datetime +from decimal import Decimal +from typing import TYPE_CHECKING, Any, cast + +import pytest +from psycopg import Connection, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.concurrency import FencingToken +from dander.providers import ProviderKind, default_provider_registry +from dander.providers.postgresql.writer import PostgreSQLWriteError +from dander.telemetry import TelemetryOperation +from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Iterator + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +def test_postgresql_warehouse_registration_is_lazy_and_explicit() -> None: + module_name = "dander.providers.postgresql.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": "postgresql", + "database": "dander_test", + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + }, + ) + + assert module_name not in sys.modules + assert config.model_dump(mode="json") == { + "provider": "postgresql", + "database": "dander_test", + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "pool_min_size": 1, + "pool_max_size": 5, + "pool_timeout_seconds": 10.0, + "statement_timeout_ms": 300_000, + "lock_timeout_ms": 30_000, + "idle_transaction_timeout_ms": 60_000, + } + + from dander.providers.postgresql.runtime import _required_sslmode + + assert _required_sslmode("postgresql://example.invalid/db") == "require" + assert _required_sslmode("postgresql://example.invalid/db?sslmode=disable") == "require" + assert _required_sslmode("postgresql://example.invalid/db?sslmode=verify-full") == "verify-full" + + +@pytest.fixture +def postgresql_warehouse() -> Iterator[tuple[WarehouseRuntime, PostgreSQLPool, str, str]]: + dsn = os.environ.get("DANDER_TEST_POSTGRES_DSN") + if not dsn: + pytest.skip("DANDER_TEST_POSTGRES_DSN is not configured") + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=5, + timeout=2, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + with pool.connection() as connection: + row = connection.execute("SELECT current_database() AS database").fetchone() + assert row is not None + database = cast("str", row["database"]) + schema_name = f"dander_wh_{uuid.uuid4().hex}" + with pool.connection() as connection: + connection.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema_name))) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": "postgresql", + "database": database, + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "statement_timeout_ms": 30_000, + "lock_timeout_ms": 5_000, + "idle_transaction_timeout_ms": 10_000, + }, + ) + runtime = registry.build(ProviderKind.WAREHOUSE, config, context={"pool": pool}) + assert isinstance(runtime, WarehouseRuntime) + try: + yield runtime, pool, database, schema_name + finally: + with pool.connection() as connection: + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_name)) + ) + pool.close() + + +def test_postgresql_runtime_exposes_codec_schema_capabilities_and_telemetry( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, _pool, database, _schema_name = postgresql_warehouse + relation = RelationRef(catalog=database, namespace="raw", name="records") + schema = runtime.schema_mapper.canonical_schema( + [ + WriteField(name="id", data_type="INT64", mode="REQUIRED"), + WriteField(name="payload", data_type="JSON"), + WriteField(name="tags", data_type="STRING", mode="REPEATED"), + ] + ) + + class _Cursor: + rowcount = 9 + + telemetry = runtime.telemetry.operation(_Cursor(), operation=TelemetryOperation.LOAD) + + assert runtime.relation_codec.render(relation) == '"raw"."records"' + assert schema.fields[0].data_type.kind is LogicalTypeKind.INTEGER + assert schema.fields[1].data_type.kind is LogicalTypeKind.JSON + assert schema.fields[2].data_type.kind is LogicalTypeKind.ARRAY + assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) + assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) + assert runtime.capabilities.supports_transforms is False + assert telemetry.rows_affected == 9 + + +def test_postgresql_scd1_streams_copy_replays_and_evolves_nullable_columns( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + base_schema = ( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + WriteField(name="amount", data_type="NUMERIC"), + WriteField(name="observed_at", data_type="TIMESTAMP"), + WriteField(name="payload", data_type="JSON"), + WriteField(name="tags", data_type="STRING", mode="REPEATED"), + ) + lease = FencingToken( + lease_table=None, + pipeline_id="postgresql_records", + run_id="run-one", + token=1, + authority_id="postgresql:test-state", + ) + relation = RelationRef(catalog=database, namespace=schema_name, name="records") + publication = runtime.target_fence.claim(relation, lease) + target = WriteTarget( + project=database, + dataset=schema_name, + table="records", + business_key=("id",), + schema=base_schema, + publication_fence=publication, + ) + unfenced = WriteTarget( + project=database, + dataset=schema_name, + table="records", + business_key=("id",), + schema=base_schema, + ) + with pytest.raises(PostgreSQLWriteError, match="require a destination target fence"): + writer.write((), unfenced) + + def initial_records() -> Iterator[dict[str, object]]: + yield { + "id": "one", + "label": "old", + "amount": Decimal("10.000000000"), + "observed_at": datetime(2026, 8, 8, 20, 0, tzinfo=UTC), + "payload": {"active": True}, + "tags": ["first"], + } + yield { + "id": "one", + "label": "new", + "amount": Decimal("11.000000000"), + "observed_at": datetime(2026, 8, 8, 20, 1, tzinfo=UTC), + "payload": {"active": True}, + "tags": ["latest"], + } + yield { + "id": "two", + "label": "second", + "amount": Decimal("12.000000000"), + "observed_at": datetime(2026, 8, 8, 20, 2, tzinfo=UTC), + "payload": {"active": False}, + "tags": [], + } + + assert writer.write(initial_records(), target) == 2 + assert ( + writer.write( + [ + { + "id": "one", + "label": "replayed", + "amount": Decimal("13.000000000"), + "observed_at": datetime(2026, 8, 8, 20, 3, tzinfo=UTC), + "payload": {"active": True}, + "tags": ["replay"], + } + ], + target, + ) + == 1 + ) + + evolved = WriteTarget( + project=database, + dataset=schema_name, + table="records", + business_key=("id",), + schema=(*base_schema, WriteField(name="note", data_type="STRING")), + publication_fence=publication, + ) + assert ( + writer.write( + [ + { + "id": "three", + "label": "third", + "amount": Decimal("14.000000000"), + "observed_at": datetime(2026, 8, 8, 20, 4, tzinfo=UTC), + "payload": {"active": True}, + "tags": ["additive"], + "note": "new column", + } + ], + evolved, + ) + == 1 + ) + + drifted = WriteTarget( + project=database, + dataset=schema_name, + table="records", + business_key=("id",), + schema=( + base_schema[0], + WriteField(name="label", data_type="INT64"), + *base_schema[2:], + WriteField(name="note", data_type="STRING"), + ), + publication_fence=publication, + ) + with pytest.raises(PostgreSQLWriteError, match="type drift.*label"): + writer.write((), drifted) + + with pool.connection() as connection: + rows = connection.execute( + sql.SQL("SELECT id, label, note FROM {} ORDER BY id").format( + sql.Identifier(schema_name, "records") + ) + ).fetchall() + typed = connection.execute( + sql.SQL("SELECT amount, observed_at, payload, tags FROM {} WHERE id = 'one'").format( + sql.Identifier(schema_name, "records") + ) + ).fetchone() + ledger = connection.execute( + sql.SQL( + "SELECT status, run_id, fencing_token FROM {} " + "WHERE target_id = %s AND pipeline_id = %s" + ).format(sql.Identifier(schema_name, "dander_target_commits")), + (".".join(relation.coordinates), "postgresql_records"), + ).fetchone() + staging = connection.execute( + "SELECT COUNT(*) AS count FROM pg_catalog.pg_class WHERE relname LIKE 'dander_stage_%'" + ).fetchone() + assert rows == [ + {"id": "one", "label": "replayed", "note": None}, + {"id": "three", "label": "third", "note": "new column"}, + {"id": "two", "label": "second", "note": None}, + ] + assert typed == { + "amount": Decimal("13.000000000"), + "observed_at": datetime(2026, 8, 8, 20, 3, tzinfo=UTC), + "payload": {"active": True}, + "tags": ["replay"], + } + assert ledger == {"status": "committed", "run_id": "run-one", "fencing_token": 1} + assert staging == {"count": 0} diff --git a/tickets/DANDER-98-postgresql-warehouse.md b/tickets/DANDER-98-postgresql-warehouse.md new file mode 100644 index 00000000..8a699474 --- /dev/null +++ b/tickets/DANDER-98-postgresql-warehouse.md @@ -0,0 +1,39 @@ +--- +id: DANDER-98 +title: Implement bounded PostgreSQL SCD1 warehouse writes +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-97] +created: 2026-08-08 +--- + +## Context + +PostgreSQL is the reference portable warehouse. Its first vertical slice must prove bounded, +schema-safe destination publication before profile selection, transforms, or Kubernetes expand the +surface area. + +## Acceptance Criteria + +- [x] Register a lazy PostgreSQL warehouse provider without importing Psycopg in base installs. +- [x] Keep the DSN outside manifests and require TLS for runtime-created connections. +- [x] Map canonical schema v1 to PostgreSQL scalar, array, record, and JSON types. +- [x] Stream bounded batches through `COPY` without materializing the endpoint. +- [x] Preserve deterministic last-record-wins SCD1 behavior within and across batches. +- [x] Verify and commit destination fencing in the same transaction as target DML. +- [x] Drop handled staging at transaction completion and leave no permanent staging relation. +- [x] Permit only explicitly declared nullable additive columns and reject other drift. +- [x] Prove replay, empty batches, typed values, schema evolution, and cleanup on PostgreSQL 15. +- [x] Keep profile selection and PostgreSQL transforms explicitly out of this slice. + +## Design + +The existing `PipelineRunner` owns endpoint batching. `PostgreSQLScd1Writer` accepts one bounded +batch, streams it to a temporary table with an ordinal, reduces duplicate keys with `DISTINCT ON`, +and performs one fenced `INSERT ... ON CONFLICT`. A selected PostgreSQL runtime exposes only SCD1 +and `COPY` capabilities; unsupported sandbox and transform requests fail clearly. + +## Review Log + +No Terraform, cloud resource, retained project, or public support claim changes in this ticket. From 5f17c72c38c90daa5171ed2ed5d87fb9a84ae080 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 20:05:11 -0400 Subject: [PATCH 050/163] feat: add PostgreSQL transform execution --- HANDOFF.md | 43 +- docs/decisions.md | 12 + docs/platform-profiles.md | 10 +- src/dander/providers/postgresql/fence.py | 39 +- src/dander/providers/postgresql/runtime.py | 27 +- src/dander/providers/postgresql/transform.py | 368 ++++++++++++++++++ src/dander/transform/project.py | 87 +++-- .../test_postgresql_transform_runtime.py | 331 ++++++++++++++++ .../test_postgresql_warehouse_runtime.py | 2 +- tests/transform/test_project.py | 19 + tickets/DANDER-99-postgresql-transforms.md | 37 ++ 11 files changed, 909 insertions(+), 66 deletions(-) create mode 100644 src/dander/providers/postgresql/transform.py create mode 100644 tests/providers/test_postgresql_transform_runtime.py create mode 100644 tickets/DANDER-99-postgresql-transforms.md diff --git a/HANDOFF.md b/HANDOFF.md index 37931388..09f90a98 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,39 @@ ## Finished -- Added the lazy PostgreSQL warehouse runtime and explicit SCD1/COPY capability declaration. -- Implemented bounded `COPY` staging and deterministic last-record-wins upserts. -- Bound every PostgreSQL target mutation to the destination target-fence transaction. -- Added canonical PostgreSQL type mapping and narrow nullable-column evolution. -- Added PostgreSQL 15 live conformance coverage without enabling a hosted profile. +- Made transform-project relation rendering target-dialect aware without changing BigQuery defaults. +- Added fenced PostgreSQL table, view, and incremental materializations. +- Added PostgreSQL-native not-null, unique, accepted-values, and relationship assertions. +- Wired PostgreSQL transform construction into the warehouse capability bundle. +- Added live PostgreSQL 15 replay, stale-owner, and sanitized-failure coverage. ## Try It -Set `DANDER_TEST_POSTGRES_DSN` to a disposable PostgreSQL 15+ database, then run the two -PostgreSQL provider test modules. Tests create and remove isolated schemas. +Set `DANDER_TEST_POSTGRES_DSN` to a disposable PostgreSQL 15+ database and run +`tests/providers/test_postgresql_transform_runtime.py`. ## Checks -- Eleven PostgreSQL 15 state and warehouse tests passed against a local disposable container. -- All 1,004 tests, repository-wide Ruff/formatting, and strict mypy passed locally. -- Dependency audit found no known vulnerabilities; wheel and sdist inspection/install passed. -- GCP, AWS/Fargate, and cross-cloud Terraform formatting, tests, and validation passed. -- The runtime-all container built successfully and passed runtime conformance. +- Twenty-nine focused PostgreSQL and transform tests passed locally. +- All 1,006 tests, repository-wide Ruff/formatting, and strict mypy passed locally. +- Wheel and sdist inspection passed and includes the PostgreSQL transform adapter. +- Protected CI remains to run before merge. ## Decisions -- Dander owns endpoint batching; PostgreSQL streams each supplied batch with `COPY`. -- Temporary staging is transaction-scoped with `ON COMMIT DROP`. -- Profile selection, transforms, assertions, and Kubernetes remain separate slices. +- PostgreSQL table replacement keeps a stable table and uses fenced truncate/insert in one transaction. +- Every model output has its own destination target-fence claim. +- PostgreSQL graphs and selectable hosted profiles remain separate slices. ## Remaining -- Run protected CI and merge the focused pull request. -- Add version 2 PostgreSQL profile selection and bind state fences in runtime assembly. -- Implement PostgreSQL transforms and assertions against the portable SQL contract. -- Add the Kubernetes launcher after the warehouse composition is complete. +- Run all repository checks and protected CI. +- Add version 2 PostgreSQL profile selection and runtime target-claim assembly. +- Add Kubernetes/Helm projection and an existing-cluster verification path. +- Run the PostgreSQL native and cross-backend matrices. ## Review First -- `src/dander/providers/postgresql/writer.py` -- `src/dander/providers/postgresql/runtime.py` -- `tests/providers/test_postgresql_warehouse_runtime.py` +- `src/dander/providers/postgresql/transform.py` +- `src/dander/transform/project.py` +- `tests/providers/test_postgresql_transform_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index b4828257..df746c16 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -801,3 +801,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - PostgreSQL types derive from canonical schema v1. Only declared nullable additions evolve automatically; extra columns, type or nullability drift, required additions, and malformed records fail before target DML. Profile selection and transforms remain separate milestones. + +## 2026-08-08 — PostgreSQL materializations use stable relations and transactional fencing + +- Portable and explicitly PostgreSQL-authored models render through the existing transform + project. A PostgreSQL project uses database-local `schema.relation` names without changing the + default BigQuery compilation path. +- Table materialization keeps a stable relation and transactionally performs create-if-absent, + truncate, and insert behind the exact target fence. Views use transactional + `CREATE OR REPLACE`; incremental models use a unique index and deterministic + `INSERT ... ON CONFLICT`. +- Generic not-null, unique, accepted-values, and relationship assertions share the model metadata + but render PostgreSQL-native SQL. Graph execution and profile selection remain out of scope. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index c4ddd3f0..8606f5e2 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -120,7 +120,9 @@ publication inside a transactionally verified destination fence. Temporary stagi `ON COMMIT DROP`; nullable top-level columns may be added when additive evolution is selected. The configured database must already exist. Runtime-created connections require TLS, and the DSN -is supplied only through the configured environment variable. The first slice supports PostgreSQL -15+, SCD1, and canonical scalar/array/JSON mappings. Transform execution, assertions, launcher -projection, and version 2 profile selection remain separate follow-up work; until then this adapter -must not be represented as an end-to-end supported PostgreSQL deployment. +is supplied only through the configured environment variable. The adapter supports PostgreSQL +15+, SCD1 ingestion, canonical scalar/array/JSON mappings, portable or PostgreSQL-exact model SQL, +table/view/incremental materialization, and the four generic assertions. Every materialization is +claimed and published through its destination target fence. Graph execution, launcher projection, +and version 2 profile selection remain separate follow-up work; until then this adapter must not be +represented as an end-to-end supported PostgreSQL deployment. diff --git a/src/dander/providers/postgresql/fence.py b/src/dander/providers/postgresql/fence.py index 8ef0902a..0766baf6 100644 --- a/src/dander/providers/postgresql/fence.py +++ b/src/dander/providers/postgresql/fence.py @@ -12,6 +12,8 @@ from dander.warehouse.runtime import PreparedWarehouseStatement if TYPE_CHECKING: + from collections.abc import Sequence + from dander.warehouse.contracts import RelationRef PostgreSQLPool = ConnectionPool[Connection[dict[str, object]]] @@ -69,14 +71,9 @@ def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: """Return DML plus the exact transactional fence operations it requires.""" finalizer = statement.strip().removesuffix(";") - table = _postgresql_fence_identifier(fence.fence_table) return PreparedWarehouseStatement( sql=finalizer, - options=PostgreSQLFencePlan( - touch_sql=_postgresql_target_touch_sql(table), - commit_sql=_postgresql_target_commit_sql(table), - parameters=_postgresql_claim_parameters(fence), - ), + options=self._plan(fence), ) def execute_dml( @@ -98,6 +95,36 @@ def execute_dml( raise TargetFenceLostError("Dander destination fence lost during publication") return result + def execute_statements( + self, + connection: Connection[dict[str, object]], + statements: Sequence[sql.SQL | sql.Composed], + fence: TargetFence, + ) -> Cursor[dict[str, object]]: + """Publish one ordered PostgreSQL statement group behind an exact target fence.""" + if not statements: + raise ValueError("PostgreSQL fenced publication requires at least one statement") + plan = self._plan(fence) + with connection.transaction(): + if connection.execute(plan.touch_sql, plan.parameters).fetchone() is None: + raise TargetFenceLostError("Dander destination fence lost before publication") + result: Cursor[dict[str, object]] | None = None + for statement in statements: + result = connection.execute(statement) + if connection.execute(plan.commit_sql, plan.parameters).fetchone() is None: + raise TargetFenceLostError("Dander destination fence lost during publication") + assert result is not None + return result + + @staticmethod + def _plan(fence: TargetFence) -> PostgreSQLFencePlan: + table = _postgresql_fence_identifier(fence.fence_table) + return PostgreSQLFencePlan( + touch_sql=_postgresql_target_touch_sql(table), + commit_sql=_postgresql_target_commit_sql(table), + parameters=_postgresql_claim_parameters(fence), + ) + def _postgresql_target_fence_table_sql(table: sql.Identifier) -> sql.Composed: return sql.SQL( diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py index 33e82032..fc685696 100644 --- a/src/dander/providers/postgresql/runtime.py +++ b/src/dander/providers/postgresql/runtime.py @@ -13,6 +13,7 @@ from dander.providers.postgresql.config import PostgreSQLWarehouseConfig from dander.providers.postgresql.fence import PostgreSQLTargetFence +from dander.providers.postgresql.transform import PostgreSQLTransformRunner from dander.providers.postgresql.writer import PostgreSQLScd1Writer, PostgreSQLTimeouts from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.telemetry import OperationTelemetry, TelemetryOperation @@ -94,17 +95,29 @@ def build_ingestion_writer( @dataclass(frozen=True, slots=True) class PostgreSQLTransformFactory: - """Fail clearly until the PostgreSQL transform runner lands.""" + """Construct PostgreSQL model execution while rejecting graph plans.""" + + database: str + pool: PostgreSQLPool + target_fence: PostgreSQLTargetFence + timeouts: PostgreSQLTimeouts def build_transform_runner( self, *, graph_plan: object | None, build_models: bool, - ) -> None: - if graph_plan is not None or build_models: - raise ValueError("PostgreSQL transforms are not available in this adapter slice") - return None + ) -> PostgreSQLTransformRunner | None: + if graph_plan is not None: + raise ValueError("PostgreSQL graph execution is not available") + if not build_models: + return None + return PostgreSQLTransformRunner( + database=self.database, + pool=self.pool, + target_fence=self.target_fence, + timeouts=self.timeouts, + ) @dataclass(frozen=True, slots=True) @@ -135,7 +148,7 @@ def operation( schema_contract_version=1, write_modes=frozenset({WriteMode.SCD1}), transports=frozenset({WriteTransport.COPY}), - supports_transforms=False, + supports_transforms=True, supports_graphs=False, supports_target_fencing=True, ) @@ -188,7 +201,7 @@ def build_postgresql_warehouse( relation_codec=PostgreSQLRelationCodec(config.database), schema_mapper=PostgreSQLSchemaMapper(), writers=PostgreSQLWriterFactory(config.database, pool, target_fence, timeouts), - transforms=PostgreSQLTransformFactory(), + transforms=PostgreSQLTransformFactory(config.database, pool, target_fence, timeouts), target_fence=target_fence, telemetry=PostgreSQLTelemetry(), capabilities=POSTGRESQL_CAPABILITIES, diff --git a/src/dander/providers/postgresql/transform.py b/src/dander/providers/postgresql/transform.py new file mode 100644 index 00000000..ec42c3c5 --- /dev/null +++ b/src/dander/providers/postgresql/transform.py @@ -0,0 +1,368 @@ +"""PostgreSQL materialization and generic assertion execution.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from psycopg import Connection, sql +from psycopg_pool import ConnectionPool + +from dander.providers.postgresql.writer import PostgreSQLTimeouts, _set_timeouts +from dander.transform import ( + SqlDialect, + TransformModel, + TransformProject, + TransformProjectError, + TransformRunError, + TransformRunResult, +) +from dander.transform.model import Materialization +from dander.warehouse import RelationRef + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from pathlib import Path + + from dander.concurrency import OwnershipGuard, TargetFence + from dander.providers.postgresql.fence import PostgreSQLTargetFence + from dander.transform.config import GenericTestMetadata + +PostgreSQLRow = dict[str, object] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +@dataclass(frozen=True, slots=True) +class _PostgreSQLAssertion: + name: str + statement: sql.SQL | sql.Composed + parameters: tuple[object, ...] = () + + +class PostgreSQLTransformRunner: + """Build provider-exact or portable models in PostgreSQL 15 or newer.""" + + def __init__( + self, + *, + database: str, + pool: PostgreSQLPool, + target_fence: PostgreSQLTargetFence, + timeouts: PostgreSQLTimeouts, + ) -> None: + self._database = database + self._pool = pool + self._target_fence = target_fence + self._timeouts = timeouts + + def build( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Materialize selected models behind per-target destination fencing.""" + if ownership is None or ownership.fence is None: + raise TransformRunError("PostgreSQL hosted transforms require active lease ownership") + project = TransformProject.load( + models_dir, + project_id=self._database, + target_dialect=SqlDialect.POSTGRES, + ) + models = project.ordered(selected) + compiled = [ + (model, _materialization_statements(model, project.compile(model))) for model in models + ] + assertions = [ + assertion for model in models for assertion in _compile_assertions(project, model) + ] + for model, statements in compiled: + ownership.verify() + relation = _relation_ref(project, model) + publication = self._target_fence.claim(relation, ownership.fence) + self._publish(statements, publication) + self._run_assertions(assertions, ownership=ownership) + return TransformRunResult( + models=tuple(model.name for model in models), + assertions=len(assertions), + ) + + def test( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ) -> TransformRunResult: + """Evaluate assertions against already materialized PostgreSQL relations.""" + project = TransformProject.load( + models_dir, + project_id=self._database, + target_dialect=SqlDialect.POSTGRES, + ) + models = project.ordered(selected) + assertions = [ + assertion for model in models for assertion in _compile_assertions(project, model) + ] + self._run_assertions(assertions) + return TransformRunResult( + models=tuple(model.name for model in models), + assertions=len(assertions), + ) + + def _publish( + self, + statements: Sequence[sql.SQL | sql.Composed], + publication: TargetFence, + ) -> None: + with self._pool.connection() as connection, connection.transaction(): + _set_timeouts(connection, self._timeouts) + self._target_fence.execute_statements(connection, statements, publication) + + def _run_assertions( + self, + assertions: Iterable[_PostgreSQLAssertion], + *, + ownership: OwnershipGuard | None = None, + ) -> None: + failures: list[str] = [] + with self._pool.connection() as connection, connection.transaction(): + _set_timeouts(connection, self._timeouts) + for assertion in assertions: + if ownership is not None: + ownership.verify() + row = connection.execute(assertion.statement, assertion.parameters).fetchone() + if row is None: + raise TransformRunError( + f"Assertion returned an invalid result: {assertion.name}" + ) + raw_count = row.get("failures") + if isinstance(raw_count, bool) or not isinstance(raw_count, int): + raise TransformRunError( + f"Assertion returned an invalid result: {assertion.name}" + ) + if raw_count > 0: + failures.append(assertion.name) + if failures: + raise TransformRunError(f"Data tests failed: {', '.join(failures)}") + + +def _materialization_statements( + model: TransformModel, + query: str, +) -> tuple[sql.SQL | sql.Composed, ...]: + relation = _relation_identifier(model) + query_sql = sql.SQL(query) + columns = tuple(column.name for column in model.metadata.columns) + selected = sql.SQL(", ").join(sql.Identifier(column) for column in columns) + source_selected = sql.SQL(", ").join( + sql.SQL("source.{}").format(sql.Identifier(column)) for column in columns + ) + match model.metadata.materialization: + case Materialization.VIEW: + return (sql.SQL("CREATE OR REPLACE VIEW {} AS {}").format(relation, query_sql),) + case Materialization.TABLE: + create = sql.SQL( + "CREATE TABLE IF NOT EXISTS {} AS SELECT {} FROM ({}) AS source WITH NO DATA" + ).format(relation, source_selected, query_sql) + replace = sql.SQL("TRUNCATE TABLE {}").format(relation) + insert = sql.SQL("INSERT INTO {} ({}) SELECT {} FROM ({}) AS source").format( + relation, + selected, + source_selected, + query_sql, + ) + return create, replace, insert + case Materialization.INCREMENTAL: + return _incremental_statements(model, query) + + +def _incremental_statements( + model: TransformModel, + query: str, +) -> tuple[sql.SQL | sql.Composed, ...]: + unique_key = tuple(model.metadata.unique_key) + cursor = model.metadata.incremental_cursor + if not unique_key or cursor is None: + raise TransformProjectError("Incremental materialization metadata is incomplete") + relation = _relation_identifier(model) + query_sql = sql.SQL(query) + columns = tuple(column.name for column in model.metadata.columns) + selected = sql.SQL(", ").join(sql.Identifier(column) for column in columns) + source_selected = sql.SQL(", ").join( + sql.SQL("source.{}").format(sql.Identifier(column)) for column in columns + ) + incoming_selected = sql.SQL(", ").join( + sql.SQL("incoming.{}").format(sql.Identifier(column)) for column in columns + ) + keys = sql.SQL(", ").join(sql.Identifier(key) for key in unique_key) + source_keys = sql.SQL(", ").join( + sql.SQL("source.{}").format(sql.Identifier(key)) for key in unique_key + ) + ordering = sql.SQL(", ").join( + [ + *(sql.SQL("source.{}").format(sql.Identifier(key)) for key in unique_key), + sql.SQL("source.{} DESC").format(sql.Identifier(cursor)), + sql.SQL("to_jsonb(source)::text DESC"), + ] + ) + mutable = tuple(column for column in columns if column not in unique_key) + conflict = ( + sql.SQL("DO UPDATE SET {}").format( + sql.SQL(", ").join( + sql.SQL("{} = EXCLUDED.{}").format( + sql.Identifier(column), + sql.Identifier(column), + ) + for column in mutable + ) + ) + if mutable + else sql.SQL("DO NOTHING") + ) + create = sql.SQL( + "CREATE TABLE IF NOT EXISTS {} AS SELECT {} FROM ({}) AS source WITH NO DATA" + ).format(relation, source_selected, query_sql) + index = sql.SQL("CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} ({})").format( + sql.Identifier(_unique_index_name(model)), + relation, + keys, + ) + insert = sql.SQL( + "INSERT INTO {} ({}) SELECT {} FROM (" + "SELECT DISTINCT ON ({}) {} FROM ({}) AS source " + "WHERE NOT EXISTS (SELECT 1 FROM {}) OR source.{} >= (SELECT MAX({}) FROM {}) " + "ORDER BY {}) AS incoming ON CONFLICT ({}) {}" + ).format( + relation, + selected, + incoming_selected, + source_keys, + source_selected, + query_sql, + relation, + sql.Identifier(cursor), + sql.Identifier(cursor), + relation, + ordering, + keys, + conflict, + ) + return create, index, insert + + +def _compile_assertions( + project: TransformProject, + model: TransformModel, +) -> tuple[_PostgreSQLAssertion, ...]: + relation = _relation_identifier(model) + assertions: list[_PostgreSQLAssertion] = [] + for test in model.metadata.tests: + assertions.extend(_assertions_for_test(project, model.name, relation, test)) + return tuple(assertions) + + +def _assertions_for_test( + project: TransformProject, + model_name: str, + relation: sql.Identifier, + test: GenericTestMetadata, +) -> list[_PostgreSQLAssertion]: + column = sql.Identifier(test.column) + assertions: list[_PostgreSQLAssertion] = [] + if test.not_null: + assertions.append( + _PostgreSQLAssertion( + name=f"{model_name}.{test.column}.not_null", + statement=sql.SQL( + "SELECT COUNT(*) FILTER (WHERE {} IS NULL) AS failures FROM {}" + ).format(column, relation), + ) + ) + if test.unique: + assertions.append( + _PostgreSQLAssertion( + name=f"{model_name}.{test.column}.unique", + statement=sql.SQL( + "SELECT COUNT(*) AS failures FROM (" + "SELECT {} FROM {} WHERE {} IS NOT NULL GROUP BY {} HAVING COUNT(*) > 1" + ") AS duplicates" + ).format(column, relation, column, column), + ) + ) + if test.accepted_values is not None: + placeholders = sql.SQL(", ").join(sql.Placeholder() for _ in test.accepted_values) + assertions.append( + _PostgreSQLAssertion( + name=f"{model_name}.{test.column}.accepted_values", + statement=sql.SQL( + "SELECT COUNT(*) FILTER (WHERE {} IS NOT NULL AND {} NOT IN ({})) " + "AS failures FROM {}" + ).format(column, column, placeholders, relation), + parameters=tuple(test.accepted_values), + ) + ) + if test.relationships is not None: + if test.relationships.to in project.models: + parent_model = project.models[test.relationships.to] + parent_columns = {column.name for column in parent_model.metadata.columns} + if test.relationships.field not in parent_columns: + raise TransformProjectError( + "Relationship test references an undeclared parent column: " + f"{test.relationships.to}.{test.relationships.field}" + ) + parent = _relation_identifier_for_ref(project, test.relationships.to) + parent_field = sql.Identifier(test.relationships.field) + assertions.append( + _PostgreSQLAssertion( + name=f"{model_name}.{test.column}.relationships", + statement=sql.SQL( + "SELECT COUNT(*) AS failures FROM {} AS child " + "LEFT JOIN {} AS parent ON child.{} = parent.{} " + "WHERE child.{} IS NOT NULL AND parent.{} IS NULL" + ).format( + relation, + parent, + column, + parent_field, + column, + parent_field, + ), + ) + ) + return assertions + + +def _relation_ref(project: TransformProject, model: TransformModel) -> RelationRef: + return RelationRef( + catalog=project.project_id, + namespace=model.metadata.dataset, + name=model.name, + ) + + +def _relation_identifier(model: TransformModel) -> sql.Identifier: + return sql.Identifier(model.metadata.dataset, model.name) + + +def _relation_identifier_for_ref( + project: TransformProject, + reference: str, +) -> sql.Identifier: + if reference in project.models: + model = project.models[reference] + return _relation_identifier(model) + if reference.startswith("raw_"): + return sql.Identifier("raw", reference.removeprefix("raw_")) + raise TransformProjectError(f"Unknown model reference: {reference}") + + +def _unique_index_name(model: TransformModel) -> str: + digest = hashlib.sha256( + f"{model.metadata.dataset}.{model.name}:{','.join(model.metadata.unique_key)}".encode() + ).hexdigest()[:10] + return f"dander_uq_{model.name}"[:51] + f"_{digest}" + + +__all__ = ["PostgreSQLTransformRunner"] diff --git a/src/dander/transform/project.py b/src/dander/transform/project.py index 97152eb0..a7d88e32 100644 --- a/src/dander/transform/project.py +++ b/src/dander/transform/project.py @@ -19,6 +19,7 @@ from pathlib import Path _PROJECT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +_POSTGRESQL_CATALOG = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _RAW_PREFIX = "raw_" @@ -45,9 +46,25 @@ def name(self) -> str: class TransformProject: """Discovered transform models and their project-scoped relation resolver.""" - def __init__(self, *, project_id: str, models: Iterable[TransformModel]) -> None: - if not _PROJECT_ID.fullmatch(project_id): - raise TransformProjectError("Invalid GCP project id") + def __init__( + self, + *, + project_id: str, + models: Iterable[TransformModel], + target_dialect: SqlDialect | str = SqlDialect.BIGQUERY, + ) -> None: + try: + self.target_dialect = SqlDialect(target_dialect) + except ValueError as error: + raise TransformProjectError(f"Unknown target SQL dialect: {target_dialect}") from error + if self.target_dialect is SqlDialect.BIGQUERY: + if not _PROJECT_ID.fullmatch(project_id): + raise TransformProjectError("Invalid GCP project id") + elif self.target_dialect is SqlDialect.POSTGRES: + if not _POSTGRESQL_CATALOG.fullmatch(project_id): + raise TransformProjectError("Invalid PostgreSQL database name") + elif not _IDENTIFIER.fullmatch(project_id): + raise TransformProjectError("Invalid warehouse catalog identifier") indexed: dict[str, TransformModel] = {} for model in models: if model.name in indexed: @@ -60,12 +77,18 @@ def __init__(self, *, project_id: str, models: Iterable[TransformModel]) -> None self._validate_references() @classmethod - def load(cls, models_dir: Path, *, project_id: str) -> TransformProject: + def load( + cls, + models_dir: Path, + *, + project_id: str, + target_dialect: SqlDialect | str = SqlDialect.BIGQUERY, + ) -> TransformProject: """Discover every SQL model beneath a directory and load its YAML sidecar. Args: models_dir: Root containing model SQL and YAML files. - project_id: GCP project used to qualify compiled relations. + project_id: Warehouse catalog used to qualify or validate compiled relations. Returns: A validated transform project. @@ -98,7 +121,7 @@ def load(cls, models_dir: Path, *, project_id: str) -> TransformProject: sql_path=sql_path, ) ) - return cls(project_id=project_id, models=discovered) + return cls(project_id=project_id, models=discovered, target_dialect=target_dialect) def ordered(self, selected: Iterable[str] | None = None) -> tuple[TransformModel, ...]: """Return selected models plus model dependencies in topological order. @@ -142,31 +165,44 @@ def visit(name: str) -> None: visit(root) return tuple(ordered) - def relation_for_model(self, model: TransformModel) -> str: + def relation_for_model( + self, + model: TransformModel, + *, + target_dialect: SqlDialect | str | None = None, + ) -> str: """Return a quoted fully-qualified output relation.""" - return self._relation(model.metadata.dataset, model.name) + dialect = self._resolved_target(target_dialect) + if dialect is SqlDialect.BIGQUERY: + return self._relation(model.metadata.dataset, model.name) + return self._relation_for_dialect(model.metadata.dataset, model.name, dialect=dialect) - def relation_for_ref(self, reference: str) -> str: + def relation_for_ref( + self, + reference: str, + *, + target_dialect: SqlDialect | str | None = None, + ) -> str: """Resolve a model reference or conventional `raw_
` source reference.""" + dialect = self._resolved_target(target_dialect) if reference in self.models: - return self.relation_for_model(self.models[reference]) + return self.relation_for_model(self.models[reference], target_dialect=dialect) if reference.startswith(_RAW_PREFIX): table = reference.removeprefix(_RAW_PREFIX) if _IDENTIFIER.fullmatch(table): - return self._relation("raw", table) + if dialect is SqlDialect.BIGQUERY: + return self._relation("raw", table) + return self._relation_for_dialect("raw", table, dialect=dialect) raise TransformProjectError(f"Unknown model reference: {reference}") def compile( self, model: TransformModel, *, - target_dialect: SqlDialect | str = SqlDialect.BIGQUERY, + target_dialect: SqlDialect | str | None = None, ) -> str: """Render refs and validate one exact or portable read-only model query.""" - try: - target = SqlDialect(target_dialect) - except ValueError as error: - raise TransformProjectError(f"Unknown target SQL dialect: {target_dialect}") from error + target = self._resolved_target(target_dialect) if target is SqlDialect.PORTABLE: raise TransformProjectError("portable is an authored SQL contract, not a target") authored = model.metadata.dialect @@ -226,16 +262,15 @@ def _validate_references(self) -> None: self.relation_for_ref(reference) def _relation_for_ref(self, reference: str, *, dialect: SqlDialect) -> str: - if dialect is SqlDialect.BIGQUERY: - return self.relation_for_ref(reference) - if reference in self.models: - model = self.models[reference] - return self._relation_for_dialect(model.metadata.dataset, model.name, dialect=dialect) - if reference.startswith(_RAW_PREFIX): - table = reference.removeprefix(_RAW_PREFIX) - if _IDENTIFIER.fullmatch(table): - return self._relation_for_dialect("raw", table, dialect=dialect) - raise TransformProjectError(f"Unknown model reference: {reference}") + return self.relation_for_ref(reference, target_dialect=dialect) + + def _resolved_target(self, target: SqlDialect | str | None) -> SqlDialect: + if target is None: + return self.target_dialect + try: + return SqlDialect(target) + except ValueError as error: + raise TransformProjectError(f"Unknown target SQL dialect: {target}") from error def _relation(self, dataset: str, table: str) -> str: if not _IDENTIFIER.fullmatch(dataset) or not _IDENTIFIER.fullmatch(table): diff --git a/tests/providers/test_postgresql_transform_runtime.py b/tests/providers/test_postgresql_transform_runtime.py new file mode 100644 index 00000000..caed7383 --- /dev/null +++ b/tests/providers/test_postgresql_transform_runtime.py @@ -0,0 +1,331 @@ +"""Live PostgreSQL transform, assertion, and publication-fence conformance.""" + +from __future__ import annotations + +import os +import uuid +from dataclasses import dataclass +from textwrap import dedent, indent +from typing import TYPE_CHECKING, Any, cast + +import pytest +from psycopg import Connection, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.concurrency import FencingToken, TargetFenceLostError +from dander.providers import ProviderKind, default_provider_registry +from dander.providers.postgresql.transform import PostgreSQLTransformRunner +from dander.transform import TransformRunError +from dander.warehouse import WarehouseRuntime + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +@dataclass +class _Ownership: + fence: FencingToken + verifications: int = 0 + + def verify(self) -> None: + self.verifications += 1 + + +@pytest.fixture +def postgresql_transform_runtime() -> Iterator[tuple[WarehouseRuntime, PostgreSQLPool, str, str]]: + dsn = os.environ.get("DANDER_TEST_POSTGRES_DSN") + if not dsn: + pytest.skip("DANDER_TEST_POSTGRES_DSN is not configured") + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=5, + timeout=2, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + with pool.connection() as connection: + row = connection.execute("SELECT current_database() AS database").fetchone() + assert row is not None + database = cast("str", row["database"]) + suffix = uuid.uuid4().hex[:12] + namespace = f"dander_transform_{suffix}" + with pool.connection() as connection: + connection.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(namespace))) + connection.execute("CREATE SCHEMA IF NOT EXISTS raw") + registry = default_provider_registry() + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": "postgresql", + "database": database, + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "statement_timeout_ms": 30_000, + "lock_timeout_ms": 5_000, + "idle_transaction_timeout_ms": 10_000, + }, + ) + runtime = registry.build(ProviderKind.WAREHOUSE, config, context={"pool": pool}) + assert isinstance(runtime, WarehouseRuntime) + try: + yield runtime, pool, database, namespace + finally: + with pool.connection() as connection: + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(namespace)) + ) + connection.execute( + sql.SQL("DROP TABLE IF EXISTS raw.{}").format(sql.Identifier(f"fixture_{suffix}")) + ) + connection.execute( + sql.SQL("DROP TABLE IF EXISTS raw.{}").format(sql.Identifier(f"parent_{suffix}")) + ) + pool.close() + + +def test_postgresql_builds_tables_views_incrementals_and_assertions( + tmp_path: Path, + postgresql_transform_runtime: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, namespace = postgresql_transform_runtime + suffix = namespace.removeprefix("dander_transform_") + raw_table = f"fixture_{suffix}" + parent_table = f"parent_{suffix}" + base_model = f"base_{suffix}" + view_model = f"view_{suffix}" + incremental_model = f"incremental_{suffix}" + with pool.connection() as connection: + connection.execute( + sql.SQL( + "CREATE TABLE raw.{} (id TEXT, parent_id TEXT, status TEXT, updated_at BIGINT)" + ).format(sql.Identifier(raw_table)) + ) + connection.execute( + sql.SQL("CREATE TABLE raw.{} (id TEXT PRIMARY KEY)").format( + sql.Identifier(parent_table) + ) + ) + connection.execute( + sql.SQL("INSERT INTO raw.{} (id) VALUES ('parent-one'), ('parent-two')").format( + sql.Identifier(parent_table) + ) + ) + connection.execute( + sql.SQL( + "INSERT INTO raw.{} (id, parent_id, status, updated_at) VALUES " + "('one', 'parent-one', 'open', 1), " + "('two', 'parent-two', 'closed', 2)" + ).format(sql.Identifier(raw_table)) + ) + + _write_models( + tmp_path, + namespace=namespace, + raw_table=raw_table, + parent_table=parent_table, + base_model=base_model, + view_model=view_model, + incremental_model=incremental_model, + ) + runner = runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + assert isinstance(runner, PostgreSQLTransformRunner) + ownership = _ownership(database, run_id="run-one", token=1) + + with pytest.raises(TransformRunError, match="require active lease ownership"): + runner.build(tmp_path, selected=[view_model]) + + result = runner.build( + tmp_path, + selected=[view_model, incremental_model], + ownership=ownership, + ) + + assert result.models == (base_model, view_model, incremental_model) + assert result.assertions == 4 + assert ownership.verifications == 7 + with pool.connection() as connection: + base_rows = connection.execute( + sql.SQL("SELECT id, status FROM {}.{} ORDER BY id").format( + sql.Identifier(namespace), + sql.Identifier(base_model), + ) + ).fetchall() + view_count = connection.execute( + sql.SQL("SELECT COUNT(*) AS count FROM {}.{}").format( + sql.Identifier(namespace), + sql.Identifier(view_model), + ) + ).fetchone() + commits = connection.execute( + sql.SQL( + "SELECT COUNT(*) AS count FROM {}.dander_target_commits " + "WHERE status = 'committed' AND run_id = 'run-one'" + ).format(sql.Identifier(namespace)) + ).fetchone() + assert base_rows == [ + {"id": "one", "status": "open"}, + {"id": "two", "status": "closed"}, + ] + assert view_count == {"count": 2} + assert commits == {"count": 3} + + with pool.connection() as connection: + connection.execute( + sql.SQL("UPDATE raw.{} SET status = 'closed', updated_at = 3 WHERE id = 'one'").format( + sql.Identifier(raw_table) + ) + ) + connection.execute( + sql.SQL( + "INSERT INTO raw.{} (id, parent_id, status, updated_at) " + "VALUES ('three', 'parent-one', 'open', 4)" + ).format(sql.Identifier(raw_table)) + ) + replay = runner.build( + tmp_path, + selected=[incremental_model], + ownership=_ownership(database, run_id="run-two", token=2), + ) + assert replay.models == (base_model, incremental_model) + with pool.connection() as connection: + incremental_rows = connection.execute( + sql.SQL("SELECT id, status, updated_at FROM {}.{} ORDER BY id").format( + sql.Identifier(namespace), + sql.Identifier(incremental_model), + ) + ).fetchall() + assert incremental_rows == [ + {"id": "one", "status": "closed", "updated_at": 3}, + {"id": "three", "status": "open", "updated_at": 4}, + {"id": "two", "status": "closed", "updated_at": 2}, + ] + with pytest.raises(TargetFenceLostError, match="rejected stale publication ownership"): + runner.build( + tmp_path, + selected=[incremental_model], + ownership=ownership, + ) + with pool.connection() as connection: + after_stale_attempt = connection.execute( + sql.SQL("SELECT id, status, updated_at FROM {}.{} ORDER BY id").format( + sql.Identifier(namespace), + sql.Identifier(incremental_model), + ) + ).fetchall() + assert after_stale_attempt == incremental_rows + assert runner.test(tmp_path, selected=[incremental_model]).assertions == 4 + + with pool.connection() as connection: + connection.execute( + sql.SQL("UPDATE raw.{} SET status = 'record-value-must-not-leak'").format( + sql.Identifier(raw_table) + ) + ) + with pytest.raises(TransformRunError, match=rf"{base_model}\.status\.accepted_values") as error: + runner.build( + tmp_path, + selected=[base_model], + ownership=_ownership(database, run_id="run-three", token=3), + ) + assert "record-value-must-not-leak" not in str(error.value) + + +def _ownership(database: str, *, run_id: str, token: int) -> _Ownership: + return _Ownership( + FencingToken( + lease_table=None, + pipeline_id="postgresql_transforms", + run_id=run_id, + token=token, + authority_id=f"postgresql:{database}-state", + ) + ) + + +def _write_models( + root: Path, + *, + namespace: str, + raw_table: str, + parent_table: str, + base_model: str, + view_model: str, + incremental_model: str, +) -> None: + root.mkdir(parents=True, exist_ok=True) + columns = dedent( + """ + - name: id + type: STRING + description: Stable fixture identifier. + - name: parent_id + type: STRING + description: Parent fixture identifier. + - name: status + type: STRING + description: Fixture status. + - name: updated_at + type: INT64 + description: Monotonic fixture cursor. + """ + ).strip() + (root / f"{base_model}.sql").write_text( + f"SELECT id, parent_id, status, updated_at FROM {{{{ ref('raw_{raw_table}') }}}}" + ) + (root / f"{base_model}.yml").write_text( + f"model: {base_model}\n" + "description: Portable PostgreSQL base fixture.\n" + "owner: data-eng\n" + "dialect: portable\n" + "materialization: table\n" + f"dataset: {namespace}\n" + "source_system: fixture\n" + "sensitivity: public\n" + "columns:\n" + f"{indent(columns, ' ')}\n" + "tests:\n" + " - column: id\n" + " not_null: true\n" + " unique: true\n" + " - column: status\n" + " accepted_values: [open, closed]\n" + " - column: parent_id\n" + " relationships:\n" + f" to: raw_{parent_table}\n" + " field: id\n" + ) + for model, materialization in ( + (view_model, "view"), + (incremental_model, "incremental"), + ): + (root / f"{model}.sql").write_text( + f"SELECT id, parent_id, status, updated_at FROM {{{{ ref('{base_model}') }}}}" + ) + incremental = ( + "unique_key: [id]\nincremental_cursor: updated_at\n" + if materialization == "incremental" + else "" + ) + (root / f"{model}.yml").write_text( + f"model: {model}\n" + f"description: Portable PostgreSQL {materialization} fixture.\n" + "owner: data-eng\n" + "dialect: portable\n" + f"materialization: {materialization}\n" + f"dataset: {namespace}\n" + "source_system: fixture\n" + "sensitivity: public\n" + f"{incremental}" + "columns:\n" + f"{indent(columns, ' ')}\n" + "tests: []\n" + ) diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py index ca4be22f..dcc9fc27 100644 --- a/tests/providers/test_postgresql_warehouse_runtime.py +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -134,7 +134,7 @@ class _Cursor: assert schema.fields[2].data_type.kind is LogicalTypeKind.ARRAY assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) - assert runtime.capabilities.supports_transforms is False + assert runtime.capabilities.supports_transforms is True assert telemetry.rows_affected == 9 diff --git a/tests/transform/test_project.py b/tests/transform/test_project.py index 33418212..bb77a675 100644 --- a/tests/transform/test_project.py +++ b/tests/transform/test_project.py @@ -107,6 +107,25 @@ def test_exact_provider_model_compiles_only_for_declared_target(tmp_path: Path) project.compile(project.models["provider_model"]) +def test_postgresql_project_uses_database_local_relations_by_default(tmp_path: Path) -> None: + _write_model( + tmp_path, + "portable_model", + "SELECT id FROM {{ ref('raw_fixture') }}", + dialect="portable", + ) + project = TransformProject.load( + tmp_path, + project_id="dander_test", + target_dialect="postgres", + ) + + model = project.models["portable_model"] + assert project.relation_for_model(model) == '"staging"."portable_model"' + assert project.relation_for_ref("raw_fixture") == '"raw"."fixture"' + assert '"raw"."fixture"' in project.compile(model) + + def test_unknown_reference_fails_during_project_load(tmp_path: Path) -> None: _write_model(tmp_path, "broken", "SELECT id FROM {{ ref('missing') }}") diff --git a/tickets/DANDER-99-postgresql-transforms.md b/tickets/DANDER-99-postgresql-transforms.md new file mode 100644 index 00000000..6285db87 --- /dev/null +++ b/tickets/DANDER-99-postgresql-transforms.md @@ -0,0 +1,37 @@ +--- +id: DANDER-99 +title: Add transactionally fenced PostgreSQL transforms +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-98] +created: 2026-08-08 +--- + +## Context + +The PostgreSQL warehouse can ingest bounded raw records, but it is not a useful Dander warehouse +until the same model DAG can materialize governed outputs and run the existing generic assertions. + +## Acceptance Criteria + +- [x] Compile portable and explicitly PostgreSQL-authored read-only models for PostgreSQL. +- [x] Render database-local quoted relations without changing BigQuery defaults. +- [x] Materialize stable tables and views inside a destination-fenced transaction. +- [x] Materialize incremental models with deterministic keyed upserts and a required unique index. +- [x] Run not-null, unique, accepted-values, and relationship assertions from existing metadata. +- [x] Reject builds without lease ownership and reject stale transform tokens before publication. +- [x] Preserve sanitized test failures without returning record values. +- [x] Prove table, view, incremental, replay, assertion, and stale-owner behavior on PostgreSQL 15. +- [x] Keep graph execution, profile selection, and Kubernetes out of this slice. + +## Design + +`PostgreSQLTransformRunner` reuses `TransformProject` with a PostgreSQL target dialect. Each model +claims its output relation, then executes an ordered provider-native statement group between the +target-fence touch and commit operations. Assertions are read-only and use bound parameters for +accepted values. + +## Review Log + +No Terraform, cloud resource, retained project, or public support claim changes in this ticket. From 01c35758c29c2c16d6b95b0970b85f0351719ef5 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 20:30:28 -0400 Subject: [PATCH 051/163] feat: compose native PostgreSQL runtime profile --- HANDOFF.md | 40 +-- docs/decisions.md | 12 + docs/known-limitations.md | 8 +- docs/platform-profiles.md | 60 ++-- src/dander/cli/main.py | 20 +- src/dander/cli/run_command.py | 53 +++- src/dander/cli/runtime_command.py | 9 +- src/dander/executor.py | 7 +- src/dander/project/config.py | 19 +- src/dander/project/portable_config.py | 56 +++- src/dander/providers/postgresql/fence.py | 3 + src/dander/providers/postgresql/transform.py | 2 + src/dander/providers/postgresql/writer.py | 1 + src/dander/runtime.py | 15 +- src/dander/transform/runner.py | 10 +- src/dander/writer/base.py | 1 + tests/cli/test_run_command.py | 18 +- tests/cli/test_runtime_cli.py | 44 +++ .../test_postgresql_native_profile.py | 279 ++++++++++++++++++ tests/project/test_portable_config.py | 61 ++++ tests/test_runtime.py | 61 +++- tickets/DANDER-100-postgresql-profile.md | 44 +++ 22 files changed, 725 insertions(+), 98 deletions(-) create mode 100644 tests/integration/test_postgresql_native_profile.py create mode 100644 tickets/DANDER-100-postgresql-profile.md diff --git a/HANDOFF.md b/HANDOFF.md index 09f90a98..fe3bc102 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Made transform-project relation rendering target-dialect aware without changing BigQuery defaults. -- Added fenced PostgreSQL table, view, and incremental materializations. -- Added PostgreSQL-native not-null, unique, accepted-values, and relationship assertions. -- Wired PostgreSQL transform construction into the warehouse capability bundle. -- Added live PostgreSQL 15 replay, stale-owner, and sanitized-failure coverage. +- Made PostgreSQL warehouse/state selectable together through a version 2 deployment. +- Added named deployment selection to `dander run` and the OCI runtime. +- Claimed required ingestion targets before extraction without changing BigQuery writer behavior. +- Made metadata compilation use the selected warehouse dialect. +- Added one complete PostgreSQL 15 ingestion, transform, metadata, replay, cursor, history, and lease proof. ## Try It Set `DANDER_TEST_POSTGRES_DSN` to a disposable PostgreSQL 15+ database and run -`tests/providers/test_postgresql_transform_runtime.py`. +`tests/integration/test_postgresql_native_profile.py`. ## Checks -- Twenty-nine focused PostgreSQL and transform tests passed locally. -- All 1,006 tests, repository-wide Ruff/formatting, and strict mypy passed locally. -- Wheel and sdist inspection passed and includes the PostgreSQL transform adapter. -- Protected CI remains to run before merge. +- All 1,010 repository tests passed locally, including the real PostgreSQL 15 integration. +- Repository-wide Ruff and strict mypy passed locally. +- Wheel/sdist inspection, source-free installation, and dependency audit passed. +- Terraform validation/tests and the full-runtime container build/conformance probe passed. ## Decisions -- PostgreSQL table replacement keeps a stable table and uses fenced truncate/insert in one transaction. -- Every model output has its own destination target-fence claim. -- PostgreSQL graphs and selectable hosted profiles remain separate slices. +- Writers explicitly declare when the neutral runner must claim a destination target fence. +- A native PostgreSQL/no-catalog/environment-secret profile needs no GCP project identifier. +- PostgreSQL-state/BigQuery-warehouse remains fail-closed until all BigQuery writers adopt target fencing. ## Remaining -- Run all repository checks and protected CI. -- Add version 2 PostgreSQL profile selection and runtime target-claim assembly. -- Add Kubernetes/Helm projection and an existing-cluster verification path. -- Run the PostgreSQL native and cross-backend matrices. +- Open, review, and merge the focused protected PR. +- Let protected CI repeat Linux container/security/secret checks. +- Add Kubernetes/Helm projection and existing-cluster verification separately. +- Run the PostgreSQL native and cross-backend benchmark matrix. ## Review First -- `src/dander/providers/postgresql/transform.py` -- `src/dander/transform/project.py` -- `tests/providers/test_postgresql_transform_runtime.py` +- `src/dander/cli/run_command.py` +- `src/dander/runtime.py` +- `tests/integration/test_postgresql_native_profile.py` diff --git a/docs/decisions.md b/docs/decisions.md index df746c16..7b67f097 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -813,3 +813,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc `INSERT ... ON CONFLICT`. - Generic not-null, unique, accepted-values, and relationship assertions share the model metadata but render PostgreSQL-native SQL. Graph execution and profile selection remain out of scope. + +## 2026-08-08 — PostgreSQL is selectable as one native runtime composition + +- Version 2 warehouse configuration is a discriminated BigQuery/PostgreSQL contract. `dander run` + and the OCI runtime select one named deployment without requiring a GCP project for a fully + PostgreSQL, no-catalog, environment-secret composition. +- A writer explicitly declares whether destination publication fencing is required. The neutral + runner claims those targets before extraction; legacy BigQuery writers retain their existing + state-side fence path unchanged. +- PostgreSQL state/warehouse and BigQuery-state/PostgreSQL-warehouse pairs are executable. + PostgreSQL-state/BigQuery-warehouse remains fail-closed until all BigQuery write modes adopt the + destination-side fence. Hosted support still requires the Kubernetes and live-profile gates. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 48c522ce..aee3d600 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -11,10 +11,10 @@ evaluate these limits before using it for an unattended system containing busine renewable credentials and live parity remain later gates. - Provider package extras and the full runtime image contain planned provider SDK dependencies, but do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. PostgreSQL - durable state is implemented and locally conformance-tested, but no PostgreSQL warehouse or - Kubernetes live profile is qualified yet. BigQuery execution with PostgreSQL state fails closed - until cross-backend destination fencing is enabled. The packaged capability manifest is the - support boundary. + state and warehouse execution are implemented and locally conformance-tested, but no Kubernetes + live profile is qualified yet. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed + until every BigQuery write mode uses destination-side fencing. The packaged capability manifest + is the support boundary. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 8606f5e2..745165b0 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -7,14 +7,15 @@ Dander version 2 separates reusable pipeline intent from environment-specific de plus runtime limits, schedules, secret references, and provider resource names. The first supported hosted profile remains BigQuery, BigQuery state, Dataplex (or no cloud -catalog), GCP Secret Manager, and Cloud Run. PostgreSQL 15 or newer has conformance adapters for -durable state and bounded SCD1 warehouse writes, but the version 2 profile schema does not yet -select a complete PostgreSQL/Kubernetes composition. A provider name outside the installed and -supported contract fails validation; adapter registration alone does not imply profile support. - -One logical project may have multiple named platforms and deployments. `dander validate` accepts -`--deployment`; the Python configuration loader accepts the same explicit selection. Other current -GCP commands require a single deployment until the shared provider factories are introduced. +catalog), GCP Secret Manager, and Cloud Run. Version 2 may select PostgreSQL 15+ for both durable +state and the warehouse, but that composition remains locally qualified until the Kubernetes +launcher and live-profile gates pass. A provider name outside the installed contract fails +validation; adapter registration alone does not imply hosted support. + +One logical project may have multiple named platforms and deployments. `dander validate` and +`dander run` accept `--deployment`; the OCI runtime's `--platform` value selects the named +deployment and may use an explicit `--platforms-config` path. Other current GCP commands require a +single deployment until their provider-neutral planning path is introduced. When a project has exactly one deployment, Dander selects it deterministically; multiple deployments never fall back to an arbitrary default. @@ -42,18 +43,13 @@ Migration refuses to overwrite an existing `dander.platforms.yaml`. Commit and r together. Existing Terraform addresses do not change merely because the equivalent configuration is represented in two files. -The current runtime continues to support only the GCP compatibility composition. The internal -factory contract provides one API-v1 registry across warehouse, state, catalog, secret, and -launcher categories. Registration loads only a small configuration model; selecting and building -a provider loads its implementation and SDK dependencies. BigQuery writer, transform, lease, -watermark, run-history, metadata-store, external catalog-publisher, and secret-store construction -now use that boundary. State migrations are explicit and versioned, but retain the existing -BigQuery table names and semantics. Dataplex keeps aspect-only updates and normalized readback; -selecting `none` loads no Dataplex implementation or credentials. GCP Secret Manager preserves its -existing environment indirection and audit behavior; environment-only resolution remains local -because Cloud Run requires managed secret references. Cloud Run execution templates now pass -through the same lazy provider boundary while delegating to the accepted projection unchanged. -The registry is a construction contract, not a support claim. +The runtime composes the selected warehouse and state through one API-v1 provider registry. +Registration loads only a small configuration model; selecting and building a provider loads its +implementation and SDK dependencies. BigQuery behavior and names remain unchanged. Dataplex keeps +aspect-only updates and normalized readback; selecting `none` loads no Dataplex implementation or +credentials. GCP Secret Manager preserves its existing environment indirection and audit behavior; +environment-only resolution remains local or launcher-injected because Cloud Run requires managed +secret references. The registry is a construction contract, not a support claim. Warehouse adapters exchange [canonical relation and schema contracts](canonical-schema.md). Existing BigQuery connector/writer declarations remain valid and expose a one-way canonical view; @@ -73,8 +69,9 @@ Select PostgreSQL only in `dander.platforms.yaml`; keep the connection string ou platforms: portable: warehouse: - provider: bigquery - location: US + provider: postgresql + database: dander + dsn_env: DANDER_POSTGRES_DSN state: provider: postgresql authority_id: postgresql:portable-state @@ -101,10 +98,10 @@ run history, and deterministic JSONB metadata snapshots. `authority_id` is a stable, non-secret identifier for this state deployment. Do not reuse it for a different database or change `authority_epoch` outside a reviewed state-backend cutover. -The current BigQuery warehouse/PostgreSQL state combination remains fail-closed at execution. -Destination-side target fencing now exists for both systems, but every execution caller must bind -the selected state's token to its warehouse target before any cross-backend combination can be -enabled. +PostgreSQL state with a PostgreSQL warehouse is executable. BigQuery state with PostgreSQL is also +transactionally fenced, although it is not a qualified hosted profile. PostgreSQL state with a +BigQuery warehouse remains fail-closed until every BigQuery write mode uses the destination-side +target fence rather than only the state-side lease transaction. Terminal `succeeded`, `failed`, and `skipped` history older than the configured retention is removed when migrations run. Active runs and `interrupted_run` records are retained. Dander @@ -113,8 +110,8 @@ migration: keep schedules paused until the destination-fence and cutover workflo ## PostgreSQL warehouse adapter -The PostgreSQL warehouse adapter is an implementation and conformance boundary, not yet a hosted -profile. It accepts declared schemas, creates database-local schemas and relations, streams each +The PostgreSQL warehouse adapter is selectable but not yet a supported hosted profile. It accepts +declared schemas, creates database-local schemas and relations, streams each bounded Dander batch through PostgreSQL `COPY`, and performs deterministic last-record-wins SCD1 publication inside a transactionally verified destination fence. Temporary staging uses `ON COMMIT DROP`; nullable top-level columns may be added when additive evolution is selected. @@ -123,6 +120,7 @@ The configured database must already exist. Runtime-created connections require is supplied only through the configured environment variable. The adapter supports PostgreSQL 15+, SCD1 ingestion, canonical scalar/array/JSON mappings, portable or PostgreSQL-exact model SQL, table/view/incremental materialization, and the four generic assertions. Every materialization is -claimed and published through its destination target fence. Graph execution, launcher projection, -and version 2 profile selection remain separate follow-up work; until then this adapter must not be -represented as an end-to-end supported PostgreSQL deployment. +claimed and published through its destination target fence. The local native-profile proof covers +bounded ingestion, replay, transforms, assertions, metadata, run history, watermarks, and leases. +Graph execution and the Kubernetes launcher remain follow-up work; until live qualification passes, +this adapter must not be represented as a supported hosted PostgreSQL deployment. diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 844b17f7..bafb9b39 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -210,6 +210,8 @@ def validate_project( @plugins_app.command("install") def install_plugins( project_config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + deployment: str | None = typer.Option(None, "--deployment"), ) -> None: """Install the manifest's exact connector-plugin package pins.""" try: @@ -1287,12 +1289,22 @@ def run( pipeline_or_source: str = typer.Argument( ..., help="Pipeline name from dander.yaml (or a legacy source name from connectors/)." ), - project: str | None = typer.Option(None, "--project", help="Override GCP_PROJECT_ID."), - dataset: str | None = typer.Option(None, "--dataset", help="Override BQ_DATASET_RAW."), + project: str | None = typer.Option( + None, + "--project", + help="Override the BigQuery/GCP project for profiles that use GCP.", + ), + dataset: str | None = typer.Option( + None, + "--dataset", + help="Override the raw warehouse dataset or schema.", + ), connectors_dir: Path = typer.Option( # noqa: B008 _DEFAULT_CONNECTORS_DIR, "--connectors-dir" ), project_config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + deployment: str | None = typer.Option(None, "--deployment"), dry_run: bool = typer.Option( False, "--dry-run", @@ -1313,7 +1325,7 @@ def run( "--batch-rows", min=1, max=100_000, - help="Maximum rows sent in one BigQuery writer request.", + help="Maximum rows sent in one bounded warehouse writer request.", ), budget_name: str = typer.Option("dander-sbx-cap", "--budget-name", hidden=True), state_path: Path = typer.Option(Path(".dander/state.db"), hidden=True), # noqa: B008 @@ -1348,6 +1360,8 @@ def run( dataset=dataset, connectors_dir=connectors_dir, project_config=project_config, + platforms_config=platforms_config, + deployment=deployment, dry_run=dry_run, sandbox=sandbox, guarded_free_tier=guarded_free_tier, diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 47d5fb12..2b495f4b 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -82,6 +82,8 @@ class RunOptions: dataset: str | None connectors_dir: Path project_config: Path + platforms_config: Path | None + deployment: str | None dry_run: bool sandbox: bool guarded_free_tier: bool @@ -111,7 +113,8 @@ class _ResolvedRun: build_models: bool publish_dataplex: bool warehouse_provider: str - warehouse_location: str + warehouse_config: dict[str, object] + warehouse_catalog: str state_provider: str state_config: dict[str, object] catalog_provider: str @@ -138,7 +141,7 @@ def execute_run( if options.dry_run: _render_dry_run(options, resolved, console=console) return None - if not resolved.project: + if _requires_gcp_project(resolved) and not resolved.project: raise ClickException("GCP project is required via --project or GCP_PROJECT_ID") _verify_safety(options, resolved) @@ -168,7 +171,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: build_models = options.build_models publish_dataplex = options.publish_dataplex warehouse_provider = "bigquery" - warehouse_location = "US" + warehouse_config: dict[str, object] = {"provider": "bigquery", "location": "US"} state_provider = "bigquery" state_config: dict[str, object] = {"provider": "bigquery"} catalog_provider = "dataplex" @@ -177,7 +180,11 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: if options.project_config.is_file(): try: - manifest = load_project_config(options.project_config) + manifest = load_project_config( + options.project_config, + platforms_path=options.platforms_config, + deployment=options.deployment, + ) plugin_registry = load_connector_plugins(manifest.plugins) pipeline = manifest.pipelines.get(options.pipeline_or_source) if pipeline is not None: @@ -196,7 +203,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: build_models = build_models or pipeline.build_models publish_dataplex = publish_dataplex or pipeline.publish_dataplex warehouse_provider = manifest.warehouse_provider - warehouse_location = manifest.platform.bigquery_location + warehouse_config = manifest.warehouse_config state_provider = manifest.state_provider state_config = manifest.state_config catalog_provider = ( @@ -220,12 +227,17 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: settings = Settings() resolved_project = options.project or settings.gcp_project_id resolved_dataset = options.dataset or settings.bq_dataset_raw + warehouse_catalog = ( + str(warehouse_config["database"]) + if warehouse_provider == "postgresql" + else resolved_project + ) if options.sandbox and options.guarded_free_tier: raise ClickException("--sandbox and --guarded-free-tier are mutually exclusive") graph_plan = _resolve_graph_plan( graph_file, config, - project=resolved_project, + project=warehouse_catalog, dataset=resolved_dataset, build_models=build_models, selected_models=selected_models, @@ -246,7 +258,8 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: build_models=build_models, publish_dataplex=publish_dataplex, warehouse_provider=warehouse_provider, - warehouse_location=warehouse_location, + warehouse_config=warehouse_config, + warehouse_catalog=warehouse_catalog, state_provider=state_provider, state_config=state_config, catalog_provider=catalog_provider, @@ -329,7 +342,7 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec source_config=resolved.source_config, ingestion=ingestion, history=stores.history, - project=resolved.project, + project=resolved.warehouse_catalog, models_dir=( resolved.graph_file.parent if resolved.graph_file is not None else options.models_dir ), @@ -383,13 +396,14 @@ def _build_ingestion_runner( source=source, writer=writer, watermarks=stores.watermarks, - project=resolved.project, + project=resolved.warehouse_catalog, dataset=resolved.dataset, resume_from_watermark=not options.sandbox, batch_rows=options.batch_rows, endpoint_names=( resolved.graph_plan.bindings.endpoint_names if resolved.graph_plan is not None else None ), + target_fence=warehouse.target_fence, ) @@ -408,10 +422,7 @@ def _build_warehouse_runtime(resolved: _ResolvedRun) -> WarehouseRuntime: try: config = registry.parse( ProviderKind.WAREHOUSE, - { - "provider": resolved.warehouse_provider, - "location": resolved.warehouse_location, - }, + resolved.warehouse_config, ) runtime = registry.build( ProviderKind.WAREHOUSE, @@ -457,11 +468,23 @@ def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: def _require_supported_state_pair(*, state_provider: str, warehouse_provider: str) -> None: if state_provider == "postgresql" and warehouse_provider == "bigquery": raise ClickException( - "PostgreSQL state with BigQuery execution is not available until the " - "cross-backend destination fence is enabled" + "PostgreSQL state with BigQuery execution remains unavailable until every " + "BigQuery write mode uses the destination-side target fence" ) +def _requires_gcp_project(resolved: _ResolvedRun) -> bool: + """Return whether one selected runtime profile needs a GCP project identifier.""" + return any( + ( + resolved.warehouse_provider == "bigquery", + resolved.state_provider == "bigquery", + resolved.catalog_provider == "dataplex", + resolved.secret_provider == "gcp_secret_manager", + ) + ) + + def build_source_adapter( config: SourceConfig, auth: AuthStrategy, diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index ec9c175c..1c352a6b 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -73,9 +73,10 @@ def execute_runtime( platform: str = typer.Option( ..., "--platform", - help="Compatibility profile; Phase 1 supports the existing 'gcp' profile.", + help="Named deployment/profile selected for this runtime execution.", ), project_config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 connectors_dir: Path = typer.Option(Path("connectors"), "--connectors-dir"), # noqa: B008 models_dir: Path = typer.Option(Path("models"), "--models-dir"), # noqa: B008 catalog_output: Path | None = typer.Option(None, "--catalog-output"), # noqa: B008 @@ -90,10 +91,6 @@ def execute_runtime( validate_runtime_contract(contract) validate_runtime_identifier(pipeline, label="pipeline id") validate_runtime_identifier(platform, label="platform") - if platform != "gcp": - raise RuntimeContractError( - "runtime contract v1 currently supports only the 'gcp' compatibility profile" - ) context = LauncherContext.from_environment() except RuntimeContractError as error: raise typer.BadParameter(str(error)) from error @@ -112,6 +109,8 @@ def execute_runtime( dataset=dataset, connectors_dir=connectors_dir, project_config=project_config, + platforms_config=platforms_config, + deployment=platform, dry_run=False, sandbox=False, guarded_free_tier=guarded_free_tier, diff --git a/src/dander/executor.py b/src/dander/executor.py index ad53a650..37a0dc64 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -12,7 +12,7 @@ from dander.runtime import PipelineRunResult from dander.state import LeaseHeartbeat, RunStage, RunStatus, classify_failure from dander.telemetry import RunTelemetry -from dander.transform import TransformProject, TransformRunResult +from dander.transform import SqlDialect, TransformProject, TransformRunResult if TYPE_CHECKING: from collections.abc import Iterable @@ -197,6 +197,11 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: transform_project = TransformProject.load( self._models_dir, project_id=self._project, + target_dialect=getattr( + self._transform_runner, + "target_dialect", + SqlDialect.BIGQUERY, + ), ) spine = MetadataSpine() compiled_assets = spine.compile( diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 71b1c95e..c19333da 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -26,6 +26,10 @@ def _default_state_config() -> dict[str, object]: return {"provider": "bigquery"} +def _default_warehouse_config() -> dict[str, object]: + return {"provider": "bigquery", "location": "US"} + + _DISTRIBUTION = re.compile(r"^[A-Za-z0-9]+(?:[-_.][A-Za-z0-9]+)*$") if TYPE_CHECKING: @@ -209,7 +213,11 @@ class DanderProject(BaseModel): pipelines: dict[str, PipelineSpec] = Field(min_length=1) platform_name: str = Field(default="gcp", exclude=True) deployment_name: str = Field(default="gcp_cloud_run", exclude=True) - warehouse_provider: Literal["bigquery"] = Field(default="bigquery", exclude=True) + warehouse_provider: Literal["bigquery", "postgresql"] = Field(default="bigquery", exclude=True) + warehouse_config: dict[str, object] = Field( + default_factory=_default_warehouse_config, + exclude=True, + ) state_provider: Literal["bigquery", "postgresql"] = Field(default="bigquery", exclude=True) state_config: dict[str, object] = Field( default_factory=_default_state_config, @@ -243,6 +251,15 @@ def validate_pipeline_ids(cls, values: dict[str, PipelineSpec]) -> dict[str, Pip @model_validator(mode="after") def validate_deployed_pipeline_ids(self) -> DanderProject: + if self.version == 1 and self.warehouse_provider == "bigquery": + self.warehouse_config = { + "provider": "bigquery", + "location": self.platform.bigquery_location, + } + if self.warehouse_config.get("provider") != self.warehouse_provider: + raise ValueError("warehouse config provider must match warehouse_provider") + if self.state_config.get("provider") != self.state_provider: + raise ValueError("state config provider must match state_provider") if self.launcher_config: configured_provider = self.launcher_config.get("provider") if configured_provider != self.launcher_provider: diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 10249240..a532a6b8 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -6,7 +6,7 @@ import re from dataclasses import dataclass from pathlib import Path -from typing import Annotated, Literal +from typing import TYPE_CHECKING, Annotated, Any, Literal import yaml from packaging.utils import canonicalize_name @@ -30,7 +30,13 @@ from dander.providers.fargate import FargateLauncherConfig # noqa: TC001 from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 -from dander.providers.postgresql import PostgreSQLStateConfig # noqa: TC001 +from dander.providers.postgresql import ( # noqa: TC001 + PostgreSQLStateConfig, + PostgreSQLWarehouseConfig, +) + +if TYPE_CHECKING: + from collections.abc import Mapping _NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") _PLUGIN_ID = re.compile(r"^[a-z][a-z0-9_]*$") @@ -138,12 +144,18 @@ def validate_pipeline_ids( ] +WarehouseSpec = Annotated[ + BigQueryWarehouseConfig | PostgreSQLWarehouseConfig, + Field(discriminator="provider"), +] + + class PlatformProfileSpec(BaseModel): """One named combination of data-plane provider selections.""" model_config = ConfigDict(extra="forbid") - warehouse: BigQueryWarehouseConfig + warehouse: WarehouseSpec state: StateSpec catalog: CatalogSpec secrets: SecretProviderSpec @@ -348,7 +360,11 @@ def _resolve( version=2, platform=PlatformSpec( region=selected.launcher.region, - bigquery_location=profile.warehouse.location, + bigquery_location=( + profile.warehouse.location + if isinstance(profile.warehouse, BigQueryWarehouseConfig) + else "US" + ), runtime=selected.runtime, safety=selected.safety, ), @@ -357,6 +373,7 @@ def _resolve( platform_name=selected.platform, deployment_name=selected_name, warehouse_provider=profile.warehouse.provider, + warehouse_config=profile.warehouse.model_dump(mode="json"), state_provider=profile.state.provider, state_config=profile.state.model_dump(mode="json"), catalog_provider=profile.catalog.provider, @@ -369,9 +386,22 @@ def _resolve( def _select_deployment(platforms: DanderPlatforms, requested: str | None) -> str: if requested is not None: - if requested not in platforms.deployments: - raise ProjectConfigError(f"Unknown deployment {requested!r}") - return requested + if requested in platforms.deployments: + return requested + profile_matches = tuple( + name + for name, deployment in platforms.deployments.items() + if deployment.platform == requested + ) + if len(profile_matches) == 1: + return profile_matches[0] + if len(profile_matches) > 1: + choices = ", ".join(sorted(profile_matches)) + raise ProjectConfigError( + f"Platform {requested!r} has multiple deployments ({choices}); " + "select one by deployment name" + ) + raise ProjectConfigError(f"Unknown deployment or platform {requested!r}") if len(platforms.deployments) != 1: choices = ", ".join(sorted(platforms.deployments)) raise ProjectConfigError( @@ -450,6 +480,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: return ( legacy.platform == migrated.platform and legacy.warehouse_provider == migrated.warehouse_provider + and legacy.warehouse_config == migrated.warehouse_config and legacy.state_provider == migrated.state_provider and legacy.state_config == migrated.state_config and legacy.catalog_provider == migrated.catalog_provider @@ -468,7 +499,12 @@ def _validation_error( *, label: str, ) -> ProjectConfigError: - locations = sorted( - {".".join(str(part) for part in issue["loc"]) or "" for issue in error.errors()} - ) + locations = sorted({_validation_location(issue) for issue in error.errors()}) return ProjectConfigError(f"Invalid {label} at {path}; check: {', '.join(locations)}") + + +def _validation_location(issue: Mapping[str, Any]) -> str: + location = ".".join(str(part) for part in issue["loc"]) or "" + if issue.get("type") == "union_tag_invalid": + return f"{location}.provider" + return location diff --git a/src/dander/providers/postgresql/fence.py b/src/dander/providers/postgresql/fence.py index 0766baf6..b4712daf 100644 --- a/src/dander/providers/postgresql/fence.py +++ b/src/dander/providers/postgresql/fence.py @@ -42,6 +42,9 @@ def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: table = sql.Identifier(target.namespace, "dander_target_commits") target_id = ".".join(target.coordinates) with self.pool.connection() as connection, connection.transaction(): + connection.execute( + sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format(sql.Identifier(target.namespace)) + ) connection.execute(_postgresql_target_fence_table_sql(table)) row = connection.execute( _postgresql_target_claim_sql(table), diff --git a/src/dander/providers/postgresql/transform.py b/src/dander/providers/postgresql/transform.py index ec42c3c5..de2bbbcf 100644 --- a/src/dander/providers/postgresql/transform.py +++ b/src/dander/providers/postgresql/transform.py @@ -43,6 +43,8 @@ class _PostgreSQLAssertion: class PostgreSQLTransformRunner: """Build provider-exact or portable models in PostgreSQL 15 or newer.""" + target_dialect = SqlDialect.POSTGRES + def __init__( self, *, diff --git a/src/dander/providers/postgresql/writer.py b/src/dander/providers/postgresql/writer.py index 8f3e5bc0..c346a6b0 100644 --- a/src/dander/providers/postgresql/writer.py +++ b/src/dander/providers/postgresql/writer.py @@ -49,6 +49,7 @@ class PostgreSQLScd1Writer(WritePattern): mode = WriteMode.SCD1 supports_batched_writes = True + requires_publication_fence = True def __init__( self, diff --git a/src/dander/runtime.py b/src/dander/runtime.py index 16f87bf7..7c1f37c8 100644 --- a/src/dander/runtime.py +++ b/src/dander/runtime.py @@ -4,7 +4,7 @@ import logging from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import date, datetime, time from decimal import Decimal, InvalidOperation from itertools import batched @@ -19,6 +19,7 @@ from dander.concurrency import OwnershipGuard from dander.ingestion.source import Endpoint, RawField, Source from dander.state.watermark import WatermarkStore + from dander.warehouse import WarehouseTargetFence from dander.writer.base import WritePattern _LOGGER = logging.getLogger(__name__) @@ -66,6 +67,7 @@ def __init__( history: RunHistoryStore | None = None, batch_rows: int = 10_000, endpoint_names: Iterable[str] | None = None, + target_fence: WarehouseTargetFence | None = None, ) -> None: self._source = source self._writer = writer @@ -74,6 +76,7 @@ def __init__( self._dataset = dataset self._resume_from_watermark = resume_from_watermark self._history = history + self._target_fence = target_fence if isinstance(batch_rows, bool) or not 1 <= batch_rows <= 100_000: raise ValueError("batch_rows must be an integer from 1 to 100000") self._batch_rows = batch_rows @@ -172,6 +175,16 @@ def _run_endpoint( schema=tuple(_write_field(field) for field in endpoint.raw_schema), fence=ownership.fence if ownership is not None else None, ) + if self._writer.requires_publication_fence: + if ownership is None or ownership.fence is None or self._target_fence is None: + raise RuntimeError("Hosted destination writes require target-fence ownership") + target = replace( + target, + publication_fence=self._target_fence.claim( + target.relation_ref, + ownership.fence, + ), + ) if not endpoint.raw_schema: _LOGGER.warning( "undeclared_raw_schema_deprecated", diff --git a/src/dander/transform/runner.py b/src/dander/transform/runner.py index 190d6c05..5d0faa0b 100644 --- a/src/dander/transform/runner.py +++ b/src/dander/transform/runner.py @@ -11,8 +11,12 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import OwnershipGuard, fenced_dml, fencing_job_config -from dander.transform.model import Materialization -from dander.transform.project import TransformModel, TransformProject, TransformProjectError +from dander.transform.model import Materialization, SqlDialect +from dander.transform.project import ( + TransformModel, + TransformProject, + TransformProjectError, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -68,6 +72,8 @@ class _MaterializationStatement: class BigQueryTransformRunner: """Build and test selected transform models with an injected BigQuery client.""" + target_dialect = SqlDialect.BIGQUERY + def __init__(self, *, project: str, client: _BigQueryClient | None = None) -> None: self._project = project self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) diff --git a/src/dander/writer/base.py b/src/dander/writer/base.py index 0db32726..9d60fe0a 100644 --- a/src/dander/writer/base.py +++ b/src/dander/writer/base.py @@ -93,6 +93,7 @@ class WritePattern(ABC): mode: WriteMode supports_batched_writes = False accepts_streaming_input = False + requires_publication_fence = False @abstractmethod def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> int: diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 5b9d9365..9f2b77e0 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -39,7 +39,7 @@ class _Built: class _ResolvedWarehouse(Protocol): warehouse_provider: str - warehouse_location: str + warehouse_config: dict[str, object] class _ResolvedState(Protocol): @@ -132,10 +132,11 @@ def build_transform_runner(self, **kwargs: object) -> _Built: class _Warehouse: writers = _WriterFactory() transforms = _TransformFactory() + target_fence = _Built(name="target-fence", args=(), kwargs={}) def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: assert resolved.warehouse_provider == "bigquery" - assert resolved.warehouse_location == "US" + assert resolved.warehouse_config == {"provider": "bigquery", "location": "US"} return _Warehouse() def build_state(resolved: _ResolvedState) -> StateRuntime: @@ -213,6 +214,7 @@ def build_secrets(provider_id: str) -> _Built: assert ingestion.kwargs["resume_from_watermark"] is True assert ingestion.kwargs["batch_rows"] == 10_000 assert ingestion.kwargs["watermarks"] == captured["watermarks"] + assert ingestion.kwargs["target_fence"] == _Warehouse.target_fence writer = cast("_Built", ingestion.kwargs["writer"]) assert writer.kwargs == { "sandbox": False, @@ -223,8 +225,8 @@ def build_secrets(provider_id: str) -> _Built: assert migrator.calls == 1 -def test_postgresql_state_fails_closed_before_cross_backend_fencing() -> None: - with pytest.raises(ClickException, match="cross-backend destination fence"): +def test_only_unfenced_postgresql_state_bigquery_warehouse_pair_fails_closed() -> None: + with pytest.raises(ClickException, match="BigQuery write mode"): run_module._require_supported_state_pair( state_provider="postgresql", warehouse_provider="bigquery", @@ -234,3 +236,11 @@ def test_postgresql_state_fails_closed_before_cross_backend_fencing() -> None: state_provider="bigquery", warehouse_provider="bigquery", ) + run_module._require_supported_state_pair( + state_provider="postgresql", + warehouse_provider="postgresql", + ) + run_module._require_supported_state_pair( + state_provider="bigquery", + warehouse_provider="postgresql", + ) diff --git a/tests/cli/test_runtime_cli.py b/tests/cli/test_runtime_cli.py index b087eea6..ec875541 100644 --- a/tests/cli/test_runtime_cli.py +++ b/tests/cli/test_runtime_cli.py @@ -100,11 +100,55 @@ def execute( options = cast("RunOptions", captured["options"]) assert options.pipeline_or_source == "greenhouse_jobs" assert options.project == "unit-project" + assert options.deployment == "gcp" assert options.batch_rows == 2500 assert str(options.catalog_output) == "/tmp/dander-catalog.json" assert captured["render"] is False +def test_runtime_execute_selects_named_version_two_deployment( + monkeypatch: MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + def execute(options: RunOptions, **_: object) -> PipelineExecutionResult: + captured["options"] = options + return PipelineExecutionResult( + run_id="postgres-run", + pipeline_id=options.pipeline_or_source, + ingestion=PipelineRunResult( + run_id="postgres-run", + source="fixture", + endpoints=(), + ), + models=(), + assertions=0, + assets=0, + ) + + monkeypatch.setattr(runtime_module, "execute_run", execute) + result = CliRunner().invoke( + app, + [ + "runtime", + "execute", + "--contract", + "io.dander.runtime/v1", + "--pipeline", + "postgresql_fixture", + "--platform", + "postgres_fargate", + "--platforms-config", + "/app/dander.platforms.yaml", + ], + ) + + assert result.exit_code == RuntimeExitCode.SUCCESS, result.output + options = cast("RunOptions", captured["options"]) + assert options.deployment == "postgres_fargate" + assert str(options.platforms_config) == "/app/dander.platforms.yaml" + + def test_runtime_execute_sanitizes_retryable_failure(monkeypatch: MonkeyPatch) -> None: class _Response: status_code = 429 diff --git a/tests/integration/test_postgresql_native_profile.py b/tests/integration/test_postgresql_native_profile.py new file mode 100644 index 00000000..b24c130b --- /dev/null +++ b/tests/integration/test_postgresql_native_profile.py @@ -0,0 +1,279 @@ +"""End-to-end PostgreSQL state, ingestion, transform, and metadata proof.""" + +from __future__ import annotations + +import os +import uuid +from textwrap import dedent +from typing import TYPE_CHECKING, Any, cast + +import pytest +from psycopg import Connection, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.executor import PipelineExecutor +from dander.ingestion import Endpoint, RawField, Source, SourceConfig +from dander.providers import ProviderKind, ProviderRegistry, default_provider_registry +from dander.runtime import PipelineRunner +from dander.state import RunStatus, StateRuntime +from dander.warehouse import WarehouseRuntime +from dander.writer import SchemaEvolution + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + from pathlib import Path + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +class _FixtureSource(Source): + def __init__(self, source_name: str) -> None: + super().__init__( + SourceConfig( + name=source_name, + base_url="https://example.invalid", + auth_strategy="none", + endpoints=[ + Endpoint( + name="widgets", + path="/widgets", + primary_key=["id"], + incremental_cursor="updated_at", + cursor_param="updated_after", + raw_schema=[ + RawField(name="id", data_type="STRING", mode="REQUIRED"), + RawField(name="name", data_type="STRING"), + RawField(name="updated_at", data_type="TIMESTAMP"), + ], + ) + ], + ) + ) + + def discover(self) -> Mapping[str, Any]: + return {} + + def extract(self, endpoint: str, *, since: str | None = None) -> Iterator[Mapping[str, Any]]: + assert endpoint == "widgets" + assert since in {None, "2026-08-08T12:01:00+00:00"} + yield {"id": "one", "name": "First", "updated_at": "2026-08-08T12:00:00Z"} + yield {"id": "two", "name": "Second", "updated_at": "2026-08-08T12:01:00Z"} + + +def test_postgresql_native_profile_runs_and_replays( + tmp_path: Path, +) -> None: + dsn = os.environ.get("DANDER_TEST_POSTGRES_DSN") + if not dsn: + pytest.skip("DANDER_TEST_POSTGRES_DSN is not configured") + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=5, + timeout=2, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + with pool.connection() as connection: + row = connection.execute("SELECT current_database() AS database").fetchone() + assert row is not None + database = cast("str", row["database"]) + suffix = uuid.uuid4().hex[:12] + raw_schema = "raw" + source_name = f"fixture_{suffix}" + raw_table = f"{source_name}_widgets" + model_schema = f"models_{suffix}" + state_schema = f"state_{suffix}" + registry = default_provider_registry() + state = _state_runtime(registry, pool, state_schema) + warehouse = _warehouse_runtime(registry, pool, database) + state.migrator.migrate() + _write_model( + tmp_path, + source_name=source_name, + raw_schema=raw_schema, + model_schema=model_schema, + ) + source = _FixtureSource(source_name) + writer = warehouse.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + ingestion = PipelineRunner( + source=source, + writer=writer, + watermarks=state.watermarks, + project=database, + dataset=raw_schema, + batch_rows=1, + target_fence=warehouse.target_fence, + ) + transform = warehouse.transforms.build_transform_runner( + graph_plan=None, + build_models=True, + ) + assert transform is not None + executor = PipelineExecutor( + pipeline_id="postgresql_fixture", + source_config=source.config, + ingestion=ingestion, + history=state.history, + project=database, + models_dir=tmp_path, + selected_models=("stg_fixture__widgets",), + build_models=True, + transform_runner=transform, + metadata_store=state.metadata, + leases=state.leases, + ) + + try: + first = executor.execute(run_id="postgres-profile-one") + replay = executor.execute(run_id="postgres-profile-two") + + assert first.ingestion.endpoints[0].extracted == 2 + assert replay.ingestion.endpoints[0].extracted == 2 + assert first.models == ("stg_fixture__widgets",) + assert state.watermarks.get(source_name, "widgets") == "2026-08-08T12:01:00+00:00" + recent = state.history.recent(limit=2, pipeline_id="postgresql_fixture") + assert [record.status for record in recent] == [ + RunStatus.SUCCEEDED, + RunStatus.SUCCEEDED, + ] + with pool.connection() as connection: + raw_count = connection.execute( + sql.SQL("SELECT COUNT(*) AS count FROM {}.{}").format( + sql.Identifier(raw_schema), + sql.Identifier(raw_table), + ) + ).fetchone() + model_count = connection.execute( + sql.SQL("SELECT COUNT(*) AS count FROM {}.stg_fixture__widgets").format( + sql.Identifier(model_schema) + ) + ).fetchone() + commits = connection.execute( + sql.SQL( + "SELECT COUNT(*) AS count FROM {}.dander_target_commits " + "WHERE status = 'committed' AND target_id = %s" + ).format(sql.Identifier(raw_schema)), + (f"{database}.{raw_schema}.{raw_table}",), + ).fetchone() + lease = connection.execute( + sql.SQL( + "SELECT run_id FROM {}.dander_pipeline_leases " + "WHERE pipeline_id = 'postgresql_fixture'" + ).format(sql.Identifier(state_schema)) + ).fetchone() + assert raw_count == {"count": 2} + assert model_count == {"count": 2} + assert commits == {"count": 1} + assert lease == {"run_id": None} + finally: + with pool.connection() as connection: + connection.execute( + sql.SQL("DROP TABLE IF EXISTS {}.{} CASCADE").format( + sql.Identifier(raw_schema), + sql.Identifier(raw_table), + ) + ) + for namespace in (model_schema, state_schema): + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(namespace)) + ) + pool.close() + + +def _state_runtime( + registry: ProviderRegistry, + pool: PostgreSQLPool, + schema_name: str, +) -> StateRuntime: + config = registry.parse( + ProviderKind.STATE, + { + "provider": "postgresql", + "authority_id": f"postgresql:{schema_name}", + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "schema_name": schema_name, + "lease_seconds": 10, + }, + ) + runtime = registry.build( + ProviderKind.STATE, + config, + context={"pool": pool, "metadata_enabled": True}, + ) + assert isinstance(runtime, StateRuntime) + return runtime + + +def _warehouse_runtime( + registry: ProviderRegistry, + pool: PostgreSQLPool, + database: str, +) -> WarehouseRuntime: + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": "postgresql", + "database": database, + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + "statement_timeout_ms": 30_000, + "lock_timeout_ms": 5_000, + "idle_transaction_timeout_ms": 10_000, + }, + ) + runtime = registry.build(ProviderKind.WAREHOUSE, config, context={"pool": pool}) + assert isinstance(runtime, WarehouseRuntime) + return runtime + + +def _write_model( + root: Path, + *, + source_name: str, + raw_schema: str, + model_schema: str, +) -> None: + (root / "stg_fixture__widgets.sql").write_text( + f"SELECT id, name, updated_at FROM {{{{ ref('{raw_schema}_{source_name}_widgets') }}}}", + encoding="utf-8", + ) + (root / "stg_fixture__widgets.yml").write_text( + dedent( + f""" + model: stg_fixture__widgets + description: Portable PostgreSQL profile fixture. + owner: data-eng + dialect: portable + materialization: table + dataset: {model_schema} + source_system: fixture + sensitivity: public + columns: + - name: id + type: STRING + description: Stable fixture identifier. + - name: name + type: STRING + description: Fixture name. + - name: updated_at + type: TIMESTAMP + description: Source update time. + tests: + - column: id + not_null: true + unique: true + """ + ).strip() + + "\n", + encoding="utf-8", + ) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index e2feeb7b..f9b90044 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -61,11 +61,13 @@ def test_version_one_migration_is_deterministic_and_behaviorally_equivalent( project_path.write_text(first.logical_yaml, encoding="utf-8") platforms_path.write_text(first.platforms_yaml, encoding="utf-8") migrated = load_project_config(project_path) + selected_by_profile = load_project_config(project_path, deployment="gcp") assert first == second assert migrated.version == 2 assert migrated.platform_name == "gcp" assert migrated.deployment_name == "gcp_cloud_run" + assert selected_by_profile.deployment_name == "gcp_cloud_run" assert original.warehouse_provider == "bigquery" assert migrated.warehouse_provider == "bigquery" assert original.state_provider == "bigquery" @@ -103,6 +105,9 @@ def test_version_two_requires_explicit_selection_with_multiple_deployments( with pytest.raises(ProjectConfigError, match="Multiple deployments"): load_project_config(project_path) + with pytest.raises(ProjectConfigError, match="Platform 'gcp' has multiple deployments"): + load_project_config(project_path, deployment="gcp") + resolved = load_project_config(project_path, deployment="shadow_cloud_run") assert resolved.deployment_name == "shadow_cloud_run" assert set(resolved.pipelines) == {"example_records", "undeployed_records"} @@ -186,6 +191,62 @@ def test_version_two_preserves_postgresql_state_connection_reference(tmp_path: P } +def test_version_two_resolves_native_postgresql_profile(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + profile = platforms["platforms"].pop("gcp") + profile["warehouse"] = { + "provider": "postgresql", + "database": "dander_portable", + "dsn_env": "DANDER_POSTGRES_DSN", + } + profile["state"] = { + "provider": "postgresql", + "authority_id": "postgresql:portable-state", + "dsn_env": "DANDER_POSTGRES_DSN", + } + profile["catalog"] = {"provider": "none"} + profile["secrets"] = {"provider": "environment"} + platforms["platforms"]["postgres"] = profile + deployment = platforms["deployments"].pop("gcp_cloud_run") + deployment["platform"] = "postgres" + deployment["launcher"] = { + "provider": "fargate", + "region": "us-east-1", + "aws_account_id": "123456789012", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/123456789012/locations/global/" + "workloadIdentityPools/dander-aws/providers/dander-aws" + ), + "subnet_ids": ["subnet-0123456789abcdef0"], + "security_group_ids": ["sg-0123456789abcdef0"], + } + platforms["deployments"]["postgres_fargate"] = deployment + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path, deployment="postgres_fargate") + + assert resolved.warehouse_provider == "postgresql" + assert resolved.warehouse_config == { + "provider": "postgresql", + "database": "dander_portable", + "dsn_env": "DANDER_POSTGRES_DSN", + "pool_min_size": 1, + "pool_max_size": 5, + "pool_timeout_seconds": 10.0, + "statement_timeout_ms": 300_000, + "lock_timeout_ms": 30_000, + "idle_transaction_timeout_ms": 60_000, + } + assert resolved.state_provider == "postgresql" + assert resolved.catalog_provider == "none" + assert resolved.secret_provider == "environment" + + def test_cloud_run_rejects_environment_only_secrets(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/test_runtime.py b/tests/test_runtime.py index eff1c9c9..c0afedd6 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -8,6 +8,7 @@ import pytest +from dander.concurrency import FencingToken, TargetFence from dander.ingestion import load_source_config from dander.ingestion.source import Endpoint, RawField, Source, SourceConfig from dander.runtime import PipelineRunner, RawSchemaError, WatermarkConflictError @@ -17,7 +18,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping - from dander.concurrency import FencingToken + from dander.warehouse import PreparedWarehouseStatement, RelationRef class _Source(Source): @@ -95,6 +96,34 @@ def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> in return len(batch) +class _FencedWriter(_Writer): + requires_publication_fence = True + + def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> int: + assert target.publication_fence is not None + return super().write(records, target) + + +class _TargetFence: + def __init__(self, events: list[str]) -> None: + self._events = events + + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + self._events.append("claim") + return TargetFence( + fence_table=f"{target.namespace}.dander_target_commits", + target_id=".".join(target.coordinates), + authority_id=fence.resolved_authority_id, + authority_epoch=fence.authority_epoch, + pipeline_id=fence.pipeline_id, + run_id=fence.run_id, + token=fence.token, + ) + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: + raise NotImplementedError + + class _DeclaredSource(Source): def __init__(self, records: list[Mapping[str, Any]]) -> None: super().__init__( @@ -201,6 +230,20 @@ def verify(self) -> None: raise LeaseLostError("Pipeline lease ownership was lost") +class _FencedOwnership: + fence = FencingToken( + lease_table=None, + authority_id="postgresql:test-state", + authority_epoch=1, + pipeline_id="example_pipeline", + run_id="run-fenced", + token=7, + ) + + def verify(self) -> None: + return None + + class _History(RunHistoryStore): def __init__(self) -> None: self.started: tuple[str, str] | None = None @@ -261,6 +304,22 @@ def test_runner_commits_maximum_cursor_after_write() -> None: assert result.endpoints[0].affected == 2 +def test_runner_claims_required_destination_before_extraction() -> None: + events: list[str] = [] + runner = PipelineRunner( + source=_Source(events), + writer=_FencedWriter(events), + watermarks=_Watermarks(events), + project="dander_test", + dataset="raw", + target_fence=_TargetFence(events), + ) + + runner.run(run_id="run-fenced", ownership=_FencedOwnership()) + + assert events == ["get", "claim", "extract", "write", "set"] + + def test_runner_extracts_only_selected_configured_endpoints() -> None: extracted: list[str] = [] diff --git a/tickets/DANDER-100-postgresql-profile.md b/tickets/DANDER-100-postgresql-profile.md new file mode 100644 index 00000000..70575664 --- /dev/null +++ b/tickets/DANDER-100-postgresql-profile.md @@ -0,0 +1,44 @@ +--- +id: DANDER-100 +title: Compose the native PostgreSQL runtime profile +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-96, DANDER-97, DANDER-98, DANDER-99] +created: 2026-08-08 +--- + +## Context + +PostgreSQL state, destination fencing, bounded SCD1 ingestion, and transforms existed as isolated +capabilities. A version 2 deployment could not yet select and execute them as one Dander pipeline. + +## Acceptance Criteria + +- [x] Select BigQuery or PostgreSQL warehouse configuration through the version 2 profile schema. +- [x] Let `dander run` and `dander runtime execute` select one named deployment explicitly. +- [x] Require a GCP project only when the selected providers actually use GCP. +- [x] Claim PostgreSQL ingestion targets before extraction and preserve legacy BigQuery behavior. +- [x] Compile metadata with the selected warehouse dialect. +- [x] Run bounded ingestion, replay, transforms, assertions, metadata, history, cursors, and leases + together against PostgreSQL 15. +- [x] Keep PostgreSQL-state/BigQuery-warehouse execution fail-closed until its writers adopt the + destination-side target fence. +- [x] Make no Terraform, cloud-resource, or hosted-support change. + +## Design + +The resolved project retains the non-secret warehouse provider block and separates the warehouse +catalog from the optional GCP project. `PipelineRunner` claims targets only for writers that declare +the destination-fence requirement. The selected transform runner supplies its SQL dialect to the +metadata compiler. + +## Implementation Notes + +The PostgreSQL native integration uses one disposable database with distinct raw, model, and state +schemas. It forces one-row batches and replays the inclusive cursor boundary without duplicates. + +## Review Log + +Protected CI and pull-request review remain before merge. Kubernetes/Helm is the next separate +Phase 4 slice. From d6300d4cbe08dd91b01468aa48f06643e70b8a81 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 21:18:09 -0400 Subject: [PATCH 052/163] feat: add existing-cluster Kubernetes launcher --- .github/workflows/ci.yml | 6 + HANDOFF.md | 45 +- docs/decisions.md | 9 + docs/execution-projection.md | 7 + docs/known-limitations.md | 9 +- docs/kubernetes.md | 108 ++++ docs/platform-profiles.md | 6 +- infra/README.md | 1 + infra/kubernetes/chart/dander/Chart.yaml | 7 + infra/kubernetes/chart/dander/README.md | 20 + .../chart/dander/templates/NOTES.txt | 9 + .../chart/dander/templates/_helpers.tpl | 13 + .../chart/dander/templates/configmap.yaml | 14 + .../chart/dander/templates/cronjobs.yaml | 83 +++ .../chart/dander/templates/rbac.yaml | 30 + .../dander/templates/serviceaccount.yaml | 13 + .../chart/dander/values.schema.json | 106 ++++ infra/kubernetes/chart/dander/values.yaml | 27 + pyproject.toml | 10 + scripts/check_distribution.py | 4 + src/dander/cli/kubernetes_command.py | 90 +++ src/dander/cli/main.py | 2 + src/dander/deployment/__init__.py | 2 + src/dander/deployment/projection.py | 16 + src/dander/deployment/runtime.py | 1 + src/dander/project/config.py | 4 +- src/dander/project/portable_config.py | 3 +- src/dander/providers/cloud_run/runtime.py | 3 + src/dander/providers/defaults.py | 9 + src/dander/providers/fargate/runtime.py | 3 + src/dander/providers/kubernetes/__init__.py | 5 + src/dander/providers/kubernetes/chart.py | 113 ++++ src/dander/providers/kubernetes/config.py | 91 ++++ src/dander/providers/kubernetes/operations.py | 514 ++++++++++++++++++ src/dander/providers/kubernetes/runtime.py | 191 +++++++ tests/cli/test_kubernetes_cli.py | 86 +++ .../connectors/records.yaml | 11 + .../kubernetes-project/dander.platforms.yaml | 42 ++ tests/fixtures/kubernetes-project/dander.yaml | 6 + tests/fixtures/kubernetes-values.yaml | 68 +++ tests/infra/test_kubernetes_chart.py | 323 +++++++++++ tests/project/test_scaffold.py | 1 + tests/providers/test_kubernetes_runtime.py | 143 +++++ .../DANDER-101-kubernetes-helm-launcher.md | 37 ++ 44 files changed, 2262 insertions(+), 29 deletions(-) create mode 100644 docs/kubernetes.md create mode 100644 infra/kubernetes/chart/dander/Chart.yaml create mode 100644 infra/kubernetes/chart/dander/README.md create mode 100644 infra/kubernetes/chart/dander/templates/NOTES.txt create mode 100644 infra/kubernetes/chart/dander/templates/_helpers.tpl create mode 100644 infra/kubernetes/chart/dander/templates/configmap.yaml create mode 100644 infra/kubernetes/chart/dander/templates/cronjobs.yaml create mode 100644 infra/kubernetes/chart/dander/templates/rbac.yaml create mode 100644 infra/kubernetes/chart/dander/templates/serviceaccount.yaml create mode 100644 infra/kubernetes/chart/dander/values.schema.json create mode 100644 infra/kubernetes/chart/dander/values.yaml create mode 100644 src/dander/cli/kubernetes_command.py create mode 100644 src/dander/providers/kubernetes/__init__.py create mode 100644 src/dander/providers/kubernetes/chart.py create mode 100644 src/dander/providers/kubernetes/config.py create mode 100644 src/dander/providers/kubernetes/operations.py create mode 100644 src/dander/providers/kubernetes/runtime.py create mode 100644 tests/cli/test_kubernetes_cli.py create mode 100644 tests/fixtures/kubernetes-project/connectors/records.yaml create mode 100644 tests/fixtures/kubernetes-project/dander.platforms.yaml create mode 100644 tests/fixtures/kubernetes-project/dander.yaml create mode 100644 tests/fixtures/kubernetes-values.yaml create mode 100644 tests/infra/test_kubernetes_chart.py create mode 100644 tests/providers/test_kubernetes_runtime.py create mode 100644 tickets/DANDER-101-kubernetes-helm-launcher.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c1ea061..535cea3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,12 @@ jobs: terraform -chdir=aws validate terraform -chdir=aws/modules/fargate init -backend=false -input=false terraform -chdir=aws/modules/fargate test + - name: Validate Kubernetes Helm chart + run: | + helm lint kubernetes/chart/dander --values ../tests/fixtures/kubernetes-values.yaml + helm template dander kubernetes/chart/dander \ + --namespace dander-test \ + --values ../tests/fixtures/kubernetes-values.yaml >/tmp/dander-kubernetes.yaml - name: Validate cross-cloud feasibility roots run: | terraform fmt -check -recursive ../acceptance/cloud-portability/phase1b diff --git a/HANDOFF.md b/HANDOFF.md index fe3bc102..0adc7d25 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,42 @@ ## Finished -- Made PostgreSQL warehouse/state selectable together through a version 2 deployment. -- Added named deployment selection to `dander run` and the OCI runtime. -- Claimed required ingestion targets before extraction without changing BigQuery writer behavior. -- Made metadata compilation use the selected warehouse dialect. -- Added one complete PostgreSQL 15 ingestion, transform, metadata, replay, cursor, history, and lease proof. +- Added a lazy Kubernetes launcher for version 2 named deployments, using the deployment name + as the unambiguous runtime selector. +- Packaged a Helm chart for existing Kubernetes 1.27+ clusters in every generated project. +- Added non-mutating `dander kubernetes plan` and read-only drift verification. +- Rendered bounded CronJobs with external Secret refs, `Forbid` overlap, explicit resources, + deadlines/retries, read-only pods, history limits, and Job TTL cleanup. +- Documented configuration, manual runs, upgrades, rollback, uninstall, and the unqualified boundary. ## Try It -Set `DANDER_TEST_POSTGRES_DSN` to a disposable PostgreSQL 15+ database and run -`tests/integration/test_postgresql_native_profile.py`. +Run `dander kubernetes plan --deployment NAME --container-image REGISTRY/IMAGE@sha256:DIGEST`, +review both saved artifacts, then use the printed Helm command only against an approved cluster. ## Checks -- All 1,010 repository tests passed locally, including the real PostgreSQL 15 integration. -- Repository-wide Ruff and strict mypy passed locally. -- Wheel/sdist inspection, source-free installation, and dependency audit passed. -- Terraform validation/tests and the full-runtime container build/conformance probe passed. +- All 1,022 tests passed with PostgreSQL 15; repository Ruff and strict mypy passed. +- Helm lint/template passed, including optional RBAC and ConfigMap-retention rendering. +- A disposable Kind cluster passed install, read-only verify, manual Job/duplicate rejection, + upgrade, rollback, and uninstall; its external Secret survived and the cluster was deleted. +- Wheel/sdist inspection, two source-free installs/scaffolds, dependency audit, Terraform + validation/tests, full-runtime container build, and read-only runtime conformance passed. ## Decisions -- Writers explicitly declare when the neutral runner must claim a destination target fence. -- A native PostgreSQL/no-catalog/environment-secret profile needs no GCP project identifier. -- PostgreSQL-state/BigQuery-warehouse remains fail-closed until all BigQuery writers adopt target fencing. +- Kubernetes targets an existing cluster; Dander does not create clusters or own external Secrets. +- Operator-managed environment injection is the first Secret path; cloud identity remains metadata. +- Kubernetes/PostgreSQL remains unsupported until the separate live end-to-end profile gate passes. ## Remaining -- Open, review, and merge the focused protected PR. -- Let protected CI repeat Linux container/security/secret checks. -- Add Kubernetes/Helm projection and existing-cluster verification separately. -- Run the PostgreSQL native and cross-backend benchmark matrix. +- Open and merge the focused protected PR after Linux CI and security scans pass. +- Run the native and cross-backend PostgreSQL matrix and bounded/concurrent benchmarks separately. +- Qualify one real existing-cluster PostgreSQL pipeline before changing the support manifest. ## Review First -- `src/dander/cli/run_command.py` -- `src/dander/runtime.py` -- `tests/integration/test_postgresql_native_profile.py` +- `src/dander/providers/kubernetes/operations.py` +- `infra/kubernetes/chart/dander/templates/cronjobs.yaml` +- `docs/kubernetes.md` diff --git a/docs/decisions.md b/docs/decisions.md index 7b67f097..b1ea0004 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -825,3 +825,12 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - PostgreSQL state/warehouse and BigQuery-state/PostgreSQL-warehouse pairs are executable. PostgreSQL-state/BigQuery-warehouse remains fail-closed until all BigQuery write modes adopt the destination-side fence. Hosted support still requires the Kubernetes and live-profile gates. +## 2026-08-08 — Kubernetes targets existing clusters through a versioned Helm chart + +- **Boundary:** Dander renders a named deployment into one packaged chart; it does not create a + cluster, database, registry, external Secret, observability stack, or cloud-specific identity. +- **Safety:** CronJobs forbid overlap, Jobs have bounded deadlines/retries and TTL cleanup, and the + runtime uses a read-only non-root pod with explicit resources. Durable Dander leases remain the + final concurrency defense. +- **Qualification:** Plan rendering and read-only cluster verification are implemented. Kubernetes + plus native PostgreSQL remains unsupported until an existing-cluster end-to-end proof passes. diff --git a/docs/execution-projection.md b/docs/execution-projection.md index f202742a..4c4ba85c 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -38,3 +38,10 @@ hour. This provider projection is an internal construction gate: provisioning, controller lifecycle, renewable credential supply, and live acceptance must pass before Fargate is a supported launcher. + +The Kubernetes provider projects a selected named profile into a packaged Helm chart for an +existing Kubernetes 1.27+ cluster. Its immutable template uses stdout, operator-owned Secret key +references, optional cloud-neutral ServiceAccount annotations, explicit pod resources, schedules, +deadlines, and bounded retries. The chart adds `Forbid` schedule concurrency and completed-Job TTL +cleanup while Dander leases remain the final overlap defense. Rendering and read-only verification +do not qualify a hosted Kubernetes profile; a real cluster and PostgreSQL acceptance run remain. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index aee3d600..72eabbf6 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -11,10 +11,11 @@ evaluate these limits before using it for an unattended system containing busine renewable credentials and live parity remain later gates. - Provider package extras and the full runtime image contain planned provider SDK dependencies, but do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. PostgreSQL - state and warehouse execution are implemented and locally conformance-tested, but no Kubernetes - live profile is qualified yet. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed - until every BigQuery write mode uses destination-side fencing. The packaged capability manifest - is the support boundary. + state and warehouse execution are implemented and locally conformance-tested. A packaged Helm + chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live + profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse + execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The + packaged capability manifest is the support boundary. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 00000000..59731dc3 --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,108 @@ +# Kubernetes existing-cluster launcher + +Dander packages a versioned Helm chart for an existing Kubernetes 1.27+ cluster. This is a +locally validated portability path, not yet a supported hosted profile. The chart does not create +a cluster, database, registry, external Secret, monitoring stack, or cloud identity. + +## Configure one deployment + +Keep logical pipelines in the version 2 `dander.yaml`. Select the native PostgreSQL profile and +Kubernetes launcher in `dander.platforms.yaml`: + +```yaml +version: 1 +platforms: + postgres: + warehouse: + provider: postgresql + database: dander + dsn_env: DANDER_POSTGRES_DSN + state: + provider: postgresql + authority_id: kubernetes:analytics + dsn_env: DANDER_POSTGRES_DSN + catalog: + provider: none + secrets: + provider: environment + +deployments: + analytics_cluster: + platform: postgres + launcher: + provider: kubernetes + context: my-existing-context + namespace: dander + release_name: dander + service_account_name: dander-runtime + existing_secret_name: dander-runtime + workload_identity_annotations: {} + pod_labels: + team: analytics + runtime: + cpu: 1 + memory: 512Mi + timeout_seconds: 900 + max_retries: 1 + batch_rows: 10000 + safety: + require_guarded_free_tier: false + pipelines: + greenhouse_jobs: + schedule: "0 9 * * *" + time_zone: America/New_York + paused: true + secret_bindings: + DANDER_POSTGRES_DSN: postgres-dsn +``` + +`environment` means the launcher injects an environment variable from an operator-owned +Kubernetes Secret. The manifest stores only the key name. Create the Secret separately in the +selected namespace; Helm never owns or reads its value. Workload-identity annotations and pod +labels are plain Kubernetes metadata, so the chart does not encode GKE, EKS, AKS, or OKE policy. + +## Plan before install or upgrade + +Use an immutable image digest. Planning lints the packaged chart and saves the exact rendered +values and manifests without contacting the cluster: + +```bash +dander kubernetes plan \ + --deployment analytics_cluster \ + --container-image registry.example/dander/runtime@sha256:DIGEST +``` + +Review both files under `.dander/kubernetes/`. The command prints the exact `helm upgrade +--install` command as the next mutating step; run it only with an identity authorized for the +selected context and namespace. New schedules should remain paused through their first manual run. + +After installation, compare the ServiceAccount, ConfigMap, CronJobs, schedule/pause settings, +overlap policy, and image digest with read-only calls: + +```bash +dander kubernetes verify \ + --deployment analytics_cluster \ + --expected-image registry.example/dander/runtime@sha256:DIGEST +``` + +## Operate and recover + +Create a manual execution from the paused CronJob so it uses the same pod template: + +```bash +kubectl --context my-existing-context --namespace dander \ + create job --from=cronjob/dander-greenhouse-jobs dander-greenhouse-manual-001 +kubectl --context my-existing-context --namespace dander \ + logs --follow job/dander-greenhouse-manual-001 +``` + +Repeated creation with the same Job name fails rather than starting a duplicate. CronJobs use +`concurrencyPolicy: Forbid`; Dander's durable lease remains the final defense against manual or +cross-launcher overlap. Jobs use `restartPolicy: Never`, bounded retry/deadline settings, a +read-only filesystem with writable bounded `/tmp`, explicit requests/limits, SIGTERM handling, +history limits, and completed-Job TTL cleanup. + +Use `helm history` and `helm rollback` for a reviewed chart rollback. A normal uninstall removes +Helm-owned ServiceAccounts, ConfigMaps, CronJobs, and optional RBAC, but never the referenced +Secret, database, Dander state, or warehouse data. `configMap.keepOnUninstall` is an explicit +escape hatch and defaults to `false`; retained objects become the operator's responsibility. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 745165b0..5d716d04 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -122,5 +122,7 @@ is supplied only through the configured environment variable. The adapter suppor table/view/incremental materialization, and the four generic assertions. Every materialization is claimed and published through its destination target fence. The local native-profile proof covers bounded ingestion, replay, transforms, assertions, metadata, run history, watermarks, and leases. -Graph execution and the Kubernetes launcher remain follow-up work; until live qualification passes, -this adapter must not be represented as a supported hosted PostgreSQL deployment. +Graph execution remains follow-up work. The packaged existing-cluster Kubernetes/Helm launcher can +render and verify this profile without cloud-specific identity assumptions, but no live cluster +qualification has passed. Until that acceptance does, this adapter must not be represented as a +supported hosted PostgreSQL deployment. See [Kubernetes existing-cluster launcher](kubernetes.md). diff --git a/infra/README.md b/infra/README.md index 875392ad..65c09d4a 100644 --- a/infra/README.md +++ b/infra/README.md @@ -13,6 +13,7 @@ call sites (mirrors the `SecretStoreProvider` / `ComputeProvider` abstractions i | `modules/secret-manager` | Named secret containers and per-secret runtime access; never secret values. **Implemented.** | | `modules/github-wif` | Repository/ref-scoped GitHub OIDC and a keyless deployment identity. **Implemented.** | | `modules/cost-guard` | Project budget, Pub/Sub, and simulation-first Gen 2 billing kill switch. **Implemented.** | +| `kubernetes/chart/dander` | Versioned Helm chart for an existing conforming cluster. **Locally validated; not live-qualified.** | The main root always calls `modules/bigquery` and can opt into the remaining workload modules. The one-time `infra/bootstrap-admin` root creates the remote-state bucket, the Artifact Registry diff --git a/infra/kubernetes/chart/dander/Chart.yaml b/infra/kubernetes/chart/dander/Chart.yaml new file mode 100644 index 00000000..0526a269 --- /dev/null +++ b/infra/kubernetes/chart/dander/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: dander +description: Existing-cluster launcher for immutable Dander pipeline images +type: application +version: 0.1.0 +appVersion: "0.7.0" +kubeVersion: ">=1.27.0-0" diff --git a/infra/kubernetes/chart/dander/README.md b/infra/kubernetes/chart/dander/README.md new file mode 100644 index 00000000..2eae8478 --- /dev/null +++ b/infra/kubernetes/chart/dander/README.md @@ -0,0 +1,20 @@ +# Dander existing-cluster chart + +This chart deploys immutable Dander runtime images to an existing Kubernetes 1.27+ cluster. It +does not create a cluster, database, external secret, registry, monitoring stack, or cloud identity. + +Generate reviewed values and manifests with `dander kubernetes plan`. Install only after review: + +```console +helm upgrade --install dander infra/kubernetes/chart/dander \ + --namespace dander --create-namespace --values .dander/kubernetes/profile.values.yaml +``` + +The chart defaults to `concurrencyPolicy: Forbid`, `restartPolicy: Never`, explicit resource +requests/limits, bounded retries/deadlines, job-history limits, and completed-Job TTL cleanup. +Existing Secret values are referenced by name and key; they are never placed in Helm values. + +Helm owns the ServiceAccount, ConfigMap, CronJobs, and optional read-only RBAC. It never owns the +referenced Secret. A ConfigMap may explicitly opt into `helm.sh/resource-policy: keep`; the default +is normal uninstall cleanup. Use `helm rollback` for chart rollback and review `helm diff` or a +server-side dry-run before upgrades where those tools are available. diff --git a/infra/kubernetes/chart/dander/templates/NOTES.txt b/infra/kubernetes/chart/dander/templates/NOTES.txt new file mode 100644 index 00000000..070648ce --- /dev/null +++ b/infra/kubernetes/chart/dander/templates/NOTES.txt @@ -0,0 +1,9 @@ +The Dander schedules were rendered with concurrencyPolicy=Forbid. Dander leases remain the final +overlap defense. + +Create a one-time manual execution from a rendered CronJob: + kubectl --context {{ .Values.context | default "CURRENT_CONTEXT" }} --namespace {{ .Release.Namespace }} \ + create job --from=cronjob/PIPELINE_RESOURCE_NAME MANUAL_JOB_NAME + +Inspect standard output: + kubectl --namespace {{ .Release.Namespace }} logs job/MANUAL_JOB_NAME diff --git a/infra/kubernetes/chart/dander/templates/_helpers.tpl b/infra/kubernetes/chart/dander/templates/_helpers.tpl new file mode 100644 index 00000000..a8e18056 --- /dev/null +++ b/infra/kubernetes/chart/dander/templates/_helpers.tpl @@ -0,0 +1,13 @@ +{{- define "dander.labels" -}} +app.kubernetes.io/name: {{ .Chart.Name | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | quote }} +dander.io/deployment: {{ .Values.deployment | quote }} +dander.io/profile: {{ .Values.profile | quote }} +{{- end }} + +{{- define "dander.serviceAccountName" -}} +{{- required "serviceAccount.name is required" .Values.serviceAccount.name -}} +{{- end }} diff --git a/infra/kubernetes/chart/dander/templates/configmap.yaml b/infra/kubernetes/chart/dander/templates/configmap.yaml new file mode 100644 index 00000000..2c281851 --- /dev/null +++ b/infra/kubernetes/chart/dander/templates/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-config" .Release.Name | trunc 63 | trimSuffix "-" }} + labels: + {{- include "dander.labels" . | nindent 4 }} + {{- if .Values.configMap.keepOnUninstall }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +data: + DANDER_DEPLOYMENT: {{ .Values.deployment | quote }} + DANDER_PLATFORM_PROFILE: {{ .Values.profile | quote }} + DANDER_IMAGE_REFERENCE: {{ .Values.image.reference | quote }} diff --git a/infra/kubernetes/chart/dander/templates/cronjobs.yaml b/infra/kubernetes/chart/dander/templates/cronjobs.yaml new file mode 100644 index 00000000..f055f88a --- /dev/null +++ b/infra/kubernetes/chart/dander/templates/cronjobs.yaml @@ -0,0 +1,83 @@ +{{- range $pipelineID, $pipeline := .Values.pipelines }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $pipeline.resourceName }} + labels: + {{- include "dander.labels" $ | nindent 4 }} + dander.io/pipeline: {{ $pipelineID | quote }} + {{- with $pipeline.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + schedule: {{ $pipeline.schedule | quote }} + timeZone: {{ $pipeline.timeZone | quote }} + suspend: {{ $pipeline.suspend }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: {{ $pipeline.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ $pipeline.failedJobsHistoryLimit }} + jobTemplate: + metadata: + labels: + {{- include "dander.labels" $ | nindent 8 }} + dander.io/pipeline: {{ $pipelineID | quote }} + spec: + backoffLimit: {{ $pipeline.backoffLimit }} + activeDeadlineSeconds: {{ $pipeline.activeDeadlineSeconds }} + ttlSecondsAfterFinished: {{ $pipeline.ttlSecondsAfterFinished }} + template: + metadata: + labels: + {{- include "dander.labels" $ | nindent 12 }} + dander.io/pipeline: {{ $pipelineID | quote }} + {{- with $.Values.podLabels }} + {{- toYaml . | nindent 12 }} + {{- end }} + annotations: + dander.io/image-digest: {{ $.Values.image.reference | quote }} + {{- with $.Values.podAnnotations }} + {{- toYaml . | nindent 12 }} + {{- end }} + spec: + serviceAccountName: {{ include "dander.serviceAccountName" $ }} + automountServiceAccountToken: {{ $.Values.rbac.create }} + restartPolicy: Never + terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: runtime + image: {{ $.Values.image.reference | quote }} + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + args: + {{- toYaml $pipeline.command | nindent 16 }} + env: + {{- range $item := $pipeline.environment }} + - name: {{ $item.name }} + value: {{ $item.value | quote }} + {{- end }} + {{- range $item := $pipeline.secretEnvironment }} + - name: {{ $item.name }} + valueFrom: + secretKeyRef: + name: {{ required "existingSecret.name is required by secret bindings" $.Values.existingSecret.name }} + key: {{ $item.key }} + {{- end }} + resources: + {{- toYaml $pipeline.resources | nindent 16 }} + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + sizeLimit: {{ index $pipeline.resources.limits "ephemeral-storage" }} +--- +{{- end }} diff --git a/infra/kubernetes/chart/dander/templates/rbac.yaml b/infra/kubernetes/chart/dander/templates/rbac.yaml new file mode 100644 index 00000000..f2b3078e --- /dev/null +++ b/infra/kubernetes/chart/dander/templates/rbac.yaml @@ -0,0 +1,30 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-runtime-read" .Release.Name | trunc 63 | trimSuffix "-" }} + labels: + {{- include "dander.labels" . | nindent 4 }} +rules: + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-runtime-read" .Release.Name | trunc 63 | trimSuffix "-" }} + labels: + {{- include "dander.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "dander.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-runtime-read" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/infra/kubernetes/chart/dander/templates/serviceaccount.yaml b/infra/kubernetes/chart/dander/templates/serviceaccount.yaml new file mode 100644 index 00000000..5dbe3ba1 --- /dev/null +++ b/infra/kubernetes/chart/dander/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "dander.serviceAccountName" . }} + labels: + {{- include "dander.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.rbac.create }} +{{- end }} diff --git a/infra/kubernetes/chart/dander/values.schema.json b/infra/kubernetes/chart/dander/values.schema.json new file mode 100644 index 00000000..330c08a8 --- /dev/null +++ b/infra/kubernetes/chart/dander/values.schema.json @@ -0,0 +1,106 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": ["deployment", "profile", "image", "serviceAccount", "pipelines"], + "properties": { + "deployment": {"type": "string", "minLength": 1}, + "profile": {"type": "string", "pattern": "^[a-z][a-z0-9_-]{0,62}$"}, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["reference"], + "properties": { + "reference": { + "type": "string", + "pattern": "^[a-z0-9.-]+/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" + } + } + }, + "serviceAccount": { + "type": "object", + "additionalProperties": false, + "required": ["create", "name", "annotations"], + "properties": { + "create": {"type": "boolean"}, + "name": {"type": "string", "minLength": 1}, + "annotations": {"type": "object", "additionalProperties": {"type": "string"}} + } + }, + "existingSecret": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": {"name": {"type": "string"}} + }, + "podLabels": {"type": "object", "additionalProperties": {"type": "string"}}, + "podAnnotations": {"type": "object", "additionalProperties": {"type": "string"}}, + "terminationGracePeriodSeconds": {"type": "integer", "minimum": 1, "maximum": 600}, + "configMap": { + "type": "object", + "additionalProperties": false, + "required": ["keepOnUninstall"], + "properties": {"keepOnUninstall": {"type": "boolean"}} + }, + "rbac": { + "type": "object", + "additionalProperties": false, + "required": ["create"], + "properties": {"create": {"type": "boolean"}} + }, + "pipelines": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"$ref": "#/definitions/pipeline"} + } + }, + "definitions": { + "nameValue": { + "type": "object", + "additionalProperties": false, + "required": ["name", "value"], + "properties": {"name": {"type": "string"}, "value": {"type": "string"}} + }, + "secretValue": { + "type": "object", + "additionalProperties": false, + "required": ["name", "key"], + "properties": {"name": {"type": "string"}, "key": {"type": "string"}} + }, + "resources": { + "type": "object", + "additionalProperties": false, + "required": ["requests", "limits"], + "properties": { + "requests": {"type": "object", "additionalProperties": {"type": "string"}}, + "limits": {"type": "object", "additionalProperties": {"type": "string"}} + } + }, + "pipeline": { + "type": "object", + "additionalProperties": false, + "required": [ + "resourceName", "command", "environment", "secretEnvironment", "schedule", + "timeZone", "suspend", "backoffLimit", "activeDeadlineSeconds", + "ttlSecondsAfterFinished", "successfulJobsHistoryLimit", + "failedJobsHistoryLimit", "resources", "labels" + ], + "properties": { + "resourceName": {"type": "string", "maxLength": 52}, + "command": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "environment": {"type": "array", "items": {"$ref": "#/definitions/nameValue"}}, + "secretEnvironment": {"type": "array", "items": {"$ref": "#/definitions/secretValue"}}, + "schedule": {"type": "string", "minLength": 1}, + "timeZone": {"type": "string", "minLength": 1}, + "suspend": {"type": "boolean"}, + "backoffLimit": {"type": "integer", "minimum": 0, "maximum": 10}, + "activeDeadlineSeconds": {"type": "integer", "minimum": 1}, + "ttlSecondsAfterFinished": {"type": "integer", "minimum": 60}, + "successfulJobsHistoryLimit": {"type": "integer", "minimum": 0}, + "failedJobsHistoryLimit": {"type": "integer", "minimum": 0}, + "resources": {"$ref": "#/definitions/resources"}, + "labels": {"type": "object", "additionalProperties": {"type": "string"}} + } + } + } +} diff --git a/infra/kubernetes/chart/dander/values.yaml b/infra/kubernetes/chart/dander/values.yaml new file mode 100644 index 00000000..b38bb9b0 --- /dev/null +++ b/infra/kubernetes/chart/dander/values.yaml @@ -0,0 +1,27 @@ +deployment: "" +profile: "" + +image: + reference: "" + +serviceAccount: + create: true + name: dander-runtime + annotations: {} + +existingSecret: + name: "" + +podLabels: {} +podAnnotations: {} +terminationGracePeriodSeconds: 120 + +configMap: + keepOnUninstall: false + +# Dander does not need Kubernetes API access while running. Leave this false unless +# an operator intentionally wants the runtime identity to inspect its own Jobs. +rbac: + create: false + +pipelines: {} diff --git a/pyproject.toml b/pyproject.toml index 192e658f..5e4cbe93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,16 @@ packages = ["src/dander"] "infra/functions/stop_billing/handler.py" = "dander/templates/project/infra/functions/stop_billing/handler.py" "infra/functions/stop_billing/main.py" = "dander/templates/project/infra/functions/stop_billing/main.py" "infra/functions/stop_billing/requirements.txt" = "dander/templates/project/infra/functions/stop_billing/requirements.txt" +"infra/kubernetes/chart/dander/Chart.yaml" = "dander/templates/project/infra/kubernetes/chart/dander/Chart.yaml" +"infra/kubernetes/chart/dander/README.md" = "dander/templates/project/infra/kubernetes/chart/dander/README.md" +"infra/kubernetes/chart/dander/templates/NOTES.txt" = "dander/templates/project/infra/kubernetes/chart/dander/templates/NOTES.txt" +"infra/kubernetes/chart/dander/templates/_helpers.tpl" = "dander/templates/project/infra/kubernetes/chart/dander/templates/_helpers.tpl" +"infra/kubernetes/chart/dander/templates/configmap.yaml" = "dander/templates/project/infra/kubernetes/chart/dander/templates/configmap.yaml" +"infra/kubernetes/chart/dander/templates/cronjobs.yaml" = "dander/templates/project/infra/kubernetes/chart/dander/templates/cronjobs.yaml" +"infra/kubernetes/chart/dander/templates/rbac.yaml" = "dander/templates/project/infra/kubernetes/chart/dander/templates/rbac.yaml" +"infra/kubernetes/chart/dander/templates/serviceaccount.yaml" = "dander/templates/project/infra/kubernetes/chart/dander/templates/serviceaccount.yaml" +"infra/kubernetes/chart/dander/values.schema.json" = "dander/templates/project/infra/kubernetes/chart/dander/values.schema.json" +"infra/kubernetes/chart/dander/values.yaml" = "dander/templates/project/infra/kubernetes/chart/dander/values.yaml" "infra/main.tf" = "dander/templates/project/infra/main.tf" "infra/modules/bigquery/main.tf" = "dander/templates/project/infra/modules/bigquery/main.tf" "infra/modules/bigquery/outputs.tf" = "dander/templates/project/infra/modules/bigquery/outputs.tf" diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index e9d248ba..8aa5373d 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -18,6 +18,9 @@ "dander/templates/project/README.md", "dander/templates/project/connectors/greenhouse_job_board.yaml", "dander/templates/project/dander.yaml", + "dander/templates/project/infra/kubernetes/chart/dander/Chart.yaml", + "dander/templates/project/infra/kubernetes/chart/dander/templates/cronjobs.yaml", + "dander/templates/project/infra/kubernetes/chart/dander/values.schema.json", "dander/templates/project/infra/main.tf", "dander/templates/project/models/staging/stg_greenhouse__jobs.sql", "dander/templates/project/models/staging/stg_greenhouse__jobs.yml", @@ -39,6 +42,7 @@ "README.md", "examples/salesforce/dander.yaml", "infra/main.tf", + "infra/kubernetes/chart/dander/Chart.yaml", "pyproject.toml", "src/dander/project/scaffold.py", } diff --git a/src/dander/cli/kubernetes_command.py b/src/dander/cli/kubernetes_command.py new file mode 100644 index 00000000..371fde8f --- /dev/null +++ b/src/dander/cli/kubernetes_command.py @@ -0,0 +1,90 @@ +"""Existing-cluster Kubernetes planning and read-only verification commands.""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +import typer +from click import ClickException +from rich.console import Console + +from dander.providers.kubernetes.operations import ( + KubernetesOperationError, + KubernetesOperations, +) + +console = Console() +kubernetes_app = typer.Typer( + help="Render and verify manifest-bound Kubernetes deployments on existing clusters." +) +_DEFAULT_PROJECT_CONFIG = Path("dander.yaml") +_DEFAULT_OUTPUT_DIR = Path(".dander/kubernetes") + + +@kubernetes_app.command("plan") +def kubernetes_plan( + deployment: str = typer.Option(..., "--deployment"), + container_image: str = typer.Option( + ..., "--container-image", help="Immutable runtime image ending in @sha256 digest." + ), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + output_dir: Path = typer.Option(_DEFAULT_OUTPUT_DIR, "--output-dir"), # noqa: B008 +) -> None: + """Lint and render a saved Helm plan without contacting or mutating a cluster.""" + try: + plan = KubernetesOperations().plan( + config=config, + deployment=deployment, + image=container_image, + output_dir=output_dir, + platforms_config=platforms_config, + ) + except KubernetesOperationError as error: + raise ClickException(str(error)) from error + console.print(f"[green]Kubernetes release rendered.[/green] Manifests: {plan.manifests}") + console.print(f"Review values: {plan.values}") + next_command = ( + "helm", + "--kube-context", + plan.context, + "upgrade", + "--install", + plan.release_name, + str(plan.chart), + "--namespace", + plan.namespace, + "--create-namespace", + "--values", + str(plan.values), + ) + console.print("Next after review: " + shlex.join(next_command), soft_wrap=True) + + +@kubernetes_app.command("verify") +def kubernetes_verify( + deployment: str = typer.Option(..., "--deployment"), + expected_image: str = typer.Option(..., "--expected-image"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 +) -> None: + """Compare one installed release to its manifest using read-only kubectl calls.""" + try: + verification = KubernetesOperations().verify( + config=config, + deployment=deployment, + expected_image=expected_image, + platforms_config=platforms_config, + ) + except KubernetesOperationError as error: + raise ClickException(str(error)) from error + console.print_json(data=verification.as_dict()) + + +def register_kubernetes_commands(app: typer.Typer) -> None: + """Register the namespaced Kubernetes command group.""" + app.add_typer(kubernetes_app, name="kubernetes") + + +__all__ = ["register_kubernetes_commands"] diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index bafb9b39..0720fee7 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -51,6 +51,7 @@ execute_platform_bootstrap, resolve_platform_config, ) +from dander.cli.kubernetes_command import register_kubernetes_commands from dander.cli.provider_runtime import build_catalog_publisher, build_secret_store from dander.cli.run_command import ( RunOptions, @@ -133,6 +134,7 @@ app.add_typer(connector_app, name="connector") app.add_typer(runtime_app, name="runtime") register_aws_commands(app) +register_kubernetes_commands(app) console = Console() _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") _DEFAULT_CONNECTORS_DIR = Path("connectors") diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index da84a3b7..ba122abb 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -4,6 +4,7 @@ CLOUD_RUN_CAPABILITIES, EXECUTION_PROJECTION_SCHEMA, FARGATE_CAPABILITIES, + KUBERNETES_CAPABILITIES, ExecutionProjectionError, ExecutionRequest, ExecutionTemplate, @@ -27,6 +28,7 @@ "ExecutionRequest", "ExecutionTemplate", "ExecutionTemplateFactory", + "KUBERNETES_CAPABILITIES", "LauncherCapabilities", "LauncherRuntime", "NetworkPlacement", diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index 06828643..b035a40c 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -364,6 +364,22 @@ class LauncherCapabilities: ) +KUBERNETES_CAPABILITIES = LauncherCapabilities( + launcher="kubernetes", + cpu_millis=frozenset({1_000, 2_000, 4_000, 6_000, 8_000}), + minimum_memory_mib=128, + maximum_memory_mib=1_048_576, + maximum_deadline_seconds=86_400, + maximum_launcher_retries=10, + maximum_task_count=1, + maximum_parallelism=1, + supports_ephemeral_storage=True, + supports_schedules=True, + supports_time_zones=True, + supports_network_placement=False, +) + + def validate_launcher_projection( template: ExecutionTemplate, capabilities: LauncherCapabilities, diff --git a/src/dander/deployment/runtime.py b/src/dander/deployment/runtime.py index 17a3e4df..9b19f317 100644 --- a/src/dander/deployment/runtime.py +++ b/src/dander/deployment/runtime.py @@ -28,6 +28,7 @@ def build( batch_rows: int, require_guarded_free_tier: bool, alert_target: str | None, + profile_id: str = "gcp", ) -> dict[str, ExecutionTemplate]: """Return validated execution templates keyed by pipeline ID.""" ... diff --git a/src/dander/project/config.py b/src/dander/project/config.py index c19333da..32f85331 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -228,7 +228,9 @@ class DanderProject(BaseModel): default="gcp_secret_manager", exclude=True, ) - launcher_provider: Literal["cloud_run", "fargate"] = Field(default="cloud_run", exclude=True) + launcher_provider: Literal["cloud_run", "fargate", "kubernetes"] = Field( + default="cloud_run", exclude=True + ) launcher_config: dict[str, object] = Field(default_factory=dict, exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index a532a6b8..2c714cec 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -29,6 +29,7 @@ from dander.providers.environment_secrets import EnvironmentSecretConfig # noqa: TC001 from dander.providers.fargate import FargateLauncherConfig # noqa: TC001 from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 +from dander.providers.kubernetes import KubernetesLauncherConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 from dander.providers.postgresql import ( # noqa: TC001 PostgreSQLStateConfig, @@ -133,7 +134,7 @@ def validate_pipeline_ids( LauncherSpec = Annotated[ - CloudRunLauncherConfig | FargateLauncherConfig, + CloudRunLauncherConfig | FargateLauncherConfig | KubernetesLauncherConfig, Field(discriminator="provider"), ] diff --git a/src/dander/providers/cloud_run/runtime.py b/src/dander/providers/cloud_run/runtime.py index a0ba2f79..379457f2 100644 --- a/src/dander/providers/cloud_run/runtime.py +++ b/src/dander/providers/cloud_run/runtime.py @@ -37,8 +37,11 @@ def build( batch_rows: int, require_guarded_free_tier: bool, alert_target: str | None, + profile_id: str = "gcp", ) -> dict[str, ExecutionTemplate]: """Build byte-equivalent Cloud Run templates through the existing projector.""" + if profile_id != "gcp": + raise ValueError("Cloud Run compatibility projection requires profile_id='gcp'") return build_gcp_execution_templates( pipelines, image=image, diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index b74fd279..4f7cdbce 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -7,6 +7,7 @@ from dander.providers.environment_secrets import EnvironmentSecretConfig from dander.providers.fargate import FargateLauncherConfig from dander.providers.gcp_secret_manager import GcpSecretManagerConfig +from dander.providers.kubernetes import KubernetesLauncherConfig from dander.providers.no_catalog import NoCatalogConfig from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -103,4 +104,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.fargate.runtime:FARGATE_LAUNCHER_FACTORY" ), ) + registry.register( + kind=ProviderKind.LAUNCHER, + provider_id="kubernetes", + config_model=KubernetesLauncherConfig, + load_factory=lazy_provider_factory( + "dander.providers.kubernetes.runtime:KUBERNETES_LAUNCHER_FACTORY" + ), + ) return registry diff --git a/src/dander/providers/fargate/runtime.py b/src/dander/providers/fargate/runtime.py index 1a2ce7ab..0f5cd9bf 100644 --- a/src/dander/providers/fargate/runtime.py +++ b/src/dander/providers/fargate/runtime.py @@ -51,8 +51,11 @@ def build( batch_rows: int, require_guarded_free_tier: bool, alert_target: str | None, + profile_id: str = "gcp", ) -> dict[str, ExecutionTemplate]: """Build fail-closed Fargate templates for the portable BigQuery proof.""" + if profile_id != "gcp": + raise ExecutionProjectionError("Fargate compatibility projection requires gcp") if require_guarded_free_tier: raise ExecutionProjectionError("Fargate cannot run the GCP guarded-free-tier preflight") if _GCP_PROJECT.fullmatch(project) is None: diff --git a/src/dander/providers/kubernetes/__init__.py b/src/dander/providers/kubernetes/__init__.py new file mode 100644 index 00000000..cadef1d0 --- /dev/null +++ b/src/dander/providers/kubernetes/__init__.py @@ -0,0 +1,5 @@ +"""Kubernetes launcher configuration and packaged Helm projection.""" + +from dander.providers.kubernetes.config import KubernetesLauncherConfig + +__all__ = ["KubernetesLauncherConfig"] diff --git a/src/dander/providers/kubernetes/chart.py b/src/dander/providers/kubernetes/chart.py new file mode 100644 index 00000000..efde605d --- /dev/null +++ b/src/dander/providers/kubernetes/chart.py @@ -0,0 +1,113 @@ +"""Deterministic Helm values for Kubernetes execution projections.""" + +from __future__ import annotations + +import hashlib +import re +from typing import TYPE_CHECKING + +from dander.deployment import ExecutionProjectionError, ExecutionTemplate + +if TYPE_CHECKING: + from collections.abc import Mapping + + from dander.providers.kubernetes.config import KubernetesLauncherConfig + +_RESOURCE_NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,50}[a-z0-9])?$") + + +def build_helm_values( + config: KubernetesLauncherConfig, + templates: Mapping[str, ExecutionTemplate], + *, + deployment_name: str, + platform_name: str, +) -> dict[str, object]: + """Render non-secret, deterministic chart inputs for one deployment.""" + if not templates: + raise ExecutionProjectionError("Kubernetes deployment requires at least one pipeline") + images = {template.image for template in templates.values()} + selectors = {template.profile_id for template in templates.values()} + if len(images) != 1 or len(selectors) != 1: + raise ExecutionProjectionError("Kubernetes pipelines must share one image and deployment") + pipelines: dict[str, object] = {} + for pipeline_id, template in sorted(templates.items()): + if template.launcher != "kubernetes": + raise ExecutionProjectionError("Helm values require Kubernetes templates") + secret_environment = [ + {"name": name, "key": reference.reference.removeprefix("env://")} + for name, reference in template.secret_bindings + ] + labels = { + key.replace("_", "-"): _label_value(value) + for key, value in template.labels + if key not in {"image_digest", "dander_version"} + } + pipelines[pipeline_id] = { + "resourceName": kubernetes_resource_name(config.release_name, pipeline_id), + "command": list(template.command), + "environment": [{"name": name, "value": value} for name, value in template.environment], + "secretEnvironment": secret_environment, + "schedule": template.schedule.expression, + "timeZone": template.schedule.time_zone, + "suspend": template.schedule.paused, + "backoffLimit": template.resources.launcher_retry_count, + "activeDeadlineSeconds": template.resources.deadline_seconds, + "ttlSecondsAfterFinished": config.ttl_seconds_after_finished, + "successfulJobsHistoryLimit": config.successful_jobs_history_limit, + "failedJobsHistoryLimit": config.failed_jobs_history_limit, + "resources": { + "requests": { + "cpu": f"{template.resources.cpu_millis}m", + "memory": f"{template.resources.memory_mib}Mi", + "ephemeral-storage": (f"{template.resources.ephemeral_storage_mib or 1}Mi"), + }, + "limits": { + "cpu": f"{template.resources.cpu_millis}m", + "memory": f"{template.resources.memory_mib}Mi", + "ephemeral-storage": (f"{template.resources.ephemeral_storage_mib or 1}Mi"), + }, + }, + "labels": dict(sorted(labels.items())), + } + image = next(iter(images)) + return { + "deployment": deployment_name, + "profile": platform_name, + "image": {"reference": image}, + "serviceAccount": { + "create": True, + "name": config.service_account_name, + "annotations": config.workload_identity_annotations, + }, + "existingSecret": {"name": config.existing_secret_name or ""}, + "podLabels": config.pod_labels, + "terminationGracePeriodSeconds": config.termination_grace_period_seconds, + "configMap": {"keepOnUninstall": False}, + "rbac": {"create": False}, + "pipelines": pipelines, + } + + +def kubernetes_resource_name(release_name: str, pipeline_id: str) -> str: + """Return one stable CronJob-safe name no longer than 52 characters.""" + base = re.sub(r"[^a-z0-9-]", "-", f"{release_name}-{pipeline_id}".lower()).strip("-") + if len(base) <= 52 and _RESOURCE_NAME.fullmatch(base): + return base + digest = hashlib.sha256(base.encode()).hexdigest()[:8] + prefix = base[:43].rstrip("-") + name = f"{prefix}-{digest}" + if _RESOURCE_NAME.fullmatch(name) is None: + raise ExecutionProjectionError("pipeline cannot produce a safe Kubernetes resource name") + return name + + +def _label_value(value: str) -> str: + normalized = re.sub(r"[^A-Za-z0-9_.-]", "-", value).strip("-_.") + if len(normalized) <= 63: + return normalized + digest = hashlib.sha256(value.encode()).hexdigest()[:8] + return f"{normalized[:54].rstrip('-_.')}-{digest}" + + +__all__ = ["build_helm_values", "kubernetes_resource_name"] diff --git a/src/dander/providers/kubernetes/config.py b/src/dander/providers/kubernetes/config.py new file mode 100644 index 00000000..c951de28 --- /dev/null +++ b/src/dander/providers/kubernetes/config.py @@ -0,0 +1,91 @@ +"""Dependency-light configuration for an existing Kubernetes cluster.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +_DNS_LABEL = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") +_DNS_SUBDOMAIN = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$") +_LABEL_NAME = re.compile( + r"^(?:[a-z0-9](?:[a-z0-9.-]{0,61}[a-z0-9])?/)?" + r"[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,61}[A-Za-z0-9])?$" +) +_LABEL_VALUE = re.compile(r"^(?:[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,61}[A-Za-z0-9])?)?$") + + +class KubernetesLauncherConfig(BaseModel): + """Select one existing Kubernetes context and namespace.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["kubernetes"] + context: str = Field(min_length=1, max_length=253) + namespace: str = Field(default="dander", min_length=1, max_length=63) + release_name: str = Field(default="dander", min_length=1, max_length=53) + service_account_name: str = Field(default="dander-runtime", min_length=1, max_length=63) + existing_secret_name: str | None = Field(default=None, min_length=1, max_length=253) + workload_identity_annotations: dict[str, str] = Field(default_factory=dict) + pod_labels: dict[str, str] = Field(default_factory=dict) + ephemeral_storage_mib: int = Field(default=1_024, ge=1, le=1_048_576) + successful_jobs_history_limit: int = Field(default=3, ge=0, le=100) + failed_jobs_history_limit: int = Field(default=3, ge=0, le=100) + ttl_seconds_after_finished: int = Field(default=3_600, ge=60, le=604_800) + termination_grace_period_seconds: int = Field(default=120, ge=1, le=600) + + @property + def region(self) -> str: + """Satisfy the shared launcher location surface without inventing a cloud region.""" + return "cluster" + + @field_validator( + "namespace", + "release_name", + "service_account_name", + ) + @classmethod + def validate_dns_labels(cls, value: str) -> str: + if _DNS_LABEL.fullmatch(value) is None: + raise ValueError("Kubernetes names must be valid DNS labels") + return value + + @field_validator("existing_secret_name") + @classmethod + def validate_secret_name(cls, value: str | None) -> str | None: + if value is not None and _DNS_SUBDOMAIN.fullmatch(value) is None: + raise ValueError("Kubernetes Secret names must be valid DNS subdomains") + return value + + @field_validator("context") + @classmethod + def validate_context(cls, value: str) -> str: + if value != value.strip() or any(character.isspace() for character in value): + raise ValueError("Kubernetes context must not contain whitespace") + return value + + @field_validator("workload_identity_annotations") + @classmethod + def validate_annotations(cls, values: dict[str, str]) -> dict[str, str]: + if any( + _LABEL_NAME.fullmatch(key) is None + or len(value) > 4_096 + or "\n" in value + or "\r" in value + for key, value in values.items() + ): + raise ValueError("workload identity annotations are invalid") + return dict(sorted(values.items())) + + @field_validator("pod_labels") + @classmethod + def validate_labels(cls, values: dict[str, str]) -> dict[str, str]: + if any( + _LABEL_NAME.fullmatch(key) is None + or _LABEL_VALUE.fullmatch(value) is None + or key.startswith(("app.kubernetes.io/", "dander.io/", "helm.sh/")) + for key, value in values.items() + ): + raise ValueError("pod labels are invalid") + return dict(sorted(values.items())) diff --git a/src/dander/providers/kubernetes/operations.py b/src/dander/providers/kubernetes/operations.py new file mode 100644 index 00000000..b63dcdab --- /dev/null +++ b/src/dander/providers/kubernetes/operations.py @@ -0,0 +1,514 @@ +"""Read-only planning and verification for existing Kubernetes clusters.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from dataclasses import asdict, dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, Protocol, cast + +import yaml + +from dander.bootstrap.terraform import build_launcher_runtime +from dander.project import ProjectConfigError, load_project_config +from dander.providers.kubernetes.chart import build_helm_values +from dander.providers.kubernetes.config import KubernetesLauncherConfig + +if TYPE_CHECKING: + from collections.abc import Sequence + + from dander.project import DanderProject + + +class KubernetesOperationError(RuntimeError): + """Raised when a Kubernetes plan or read-only verification fails.""" + + +class _Runner(Protocol): + def __call__( + self, + args: Sequence[str], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _subprocess_runner( + args: Sequence[str], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + ) + + +@dataclass(frozen=True, slots=True) +class KubernetesPlan: + """Saved Helm values and manifests produced without cluster mutation.""" + + chart: Path + values: Path + manifests: Path + release_name: str + namespace: str + context: str + + +@dataclass(frozen=True, slots=True) +class KubernetesVerification: + """Sanitized read-only comparison of one installed release.""" + + context: str + namespace: str + service_account: str + config_map: str + pipelines: tuple[str, ...] + image: str + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +class KubernetesOperations: + """Render Helm artifacts and verify a selected existing-cluster release.""" + + def __init__(self, *, runner: _Runner | None = None) -> None: + self._runner = runner or _subprocess_runner + + def plan( + self, + *, + config: Path, + deployment: str, + image: str, + output_dir: Path, + platforms_config: Path | None = None, + ) -> KubernetesPlan: + """Lint and render the chart into saved review artifacts without an apply.""" + project_config = config.expanduser().resolve() + try: + _manifest, launcher_config, values = _resolve_release( + project_config=project_config, + platforms_config=platforms_config, + deployment=deployment, + image=image, + ) + except (ProjectConfigError, ValueError) as error: + raise KubernetesOperationError(str(error)) from error + chart = project_config.parent / "infra" / "kubernetes" / "chart" / "dander" + if not (chart / "Chart.yaml").is_file(): + raise KubernetesOperationError(f"Packaged Dander Helm chart is missing: {chart}") + destination = output_dir.expanduser().resolve() + destination.mkdir(parents=True, exist_ok=True) + values_path = destination / f"{deployment}.values.yaml" + manifests_path = destination / f"{deployment}.manifests.yaml" + _atomic_write(values_path, yaml.safe_dump(values, sort_keys=False, width=100)) + self._execute( + ("helm", "lint", str(chart), "--values", str(values_path)), + cwd=project_config.parent, + ) + rendered = self._execute( + ( + "helm", + "template", + launcher_config.release_name, + str(chart), + "--namespace", + launcher_config.namespace, + "--values", + str(values_path), + ), + cwd=project_config.parent, + ) + _atomic_write(manifests_path, rendered) + return KubernetesPlan( + chart=chart, + values=values_path, + manifests=manifests_path, + release_name=launcher_config.release_name, + namespace=launcher_config.namespace, + context=launcher_config.context, + ) + + def verify( + self, + *, + config: Path, + deployment: str, + expected_image: str, + platforms_config: Path | None = None, + ) -> KubernetesVerification: + """Verify service account, config, CronJobs, schedules, and image read-only.""" + project_config = config.expanduser().resolve() + try: + manifest, launcher, expected_values = _resolve_release( + project_config=project_config, + platforms_config=platforms_config, + deployment=deployment, + image=expected_image, + ) + except (ProjectConfigError, ValueError) as error: + raise KubernetesOperationError(str(error)) from error + prefix = ( + "kubectl", + "--context", + launcher.context, + "--namespace", + launcher.namespace, + ) + service_account = self._json( + (*prefix, "get", "serviceaccount", launcher.service_account_name, "-o", "json"), + cwd=project_config.parent, + ) + _verify_service_account(service_account, launcher) + config_map = f"{launcher.release_name}-config" + config_payload = self._json( + (*prefix, "get", "configmap", config_map, "-o", "json"), + cwd=project_config.parent, + ) + _verify_config_map( + config_payload, + deployment=manifest.deployment_name, + profile=manifest.platform_name, + image=expected_image, + ) + secret = expected_values.get("existingSecret") + secret_name = secret.get("name") if isinstance(secret, dict) else None + if secret_name: + self._execute( + (*prefix, "get", "secret", str(secret_name), "-o", "json"), + cwd=project_config.parent, + ) + payload = self._json( + ( + *prefix, + "get", + "cronjobs", + "-l", + f"app.kubernetes.io/instance={launcher.release_name}", + "-o", + "json", + ), + cwd=project_config.parent, + ) + items = payload.get("items") + if not isinstance(items, list): + raise KubernetesOperationError("kubectl returned an invalid CronJob list") + observed: dict[str, dict[str, object]] = {} + for item in items: + if not isinstance(item, dict): + raise KubernetesOperationError("kubectl returned an invalid CronJob") + metadata = item.get("metadata") + if not isinstance(metadata, dict) or not isinstance(metadata.get("name"), str): + raise KubernetesOperationError("kubectl returned a CronJob without a name") + labels = metadata.get("labels") + pipeline_id = labels.get("dander.io/pipeline") if isinstance(labels, dict) else None + if not isinstance(pipeline_id, str): + raise KubernetesOperationError("deployed CronJob is missing its pipeline label") + observed[pipeline_id] = item + expected_ids = set(manifest.terraform_pipelines()) + if set(observed) != expected_ids: + raise KubernetesOperationError("deployed CronJobs do not match the selected pipelines") + expected_pipelines = expected_values.get("pipelines") + if not isinstance(expected_pipelines, dict): + raise KubernetesOperationError("rendered Kubernetes pipeline values are invalid") + for pipeline_id, item in observed.items(): + expected_pipeline = expected_pipelines.get(pipeline_id) + if not isinstance(expected_pipeline, dict): + raise KubernetesOperationError("rendered Kubernetes pipeline is invalid") + spec = item.get("spec") + if not isinstance(spec, dict): + raise KubernetesOperationError("deployed CronJob has an invalid spec") + _verify_cronjob(spec, expected_pipeline, launcher, expected_image) + return KubernetesVerification( + context=launcher.context, + namespace=launcher.namespace, + service_account=launcher.service_account_name, + config_map=config_map, + pipelines=tuple(sorted(observed)), + image=expected_image, + ) + + def _json(self, args: Sequence[str], *, cwd: Path) -> dict[str, object]: + output = self._execute(args, cwd=cwd) + try: + payload = json.loads(output) + except json.JSONDecodeError as error: + raise KubernetesOperationError("kubectl returned invalid JSON") from error + if not isinstance(payload, dict): + raise KubernetesOperationError("kubectl returned an invalid response") + return payload + + def _execute(self, args: Sequence[str], *, cwd: Path) -> str: + executable = args[0] + if shutil.which(executable) is None and self._runner is _subprocess_runner: + raise KubernetesOperationError(f"{executable} is not installed or not on PATH") + try: + completed = self._runner( + args, + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise KubernetesOperationError( + f"{executable} is not installed or not on PATH" + ) from error + except subprocess.CalledProcessError as error: + detail = (error.stderr or "").strip().splitlines() + summary = detail[-1][:300] if detail else "command failed" + raise KubernetesOperationError(f"{executable} failed: {summary}") from error + return completed.stdout + + +def _resolve_release( + *, + project_config: Path, + platforms_config: Path | None, + deployment: str, + image: str, +) -> tuple[DanderProject, KubernetesLauncherConfig, dict[str, object]]: + manifest = load_project_config( + project_config, + platforms_path=platforms_config, + deployment=deployment, + ) + if manifest.launcher_provider != "kubernetes": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select launcher.provider='kubernetes'" + ) + if manifest.secret_provider != "environment": + raise ProjectConfigError( + "Kubernetes currently requires secrets.provider='environment' for " + "operator-managed Secret references" + ) + manifest.validate_references(project_config.parent) + launcher_config = KubernetesLauncherConfig.model_validate(manifest.resolved_launcher_config()) + launcher = build_launcher_runtime(launcher_config=manifest.resolved_launcher_config()) + templates = launcher.templates.build( + manifest.terraform_pipelines(), + image=image, + project="", + cpu=manifest.platform.runtime.cpu, + memory=manifest.platform.runtime.memory, + deadline_seconds=manifest.platform.runtime.timeout_seconds, + launcher_retry_count=manifest.platform.runtime.max_retries, + batch_rows=manifest.platform.runtime.batch_rows, + require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, + alert_target=None, + profile_id=manifest.deployment_name, + ) + values = build_helm_values( + launcher_config, + templates, + deployment_name=manifest.deployment_name, + platform_name=manifest.platform_name, + ) + return manifest, launcher_config, values + + +def _atomic_write(path: Path, content: str) -> None: + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as temporary: + temporary.write(content) + temporary_path = Path(temporary.name) + os.chmod(temporary_path, 0o600) + temporary_path.replace(path) + + +def _verify_service_account( + payload: dict[str, object], + launcher: KubernetesLauncherConfig, +) -> None: + metadata = _mapping(payload.get("metadata"), "deployed ServiceAccount has invalid metadata") + annotations = _mapping( + metadata.get("annotations", {}), + "deployed ServiceAccount annotations are invalid", + ) + if any( + annotations.get(key) != value + for key, value in launcher.workload_identity_annotations.items() + ): + raise KubernetesOperationError("deployed ServiceAccount annotations do not match") + if payload.get("automountServiceAccountToken") is not False: + raise KubernetesOperationError("deployed ServiceAccount unexpectedly mounts API tokens") + + +def _verify_config_map( + payload: dict[str, object], + *, + deployment: str, + profile: str, + image: str, +) -> None: + data = _mapping(payload.get("data"), "deployed ConfigMap data is invalid") + expected = { + "DANDER_DEPLOYMENT": deployment, + "DANDER_PLATFORM_PROFILE": profile, + "DANDER_IMAGE_REFERENCE": image, + } + if any(data.get(key) != value for key, value in expected.items()): + raise KubernetesOperationError("deployed ConfigMap does not match the selected deployment") + + +def _verify_cronjob( + spec: dict[str, object], + expected: dict[str, object], + launcher: KubernetesLauncherConfig, + image: str, +) -> None: + scalar_fields = ( + "schedule", + "timeZone", + "suspend", + "successfulJobsHistoryLimit", + "failedJobsHistoryLimit", + ) + if any(spec.get(field) != expected.get(field) for field in scalar_fields): + raise KubernetesOperationError("deployed CronJob schedule or history does not match") + if spec.get("concurrencyPolicy") != "Forbid": + raise KubernetesOperationError("deployed CronJob does not forbid overlap") + job_template = _mapping(spec.get("jobTemplate"), "deployed CronJob has an invalid Job template") + job_spec = _mapping(job_template.get("spec"), "deployed CronJob has an invalid Job spec") + if any( + job_spec.get(field) != expected.get(field) + for field in ("backoffLimit", "activeDeadlineSeconds", "ttlSecondsAfterFinished") + ): + raise KubernetesOperationError("deployed Job retry, deadline, or TTL does not match") + pod_template = _mapping( + job_spec.get("template"), "deployed CronJob has an invalid pod template" + ) + pod_spec = _mapping(pod_template.get("spec"), "deployed CronJob has an invalid pod spec") + if ( + pod_spec.get("restartPolicy") != "Never" + or pod_spec.get("serviceAccountName") != launcher.service_account_name + or pod_spec.get("automountServiceAccountToken") is not False + or pod_spec.get("terminationGracePeriodSeconds") + != launcher.termination_grace_period_seconds + ): + raise KubernetesOperationError("deployed pod lifecycle or identity does not match") + containers = pod_spec.get("containers") + if not isinstance(containers, list) or len(containers) != 1: + raise KubernetesOperationError("deployed CronJob must have one runtime container") + container = _mapping(containers[0], "deployed runtime container is invalid") + if container.get("image") != image or container.get("args") != expected.get("command"): + raise KubernetesOperationError("deployed runtime image or command does not match") + _verify_environment(container, expected, launcher) + _verify_resources(container, expected) + + +def _verify_environment( + container: dict[str, object], + expected: dict[str, object], + launcher: KubernetesLauncherConfig, +) -> None: + environment = container.get("env") + if not isinstance(environment, list): + raise KubernetesOperationError("deployed runtime environment is invalid") + observed_values: dict[str, str] = {} + observed_secrets: dict[str, tuple[str, str]] = {} + for raw_item in environment: + item = _mapping(raw_item, "deployed runtime environment is invalid") + name = item.get("name") + if not isinstance(name, str): + raise KubernetesOperationError("deployed runtime environment name is invalid") + value = item.get("value") + if isinstance(value, str): + observed_values[name] = value + continue + value_from = _mapping(item.get("valueFrom"), "deployed runtime secret reference is invalid") + secret_ref = _mapping( + value_from.get("secretKeyRef"), + "deployed runtime secret reference is invalid", + ) + secret_name = secret_ref.get("name") + secret_key = secret_ref.get("key") + if not isinstance(secret_name, str) or not isinstance(secret_key, str): + raise KubernetesOperationError("deployed runtime secret reference is invalid") + observed_secrets[name] = (secret_name, secret_key) + expected_values = _expected_pairs(expected.get("environment"), value_key="value") + expected_keys = _expected_pairs(expected.get("secretEnvironment"), value_key="key") + expected_secrets = { + name: (str(launcher.existing_secret_name), key) for name, key in expected_keys.items() + } + if observed_values != expected_values or observed_secrets != expected_secrets: + raise KubernetesOperationError("deployed runtime environment references do not match") + + +def _expected_pairs(value: object, *, value_key: str) -> dict[str, str]: + if not isinstance(value, list): + raise KubernetesOperationError("rendered Kubernetes environment is invalid") + result: dict[str, str] = {} + for raw_item in value: + item = _mapping(raw_item, "rendered Kubernetes environment is invalid") + name = item.get("name") + item_value = item.get(value_key) + if not isinstance(name, str) or not isinstance(item_value, str): + raise KubernetesOperationError("rendered Kubernetes environment is invalid") + result[name] = item_value + return result + + +def _verify_resources(container: dict[str, object], expected: dict[str, object]) -> None: + resources = _mapping(container.get("resources"), "deployed runtime resources are invalid") + expected_resources = _mapping( + expected.get("resources"), "rendered runtime resources are invalid" + ) + for group in ("requests", "limits"): + actual_group = _mapping(resources.get(group), "deployed runtime resources are invalid") + expected_group = _mapping( + expected_resources.get(group), "rendered runtime resources are invalid" + ) + for name, value in expected_group.items(): + if _resource_quantity(name, actual_group.get(name)) != _resource_quantity(name, value): + raise KubernetesOperationError("deployed runtime resources do not match") + + +def _resource_quantity(name: str, value: object) -> int: + if not isinstance(value, str): + raise KubernetesOperationError("deployed runtime resource quantity is invalid") + try: + if name == "cpu": + return int(value.removesuffix("m")) if value.endswith("m") else int(value) * 1_000 + if value.endswith("Mi"): + return int(value.removesuffix("Mi")) + if value.endswith("Gi"): + return int(value.removesuffix("Gi")) * 1_024 + except ValueError as error: + raise KubernetesOperationError("deployed runtime resource quantity is invalid") from error + raise KubernetesOperationError("deployed runtime resource quantity is unsupported") + + +def _mapping(value: object, message: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise KubernetesOperationError(message) + return cast("dict[str, object]", value) + + +__all__ = [ + "KubernetesOperationError", + "KubernetesOperations", + "KubernetesPlan", + "KubernetesVerification", +] diff --git a/src/dander/providers/kubernetes/runtime.py b/src/dander/providers/kubernetes/runtime.py new file mode 100644 index 00000000..ce455eac --- /dev/null +++ b/src/dander/providers/kubernetes/runtime.py @@ -0,0 +1,191 @@ +"""Existing-cluster Kubernetes execution projection.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander import __version__ +from dander.deployment.projection import ( + EXECUTION_PROJECTION_SCHEMA, + KUBERNETES_CAPABILITIES, + ExecutionProjectionError, + ExecutionTemplate, + NetworkPlacement, + ObservabilityProjection, + ResourceProjection, + ScheduleProjection, + SecretReference, + validate_launcher_projection, +) +from dander.deployment.runtime import LauncherRuntime +from dander.providers.kubernetes.config import KubernetesLauncherConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.runtime_contract import RUNTIME_CONTRACT + +if TYPE_CHECKING: + from pydantic import BaseModel + +_PROFILE = re.compile(r"^[a-z][a-z0-9_-]{0,62}$") + + +@dataclass(frozen=True, slots=True) +class KubernetesTemplateFactory: + """Project one selected profile into Kubernetes Jobs and CronJobs.""" + + config: KubernetesLauncherConfig + + def build( + self, + pipelines: Mapping[str, Mapping[str, object]], + *, + image: str, + project: str, + cpu: int, + memory: str, + deadline_seconds: int, + launcher_retry_count: int, + batch_rows: int, + require_guarded_free_tier: bool, + alert_target: str | None, + profile_id: str = "gcp", + ) -> dict[str, ExecutionTemplate]: + """Build immutable templates without contacting the selected cluster.""" + del project + if _PROFILE.fullmatch(profile_id) is None: + raise ExecutionProjectionError("invalid Kubernetes deployment selector") + if require_guarded_free_tier: + raise ExecutionProjectionError( + "Kubernetes cannot run the GCP guarded-free-tier preflight" + ) + if alert_target is not None: + raise ExecutionProjectionError( + "Kubernetes uses cluster observability and cannot provision an alert target" + ) + memory_mib = _memory_mib(memory) + templates: dict[str, ExecutionTemplate] = {} + for pipeline_id, pipeline in sorted(pipelines.items()): + secret_env = pipeline["secret_env"] + if not isinstance(secret_env, Mapping): + raise ExecutionProjectionError("pipeline secret bindings are invalid") + if secret_env and self.config.existing_secret_name is None: + raise ExecutionProjectionError( + "Kubernetes secret bindings require launcher.existing_secret_name" + ) + command: tuple[str, ...] = ( + "runtime", + "execute", + "--contract", + RUNTIME_CONTRACT, + "--pipeline", + pipeline_id, + "--platform", + profile_id, + "--config", + "/app/dander.yaml", + "--models-dir", + "/app/models", + "--batch-rows", + str(batch_rows), + ) + if bool(pipeline["build_models"]): + command = (*command, "--catalog-output", "/tmp/dander-catalog.json") + identity = ( + f"kubernetes://{self.config.namespace}/" + f"serviceaccounts/{self.config.service_account_name}" + ) + labels = { + "dander_version": __version__, + "image_digest": image.rsplit("@", maxsplit=1)[-1], + "pipeline": pipeline_id, + "profile": profile_id, + } + template = ExecutionTemplate( + schema=EXECUTION_PROJECTION_SCHEMA, + contract=RUNTIME_CONTRACT, + pipeline_id=pipeline_id, + profile_id=profile_id, + launcher="kubernetes", + image=image, + command=command, + configuration_reference="/app/dander.yaml", + environment=tuple( + sorted( + { + "DANDER_IMAGE_DIGEST": image.rsplit("@", maxsplit=1)[-1], + "DANDER_LAUNCHER": "kubernetes", + "DANDER_PRINCIPAL": identity, + }.items() + ) + ), + secret_bindings=tuple( + ( + str(environment_name), + SecretReference(provider="environment", reference=f"env://{secret_id}"), + ) + for environment_name, secret_id in sorted(secret_env.items()) + ), + workload_identity=identity, + resources=ResourceProjection( + cpu_millis=cpu * 1_000, + memory_mib=memory_mib, + ephemeral_storage_mib=self.config.ephemeral_storage_mib, + deadline_seconds=deadline_seconds, + runtime_retry_count=0, + launcher_retry_count=launcher_retry_count, + ), + schedule=ScheduleProjection( + task_count=1, + maximum_parallelism=1, + expression=str(pipeline["schedule"]), + time_zone=str(pipeline["time_zone"]), + paused=bool(pipeline["paused"]), + ), + network=NetworkPlacement(), + labels=tuple(sorted(labels.items())), + observability=ObservabilityProjection( + log_destination="stdout", + metric_namespace="kubernetes.io", + alert_target=None, + retention_days=None, + ), + ) + validate_launcher_projection(template, KUBERNETES_CAPABILITIES) + templates[pipeline_id] = template + return templates + + +def build_kubernetes_launcher( + config: BaseModel, + context: Mapping[str, object], +) -> LauncherRuntime: + """Build the Kubernetes launcher only after explicit selection.""" + del context + if not isinstance(config, KubernetesLauncherConfig): + raise TypeError("Kubernetes launcher factory received the wrong configuration") + return LauncherRuntime( + provider_id="kubernetes", + region=config.region, + templates=KubernetesTemplateFactory(config), + capabilities=KUBERNETES_CAPABILITIES, + ) + + +def _memory_mib(value: str) -> int: + match = re.fullmatch(r"([1-9][0-9]*)(Mi|Gi)", value) + if match is None: + raise ExecutionProjectionError("runtime memory must use Mi or Gi") + quantity = int(match.group(1)) + return quantity if match.group(2) == "Mi" else quantity * 1_024 + + +KUBERNETES_LAUNCHER_FACTORY: ProviderFactory[LauncherRuntime] = ProviderFactory( + kind=ProviderKind.LAUNCHER, + provider_id="kubernetes", + api_version=PROVIDER_API_VERSION, + build=build_kubernetes_launcher, +) + +__all__ = ["KUBERNETES_LAUNCHER_FACTORY"] diff --git a/tests/cli/test_kubernetes_cli.py b/tests/cli/test_kubernetes_cli.py new file mode 100644 index 00000000..140f8e3d --- /dev/null +++ b/tests/cli/test_kubernetes_cli.py @@ -0,0 +1,86 @@ +"""Kubernetes CLI keeps plans non-mutating and verification read-only.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from typer.testing import CliRunner + +from dander.cli.main import app +from dander.providers.kubernetes.operations import KubernetesPlan, KubernetesVerification + +if TYPE_CHECKING: + from pathlib import Path + + from pytest import MonkeyPatch + + +def test_kubernetes_plan_prints_reviewed_helm_command( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + chart = tmp_path / "chart" + values = tmp_path / "deployment.values.yaml" + manifests = tmp_path / "deployment.manifests.yaml" + + def plan(_operations: object, **_: object) -> KubernetesPlan: + return KubernetesPlan( + chart=chart, + values=values, + manifests=manifests, + release_name="dander", + namespace="dander-test", + context="kind-dander", + ) + + monkeypatch.setattr("dander.cli.kubernetes_command.KubernetesOperations.plan", plan) + result = CliRunner().invoke( + app, + [ + "kubernetes", + "plan", + "--deployment", + "local_cluster", + "--container-image", + "ghcr.io/example/dander@sha256:" + "a" * 64, + ], + ) + + assert result.exit_code == 0, result.output + assert manifests.name in result.output + assert "helm --kube-context kind-dander upgrade --install dander" in " ".join( + result.output.split() + ) + assert "--namespace dander-test" in " ".join(result.output.split()) + + +def test_kubernetes_verify_prints_only_sanitized_result( + monkeypatch: MonkeyPatch, +) -> None: + def verify(_operations: object, **_: object) -> KubernetesVerification: + return KubernetesVerification( + context="kind-dander", + namespace="dander-test", + service_account="dander-runtime", + config_map="dander-config", + pipelines=("records",), + image="ghcr.io/example/dander@sha256:" + "b" * 64, + ) + + monkeypatch.setattr("dander.cli.kubernetes_command.KubernetesOperations.verify", verify) + result = CliRunner().invoke( + app, + [ + "kubernetes", + "verify", + "--deployment", + "local_cluster", + "--expected-image", + "ghcr.io/example/dander@sha256:" + "b" * 64, + ], + ) + + assert result.exit_code == 0, result.output + assert '"pipelines": [' in result.output + assert '"records"' in result.output + assert "secret" not in result.output.lower() diff --git a/tests/fixtures/kubernetes-project/connectors/records.yaml b/tests/fixtures/kubernetes-project/connectors/records.yaml new file mode 100644 index 00000000..f9e363d5 --- /dev/null +++ b/tests/fixtures/kubernetes-project/connectors/records.yaml @@ -0,0 +1,11 @@ +name: records +base_url: https://example.invalid +auth_strategy: none +endpoints: + - name: records + path: /records + primary_key: [id] + raw_schema: + - name: id + type: STRING + mode: REQUIRED diff --git a/tests/fixtures/kubernetes-project/dander.platforms.yaml b/tests/fixtures/kubernetes-project/dander.platforms.yaml new file mode 100644 index 00000000..d04c97e4 --- /dev/null +++ b/tests/fixtures/kubernetes-project/dander.platforms.yaml @@ -0,0 +1,42 @@ +version: 1 +platforms: + postgres: + warehouse: + provider: postgresql + database: dander + state: + provider: postgresql + authority_id: kubernetes:test + catalog: + provider: none + secrets: + provider: environment +deployments: + test_cluster: + platform: postgres + launcher: + provider: kubernetes + context: kind-dander-portability + namespace: dander-test + release_name: dander + service_account_name: dander-runtime + existing_secret_name: dander-runtime + workload_identity_annotations: + example.com/workload-identity: dander-test + pod_labels: + team: analytics + runtime: + cpu: 1 + memory: 512Mi + timeout_seconds: 300 + max_retries: 1 + batch_rows: 2 + safety: + require_guarded_free_tier: false + pipelines: + records: + schedule: "0 9 * * *" + time_zone: America/New_York + paused: true + secret_bindings: + DANDER_POSTGRES_DSN: postgres-dsn diff --git a/tests/fixtures/kubernetes-project/dander.yaml b/tests/fixtures/kubernetes-project/dander.yaml new file mode 100644 index 00000000..0a18217b --- /dev/null +++ b/tests/fixtures/kubernetes-project/dander.yaml @@ -0,0 +1,6 @@ +version: 2 +pipelines: + records: + source: records + models: [] + build_models: false diff --git a/tests/fixtures/kubernetes-values.yaml b/tests/fixtures/kubernetes-values.yaml new file mode 100644 index 00000000..e1cfc1f4 --- /dev/null +++ b/tests/fixtures/kubernetes-values.yaml @@ -0,0 +1,68 @@ +deployment: test_cluster +profile: postgres +image: + reference: ghcr.io/example/dander@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +serviceAccount: + create: true + name: dander-runtime + annotations: + example.com/workload-identity: dander-test +existingSecret: + name: dander-runtime +podLabels: + team: analytics +podAnnotations: + prometheus.io/scrape: "false" +terminationGracePeriodSeconds: 120 +configMap: + keepOnUninstall: false +rbac: + create: false +pipelines: + records: + resourceName: dander-records + command: + - runtime + - execute + - --contract + - io.dander.runtime/v1 + - --pipeline + - records + - --platform + - test_cluster + - --config + - /app/dander.yaml + - --models-dir + - /app/models + - --batch-rows + - "2" + environment: + - name: DANDER_IMAGE_DIGEST + value: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + - name: DANDER_LAUNCHER + value: kubernetes + - name: DANDER_PRINCIPAL + value: kubernetes://dander-test/serviceaccounts/dander-runtime + secretEnvironment: + - name: DANDER_POSTGRES_DSN + key: postgres-dsn + schedule: "0 9 * * *" + timeZone: America/New_York + suspend: true + backoffLimit: 1 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 3600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + resources: + requests: + cpu: 1000m + memory: 512Mi + ephemeral-storage: 1024Mi + limits: + cpu: 1000m + memory: 512Mi + ephemeral-storage: 1024Mi + labels: + pipeline: records + profile: postgres diff --git a/tests/infra/test_kubernetes_chart.py b/tests/infra/test_kubernetes_chart.py new file mode 100644 index 00000000..c6b8af83 --- /dev/null +++ b/tests/infra/test_kubernetes_chart.py @@ -0,0 +1,323 @@ +"""Packaged Helm chart safety and deterministic-value coverage.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import yaml + +from dander.providers.kubernetes.operations import ( + KubernetesOperationError, + KubernetesOperations, +) + +ROOT = Path(__file__).parents[2] +IMAGE = "ghcr.io/example/dander@sha256:" + "b" * 64 + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class _Runner: + def __init__(self, responses: dict[tuple[str, ...], list[str]] | None = None) -> None: + self.commands: list[tuple[str, ...]] = [] + self.responses = responses or {} + + def __call__( + self, + args: Sequence[str], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + del cwd, check, capture_output, text + command = tuple(args) + self.commands.append(command) + for prefix, responses in self.responses.items(): + if command[: len(prefix)] == prefix: + return subprocess.CompletedProcess(command, 0, stdout=responses.pop(0), stderr="") + if command[:2] == ("helm", "lint"): + return subprocess.CompletedProcess(command, 0, stdout="lint ok", stderr="") + if command[:2] == ("helm", "template"): + return subprocess.CompletedProcess( + command, + 0, + stdout="apiVersion: v1\nkind: ConfigMap\n", + stderr="", + ) + raise AssertionError(f"Unexpected command: {command}") + + +def _project(tmp_path: Path) -> tuple[Path, Path]: + project = tmp_path / "dander.yaml" + platforms = tmp_path / "dander.platforms.yaml" + project.write_text( + """ +version: 2 +pipelines: + records: + source: records + models: [] + build_models: false +""".lstrip(), + encoding="utf-8", + ) + platforms.write_text( + """ +version: 1 +platforms: + postgres: + warehouse: + provider: postgresql + database: dander + state: + provider: postgresql + authority_id: kubernetes:test + catalog: + provider: none + secrets: + provider: environment +deployments: + local_cluster: + platform: postgres + launcher: + provider: kubernetes + context: kind-dander + namespace: dander-test + release_name: dander + service_account_name: dander-runtime + existing_secret_name: dander-runtime + runtime: + cpu: 1 + memory: 512Mi + timeout_seconds: 300 + max_retries: 1 + batch_rows: 2 + safety: + require_guarded_free_tier: false + pipelines: + records: + schedule: "0 9 * * *" + time_zone: America/New_York + paused: true + secret_bindings: + DANDER_POSTGRES_DSN: postgres-dsn +""".lstrip(), + encoding="utf-8", + ) + connectors = tmp_path / "connectors" + connectors.mkdir() + (connectors / "records.yaml").write_text( + """ +name: records +base_url: https://example.invalid +auth_strategy: none +endpoints: + - name: records + path: /records + primary_key: [id] + raw_schema: + - {name: id, type: STRING, mode: REQUIRED} +""".lstrip(), + encoding="utf-8", + ) + shutil.copytree( + ROOT / "infra" / "kubernetes", + tmp_path / "infra" / "kubernetes", + ) + return project, platforms + + +def _cluster_responses(cronjobs: list[dict[str, object]]) -> list[str]: + return [ + json.dumps( + { + "metadata": { + "name": "dander-runtime", + "annotations": {}, + }, + "automountServiceAccountToken": False, + } + ), + json.dumps( + { + "metadata": {"name": "dander-config"}, + "data": { + "DANDER_DEPLOYMENT": "local_cluster", + "DANDER_PLATFORM_PROFILE": "postgres", + "DANDER_IMAGE_REFERENCE": IMAGE, + }, + } + ), + json.dumps({"metadata": {"name": "dander-runtime"}}), + json.dumps({"items": cronjobs}), + ] + + +def test_plan_saves_non_secret_values_and_rendered_manifests(tmp_path: Path) -> None: + project, platforms = _project(tmp_path) + runner = _Runner() + plan = KubernetesOperations(runner=runner).plan( + config=project, + platforms_config=platforms, + deployment="local_cluster", + image=IMAGE, + output_dir=tmp_path / "plans", + ) + + values = yaml.safe_load(plan.values.read_text(encoding="utf-8")) + pipeline = values["pipelines"]["records"] + assert values["image"]["reference"] == IMAGE + assert values["existingSecret"]["name"] == "dander-runtime" + assert pipeline["secretEnvironment"] == [{"name": "DANDER_POSTGRES_DSN", "key": "postgres-dsn"}] + assert pipeline["suspend"] is True + assert pipeline["backoffLimit"] == 1 + assert pipeline["activeDeadlineSeconds"] == 300 + assert pipeline["resources"]["requests"] == pipeline["resources"]["limits"] + assert plan.manifests.read_text(encoding="utf-8").startswith("apiVersion") + assert plan.values.stat().st_mode & 0o777 == 0o600 + assert "postgresql://" not in plan.values.read_text(encoding="utf-8") + assert [command[:2] for command in runner.commands] == [ + ("helm", "lint"), + ("helm", "template"), + ] + + +def test_verify_is_read_only_and_checks_schedule_pause_image_and_overlap(tmp_path: Path) -> None: + project, platforms = _project(tmp_path) + cronjob = { + "metadata": {"name": "dander-records", "labels": {"dander.io/pipeline": "records"}}, + "spec": { + "schedule": "0 9 * * *", + "timeZone": "America/New_York", + "suspend": True, + "concurrencyPolicy": "Forbid", + "successfulJobsHistoryLimit": 3, + "failedJobsHistoryLimit": 3, + "jobTemplate": { + "spec": { + "backoffLimit": 1, + "activeDeadlineSeconds": 300, + "ttlSecondsAfterFinished": 3_600, + "template": { + "spec": { + "restartPolicy": "Never", + "serviceAccountName": "dander-runtime", + "automountServiceAccountToken": False, + "terminationGracePeriodSeconds": 120, + "containers": [ + { + "name": "runtime", + "image": IMAGE, + "args": [ + "runtime", + "execute", + "--contract", + "io.dander.runtime/v1", + "--pipeline", + "records", + "--platform", + "local_cluster", + "--config", + "/app/dander.yaml", + "--models-dir", + "/app/models", + "--batch-rows", + "2", + ], + "env": [ + { + "name": "DANDER_IMAGE_DIGEST", + "value": "sha256:" + "b" * 64, + }, + {"name": "DANDER_LAUNCHER", "value": "kubernetes"}, + { + "name": "DANDER_PRINCIPAL", + "value": ( + "kubernetes://dander-test/" + "serviceaccounts/dander-runtime" + ), + }, + { + "name": "DANDER_POSTGRES_DSN", + "valueFrom": { + "secretKeyRef": { + "name": "dander-runtime", + "key": "postgres-dsn", + } + }, + }, + ], + "resources": { + "requests": { + "cpu": "1", + "memory": "512Mi", + "ephemeral-storage": "1Gi", + }, + "limits": { + "cpu": "1", + "memory": "512Mi", + "ephemeral-storage": "1Gi", + }, + }, + } + ], + } + }, + } + }, + }, + } + runner = _Runner({("kubectl", "--context", "kind-dander"): _cluster_responses([cronjob])}) + + result = KubernetesOperations(runner=runner).verify( + config=project, + platforms_config=platforms, + deployment="local_cluster", + expected_image=IMAGE, + ) + + assert result.pipelines == ("records",) + assert result.image == IMAGE + assert all(command[-2:] == ("-o", "json") for command in runner.commands) + assert not any( + command[-2:] in {("create", "job"), ("delete", "job")} for command in runner.commands + ) + + +def test_verify_fails_closed_on_drift(tmp_path: Path) -> None: + project, platforms = _project(tmp_path) + runner = _Runner({("kubectl", "--context", "kind-dander"): _cluster_responses([])}) + + with pytest.raises(KubernetesOperationError, match="do not match"): + KubernetesOperations(runner=runner).verify( + config=project, + platforms_config=platforms, + deployment="local_cluster", + expected_image=IMAGE, + ) + + +def test_chart_encodes_job_safety_and_no_secret_ownership() -> None: + chart = ROOT / "infra" / "kubernetes" / "chart" / "dander" + cronjobs = (chart / "templates" / "cronjobs.yaml").read_text(encoding="utf-8") + service_account = (chart / "templates" / "serviceaccount.yaml").read_text(encoding="utf-8") + + assert "concurrencyPolicy: Forbid" in cronjobs + assert "restartPolicy: Never" in cronjobs + assert "activeDeadlineSeconds" in cronjobs + assert "ttlSecondsAfterFinished" in cronjobs + assert "secretKeyRef:" in cronjobs + assert "readOnlyRootFilesystem: true" in cronjobs + assert 'drop: ["ALL"]' in cronjobs + assert "sizeLimit:" in cronjobs + assert "automountServiceAccountToken: {{ $.Values.rbac.create }}" in cronjobs + assert "automountServiceAccountToken: {{ .Values.rbac.create }}" in service_account + assert not list((chart / "templates").glob("*secret*")) diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index af953b70..def88fdb 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -23,6 +23,7 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: assert manifest.version == 2 assert manifest.platform_name == "gcp" assert manifest.deployment_name == "gcp_cloud_run" + assert (project / "infra" / "kubernetes" / "chart" / "dander" / "Chart.yaml").is_file() dockerfile = (project / "Dockerfile").read_text(encoding="utf-8") assert f"ARG DANDER_VERSION={__version__}" in dockerfile assert "apt-get install --yes --no-install-recommends libpq5" in dockerfile diff --git a/tests/providers/test_kubernetes_runtime.py b/tests/providers/test_kubernetes_runtime.py new file mode 100644 index 00000000..644cafe1 --- /dev/null +++ b/tests/providers/test_kubernetes_runtime.py @@ -0,0 +1,143 @@ +"""Kubernetes launcher projection and fail-closed configuration tests.""" + +from __future__ import annotations + +import sys + +import pytest +from pydantic import ValidationError + +from dander.deployment import ExecutionProjectionError, LauncherRuntime +from dander.providers import ProviderKind, default_provider_registry +from dander.providers.kubernetes import KubernetesLauncherConfig + +_IMAGE = "ghcr.io/example/dander@sha256:" + "a" * 64 +_PIPELINES: dict[str, dict[str, object]] = { + "salesforce_crm": { + "runtime_service_account_id": "dander-runtime", + "build_models": True, + "schedule": "0 9 * * *", + "time_zone": "America/New_York", + "paused": True, + "secret_env": {"DANDER_POSTGRES_DSN": "postgres-dsn"}, + } +} + + +def _runtime(*, secret_name: str | None = "dander-runtime") -> LauncherRuntime: + registry = default_provider_registry() + config = registry.parse( + ProviderKind.LAUNCHER, + { + "provider": "kubernetes", + "context": "kind-dander", + "namespace": "dander-test", + "release_name": "dander", + "service_account_name": "dander-runtime", + "existing_secret_name": secret_name, + "workload_identity_annotations": { + "eks.amazonaws.com/role-arn": "arn:aws:iam::123456789012:role/dander" + }, + "pod_labels": {"team": "analytics"}, + }, + ) + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + return runtime + + +def test_kubernetes_factory_is_lazy_and_projects_native_profile() -> None: + module_name = "dander.providers.kubernetes.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.LAUNCHER, + { + "provider": "kubernetes", + "context": "kind-dander", + "existing_secret_name": "dander-runtime", + "pod_labels": {"team": "analytics"}, + }, + ) + + assert module_name not in sys.modules + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + assert module_name in sys.modules + template = runtime.templates.build( + _PIPELINES, + image=_IMAGE, + project="", + cpu=1, + memory="512Mi", + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=100, + require_guarded_free_tier=False, + alert_target=None, + profile_id="postgres", + )["salesforce_crm"] + + assert template.launcher == "kubernetes" + assert template.profile_id == "postgres" + assert template.workload_identity == "kubernetes://dander/serviceaccounts/dander-runtime" + assert template.command[:8] == ( + "runtime", + "execute", + "--contract", + "io.dander.runtime/v1", + "--pipeline", + "salesforce_crm", + "--platform", + "postgres", + ) + assert template.resources.ephemeral_storage_mib == 1_024 + assert template.observability.log_destination == "stdout" + assert dict(template.secret_bindings)["DANDER_POSTGRES_DSN"].reference == ("env://postgres-dsn") + assert dict(template.labels)["profile"] == "postgres" + + +@pytest.mark.parametrize( + ("secret_name", "guarded", "alert_target", "message"), + [ + (None, False, None, "existing_secret_name"), + ("dander-runtime", True, None, "guarded-free-tier"), + ("dander-runtime", False, "operator@example.invalid", "alert target"), + ], +) +def test_kubernetes_rejects_intent_the_chart_cannot_honor( + secret_name: str | None, + guarded: bool, + alert_target: str | None, + message: str, +) -> None: + with pytest.raises(ExecutionProjectionError, match=message): + _runtime(secret_name=secret_name).templates.build( + _PIPELINES, + image=_IMAGE, + project="", + cpu=1, + memory="512Mi", + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=100, + require_guarded_free_tier=guarded, + alert_target=alert_target, + profile_id="postgres", + ) + + +@pytest.mark.parametrize( + "config", + [ + {"provider": "kubernetes", "context": "kind-test", "namespace": "not.valid"}, + { + "provider": "kubernetes", + "context": "kind-test", + "pod_labels": {"app.kubernetes.io/name": "override"}, + }, + ], +) +def test_kubernetes_rejects_invalid_or_reserved_metadata(config: dict[str, object]) -> None: + with pytest.raises(ValidationError): + KubernetesLauncherConfig.model_validate(config) diff --git a/tickets/DANDER-101-kubernetes-helm-launcher.md b/tickets/DANDER-101-kubernetes-helm-launcher.md new file mode 100644 index 00000000..f351687a --- /dev/null +++ b/tickets/DANDER-101-kubernetes-helm-launcher.md @@ -0,0 +1,37 @@ +--- +id: DANDER-101 +title: Add the existing-cluster Kubernetes and Helm launcher +status: in-review +component: python +epic: cloud-portability +depends_on: [DANDER-89, DANDER-100] +created: 2026-08-08 +--- + +## Context + +The execution projection and native PostgreSQL profile had no cloud-neutral scheduled launcher. +Kubernetes must target an existing cluster without becoming another cluster-provisioning program. + +## Acceptance Criteria + +- [x] Register a validated, lazily loaded Kubernetes launcher without cloud-specific SDK imports. +- [x] Package one versioned Helm chart in source-free projects. +- [x] Render ConfigMap, ServiceAccount, CronJobs/Job templates, optional RBAC, external Secret + references, resources, deadlines, retries, history limits, and completed-Job TTL. +- [x] Default schedules to `Forbid` overlap and pods to `restartPolicy: Never`. +- [x] Accept cloud-neutral ServiceAccount annotations and pod labels. +- [x] Add non-mutating `dander kubernetes plan` and read-only deployment verification. +- [x] Validate the chart with Helm in protected CI. +- [x] Keep cluster creation, chart installation, live cloud mutation, and support claims out of scope. + +## Design + +Version 2 owns the context, namespace, release, ServiceAccount, external Secret name, and metadata. +The provider maps `io.dander.execution/v1` into deterministic non-secret Helm values. The chart +uses stdout and native cluster primitives instead of bundling an observability or identity system. + +## Review Log + +Local Helm rendering and protected CI remain before merge. A disposable existing-cluster +PostgreSQL run belongs to the separate Phase 4 profile-acceptance slice. From ea7403da7d9145b90c80b64afe0ac1e959272bda Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 21:21:29 -0400 Subject: [PATCH 053/163] fix: keep Kubernetes plan paths unwrapped --- src/dander/cli/kubernetes_command.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dander/cli/kubernetes_command.py b/src/dander/cli/kubernetes_command.py index 371fde8f..749101a8 100644 --- a/src/dander/cli/kubernetes_command.py +++ b/src/dander/cli/kubernetes_command.py @@ -43,8 +43,11 @@ def kubernetes_plan( ) except KubernetesOperationError as error: raise ClickException(str(error)) from error - console.print(f"[green]Kubernetes release rendered.[/green] Manifests: {plan.manifests}") - console.print(f"Review values: {plan.values}") + console.print( + f"[green]Kubernetes release rendered.[/green] Manifests: {plan.manifests}", + soft_wrap=True, + ) + console.print(f"Review values: {plan.values}", soft_wrap=True) next_command = ( "helm", "--kube-context", From 99afe9a8c77ed986fdebf2905842e5c3889b1892 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 21:40:38 -0400 Subject: [PATCH 054/163] test: publish PostgreSQL backend matrix evidence --- HANDOFF.md | 45 +- docs/compatibility-matrix.md | 23 + docs/decisions.md | 11 + docs/known-limitations.md | 4 +- docs/platform-profiles.md | 5 +- docs/postgresql-benchmarks.md | 37 ++ pyproject.toml | 5 + scripts/__init__.py | 1 + scripts/benchmarks/__init__.py | 1 + scripts/benchmarks/postgresql.py | 465 ++++++++++++++++++ scripts/check_distribution.py | 4 + src/dander/cli/run_command.py | 15 +- src/dander/cli/runtime_command.py | 11 + src/dander/compatibility.py | 129 +++++ src/dander/runtime-compatibility.json | 29 ++ tests/cli/test_run_command.py | 10 +- .../portability/test_postgresql_benchmark.py | 43 ++ .../test_state_warehouse_matrix.py | 138 ++++++ tests/test_compatibility.py | 60 +++ ...DANDER-102-postgresql-matrix-benchmarks.md | 23 + 20 files changed, 1022 insertions(+), 37 deletions(-) create mode 100644 docs/compatibility-matrix.md create mode 100644 docs/postgresql-benchmarks.md create mode 100644 scripts/__init__.py create mode 100644 scripts/benchmarks/__init__.py create mode 100644 scripts/benchmarks/postgresql.py create mode 100644 src/dander/compatibility.py create mode 100644 src/dander/runtime-compatibility.json create mode 100644 tests/portability/test_postgresql_benchmark.py create mode 100644 tests/portability/test_state_warehouse_matrix.py create mode 100644 tests/test_compatibility.py create mode 100644 tickets/DANDER-102-postgresql-matrix-benchmarks.md diff --git a/HANDOFF.md b/HANDOFF.md index 0adc7d25..d6f850e6 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,42 +2,39 @@ ## Finished -- Added a lazy Kubernetes launcher for version 2 named deployments, using the deployment name - as the unambiguous runtime selector. -- Packaged a Helm chart for existing Kubernetes 1.27+ clusters in every generated project. -- Added non-mutating `dander kubernetes plan` and read-only drift verification. -- Rendered bounded CronJobs with external Secret refs, `Forbid` overlap, explicit resources, - deadlines/retries, read-only pods, history limits, and Job TTL cleanup. -- Documented configuration, manual runs, upgrades, rollback, uninstall, and the unqualified boundary. +- Published the four-way BigQuery/PostgreSQL state/warehouse matrix through the installed CLI. +- Centralized fail-closed backend-pair selection without changing the existing executable pairs. +- Proved a BigQuery-issued authority/token fences PostgreSQL publication and rejects stale writes. +- Added a real PostgreSQL bounded-batch/concurrency/staging benchmark with honest qualification. +- Documented the matrix, measured local smoke, and remaining live qualification boundary. ## Try It -Run `dander kubernetes plan --deployment NAME --container-image REGISTRY/IMAGE@sha256:DIGEST`, -review both saved artifacts, then use the printed Helm command only against an approved cluster. +Run `dander runtime compatibility`. With a disposable PostgreSQL DSN, run +`uv run python -m scripts.benchmarks.postgresql --rows 100000 --payload-bytes 1024`. ## Checks -- All 1,022 tests passed with PostgreSQL 15; repository Ruff and strict mypy passed. -- Helm lint/template passed, including optional RBAC and ConfigMap-retention rendering. -- A disposable Kind cluster passed install, read-only verify, manual Job/duplicate rejection, - upgrade, rollback, and uninstall; its external Secret survived and the cluster was deleted. -- Wheel/sdist inspection, two source-free installs/scaffolds, dependency audit, Terraform - validation/tests, full-runtime container build, and read-only runtime conformance passed. +- All 1,031 tests passed with PostgreSQL 15; repository Ruff and strict mypy passed. +- A 50,000-row/52.4 MB logical-input smoke completed; stale publication was rejected and no + temporary staging relation remained. +- Local smoke recorded 20,998 rows/second and explicitly remained `not_evaluated` for qualification. +- Wheel/sdist inspection and a clean wheel installation/CLI matrix read passed. ## Decisions -- Kubernetes targets an existing cluster; Dander does not create clusters or own external Secrets. -- Operator-managed environment injection is the first Secret path; cloud identity remains metadata. -- Kubernetes/PostgreSQL remains unsupported until the separate live end-to-end profile gate passes. +- Matrix status and executable behavior are separate from the supported capability manifest. +- Local throughput protects regressions; only an enforced-memory run may satisfy the scale SLO. +- PostgreSQL-state/BigQuery-warehouse remains unsupported and unchanged. ## Remaining -- Open and merge the focused protected PR after Linux CI and security scans pass. -- Run the native and cross-backend PostgreSQL matrix and bounded/concurrent benchmarks separately. -- Qualify one real existing-cluster PostgreSQL pipeline before changing the support manifest. +- Open and merge the focused protected PR if Linux package, container, Terraform, and scans pass. +- Qualify Kubernetes/PostgreSQL live before promoting that profile from experimental. +- Continue with Snowflake/Redshift only from the merged matrix baseline. ## Review First -- `src/dander/providers/kubernetes/operations.py` -- `infra/kubernetes/chart/dander/templates/cronjobs.yaml` -- `docs/kubernetes.md` +- `src/dander/compatibility.py` +- `tests/portability/test_state_warehouse_matrix.py` +- `scripts/benchmarks/postgresql.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md new file mode 100644 index 00000000..d04e1e5c --- /dev/null +++ b/docs/compatibility-matrix.md @@ -0,0 +1,23 @@ +# Runtime compatibility matrix + +Dander publishes the state/warehouse combinations tested by the installed package. Read it without +provider credentials or network access: + +```bash +dander runtime compatibility +``` + +The output is the machine-readable `io.dander.runtime.compatibility/v1` contract. Selection fails +before provider construction when a pair is absent or marked `unsupported`. + +| State | Warehouse | Status | Current evidence | +|---|---|---|---| +| BigQuery | BigQuery | supported | Released GCP profile and BigQuery regression baseline | +| BigQuery | PostgreSQL | experimental | Local cross-backend token/fence publication proof | +| PostgreSQL | BigQuery | unsupported | Not every BigQuery write mode uses destination fencing | +| PostgreSQL | PostgreSQL | experimental | Local native profile, state/warehouse conformance, and benchmarks | + +`experimental` means the code path is executable and tested locally; it is not a hosted support +claim. The packaged `runtime-capabilities.json` remains the supported launcher/profile boundary. +Promotion requires the named live profile, identity, failure, rollback, and no-drift evidence from +the cloud-portability plan. Adding an adapter does not silently add a matrix row. diff --git a/docs/decisions.md b/docs/decisions.md index b1ea0004..e112aebc 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -834,3 +834,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc final concurrency defense. - **Qualification:** Plan rendering and read-only cluster verification are implemented. Kubernetes plus native PostgreSQL remains unsupported until an existing-cluster end-to-end proof passes. + +## 2026-08-08 — Backend compatibility is package-owned and qualification stays explicit + +- **Matrix:** The installed package publishes every current BigQuery/PostgreSQL state/warehouse + pair through `dander runtime compatibility`; absent and unsupported pairs fail before provider + construction. +- **Evidence:** BigQuery-state/PostgreSQL-warehouse accepts the resolved BigQuery authority in the + PostgreSQL destination fence and rejects the older token after a newer claim. PostgreSQL native + remains experimental until its Kubernetes live profile passes. +- **Scale:** The repository benchmark exercises bounded batches, independent pipeline concurrency, + stale-fence rejection, and staging cleanup. A local smoke cannot claim the controlled-memory SLO. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 72eabbf6..cc287fb3 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -15,7 +15,9 @@ evaluate these limits before using it for an unattended system containing busine chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The - packaged capability manifest is the support boundary. + package publishes this pair matrix through `dander runtime compatibility`, while the packaged + capability manifest remains the support boundary. Local PostgreSQL benchmark results are + regression evidence, not a paid or controlled-memory scale qualification. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 5d716d04..7533ec66 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -101,7 +101,8 @@ different database or change `authority_epoch` outside a reviewed state-backend PostgreSQL state with a PostgreSQL warehouse is executable. BigQuery state with PostgreSQL is also transactionally fenced, although it is not a qualified hosted profile. PostgreSQL state with a BigQuery warehouse remains fail-closed until every BigQuery write mode uses the destination-side -target fence rather than only the state-side lease transaction. +target fence rather than only the state-side lease transaction. Run `dander runtime compatibility` +to inspect this package's exact matrix; see [Runtime compatibility matrix](compatibility-matrix.md). Terminal `succeeded`, `failed`, and `skipped` history older than the configured retention is removed when migrations run. Active runs and `interrupted_run` records are retained. Dander @@ -126,3 +127,5 @@ Graph execution remains follow-up work. The packaged existing-cluster Kubernetes render and verify this profile without cloud-specific identity assumptions, but no live cluster qualification has passed. Until that acceptance does, this adapter must not be represented as a supported hosted PostgreSQL deployment. See [Kubernetes existing-cluster launcher](kubernetes.md). +The reproducible local scale harness and its non-qualification boundary are documented in +[PostgreSQL portability benchmarks](postgresql-benchmarks.md). diff --git a/docs/postgresql-benchmarks.md b/docs/postgresql-benchmarks.md new file mode 100644 index 00000000..405f882d --- /dev/null +++ b/docs/postgresql-benchmarks.md @@ -0,0 +1,37 @@ +# PostgreSQL portability benchmarks + +The repository includes one reproducible, non-sensitive local harness for PostgreSQL 15+: + +```bash +export DANDER_TEST_POSTGRES_DSN='postgresql://...' +uv run python -m scripts.benchmarks.postgresql \ + --rows 100000 \ + --payload-bytes 1024 \ + --batch-rows 1000 \ + --concurrent-pipelines 4 \ + --concurrent-rows 5000 +``` + +It streams generated rows through the real bounded COPY/SCD1 path, runs independent pipelines +through the bounded pool, proves an older target fence cannot publish, and checks that temporary +staging is gone. The JSON report contains no DSN, row value, SQL text, host name, or credential. + +## Local smoke evidence + +On 2026-08-08, the committed harness ran against the pinned PostgreSQL 15.18 container with 50,000 +rows, a 1,024-byte payload, 500-row batches, and four concurrent pipelines: + +- 52,400,000 estimated logical input bytes in 2.381146 seconds (20,998.292 rows/second); +- 8,000 concurrent rows in 0.143238 seconds (55,850.971 rows/second); +- stale publication rejected and zero temporary staging relations observed; +- peak process RSS was 161,218,560 bytes; cost was not measured. + +This is a regression smoke, not scale qualification. No controlled container memory limit was +applied, the input was not ten times a declared container limit, and the report therefore says +`qualification_status=not_evaluated`. + +For qualification, run the process inside an externally enforced container memory limit, supply +that same limit with `--qualification-memory-limit-mib`, use logical input at least ten times the +limit, and require peak RSS at or below 80 percent. Record the image digest, provider/service +shape, region, raw provider IDs, date, and approved cost ceiling beside the JSON report. A supplied +limit documents the test environment; it does not itself impose a memory limit. diff --git a/pyproject.toml b/pyproject.toml index 5e4cbe93..5480c34f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -228,10 +228,12 @@ include = [ "dander.yaml", "docs/getting-started.md", "docs/connector-plugins.md", + "docs/compatibility-matrix.md", "docs/execution-projection.md", "docs/known-limitations.md", "docs/netsuite-simulator.md", "docs/operator-soak.md", + "docs/postgresql-benchmarks.md", "docs/oci-artifact-contract.md", "docs/rollback.md", "docs/upgrading.md", @@ -240,6 +242,9 @@ include = [ "graphs", "models", "pyproject.toml", + "scripts/__init__.py", + "scripts/benchmarks/__init__.py", + "scripts/benchmarks/postgresql.py", "src", "tests", "uv.lock", diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..9024639f --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Repository-only validation and release helpers.""" diff --git a/scripts/benchmarks/__init__.py b/scripts/benchmarks/__init__.py new file mode 100644 index 00000000..3e1049c3 --- /dev/null +++ b/scripts/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Reproducible cloud-portability benchmark harnesses.""" diff --git a/scripts/benchmarks/postgresql.py b/scripts/benchmarks/postgresql.py new file mode 100644 index 00000000..5c2df3f9 --- /dev/null +++ b/scripts/benchmarks/postgresql.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""Reproducible local PostgreSQL bounded-memory and concurrency benchmark.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import resource +import sys +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass +from itertools import islice +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +from psycopg import Connection, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.providers import ProviderKind, default_provider_registry +from dander.state import StateRuntime +from dander.warehouse import RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WritePattern, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterator + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] +_REPORT_SCHEMA = "io.dander.benchmark.postgresql/v1" + + +@dataclass(frozen=True, slots=True) +class BenchmarkConfig: + """Bounded local workload controls; the DSN is deliberately not part of the report.""" + + rows: int = 100_000 + payload_bytes: int = 1_024 + batch_rows: int = 1_000 + concurrent_pipelines: int = 4 + concurrent_rows_per_pipeline: int = 5_000 + qualification_memory_limit_mib: int | None = None + + def __post_init__(self) -> None: + limits = { + "rows": 10_000_000, + "payload_bytes": 1_048_576, + "batch_rows": 100_000, + "concurrent_pipelines": 32, + "concurrent_rows_per_pipeline": 1_000_000, + } + for name, maximum in limits.items(): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + if value > maximum: + raise ValueError(f"{name} must not exceed {maximum}") + if self.batch_rows * self.payload_bytes > 256 * 1_024 * 1_024: + raise ValueError("one configured batch must not exceed 256 MiB of payload") + if self.qualification_memory_limit_mib is not None and ( + isinstance(self.qualification_memory_limit_mib, bool) + or self.qualification_memory_limit_mib <= 0 + ): + raise ValueError("qualification_memory_limit_mib must be a positive integer") + + +@dataclass(frozen=True, slots=True) +class PostgreSQLBenchmarkReport: + """Non-sensitive measurements from one local PostgreSQL benchmark execution.""" + + schema: str + provider: str + provider_version: str + python_version: str + rows: int + logical_input_bytes: int + payload_bytes: int + batch_rows: int + duration_seconds: float + throughput_rows_per_second: float + peak_rss_bytes: int + concurrent_pipelines: int + concurrent_rows: int + concurrent_duration_seconds: float + concurrent_throughput_rows_per_second: float + stale_publication_rejected: bool + temporary_staging_relations: int + qualification_status: str + qualification_reason: str + cost_status: str = "not_measured" + + def to_json(self) -> str: + return json.dumps(asdict(self), separators=(",", ":"), sort_keys=True) + + +def run_postgresql_benchmark( + dsn: str, + config: BenchmarkConfig, +) -> PostgreSQLBenchmarkReport: + """Run bounded COPY, independent concurrency, and stale-fence probes.""" + if not dsn: + raise ValueError("PostgreSQL benchmark requires a non-empty DSN") + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=max(5, config.concurrent_pipelines + 1), + timeout=10, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=10) + suffix = uuid.uuid4().hex[:12] + warehouse_schema = f"dander_benchmark_{suffix}" + state_schema = f"dander_benchmark_state_{suffix}" + with pool.connection() as connection: + version_row = connection.execute( + "SELECT current_database() AS database, current_setting('server_version') AS version" + ).fetchone() + if version_row is None: + pool.close() + raise RuntimeError("PostgreSQL benchmark could not read server metadata") + database = cast("str", version_row["database"]) + provider_version = cast("str", version_row["version"]) + warehouse = _warehouse_runtime(pool, database) + state = _state_runtime(pool, state_schema) + state.migrator.migrate() + + try: + peak_before = _peak_rss_bytes() + started = time.perf_counter() + bounded_rows = _write_bounded( + warehouse, + database=database, + schema=warehouse_schema, + rows=config.rows, + payload_bytes=config.payload_bytes, + batch_rows=config.batch_rows, + ) + bounded_duration = max(time.perf_counter() - started, 1e-9) + peak_after = _peak_rss_bytes() + concurrent_rows, concurrent_duration = _write_concurrently( + warehouse, + state, + database=database, + schema=warehouse_schema, + pipelines=config.concurrent_pipelines, + rows_per_pipeline=config.concurrent_rows_per_pipeline, + batch_rows=config.batch_rows, + ) + stale_rejected = _reject_stale_publication( + warehouse, + state, + database=database, + schema=warehouse_schema, + ) + staging_count = _temporary_staging_count(pool) + logical_input = config.rows * (config.payload_bytes + 24) + qualification_status, qualification_reason = _qualification( + logical_input_bytes=logical_input, + peak_rss_bytes=max(peak_before, peak_after), + memory_limit_mib=config.qualification_memory_limit_mib, + ) + return PostgreSQLBenchmarkReport( + schema=_REPORT_SCHEMA, + provider="postgresql", + provider_version=provider_version, + python_version=platform.python_version(), + rows=bounded_rows, + logical_input_bytes=logical_input, + payload_bytes=config.payload_bytes, + batch_rows=config.batch_rows, + duration_seconds=round(bounded_duration, 6), + throughput_rows_per_second=round(bounded_rows / bounded_duration, 3), + peak_rss_bytes=max(peak_before, peak_after), + concurrent_pipelines=config.concurrent_pipelines, + concurrent_rows=concurrent_rows, + concurrent_duration_seconds=round(concurrent_duration, 6), + concurrent_throughput_rows_per_second=round( + concurrent_rows / max(concurrent_duration, 1e-9), 3 + ), + stale_publication_rejected=stale_rejected, + temporary_staging_relations=staging_count, + qualification_status=qualification_status, + qualification_reason=qualification_reason, + ) + finally: + with pool.connection() as connection: + for schema_name in (warehouse_schema, state_schema): + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_name)) + ) + pool.close() + + +def _warehouse_runtime(pool: PostgreSQLPool, database: str) -> WarehouseRuntime: + registry = default_provider_registry() + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": "postgresql", + "database": database, + "dsn_env": "DANDER_BENCHMARK_POSTGRES_DSN", + "statement_timeout_ms": 300_000, + "lock_timeout_ms": 30_000, + "idle_transaction_timeout_ms": 60_000, + }, + ) + runtime = registry.build(ProviderKind.WAREHOUSE, config, context={"pool": pool}) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("PostgreSQL benchmark received an invalid warehouse runtime") + return runtime + + +def _state_runtime(pool: PostgreSQLPool, schema: str) -> StateRuntime: + registry = default_provider_registry() + config = registry.parse( + ProviderKind.STATE, + { + "provider": "postgresql", + "authority_id": f"postgresql:{schema}", + "dsn_env": "DANDER_BENCHMARK_POSTGRES_DSN", + "schema_name": schema, + "lease_seconds": 30, + }, + ) + runtime = registry.build( + ProviderKind.STATE, + config, + context={"pool": pool, "metadata_enabled": False}, + ) + if not isinstance(runtime, StateRuntime): + raise TypeError("PostgreSQL benchmark received an invalid state runtime") + return runtime + + +def _write_bounded( + warehouse: WarehouseRuntime, + *, + database: str, + schema: str, + rows: int, + payload_bytes: int, + batch_rows: int, +) -> int: + writer = warehouse.writers.build_ingestion_writer( + sandbox=False, + batch_rows=batch_rows, + schema_evolution=SchemaEvolution.STRICT, + ) + relation = RelationRef(catalog=database, namespace=schema, name="bounded_records") + fence = warehouse.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="benchmark_bounded", + run_id="bounded-one", + token=1, + authority_id="postgresql:benchmark", + ), + ) + target = _target(database, schema, "bounded_records", fence) + return _write_batches( + writer, + _records(rows, payload_bytes), + target, + batch_rows=batch_rows, + ) + + +def _write_concurrently( + warehouse: WarehouseRuntime, + state: StateRuntime, + *, + database: str, + schema: str, + pipelines: int, + rows_per_pipeline: int, + batch_rows: int, +) -> tuple[int, float]: + def run(index: int) -> int: + pipeline_id = f"benchmark_concurrent_{index}" + run_id = f"concurrent-{index}" + lease = state.leases.acquire(pipeline_id, run_id) + if lease is None or lease.fence is None: + raise RuntimeError("PostgreSQL benchmark could not acquire an independent lease") + try: + table = f"concurrent_{index}" + relation = RelationRef(catalog=database, namespace=schema, name=table) + publication = warehouse.target_fence.claim(relation, lease.fence) + writer = warehouse.writers.build_ingestion_writer( + sandbox=False, + batch_rows=batch_rows, + schema_evolution=SchemaEvolution.STRICT, + ) + return _write_batches( + writer, + _records(rows_per_pipeline, 128, prefix=f"{index}-"), + _target(database, schema, table, publication), + batch_rows=batch_rows, + ) + finally: + state.leases.release(lease) + + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=pipelines) as executor: + affected = sum(executor.map(run, range(pipelines))) + return affected, max(time.perf_counter() - started, 1e-9) + + +def _reject_stale_publication( + warehouse: WarehouseRuntime, + state: StateRuntime, + *, + database: str, + schema: str, +) -> bool: + pipeline_id = "benchmark_stale" + first = state.leases.acquire(pipeline_id, "stale-one") + if first is None or first.fence is None: + raise RuntimeError("PostgreSQL benchmark could not acquire its first stale probe lease") + relation = RelationRef(catalog=database, namespace=schema, name="stale_records") + stale_publication = warehouse.target_fence.claim(relation, first.fence) + state.leases.release(first) + second = state.leases.acquire(pipeline_id, "stale-two") + if second is None or second.fence is None: + raise RuntimeError("PostgreSQL benchmark could not acquire its second stale probe lease") + current_publication = warehouse.target_fence.claim(relation, second.fence) + writer = warehouse.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.STRICT, + ) + try: + rejected = False + try: + writer.write( + _records(1, 16), + _target(database, schema, "stale_records", stale_publication), + ) + except TargetFenceLostError: + rejected = True + if rejected: + writer.write( + _records(1, 16), + _target(database, schema, "stale_records", current_publication), + ) + return rejected + finally: + state.leases.release(second) + + +def _target( + database: str, + schema: str, + table: str, + publication_fence: TargetFence, +) -> WriteTarget: + return WriteTarget( + project=database, + dataset=schema, + table=table, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING", mode="REQUIRED"), + WriteField(name="payload", data_type="STRING"), + ), + publication_fence=publication_fence, + ) + + +def _write_batches( + writer: WritePattern, + records: Iterator[dict[str, object]], + target: WriteTarget, + *, + batch_rows: int, +) -> int: + affected = 0 + while batch := list(islice(records, batch_rows)): + affected += writer.write(batch, target) + return affected + + +def _records(rows: int, payload_bytes: int, *, prefix: str = "") -> Iterator[dict[str, object]]: + padding = "x" * payload_bytes + for index in range(rows): + yield {"id": f"{prefix}{index:012d}", "payload": f"{index:012d}{padding}"} + + +def _temporary_staging_count(pool: PostgreSQLPool) -> int: + with pool.connection() as connection: + row = connection.execute( + "SELECT COUNT(*) AS count FROM pg_catalog.pg_class " + "WHERE relname LIKE 'dander_stage_%' AND relpersistence = 't'" + ).fetchone() + return int(cast("int", row["count"] if row is not None else 0)) + + +def _peak_rss_bytes() -> int: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return int(peak if sys.platform == "darwin" else peak * 1_024) + + +def _qualification( + *, + logical_input_bytes: int, + peak_rss_bytes: int, + memory_limit_mib: int | None, +) -> tuple[str, str]: + if memory_limit_mib is None: + return "not_evaluated", "No controlled container memory limit was supplied." + memory_limit_bytes = memory_limit_mib * 1_024 * 1_024 + if logical_input_bytes < memory_limit_bytes * 10: + return "failed", "Logical input was less than ten times the container memory limit." + if peak_rss_bytes > memory_limit_bytes * 0.8: + return "failed", "Peak RSS exceeded eighty percent of the container memory limit." + return "passed", "Input and peak RSS met the published bounded-memory objective." + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dsn-env", default="DANDER_TEST_POSTGRES_DSN") + parser.add_argument("--rows", type=int, default=100_000) + parser.add_argument("--payload-bytes", type=int, default=1_024) + parser.add_argument("--batch-rows", type=int, default=1_000) + parser.add_argument("--concurrent-pipelines", type=int, default=4) + parser.add_argument("--concurrent-rows", type=int, default=5_000) + parser.add_argument("--qualification-memory-limit-mib", type=int) + parser.add_argument("--output", type=Path) + return parser.parse_args() + + +def main() -> None: + arguments = _arguments() + dsn = os.environ.get(arguments.dsn_env) + if not dsn: + raise SystemExit(f"PostgreSQL DSN is required in {arguments.dsn_env}") + report = run_postgresql_benchmark( + dsn, + BenchmarkConfig( + rows=arguments.rows, + payload_bytes=arguments.payload_bytes, + batch_rows=arguments.batch_rows, + concurrent_pipelines=arguments.concurrent_pipelines, + concurrent_rows_per_pipeline=arguments.concurrent_rows, + qualification_memory_limit_mib=arguments.qualification_memory_limit_mib, + ), + ) + output = report.to_json() + "\n" + if arguments.output is not None: + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(output, encoding="utf-8") + print(output, end="") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 8aa5373d..5b3d89d4 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -12,6 +12,7 @@ ROOT = Path(__file__).resolve().parents[1] WHEEL_REQUIRED = { + "dander/runtime-compatibility.json", "dander/templates/project/.dockerignore", "dander/templates/project/.gitignore", "dander/templates/project/Dockerfile", @@ -40,10 +41,13 @@ SDIST_REQUIRED = { "LICENSE", "README.md", + "docs/compatibility-matrix.md", + "docs/postgresql-benchmarks.md", "examples/salesforce/dander.yaml", "infra/main.tf", "infra/kubernetes/chart/dander/Chart.yaml", "pyproject.toml", + "scripts/benchmarks/postgresql.py", "src/dander/project/scaffold.py", } diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 2b495f4b..c94efe79 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -17,6 +17,7 @@ SqliteMetadataStore, ) from dander.cli.provider_runtime import build_catalog_publisher, build_secret_store +from dander.compatibility import CompatibilityError, load_runtime_compatibility from dander.core.config import Settings from dander.executor import PipelineExecutionResult, PipelineExecutor from dander.ingestion import Endpoint, Source, SourceConfig, load_source_config @@ -437,7 +438,7 @@ def _build_warehouse_runtime(resolved: _ResolvedRun) -> WarehouseRuntime: def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: - _require_supported_state_pair( + _require_executable_state_pair( state_provider=resolved.state_provider, warehouse_provider=resolved.warehouse_provider, ) @@ -465,12 +466,14 @@ def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: return runtime -def _require_supported_state_pair(*, state_provider: str, warehouse_provider: str) -> None: - if state_provider == "postgresql" and warehouse_provider == "bigquery": - raise ClickException( - "PostgreSQL state with BigQuery execution remains unavailable until every " - "BigQuery write mode uses the destination-side target fence" +def _require_executable_state_pair(*, state_provider: str, warehouse_provider: str) -> None: + try: + load_runtime_compatibility().require_executable( + state=state_provider, + warehouse=warehouse_provider, ) + except CompatibilityError as error: + raise ClickException(str(error)) from error def _requires_gcp_project(resolved: _ResolvedRun) -> bool: diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index 1c352a6b..37b6335d 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -12,6 +12,7 @@ from rich.console import Console from dander.cli.run_command import RunOptions, execute_run +from dander.compatibility import CompatibilityError, load_runtime_compatibility from dander.identity import prepare_launcher_identity from dander.runtime_contract import ( RUNTIME_CONTRACT, @@ -35,6 +36,16 @@ _CONSOLE = Console() +@runtime_app.command("compatibility") +def compatibility_runtime_command() -> None: + """Print the package-owned provider compatibility matrix without provider access.""" + try: + compatibility = load_runtime_compatibility() + except CompatibilityError as error: + raise ClickException(str(error)) from error + typer.echo(compatibility.to_json()) + + @runtime_app.command("inspect") def inspect_runtime_command( project_config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 diff --git a/src/dander/compatibility.py b/src/dander/compatibility.py new file mode 100644 index 00000000..095e6f93 --- /dev/null +++ b/src/dander/compatibility.py @@ -0,0 +1,129 @@ +"""Installed runtime compatibility matrix and fail-closed pair selection.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from enum import StrEnum +from importlib import resources + +_SCHEMA = "io.dander.runtime.compatibility/v1" + + +class CompatibilityStatus(StrEnum): + """Release status for one explicitly tested provider combination.""" + + SUPPORTED = "supported" + EXPERIMENTAL = "experimental" + UNSUPPORTED = "unsupported" + + +class CompatibilityError(ValueError): + """Raised when compatibility metadata is invalid or a pair cannot execute.""" + + +@dataclass(frozen=True, slots=True) +class StateWarehousePair: + """One state/warehouse pairing published by the installed package.""" + + state: str + warehouse: str + status: CompatibilityStatus + reason: str + + def as_dict(self) -> dict[str, str]: + return { + "state": self.state, + "warehouse": self.warehouse, + "status": self.status.value, + "reason": self.reason, + } + + +@dataclass(frozen=True, slots=True) +class RuntimeCompatibility: + """Validated, deterministic compatibility report for the installed runtime.""" + + schema: str + state_warehouse_pairs: tuple[StateWarehousePair, ...] + + def to_json(self) -> str: + return json.dumps( + { + "schema": self.schema, + "state_warehouse_pairs": [pair.as_dict() for pair in self.state_warehouse_pairs], + }, + separators=(",", ":"), + sort_keys=True, + ) + + def pair(self, *, state: str, warehouse: str) -> StateWarehousePair: + for pair in self.state_warehouse_pairs: + if pair.state == state and pair.warehouse == warehouse: + return pair + raise CompatibilityError( + f"State provider {state!r} with warehouse {warehouse!r} is not in the " + "installed compatibility matrix" + ) + + def require_executable(self, *, state: str, warehouse: str) -> StateWarehousePair: + pair = self.pair(state=state, warehouse=warehouse) + if pair.status is CompatibilityStatus.UNSUPPORTED: + raise CompatibilityError(pair.reason) + return pair + + +def load_runtime_compatibility() -> RuntimeCompatibility: + """Load and validate the package-owned matrix without contacting providers.""" + try: + raw = ( + resources.files("dander") + .joinpath("runtime-compatibility.json") + .read_text(encoding="utf-8") + ) + payload = json.loads(raw) + except (OSError, json.JSONDecodeError) as error: + raise CompatibilityError("runtime compatibility matrix is unavailable") from error + if not isinstance(payload, dict) or payload.get("schema") != _SCHEMA: + raise CompatibilityError("runtime compatibility matrix is incompatible") + raw_pairs = payload.get("state_warehouse_pairs") + if not isinstance(raw_pairs, list) or not raw_pairs: + raise CompatibilityError("runtime compatibility matrix has no state/warehouse pairs") + pairs: list[StateWarehousePair] = [] + for raw_pair in raw_pairs: + if not isinstance(raw_pair, dict): + raise CompatibilityError("runtime compatibility matrix contains an invalid pair") + try: + pair = StateWarehousePair( + state=_required_text(raw_pair, "state"), + warehouse=_required_text(raw_pair, "warehouse"), + status=CompatibilityStatus(_required_text(raw_pair, "status")), + reason=_required_text(raw_pair, "reason"), + ) + except ValueError as error: + raise CompatibilityError( + "runtime compatibility matrix contains an invalid status" + ) from error + pairs.append(pair) + keys = [(pair.state, pair.warehouse) for pair in pairs] + if len(keys) != len(set(keys)) or keys != sorted(keys): + raise CompatibilityError( + "runtime compatibility pairs must be unique and sorted by state/warehouse" + ) + return RuntimeCompatibility(schema=_SCHEMA, state_warehouse_pairs=tuple(pairs)) + + +def _required_text(payload: dict[object, object], name: str) -> str: + value = payload.get(name) + if not isinstance(value, str) or not value.strip(): + raise CompatibilityError(f"runtime compatibility pair has invalid {name}") + return value + + +__all__ = [ + "CompatibilityError", + "CompatibilityStatus", + "RuntimeCompatibility", + "StateWarehousePair", + "load_runtime_compatibility", +] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json new file mode 100644 index 00000000..75e840a0 --- /dev/null +++ b/src/dander/runtime-compatibility.json @@ -0,0 +1,29 @@ +{ + "schema": "io.dander.runtime.compatibility/v1", + "state_warehouse_pairs": [ + { + "state": "bigquery", + "warehouse": "bigquery", + "status": "supported", + "reason": "Released GCP profile with destination-side BigQuery fencing." + }, + { + "state": "bigquery", + "warehouse": "postgresql", + "status": "experimental", + "reason": "Locally conformance-tested with destination-side PostgreSQL fencing; no live hosted profile is qualified." + }, + { + "state": "postgresql", + "warehouse": "bigquery", + "status": "unsupported", + "reason": "PostgreSQL state with BigQuery execution remains unavailable until every BigQuery write mode uses the destination-side target fence." + }, + { + "state": "postgresql", + "warehouse": "postgresql", + "status": "experimental", + "reason": "Local native-profile conformance passed; Kubernetes live qualification remains pending." + } + ] +} diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 9f2b77e0..2bb19f53 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -225,22 +225,22 @@ def build_secrets(provider_id: str) -> _Built: assert migrator.calls == 1 -def test_only_unfenced_postgresql_state_bigquery_warehouse_pair_fails_closed() -> None: +def test_only_unsupported_postgresql_state_bigquery_warehouse_pair_fails_closed() -> None: with pytest.raises(ClickException, match="BigQuery write mode"): - run_module._require_supported_state_pair( + run_module._require_executable_state_pair( state_provider="postgresql", warehouse_provider="bigquery", ) - run_module._require_supported_state_pair( + run_module._require_executable_state_pair( state_provider="bigquery", warehouse_provider="bigquery", ) - run_module._require_supported_state_pair( + run_module._require_executable_state_pair( state_provider="postgresql", warehouse_provider="postgresql", ) - run_module._require_supported_state_pair( + run_module._require_executable_state_pair( state_provider="bigquery", warehouse_provider="postgresql", ) diff --git a/tests/portability/test_postgresql_benchmark.py b/tests/portability/test_postgresql_benchmark.py new file mode 100644 index 00000000..e6c67176 --- /dev/null +++ b/tests/portability/test_postgresql_benchmark.py @@ -0,0 +1,43 @@ +"""Smoke the reproducible PostgreSQL scale harness without claiming live qualification.""" + +from __future__ import annotations + +import json +import os + +import pytest +from scripts.benchmarks.postgresql import BenchmarkConfig, run_postgresql_benchmark + + +def test_postgresql_benchmark_streams_and_rejects_stale_publication() -> None: + dsn = os.environ.get("DANDER_TEST_POSTGRES_DSN") + if not dsn: + pytest.skip("DANDER_TEST_POSTGRES_DSN is not configured") + + report = run_postgresql_benchmark( + dsn, + BenchmarkConfig( + rows=2_000, + payload_bytes=256, + batch_rows=100, + concurrent_pipelines=3, + concurrent_rows_per_pipeline=200, + ), + ) + payload = json.loads(report.to_json()) + + assert payload["schema"] == "io.dander.benchmark.postgresql/v1" + assert payload["provider"] == "postgresql" + assert payload["rows"] == 2_000 + assert payload["concurrent_rows"] == 600 + assert payload["stale_publication_rejected"] is True + assert payload["temporary_staging_relations"] == 0 + assert payload["qualification_status"] == "not_evaluated" + assert "postgresql://" not in report.to_json() + + +def test_benchmark_qualification_is_explicit_not_inferred() -> None: + with pytest.raises(ValueError, match="rows must be a positive integer"): + BenchmarkConfig(rows=0) + with pytest.raises(ValueError, match="batch must not exceed 256 MiB"): + BenchmarkConfig(batch_rows=100_000, payload_bytes=4_096) diff --git a/tests/portability/test_state_warehouse_matrix.py b/tests/portability/test_state_warehouse_matrix.py new file mode 100644 index 00000000..1dcb4eb8 --- /dev/null +++ b/tests/portability/test_state_warehouse_matrix.py @@ -0,0 +1,138 @@ +"""Cross-backend publication proof for the executable BigQuery/PostgreSQL pair.""" + +from __future__ import annotations + +import os +import uuid +from typing import TYPE_CHECKING, Any, cast + +import pytest +from psycopg import Connection, sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.providers import ProviderKind, default_provider_registry +from dander.warehouse import RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterator + +PostgreSQLRow = dict[str, Any] +PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] + + +@pytest.fixture +def postgresql_matrix_warehouse() -> Iterator[tuple[WarehouseRuntime, PostgreSQLPool, str, str]]: + dsn = os.environ.get("DANDER_TEST_POSTGRES_DSN") + if not dsn: + pytest.skip("DANDER_TEST_POSTGRES_DSN is not configured") + pool = cast( + "PostgreSQLPool", + ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=3, + timeout=2, + kwargs={"row_factory": dict_row}, + open=True, + ), + ) + pool.wait(timeout=5) + with pool.connection() as connection: + row = connection.execute("SELECT current_database() AS database").fetchone() + assert row is not None + database = cast("str", row["database"]) + schema = f"dander_matrix_{uuid.uuid4().hex}" + registry = default_provider_registry() + config = registry.parse( + ProviderKind.WAREHOUSE, + { + "provider": "postgresql", + "database": database, + "dsn_env": "DANDER_TEST_POSTGRES_DSN", + }, + ) + runtime = registry.build(ProviderKind.WAREHOUSE, config, context={"pool": pool}) + assert isinstance(runtime, WarehouseRuntime) + try: + yield runtime, pool, database, schema + finally: + with pool.connection() as connection: + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema)) + ) + pool.close() + + +def test_bigquery_state_token_fences_postgresql_publication( + postgresql_matrix_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + warehouse, pool, database, schema = postgresql_matrix_warehouse + relation = RelationRef(catalog=database, namespace=schema, name="records") + legacy_bigquery_lease_table = "unit-project.dander_meta._dander_lease_0123456789abcdef" + first_token = FencingToken( + lease_table=legacy_bigquery_lease_table, + pipeline_id="cross_backend", + run_id="run-one", + token=1, + ) + first_publication = warehouse.target_fence.claim(relation, first_token) + writer = warehouse.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.STRICT, + ) + target = _target(database, schema, first_publication) + + assert writer.write([{"id": "one", "label": "first"}], target) == 1 + + second_token = FencingToken( + lease_table=legacy_bigquery_lease_table, + pipeline_id="cross_backend", + run_id="run-two", + token=2, + ) + second_publication = warehouse.target_fence.claim(relation, second_token) + with pytest.raises(TargetFenceLostError, match="destination fence lost"): + writer.write([{"id": "one", "label": "stale"}], target) + assert ( + writer.write( + [{"id": "one", "label": "second"}], + _target(database, schema, second_publication), + ) + == 1 + ) + + with pool.connection() as connection: + row = connection.execute( + sql.SQL("SELECT label FROM {}.records WHERE id = 'one'").format(sql.Identifier(schema)) + ).fetchone() + fence = connection.execute( + sql.SQL( + "SELECT authority_id, run_id, fencing_token, status " + "FROM {}.dander_target_commits WHERE pipeline_id = 'cross_backend'" + ).format(sql.Identifier(schema)) + ).fetchone() + assert row == {"label": "second"} + assert fence == { + "authority_id": "bigquery:unit-project.dander_meta", + "run_id": "run-two", + "fencing_token": 2, + "status": "committed", + } + + +def _target(database: str, schema: str, fence: TargetFence) -> WriteTarget: + return WriteTarget( + project=database, + dataset=schema, + table="records", + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING", mode="REQUIRED"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=fence, + ) diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py new file mode 100644 index 00000000..5975a194 --- /dev/null +++ b/tests/test_compatibility.py @@ -0,0 +1,60 @@ +"""The installed backend matrix is explicit, deterministic, and fail-closed.""" + +from __future__ import annotations + +import json + +import pytest +from typer.testing import CliRunner + +from dander.cli.main import app +from dander.compatibility import ( + CompatibilityError, + CompatibilityStatus, + load_runtime_compatibility, +) + + +def test_state_warehouse_matrix_covers_every_current_pair() -> None: + matrix = load_runtime_compatibility() + + assert matrix.schema == "io.dander.runtime.compatibility/v1" + assert {(pair.state, pair.warehouse): pair.status for pair in matrix.state_warehouse_pairs} == { + ("bigquery", "bigquery"): CompatibilityStatus.SUPPORTED, + ("bigquery", "postgresql"): CompatibilityStatus.EXPERIMENTAL, + ("postgresql", "bigquery"): CompatibilityStatus.UNSUPPORTED, + ("postgresql", "postgresql"): CompatibilityStatus.EXPERIMENTAL, + } + + +@pytest.mark.parametrize( + ("state", "warehouse"), + [ + ("bigquery", "bigquery"), + ("bigquery", "postgresql"), + ("postgresql", "postgresql"), + ], +) +def test_executable_pairs_are_admitted(state: str, warehouse: str) -> None: + pair = load_runtime_compatibility().require_executable(state=state, warehouse=warehouse) + + assert pair.status is not CompatibilityStatus.UNSUPPORTED + + +def test_unsupported_and_unlisted_pairs_fail_with_a_reason() -> None: + matrix = load_runtime_compatibility() + + with pytest.raises(CompatibilityError, match="every BigQuery write mode"): + matrix.require_executable(state="postgresql", warehouse="bigquery") + with pytest.raises(CompatibilityError, match="not in the installed compatibility matrix"): + matrix.require_executable(state="sqlite", warehouse="postgresql") + + +def test_runtime_compatibility_cli_prints_one_deterministic_document() -> None: + result = CliRunner().invoke(app, ["runtime", "compatibility"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["schema"] == "io.dander.runtime.compatibility/v1" + assert len(payload["state_warehouse_pairs"]) == 4 + assert result.output.strip() == load_runtime_compatibility().to_json() diff --git a/tickets/DANDER-102-postgresql-matrix-benchmarks.md b/tickets/DANDER-102-postgresql-matrix-benchmarks.md new file mode 100644 index 00000000..0ddd1a57 --- /dev/null +++ b/tickets/DANDER-102-postgresql-matrix-benchmarks.md @@ -0,0 +1,23 @@ +# DANDER-102 — PostgreSQL matrix and local benchmark evidence + +## Scope + +- Publish the complete current BigQuery/PostgreSQL state/warehouse matrix from the installed CLI. +- Fail before provider construction for absent or unsupported pairs. +- Prove BigQuery-issued lease authority can fence PostgreSQL publication and reject a stale token. +- Add a reproducible local bounded-batch/concurrency benchmark without claiming live scale. + +## Acceptance + +- All four current backend pairs have one explicit status and reason. +- BigQuery-state/PostgreSQL-warehouse publishes through the PostgreSQL target fence. +- PostgreSQL-state/BigQuery-warehouse remains fail-closed. +- The benchmark reports rows, logical bytes, duration, throughput, peak RSS, concurrency, stale + rejection, staging residue, qualification status, and cost status without DSNs or row values. +- Local smoke results stay `not_evaluated` unless an enforced memory limit is supplied and both SLO + thresholds pass. + +## Exclusions + +No cloud deployment, support promotion, paid scale run, adaptive batching, new warehouse mode, or +PostgreSQL-state/BigQuery enablement belongs in this ticket. From 881508573bf0387667ef6ed464e50d7b09658b32 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 21:53:20 -0400 Subject: [PATCH 055/163] feat: add bounded columnar staging --- HANDOFF.md | 38 ++- docs/decisions.md | 9 + docs/known-limitations.md | 4 +- docs/warehouse-staging.md | 30 ++ pyproject.toml | 4 + scripts/check_distribution.py | 1 + src/dander/providers/dependencies.py | 4 +- src/dander/warehouse/__init__.py | 10 + src/dander/warehouse/staging.py | 407 +++++++++++++++++++++++++ tests/warehouse/test_staging.py | 177 +++++++++++ tickets/DANDER-103-columnar-staging.md | 21 ++ uv.lock | 6 + 12 files changed, 688 insertions(+), 23 deletions(-) create mode 100644 docs/warehouse-staging.md create mode 100644 src/dander/warehouse/staging.py create mode 100644 tests/warehouse/test_staging.py create mode 100644 tickets/DANDER-103-columnar-staging.md diff --git a/HANDOFF.md b/HANDOFF.md index d6f850e6..f1822fe9 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,37 @@ ## Finished -- Published the four-way BigQuery/PostgreSQL state/warehouse matrix through the installed CLI. -- Centralized fail-closed backend-pair selection without changing the existing executable pairs. -- Proved a BigQuery-issued authority/token fences PostgreSQL publication and rejects stale writes. -- Added a real PostgreSQL bounded-batch/concurrency/staging benchmark with honest qualification. -- Documented the matrix, measured local smoke, and remaining live qualification boundary. +- Added one bounded, compressed Parquet staging contract for Snowflake and Redshift loaders. +- Mapped canonical scalar, decimal, temporal, JSON, array, and record types explicitly to Arrow. +- Added per-part SHA-256, schema fingerprinting, row/byte counts, and deterministic manifest JSON. +- Added owner-only run directories and fail-closed cleanup limited to exact owned regular files. +- Made PyArrow an explicit Snowflake, Redshift, and full-runtime dependency. ## Try It -Run `dander runtime compatibility`. With a disposable PostgreSQL DSN, run -`uv run python -m scripts.benchmarks.postgresql --rows 100000 --payload-bytes 1024`. +Use `ParquetStagingSession` as a context manager, call `stage(records, canonical_schema)`, upload the +returned immutable parts inside the context, and let normal exit remove local artifacts. ## Checks -- All 1,031 tests passed with PostgreSQL 15; repository Ruff and strict mypy passed. -- A 50,000-row/52.4 MB logical-input smoke completed; stale publication was rejected and no - temporary staging relation remained. -- Local smoke recorded 20,998 rows/second and explicitly remained `not_evaluated` for qualification. -- Wheel/sdist inspection and a clean wheel installation/CLI matrix read passed. +- All 1,036 tests passed with PostgreSQL 15; repository Ruff and strict mypy passed. +- Staging round-trip, row/byte splitting, redaction, cleanup, and dependency tests passed. +- Wheel/sdist inspection and Snowflake/Redshift/runtime-all PyArrow metadata checks passed. ## Decisions -- Matrix status and executable behavior are separate from the supported capability manifest. -- Local throughput protects regressions; only an enforced-memory run may satisfy the scale SLO. -- PostgreSQL-state/BigQuery-warehouse remains unsupported and unchanged. +- Shared code owns local artifacts only; provider adapters own remote stages and publication. +- JSON uses deterministic compact text; provider-native semi-structured mapping remains explicit. +- A singleton oversized row may exceed the soft byte target so staging always makes progress. ## Remaining - Open and merge the focused protected PR if Linux package, container, Terraform, and scans pass. -- Qualify Kubernetes/PostgreSQL live before promoting that profile from experimental. -- Continue with Snowflake/Redshift only from the merged matrix baseline. +- Build Snowflake upload/COPY/fencing against this contract in a separate branch. +- Build Redshift S3/COPY/fencing only after the shared contract merges. ## Review First -- `src/dander/compatibility.py` -- `tests/portability/test_state_warehouse_matrix.py` -- `scripts/benchmarks/postgresql.py` +- `src/dander/warehouse/staging.py` +- `tests/warehouse/test_staging.py` +- `docs/warehouse-staging.md` diff --git a/docs/decisions.md b/docs/decisions.md index e112aebc..68d6d1b6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -845,3 +845,12 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc remains experimental until its Kubernetes live profile passes. - **Scale:** The repository benchmark exercises bounded batches, independent pipeline concurrency, stale-fence rejection, and staging cleanup. A local smoke cannot claim the controlled-memory SLO. + +## 2026-08-08 — Snowflake and Redshift share bounded Parquet artifacts, not loaders + +- **Boundary:** One run-scoped session maps canonical schema v1 to bounded, compressed Parquet + parts. Provider upload, remote stage lifecycle, `COPY`, and publication remain separate adapters. +- **Integrity:** Every part is owner-only and content-addressed with SHA-256; the deterministic + manifest exposes counts and fingerprints but no row values, credentials, or provider locations. +- **Cleanup:** Normal exit removes only the exact owned run directory. One oversized record may + exceed the logical-byte target but stays a singleton; remote adapters must enforce service limits. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index cc287fb3..fa2be3d4 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,7 +10,9 @@ evaluate these limits before using it for an unattended system containing busine task-role session into process memory and therefore caps a Fargate projection at one hour; renewable credentials and live parity remain later gates. - Provider package extras and the full runtime image contain planned provider SDK dependencies, - but do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. PostgreSQL + but do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. A shared + bounded Parquet staging contract exists for future Snowflake/Redshift bulk loaders, but no remote + upload or warehouse adapter uses it yet. PostgreSQL state and warehouse execution are implemented and locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse diff --git a/docs/warehouse-staging.md b/docs/warehouse-staging.md new file mode 100644 index 00000000..07aee237 --- /dev/null +++ b/docs/warehouse-staging.md @@ -0,0 +1,30 @@ +# Shared columnar staging + +Snowflake and Redshift bulk loaders share one local Parquet artifact contract before their +provider-specific upload and `COPY` steps. This contract does not contact a warehouse or cloud. + +`ParquetStagingSession` creates one exclusive run directory, consumes the input iterator once, and +writes Zstandard- or Snappy-compressed parts. A part is bounded by both row count and estimated +logical bytes. One individually oversized row is written alone so the iterator can always make +progress; provider adapters must still enforce their remote object limits. + +Each artifact records only: + +- its relative filename; +- row and logical/compressed byte counts; +- a SHA-256 content digest. + +The manifest also records the run ID and canonical-schema fingerprint. It never contains absolute +paths, DSNs, row values, SQL, credentials, bucket names, stages, or provider account identifiers. +Files and the run directory are owner-only. Normal context exit removes only regular files in that +exact run directory and refuses unexpected nested or symbolic paths. + +Canonical scalar, array, record, JSON, decimal, and temporal declarations map explicitly to Arrow. +JSON is stored as stable compact text so each warehouse adapter can choose its own declared native +or string representation. Decimal precision above Parquet/Arrow's 76-digit limit fails before a +file is published. Missing/extra fields, required nulls, malformed JSON, and Arrow conversion errors +produce row-indexed messages without echoing record contents. + +This is a prerequisite, not a support claim. Snowflake and Redshift remain unavailable until their +upload, idempotency, target fencing, schema, transform, telemetry, cleanup, and live-profile gates +pass independently. diff --git a/pyproject.toml b/pyproject.toml index 5480c34f..8b043082 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,10 +55,12 @@ bigquery = [ "google-cloud-bigquery-storage>=2.30", ] snowflake = [ + "pyarrow>=19,<26", "snowflake-connector-python>=3.12,<5", ] redshift = [ "boto3>=1.35,<2", + "pyarrow>=19,<26", "redshift-connector>=2.1,<3", ] postgres = [ @@ -90,6 +92,7 @@ runtime-all = [ "google-cloud-secret-manager>=2.20", "psycopg[pool]>=3.2,<4; sys_platform == 'linux'", "psycopg[binary,pool]>=3.2,<4; sys_platform != 'linux'", + "pyarrow>=19,<26", "redshift-connector>=2.1,<3", "snowflake-connector-python>=3.12,<5", ] @@ -237,6 +240,7 @@ include = [ "docs/oci-artifact-contract.md", "docs/rollback.md", "docs/upgrading.md", + "docs/warehouse-staging.md", "examples", "infra", "graphs", diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 5b3d89d4..0962bad4 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -43,6 +43,7 @@ "README.md", "docs/compatibility-matrix.md", "docs/postgresql-benchmarks.md", + "docs/warehouse-staging.md", "examples/salesforce/dander.yaml", "infra/main.tf", "infra/kubernetes/chart/dander/Chart.yaml", diff --git a/src/dander/providers/dependencies.py b/src/dander/providers/dependencies.py index 28dd0333..0f420a8b 100644 --- a/src/dander/providers/dependencies.py +++ b/src/dander/providers/dependencies.py @@ -26,11 +26,11 @@ class ProviderDependencySet: ), ProviderDependencySet( extra="snowflake", - distributions=("snowflake-connector-python",), + distributions=("pyarrow", "snowflake-connector-python"), ), ProviderDependencySet( extra="redshift", - distributions=("boto3", "redshift-connector"), + distributions=("boto3", "pyarrow", "redshift-connector"), ), ProviderDependencySet( extra="postgres", diff --git a/src/dander/warehouse/__init__.py b/src/dander/warehouse/__init__.py index 3fdd279e..332b5a71 100644 --- a/src/dander/warehouse/__init__.py +++ b/src/dander/warehouse/__init__.py @@ -26,6 +26,12 @@ WarehouseTransformRunner, WarehouseWriterFactory, ) +from dander.warehouse.staging import ( + ParquetStagingSession, + StagedArtifact, + StagingArtifactError, + StagingManifest, +) __all__ = [ "BigQuerySchemaCompatibilityError", @@ -33,11 +39,15 @@ "CanonicalType", "FieldCardinality", "LogicalTypeKind", + "ParquetStagingSession", "ProviderExtension", "PreparedWarehouseStatement", "RelationCodec", "RelationRef", "RelationSchema", + "StagedArtifact", + "StagingArtifactError", + "StagingManifest", "WarehouseCapabilities", "WarehouseRuntime", "WarehouseSchemaMapper", diff --git a/src/dander/warehouse/staging.py b/src/dander/warehouse/staging.py new file mode 100644 index 00000000..d41ee716 --- /dev/null +++ b/src/dander/warehouse/staging.py @@ -0,0 +1,407 @@ +"""Bounded, checksummed Parquet artifacts shared by staged warehouse loaders.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from typing import TYPE_CHECKING, Self, cast + +from dander.warehouse.contracts import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + RelationSchema, +) + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + from pathlib import Path + + import pyarrow as pa + +_RUN_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,126}$") +_MANIFEST_SCHEMA = "io.dander.warehouse.staging/v1" + + +class StagingArtifactError(ValueError): + """Raised when rows or local artifact lifecycle violate the staging contract.""" + + +@dataclass(frozen=True, slots=True) +class StagedArtifact: + """One immutable Parquet part and its content identity.""" + + path: Path + rows: int + logical_bytes: int + compressed_bytes: int + sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "file": self.path.name, + "rows": self.rows, + "logical_bytes": self.logical_bytes, + "compressed_bytes": self.compressed_bytes, + "sha256": self.sha256, + } + + +@dataclass(frozen=True, slots=True) +class StagingManifest: + """Deterministic, non-sensitive manifest for one run-scoped artifact set.""" + + run_id: str + schema_fingerprint: str + artifacts: tuple[StagedArtifact, ...] + schema: str = _MANIFEST_SCHEMA + + @property + def rows(self) -> int: + return sum(artifact.rows for artifact in self.artifacts) + + @property + def logical_bytes(self) -> int: + return sum(artifact.logical_bytes for artifact in self.artifacts) + + @property + def compressed_bytes(self) -> int: + return sum(artifact.compressed_bytes for artifact in self.artifacts) + + def to_json(self) -> str: + return json.dumps( + { + "schema": self.schema, + "run_id": self.run_id, + "schema_fingerprint": self.schema_fingerprint, + "rows": self.rows, + "logical_bytes": self.logical_bytes, + "compressed_bytes": self.compressed_bytes, + "artifacts": [artifact.as_dict() for artifact in self.artifacts], + }, + separators=(",", ":"), + sort_keys=True, + ) + + +class ParquetStagingSession: + """Create one exclusive run directory and remove only its owned files on exit.""" + + def __init__( + self, + root: Path, + *, + run_id: str, + max_rows_per_file: int = 100_000, + max_logical_bytes_per_file: int = 128 * 1_024 * 1_024, + compression: str = "zstd", + ) -> None: + if _RUN_ID.fullmatch(run_id) is None: + raise StagingArtifactError("staging run id is invalid") + if not 1 <= max_rows_per_file <= 1_000_000: + raise StagingArtifactError("max_rows_per_file must be between 1 and 1000000") + if not 1_024 <= max_logical_bytes_per_file <= 1_073_741_824: + raise StagingArtifactError( + "max_logical_bytes_per_file must be between 1024 and 1073741824" + ) + if compression not in {"snappy", "zstd"}: + raise StagingArtifactError("Parquet compression must be snappy or zstd") + self._root = root.expanduser().resolve() + self._directory = self._root / run_id + self._run_id = run_id + self._max_rows = max_rows_per_file + self._max_bytes = max_logical_bytes_per_file + self._compression = compression + self._entered = False + self._staged = False + + @property + def directory(self) -> Path: + return self._directory + + def __enter__(self) -> Self: + self._root.mkdir(parents=True, exist_ok=True) + try: + self._directory.mkdir(mode=0o700) + except FileExistsError as error: + raise StagingArtifactError( + f"staging directory already exists for run {self._run_id!r}" + ) from error + self._entered = True + return self + + def __exit__(self, *_: object) -> None: + self.cleanup() + + def stage( + self, + records: Iterable[Mapping[str, object]], + schema: RelationSchema, + ) -> StagingManifest: + """Consume rows once and write bounded immutable Parquet parts.""" + if not self._entered or not self._directory.is_dir(): + raise StagingArtifactError("Parquet staging session is not active") + if self._staged: + raise StagingArtifactError("Parquet staging session may stage only once") + self._staged = True + arrow_schema = _arrow_schema(schema) + artifacts: list[StagedArtifact] = [] + chunk: list[dict[str, object]] = [] + chunk_bytes = 0 + index = 0 + for row_index, record in enumerate(records): + normalized = _normalize_record(record, schema.fields, row_index=row_index) + row_bytes = _logical_size(normalized) + if chunk and ( + len(chunk) >= self._max_rows or chunk_bytes + row_bytes > self._max_bytes + ): + artifacts.append( + self._write_part( + arrow_schema, + chunk, + logical_bytes=chunk_bytes, + index=index, + ) + ) + index += 1 + chunk = [] + chunk_bytes = 0 + chunk.append(normalized) + chunk_bytes += row_bytes + if chunk: + artifacts.append( + self._write_part( + arrow_schema, + chunk, + logical_bytes=chunk_bytes, + index=index, + ) + ) + return StagingManifest( + run_id=self._run_id, + schema_fingerprint=hashlib.sha256( + schema.model_dump_json(by_alias=True).encode("utf-8") + ).hexdigest(), + artifacts=tuple(artifacts), + ) + + def cleanup(self) -> None: + """Delete only regular files inside this session's exact run directory.""" + if not self._directory.exists(): + return + for path in self._directory.iterdir(): + if not path.is_file() or path.is_symlink(): + raise StagingArtifactError("staging cleanup found an unexpected path") + path.unlink() + self._directory.rmdir() + + def _write_part( + self, + arrow_schema: pa.Schema, + records: list[dict[str, object]], + *, + logical_bytes: int, + index: int, + ) -> StagedArtifact: + import pyarrow as arrow + import pyarrow.parquet as parquet + + temporary = self._directory / f".part-{index:05d}.parquet.tmp" + try: + table = arrow.Table.from_pylist(records, schema=arrow_schema) + parquet.write_table( # type: ignore[no-untyped-call] + table, + temporary, + compression=self._compression, + use_dictionary=True, + write_statistics=True, + ) + except Exception as error: + if temporary.exists(): + temporary.unlink() + raise StagingArtifactError( + f"staging part {index} does not conform to the declared schema" + ) from error + os.chmod(temporary, 0o600) + checksum = _file_sha256(temporary) + final = self._directory / f"part-{index:05d}-{checksum[:16]}.parquet" + temporary.replace(final) + return StagedArtifact( + path=final, + rows=len(records), + logical_bytes=logical_bytes, + compressed_bytes=final.stat().st_size, + sha256=checksum, + ) + + +def _arrow_schema(schema: RelationSchema) -> pa.Schema: + try: + import pyarrow as arrow + except ModuleNotFoundError as error: + raise StagingArtifactError( + "Parquet staging requires pyarrow; install a Snowflake or Redshift runtime extra" + ) from error + return arrow.schema( + [ + arrow.field( + field.name, + _arrow_type(field.data_type), + nullable=field.cardinality is FieldCardinality.NULLABLE, + ) + for field in schema.fields + ] + ) + + +def _arrow_type(data_type: CanonicalType) -> pa.DataType: + import pyarrow as arrow + + match data_type.kind: + case LogicalTypeKind.BOOLEAN: + return arrow.bool_() + case LogicalTypeKind.INTEGER: + assert data_type.bit_width is not None + return { + 8: arrow.int8(), + 16: arrow.int16(), + 32: arrow.int32(), + 64: arrow.int64(), + }[data_type.bit_width] + case LogicalTypeKind.DECIMAL: + assert data_type.precision is not None and data_type.scale is not None + if data_type.precision <= 38: + return arrow.decimal128(data_type.precision, data_type.scale) + if data_type.precision <= 76: + return arrow.decimal256(data_type.precision, data_type.scale) + raise StagingArtifactError("Parquet decimal precision cannot exceed 76") + case LogicalTypeKind.FLOAT: + return arrow.float32() if data_type.bit_width == 32 else arrow.float64() + case LogicalTypeKind.STRING | LogicalTypeKind.JSON: + return arrow.string() + case LogicalTypeKind.BINARY: + return arrow.binary() + case LogicalTypeKind.DATE: + return arrow.date32() + case LogicalTypeKind.TIME: + assert data_type.fractional_second_precision is not None + unit = "us" if data_type.fractional_second_precision <= 6 else "ns" + return arrow.time64(unit) + case LogicalTypeKind.TIMESTAMP: + assert data_type.fractional_second_precision is not None + precision = data_type.fractional_second_precision + unit = "ms" if precision <= 3 else "us" if precision <= 6 else "ns" + return arrow.timestamp(unit, tz="UTC" if data_type.with_timezone else None) + case LogicalTypeKind.ARRAY: + assert data_type.element is not None + return arrow.list_(_arrow_type(data_type.element)) + case LogicalTypeKind.RECORD: + return arrow.struct( + [ + arrow.field( + field.name, + _arrow_type(field.data_type), + nullable=field.cardinality is FieldCardinality.NULLABLE, + ) + for field in data_type.fields + ] + ) + raise AssertionError("Unhandled canonical Parquet type") + + +def _normalize_record( + record: Mapping[str, object], + fields: tuple[CanonicalField, ...], + *, + row_index: int, +) -> dict[str, object]: + expected = {field.name for field in fields} + if set(record) != expected: + raise StagingArtifactError(f"staging record {row_index} does not match the declared schema") + return { + field.name: _normalize_value(record[field.name], field, row_index=row_index) + for field in fields + } + + +def _normalize_value(value: object, field: CanonicalField, *, row_index: int) -> object: + if value is None: + if field.cardinality is FieldCardinality.REQUIRED: + raise StagingArtifactError(f"staging record {row_index} has a null required field") + return None + return _normalize_type_value(value, field.data_type, row_index=row_index) + + +def _normalize_type_value(value: object, data_type: CanonicalType, *, row_index: int) -> object: + if data_type.kind is LogicalTypeKind.JSON: + try: + return json.dumps(value, separators=(",", ":"), sort_keys=True) + except (TypeError, ValueError) as error: + raise StagingArtifactError( + f"staging record {row_index} contains invalid JSON" + ) from error + if data_type.kind is LogicalTypeKind.ARRAY: + if not isinstance(value, (list, tuple)): + raise StagingArtifactError(f"staging record {row_index} contains an invalid array") + assert data_type.element is not None + return [ + _normalize_type_value(item, data_type.element, row_index=row_index) + if item is not None + else None + for item in value + ] + if data_type.kind is LogicalTypeKind.RECORD: + if not isinstance(value, dict): + raise StagingArtifactError(f"staging record {row_index} contains an invalid record") + return _normalize_record( + cast("Mapping[str, object]", value), + data_type.fields, + row_index=row_index, + ) + return value + + +def _logical_size(value: object) -> int: + if value is None: + return 1 + if isinstance(value, str): + return len(value.encode("utf-8")) + if isinstance(value, bytes): + return len(value) + if isinstance(value, bool): + return 1 + if isinstance(value, (int, float)): + return 8 + if isinstance(value, Decimal): + return len(str(value)) + if isinstance(value, (date, datetime, time)): + return len(value.isoformat()) + if isinstance(value, (list, tuple)): + return sum(_logical_size(item) for item in value) + if isinstance(value, dict): + return sum( + len(str(key).encode("utf-8")) + _logical_size(item) for key, item in value.items() + ) + return len(str(value).encode("utf-8")) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +__all__ = [ + "ParquetStagingSession", + "StagedArtifact", + "StagingArtifactError", + "StagingManifest", +] diff --git a/tests/warehouse/test_staging.py b/tests/warehouse/test_staging.py new file mode 100644 index 00000000..9a8175c9 --- /dev/null +++ b/tests/warehouse/test_staging.py @@ -0,0 +1,177 @@ +"""Shared Parquet staging is bounded, deterministic, sanitized, and run-scoped.""" + +from __future__ import annotations + +import hashlib +import json +import stat +from datetime import UTC, datetime +from decimal import Decimal +from typing import TYPE_CHECKING + +import pyarrow.parquet as parquet +import pytest + +from dander.warehouse import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + ParquetStagingSession, + RelationSchema, + StagingArtifactError, +) +from dander.writer import WriteField + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + +def _schema() -> RelationSchema: + return RelationSchema( + fields=( + WriteField(name="id", data_type="STRING", mode="REQUIRED").to_canonical(), + WriteField(name="amount", data_type="NUMERIC").to_canonical(), + WriteField(name="observed_at", data_type="TIMESTAMP").to_canonical(), + WriteField(name="payload", data_type="JSON").to_canonical(), + WriteField(name="tags", data_type="STRING", mode="REPEATED").to_canonical(), + CanonicalField( + name="details", + data_type=CanonicalType( + kind=LogicalTypeKind.RECORD, + fields=( + CanonicalField( + name="active", + data_type=CanonicalType(kind=LogicalTypeKind.BOOLEAN), + cardinality=FieldCardinality.REQUIRED, + ), + CanonicalField( + name="note", + data_type=CanonicalType(kind=LogicalTypeKind.STRING), + ), + ), + ), + ), + ) + ) + + +def _records(count: int) -> Iterator[dict[str, object]]: + for index in range(count): + yield { + "id": f"row-{index}", + "amount": Decimal(f"{index}.000000000"), + "observed_at": datetime(2026, 8, 8, 12, index, tzinfo=UTC), + "payload": {"index": index, "ready": True}, + "tags": ["fixture", str(index)], + "details": {"active": True, "note": None}, + } + + +def test_parquet_staging_splits_parts_checksums_and_cleans_only_its_run(tmp_path: Path) -> None: + root = tmp_path / "staging" + root.mkdir() + sibling = root / "operator-owned.txt" + sibling.write_text("keep", encoding="utf-8") + + with ParquetStagingSession( + root, + run_id="run-123", + max_rows_per_file=2, + max_logical_bytes_per_file=1_048_576, + ) as session: + manifest = session.stage(_records(5), _schema()) + + assert manifest.rows == 5 + assert len(manifest.artifacts) == 3 + assert [artifact.rows for artifact in manifest.artifacts] == [2, 2, 1] + assert manifest.logical_bytes > 0 + assert manifest.compressed_bytes > 0 + assert len(manifest.schema_fingerprint) == 64 + for index, artifact in enumerate(manifest.artifacts): + assert artifact.path.name.startswith(f"part-{index:05d}-{artifact.sha256[:16]}") + assert artifact.sha256 == hashlib.sha256(artifact.path.read_bytes()).hexdigest() + assert stat.S_IMODE(artifact.path.stat().st_mode) == 0o600 + table = parquet.read_table( # type: ignore[no-untyped-call] + [artifact.path for artifact in manifest.artifacts] + ) + assert table.num_rows == 5 + assert table.column("id").to_pylist() == [f"row-{index}" for index in range(5)] + assert json.loads(table.column("payload")[0].as_py()) == {"index": 0, "ready": True} + serialized = manifest.to_json() + assert str(root) not in serialized + assert "operator-owned" not in serialized + + assert not session.directory.exists() + assert sibling.read_text(encoding="utf-8") == "keep" + + +def test_staging_is_single_use_and_empty_input_is_explicit(tmp_path: Path) -> None: + with ParquetStagingSession(tmp_path, run_id="empty") as session: + manifest = session.stage((), _schema()) + + assert manifest.rows == 0 + assert manifest.artifacts == () + with pytest.raises(StagingArtifactError, match="only once"): + session.stage((), _schema()) + + +def test_staging_splits_on_logical_bytes_without_materializing_the_endpoint(tmp_path: Path) -> None: + schema = RelationSchema( + fields=( + WriteField(name="id", data_type="STRING", mode="REQUIRED").to_canonical(), + WriteField(name="payload", data_type="STRING").to_canonical(), + ) + ) + + def records() -> Iterator[dict[str, object]]: + for index in range(3): + yield {"id": str(index), "payload": "x" * 800} + + with ParquetStagingSession( + tmp_path, + run_id="byte-split", + max_rows_per_file=100, + max_logical_bytes_per_file=1_024, + ) as session: + manifest = session.stage(records(), schema) + + assert [artifact.rows for artifact in manifest.artifacts] == [1, 1, 1] + + +def test_staging_failure_is_sanitized_and_cleanup_runs(tmp_path: Path) -> None: + private_value = "private-record-value" + session = ParquetStagingSession(tmp_path, run_id="invalid") + + with pytest.raises(StagingArtifactError, match="record 0") as raised, session: + session.stage( + ({"id": private_value},), + _schema(), + ) + + assert private_value not in str(raised.value) + assert not session.directory.exists() + + +def test_staging_refuses_existing_run_directory_and_unsupported_decimal(tmp_path: Path) -> None: + (tmp_path / "existing").mkdir() + with ( + pytest.raises(StagingArtifactError, match="already exists"), + ParquetStagingSession(tmp_path, run_id="existing"), + ): + raise AssertionError("unreachable") + + oversized_decimal = RelationSchema( + fields=( + CanonicalField( + name="amount", + data_type=CanonicalType(kind=LogicalTypeKind.DECIMAL, precision=77, scale=2), + ), + ) + ) + with ( + pytest.raises(StagingArtifactError, match="precision cannot exceed 76"), + ParquetStagingSession(tmp_path, run_id="decimal") as session, + ): + session.stage(({"amount": Decimal("1.00")},), oversized_decimal) diff --git a/tickets/DANDER-103-columnar-staging.md b/tickets/DANDER-103-columnar-staging.md new file mode 100644 index 00000000..a24a3a2a --- /dev/null +++ b/tickets/DANDER-103-columnar-staging.md @@ -0,0 +1,21 @@ +# DANDER-103 — Shared bounded columnar staging + +## Scope + +- Map canonical schema v1 into Arrow/Parquet without importing provider SDKs at Dander startup. +- Consume one row iterator into bounded row/byte parts with Zstandard or Snappy compression. +- Record relative filenames, counts, compressed size, schema fingerprint, and SHA-256 checksums. +- Clean only the exact run-scoped local directory on normal or handled-failure exit. + +## Acceptance + +- Scalar, decimal, temporal, JSON, array, and record values round-trip through Parquet. +- Row and logical-byte thresholds split an input without endpoint-wide materialization. +- Files are owner-only and manifest JSON contains no absolute path or row value. +- Invalid schema/records fail with row index only and handled failures remove partial artifacts. +- Snowflake and Redshift extras explicitly install the shared PyArrow dependency. + +## Exclusions + +No S3/Snowflake upload, remote stage, warehouse `COPY`, target mutation, cloud credentials, support +promotion, or live provider call belongs in this ticket. diff --git a/uv.lock b/uv.lock index 92f59daf..53fb31fd 100644 --- a/uv.lock +++ b/uv.lock @@ -676,6 +676,7 @@ postgres = [ ] redshift = [ { name = "boto3" }, + { name = "pyarrow" }, { name = "redshift-connector" }, ] runtime-all = [ @@ -690,10 +691,12 @@ runtime-all = [ { name = "google-cloud-secret-manager" }, { name = "psycopg", extra = ["binary"], marker = "sys_platform != 'linux'" }, { name = "psycopg", extra = ["pool"] }, + { name = "pyarrow" }, { name = "redshift-connector" }, { name = "snowflake-connector-python" }, ] snowflake = [ + { name = "pyarrow" }, { name = "snowflake-connector-python" }, ] @@ -733,6 +736,9 @@ requires-dist = [ { name = "psycopg", extras = ["binary", "pool"], marker = "sys_platform != 'linux' and extra == 'runtime-all'", specifier = ">=3.2,<4" }, { name = "psycopg", extras = ["pool"], marker = "sys_platform == 'linux' and extra == 'postgres'", specifier = ">=3.2,<4" }, { name = "psycopg", extras = ["pool"], marker = "sys_platform == 'linux' and extra == 'runtime-all'", specifier = ">=3.2,<4" }, + { name = "pyarrow", marker = "extra == 'redshift'", specifier = ">=19,<26" }, + { name = "pyarrow", marker = "extra == 'runtime-all'", specifier = ">=19,<26" }, + { name = "pyarrow", marker = "extra == 'snowflake'", specifier = ">=19,<26" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pydantic-settings", specifier = ">=2.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3" }, From a28844c5a3f42bdb931d600d738d521919881d65 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 23:17:19 -0400 Subject: [PATCH 056/163] refactor runtime relation coordinates --- HANDOFF.md | 40 +-- docs/decisions.md | 12 + src/dander/catalog/spine.py | 59 ++-- src/dander/cli/provider_runtime.py | 11 +- src/dander/cli/run_command.py | 175 ++++++++--- src/dander/cli/runtime_command.py | 12 +- src/dander/executor.py | 27 +- src/dander/pipeline/compiler.py | 145 +++------- src/dander/pipeline/node_config.py | 44 +-- src/dander/pipeline/runtime.py | 183 +++--------- src/dander/project/portable_config.py | 2 +- src/dander/providers/bigquery/config.py | 32 ++- src/dander/providers/bigquery/graph.py | 271 ++++++++++++++++++ src/dander/providers/bigquery/runtime.py | 13 +- src/dander/providers/bigquery/state.py | 19 +- src/dander/providers/dataplex/runtime.py | 4 +- src/dander/providers/postgresql/config.py | 28 +- src/dander/providers/postgresql/runtime.py | 2 + src/dander/providers/postgresql/transform.py | 28 +- src/dander/providers/postgresql/writer.py | 22 +- src/dander/runtime.py | 34 ++- src/dander/transform/__init__.py | 19 +- src/dander/transform/config.py | 5 + src/dander/transform/project.py | 103 ++++--- src/dander/transform/result.py | 18 ++ src/dander/transform/runner.py | 34 ++- src/dander/warehouse/__init__.py | 2 + src/dander/warehouse/contracts.py | 22 +- src/dander/warehouse/runtime.py | 1 + src/dander/writer/__init__.py | 67 ++++- src/dander/writer/base.py | 65 ++++- tests/catalog/test_dataplex.py | 12 +- tests/cli/test_cli.py | 6 +- tests/cli/test_run_command.py | 105 ++++++- .../test_postgresql_native_profile.py | 28 +- tests/pipeline/test_compiler.py | 39 +++ tests/pipeline/test_runtime_bridge.py | 37 ++- tests/project/test_portable_config.py | 2 + tests/providers/test_bigquery_runtime.py | 16 ++ .../test_postgresql_warehouse_runtime.py | 25 ++ tests/test_executor.py | 33 +++ tests/test_runtime.py | 29 +- tests/transform/test_project.py | 28 ++ 43 files changed, 1343 insertions(+), 516 deletions(-) create mode 100644 src/dander/providers/bigquery/graph.py create mode 100644 src/dander/transform/result.py diff --git a/HANDOFF.md b/HANDOFF.md index f1822fe9..3d4b44a0 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,39 @@ ## Finished -- Added one bounded, compressed Parquet staging contract for Snowflake and Redshift loaders. -- Mapped canonical scalar, decimal, temporal, JSON, array, and record types explicitly to Arrow. -- Added per-part SHA-256, schema fingerprinting, row/byte counts, and deterministic manifest JSON. -- Added owner-only run directories and fail-closed cleanup limited to exact owned regular files. -- Made PyArrow an explicit Snowflake, Redshift, and full-runtime dependency. +- Made `RelationRef(catalog, namespace, name)` the dependency-light coordinate passed through run, + graph, transform, metadata, and writer orchestration. +- Moved BigQuery project/dataset and PostgreSQL database/schema translation to provider boundaries. +- Kept legacy CLI and Python compatibility inputs while translating them before neutral execution. +- Fixed non-default raw namespaces so ingestion, transforms, and metadata use the same relation. +- Kept mixed-provider state coordinates independent and rejected BigQuery-only safety/Dataplex + operations before external clients are created. ## Try It -Use `ParquetStagingSession` as a context manager, call `stage(records, canonical_schema)`, upload the -returned immutable parts inside the context, and let normal exit remove local artifacts. +Run an existing BigQuery command unchanged, or select a PostgreSQL profile. Neutral internals now +receive canonical relations; provider configuration still uses its native vocabulary. ## Checks -- All 1,036 tests passed with PostgreSQL 15; repository Ruff and strict mypy passed. -- Staging round-trip, row/byte splitting, redaction, cleanup, and dependency tests passed. -- Wheel/sdist inspection and Snowflake/Redshift/runtime-all PyArrow metadata checks passed. +- All 1,048 tests passed with PostgreSQL 15; Ruff and strict mypy passed. +- Wheel/sdist build, inspection, outside-checkout installs, generation, and validation passed. +- Source-free container conformance, proof assets, dependency audit, and Helm checks passed. +- Repository and generated-project GCP/AWS Terraform validation and module tests passed. ## Decisions -- Shared code owns local artifacts only; provider adapters own remote stages and publication. -- JSON uses deterministic compact text; provider-native semi-structured mapping remains explicit. -- A singleton oversized row may exceed the soft byte target so staging always makes progress. +- Canonical relations are the neutral contract; provider-shaped names are compatibility aliases. +- Provider configs own native-to-canonical translation; provider runtimes own translation back. +- Serialized v1/GCP catalog output and resource identities remain unchanged. ## Remaining -- Open and merge the focused protected PR if Linux package, container, Terraform, and scans pass. -- Build Snowflake upload/COPY/fencing against this contract in a separate branch. -- Build Redshift S3/COPY/fencing only after the shared contract merges. +- Run protected Linux CI, image/secret scans, and merge the focused PR if clean. +- Resume Snowflake and Redshift runtime work only after this cleanup merges. ## Review First -- `src/dander/warehouse/staging.py` -- `tests/warehouse/test_staging.py` -- `docs/warehouse-staging.md` +- `src/dander/cli/run_command.py` +- `src/dander/runtime.py` +- `src/dander/pipeline/runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 68d6d1b6..a9a4bd78 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -854,3 +854,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc manifest exposes counts and fingerprints but no row values, credentials, or provider locations. - **Cleanup:** Normal exit removes only the exact owned run directory. One oversized record may exceed the logical-byte target but stays a singleton; remote adapters must enforce service limits. + +## 2026-08-08 — Neutral orchestration carries canonical relation coordinates + +- **Boundary:** Provider-neutral run, graph, transform, metadata, and writer orchestration carries + `RelationRef(catalog, namespace, name)`. Warehouse providers own translation from their native + configuration into those coordinates and back into provider SDK or SQL vocabulary. +- **Compatibility:** Existing `--project`, `--dataset`, `project_id`, and serialized catalog fields + remain supported, but are translated at CLI or provider boundaries instead of defining the + neutral runtime contract. BigQuery resource identities and defaults remain unchanged. +- **Correctness:** A selected raw namespace is shared by ingestion, transforms, and metadata; + canonical source coordinates are preserved rather than reconstructed from BigQuery defaults. + State backends keep their own control catalog and namespace in mixed-provider compositions. diff --git a/src/dander/catalog/spine.py b/src/dander/catalog/spine.py index 8f770203..f3b4d98e 100644 --- a/src/dander/catalog/spine.py +++ b/src/dander/catalog/spine.py @@ -6,11 +6,12 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Iterable, Mapping from dander.ingestion import SourceConfig from dander.transform import TransformModel, TransformProject from dander.transform.config import GenericTestMetadata, MetricMetadata, Scalar + from dander.warehouse import RelationRef @dataclass(frozen=True) @@ -80,10 +81,7 @@ def to_manifest(self) -> dict[str, object]: class CatalogAsset: """Cloud-neutral metadata spine record for one materialized model.""" - project: str - dataset: str - name: str - relation: str + relation_ref: RelationRef description: str owner: str materialization: str @@ -94,6 +92,26 @@ class CatalogAsset: tests: tuple[TestContract, ...] metrics: tuple[MetricDefinition, ...] + @property + def project(self) -> str: + """Return the legacy BigQuery-named catalog alias.""" + return self.relation_ref.catalog + + @property + def dataset(self) -> str: + """Return the legacy BigQuery-named namespace alias.""" + return self.relation_ref.namespace + + @property + def name(self) -> str: + """Return the relation name.""" + return self.relation_ref.name + + @property + def relation(self) -> str: + """Return stable unquoted canonical coordinates for serialization.""" + return ".".join(self.relation_ref.coordinates) + def to_manifest(self) -> dict[str, object]: """Return a deterministic JSON-compatible semantic registry record.""" return { @@ -126,7 +144,7 @@ def compile( def manifest(self, assets: Iterable[CatalogAsset]) -> dict[str, object]: """Build a versioned semantic registry without volatile timestamps.""" ordered = sorted(assets, key=lambda asset: asset.relation) - projects = sorted({asset.project for asset in ordered}) + projects = sorted({asset.relation_ref.catalog for asset in ordered}) return { "schema_version": 1, "projects": projects, @@ -139,10 +157,11 @@ def pipeline_manifest( pipeline_id: str, source: SourceConfig, assets: Iterable[CatalogAsset], + source_relations: Mapping[str, RelationRef] | None = None, ) -> dict[str, object]: """Project source, model, lineage, test, and metric metadata as one snapshot.""" ordered = sorted(assets, key=lambda asset: asset.relation) - projects = sorted({asset.project for asset in ordered}) + projects = sorted({asset.relation_ref.catalog for asset in ordered}) project_id = projects[0] if len(projects) == 1 else "" return { "schema_version": 2, @@ -155,9 +174,13 @@ def pipeline_manifest( { "name": endpoint.name, "relation": ( - f"{project_id}.raw.{source.name}_{endpoint.name}" - if project_id - else f"raw.{source.name}_{endpoint.name}" + ".".join(source_relations[endpoint.name].coordinates) + if source_relations is not None + else ( + f"{project_id}.raw.{source.name}_{endpoint.name}" + if project_id + else f"raw.{source.name}_{endpoint.name}" + ) ), "primary_key": list(endpoint.primary_key), "incremental_cursor": endpoint.incremental_cursor, @@ -172,17 +195,15 @@ def _asset(self, project: TransformProject, model: TransformModel) -> CatalogAss metadata = model.metadata not_null_columns = {test.column for test in metadata.tests if test.not_null} return CatalogAsset( - project=project.project_id, - dataset=metadata.dataset, - name=model.name, - relation=_unquote(project.relation_for_model(model)), + relation_ref=project.relation_ref_for_model(model), description=metadata.description, owner=metadata.owner, materialization=metadata.materialization.value, source_system=metadata.source_system, sensitivity=metadata.sensitivity, upstream_relations=tuple( - _unquote(project.relation_for_ref(reference)) for reference in model.refs + ".".join(project.relation_ref_for_ref(reference).coordinates) + for reference in model.refs ), columns=tuple( CatalogColumn( @@ -222,17 +243,15 @@ def _test_contracts( TestContract( kind="relationships", column=test.column, - target_relation=_unquote(project.relation_for_ref(test.relationships.to)), + target_relation=".".join( + project.relation_ref_for_ref(test.relationships.to).coordinates + ), target_field=test.relationships.field, ) ) return tuple(contracts) -def _unquote(relation: str) -> str: - return relation.removeprefix("`").removesuffix("`") - - def _metric_definition(metric: MetricMetadata) -> MetricDefinition: field = metric.field calculations = { diff --git a/src/dander/cli/provider_runtime.py b/src/dander/cli/provider_runtime.py index dbd05cc2..6ec710df 100644 --- a/src/dander/cli/provider_runtime.py +++ b/src/dander/cli/provider_runtime.py @@ -11,17 +11,24 @@ def build_catalog_publisher( *, provider_id: str, - project: str, + catalog: str | None = None, + project: str | None = None, location: str, ) -> CatalogPublisher: """Build the selected external catalog publisher or fail when it is disabled.""" + if catalog is None: + catalog = project + elif project is not None and project != catalog: + raise ClickException("catalog and legacy project must match") + if catalog is None: + raise ClickException("Catalog provider requires a warehouse catalog") registry = default_provider_registry() try: config = registry.parse(ProviderKind.CATALOG, {"provider": provider_id}) runtime = registry.build( ProviderKind.CATALOG, config, - context={"project": project, "location": location}, + context={"catalog": catalog, "location": location}, ) except ProviderFactoryError as error: raise ClickException(str(error)) from error diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index c94efe79..0c2fefc5 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -60,7 +60,12 @@ WatermarkStore, ) from dander.transform import TransformProjectError, TransformRunError -from dander.warehouse import WarehouseRuntime, WarehouseTransformRunner +from dander.warehouse import ( + RelationRef, + WarehouseCoordinateConfig, + WarehouseRuntime, + WarehouseTransformRunner, +) from dander.writer import SchemaEvolution if TYPE_CHECKING: @@ -107,20 +112,43 @@ class _ResolvedRun: project_pipeline: bool graph_file: Path | None graph_plan: GraphExecutionPlan | None - project: str - dataset: str - metadata_dataset: str + gcp_project: str + catalog: str + raw_namespace: str + metadata_namespace: str + endpoint_relations: dict[str, RelationRef] + state_catalog: str | None + state_namespace: str | None selected_models: tuple[str, ...] | None build_models: bool publish_dataplex: bool warehouse_provider: str warehouse_config: dict[str, object] - warehouse_catalog: str state_provider: str state_config: dict[str, object] catalog_provider: str secret_provider: str + @property + def project(self) -> str: + """Return the v1 CLI compatibility project value.""" + return self.gcp_project + + @property + def dataset(self) -> str: + """Return the v1 CLI compatibility dataset value.""" + return self.raw_namespace + + @property + def metadata_dataset(self) -> str: + """Return the v1 BigQuery metadata namespace alias.""" + return self.metadata_namespace + + @property + def warehouse_catalog(self) -> str: + """Return the former internal warehouse-catalog shim.""" + return self.catalog + @dataclass(frozen=True, slots=True) class _ControlStores: @@ -139,10 +167,11 @@ def execute_run( ) -> PipelineExecutionResult | None: """Resolve, execute, and render one ``dander run`` request.""" resolved = _resolve_run(options) + _require_provider_compatible_options(options, resolved) if options.dry_run: _render_dry_run(options, resolved, console=console) return None - if _requires_gcp_project(resolved) and not resolved.project: + if _requires_gcp_project(resolved) and not resolved.gcp_project: raise ClickException("GCP project is required via --project or GCP_PROJECT_ID") _verify_safety(options, resolved) @@ -227,19 +256,40 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: settings = Settings() resolved_project = options.project or settings.gcp_project_id - resolved_dataset = options.dataset or settings.bq_dataset_raw - warehouse_catalog = ( - str(warehouse_config["database"]) - if warehouse_provider == "postgresql" - else resolved_project + state_catalog = (resolved_project or None) if state_provider == "bigquery" else None + state_namespace = ( + (options.dataset or settings.bq_dataset_raw) if state_provider == "bigquery" else None ) + registry = default_provider_registry() + try: + coordinate_config = registry.parse(ProviderKind.WAREHOUSE, warehouse_config) + if not isinstance(coordinate_config, WarehouseCoordinateConfig): + raise ProviderFactoryError( + f"Warehouse provider {warehouse_provider!r} cannot resolve relation coordinates" + ) + coordinate = coordinate_config.raw_relation( + "_dander_coordinate", + compatibility_catalog=resolved_project or "dander-dry-run", + compatibility_namespace=options.dataset, + default_namespace=settings.bq_dataset_raw, + ) + endpoint_relations = { + endpoint.name: coordinate_config.raw_relation( + f"{config.name}_{endpoint.name}", + compatibility_catalog=resolved_project or "dander-dry-run", + compatibility_namespace=options.dataset, + default_namespace=settings.bq_dataset_raw, + ) + for endpoint in config.endpoints + } + except (ProviderFactoryError, ValueError) as error: + raise ClickException(str(error)) from error if options.sandbox and options.guarded_free_tier: raise ClickException("--sandbox and --guarded-free-tier are mutually exclusive") graph_plan = _resolve_graph_plan( graph_file, config, - project=warehouse_catalog, - dataset=resolved_dataset, + endpoint_relations=endpoint_relations, build_models=build_models, selected_models=selected_models, catalog_output=options.catalog_output, @@ -252,15 +302,18 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: project_pipeline=project_pipeline, graph_file=graph_file, graph_plan=graph_plan, - project=resolved_project, - dataset=resolved_dataset, - metadata_dataset=settings.bq_dataset_metadata, + gcp_project=resolved_project, + catalog=coordinate.catalog, + raw_namespace=coordinate.namespace, + metadata_namespace=settings.bq_dataset_metadata, + endpoint_relations=endpoint_relations, + state_catalog=state_catalog, + state_namespace=state_namespace, selected_models=selected_models, build_models=build_models, publish_dataplex=publish_dataplex, warehouse_provider=warehouse_provider, warehouse_config=warehouse_config, - warehouse_catalog=warehouse_catalog, state_provider=state_provider, state_config=state_config, catalog_provider=catalog_provider, @@ -272,8 +325,7 @@ def _resolve_graph_plan( graph_file: Path | None, config: SourceConfig, *, - project: str, - dataset: str, + endpoint_relations: dict[str, RelationRef], build_models: bool, selected_models: Sequence[str] | None, catalog_output: Path | None, @@ -290,8 +342,7 @@ def _resolve_graph_plan( return plan_graph_execution( graph, config, - project=project or "dander-dry-run", - dataset=dataset, + endpoint_relations=endpoint_relations, ) except GraphRuntimeError as error: raise ClickException(str(error)) from error @@ -300,19 +351,30 @@ def _resolve_graph_plan( def _verify_safety(options: RunOptions, resolved: _ResolvedRun) -> None: if options.sandbox: try: - SandboxDataset().prepare(resolved.project, resolved.dataset) + SandboxDataset().prepare(resolved.gcp_project, resolved.raw_namespace) except SandboxSafetyError as error: raise ClickException(str(error)) from error elif options.guarded_free_tier: try: GuardedFreeTierVerifier().require_guarded( - resolved.project, + resolved.gcp_project, budget_name=options.budget_name, ) except SandboxSafetyError as error: raise ClickException(str(error)) from error +def _require_provider_compatible_options(options: RunOptions, resolved: _ResolvedRun) -> None: + """Reject BigQuery-only operations before any external client can be constructed.""" + if resolved.warehouse_provider != "bigquery": + if options.sandbox: + raise ClickException("--sandbox is available only with a BigQuery warehouse") + if options.guarded_free_tier: + raise ClickException("--guarded-free-tier is available only with a BigQuery warehouse") + if resolved.publish_dataplex and resolved.catalog_provider == "dataplex": + raise ClickException("Dataplex publication currently requires a BigQuery warehouse") + + def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExecutor: secrets = build_secret_store("environment" if options.sandbox else resolved.secret_provider) auth = build_auth(resolved.source_config, secrets) @@ -329,7 +391,7 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec dataplex_publisher = ( build_catalog_publisher( provider_id=resolved.catalog_provider, - project=resolved.project, + catalog=resolved.catalog, location=options.dataplex_location, ) if resolved.publish_dataplex @@ -343,7 +405,9 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec source_config=resolved.source_config, ingestion=ingestion, history=stores.history, - project=resolved.warehouse_catalog, + catalog=resolved.catalog, + raw_namespace=resolved.raw_namespace, + source_relations=resolved.endpoint_relations, models_dir=( resolved.graph_file.parent if resolved.graph_file is not None else options.models_dir ), @@ -397,8 +461,7 @@ def _build_ingestion_runner( source=source, writer=writer, watermarks=stores.watermarks, - project=resolved.warehouse_catalog, - dataset=resolved.dataset, + endpoint_relations=resolved.endpoint_relations, resume_from_watermark=not options.sandbox, batch_rows=options.batch_rows, endpoint_names=( @@ -415,6 +478,7 @@ def _build_transform_runner( return warehouse.transforms.build_transform_runner( graph_plan=resolved.graph_plan, build_models=resolved.build_models, + raw_namespace=resolved.raw_namespace, ) @@ -428,7 +492,7 @@ def _build_warehouse_runtime(resolved: _ResolvedRun) -> WarehouseRuntime: runtime = registry.build( ProviderKind.WAREHOUSE, config, - context={"project": resolved.project}, + context={"catalog": resolved.catalog}, ) except ProviderFactoryError as error: raise ClickException(str(error)) from error @@ -451,13 +515,7 @@ def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: runtime = registry.build( ProviderKind.STATE, config, - context={ - "project": resolved.project, - "raw_dataset": resolved.dataset, - "metadata_dataset": resolved.metadata_dataset, - "project_pipeline": resolved.project_pipeline, - "metadata_enabled": (resolved.project_pipeline and resolved.graph_plan is None), - }, + context=_state_runtime_context(resolved), ) except ProviderFactoryError as error: raise ClickException(str(error)) from error @@ -466,6 +524,17 @@ def _build_state_runtime(resolved: _ResolvedRun) -> StateRuntime: return runtime +def _state_runtime_context(resolved: _ResolvedRun) -> dict[str, object]: + """Keep state-provider coordinates independent from warehouse coordinates.""" + return { + "catalog": resolved.state_catalog, + "raw_namespace": resolved.state_namespace, + "metadata_namespace": resolved.metadata_namespace, + "project_pipeline": resolved.project_pipeline, + "metadata_enabled": (resolved.project_pipeline and resolved.graph_plan is None), + } + + def _require_executable_state_pair(*, state_provider: str, warehouse_provider: str) -> None: try: load_runtime_compatibility().require_executable( @@ -570,27 +639,34 @@ def build_auth( def _render_dry_run(options: RunOptions, resolved: _ResolvedRun, *, console: Console) -> None: + missing_bigquery_catalog = ( + resolved.warehouse_provider == "bigquery" and not resolved.gcp_project + ) _print_plan( resolved.source_config.name, - resolved.project, - resolved.dataset, + resolved.endpoint_relations, _selected_endpoints(resolved.source_config, resolved.graph_plan), console=console, + catalog_label="" if missing_bigquery_catalog else None, sandbox=options.sandbox, guarded_free_tier=options.guarded_free_tier, batch_rows=options.batch_rows, ) if resolved.graph_plan is not None: - _print_graph_plan(resolved.graph_plan, project=resolved.project, console=console) + _print_graph_plan( + resolved.graph_plan, + console=console, + catalog_label="" if missing_bigquery_catalog else None, + ) def _print_plan( source: str, - project: str, - dataset: str, + endpoint_relations: dict[str, RelationRef], endpoints: Sequence[Endpoint], *, console: Console, + catalog_label: str | None = None, sandbox: bool = False, guarded_free_tier: bool = False, batch_rows: int = 10_000, @@ -607,9 +683,15 @@ def _print_plan( mode = "SCD1 (guarded billing)" else: mode = "SCD1" + relation = endpoint_relations[endpoint.name] + coordinates = ( + (catalog_label, relation.namespace, relation.name) + if catalog_label is not None + else relation.coordinates + ) table.add_row( endpoint.name, - f"{project or ''}.{dataset}.{source}_{endpoint.name}", + ".".join(coordinates), mode, ) console.print(table) @@ -629,8 +711,8 @@ def _selected_endpoints( def _print_graph_plan( plan: GraphExecutionPlan, *, - project: str, console: Console, + catalog_label: str | None = None, ) -> None: """Render the compiled target portion of a credential-free graph plan.""" table = Table(title="PipelineGraph targets") @@ -638,10 +720,15 @@ def _print_graph_plan( table.add_column("Target") table.add_column("Mode") for target in plan.targets: - target_project = project or "" + relation = target.target.relation_ref + coordinates = ( + (catalog_label, relation.namespace, relation.name) + if catalog_label is not None + else relation.coordinates + ) table.add_row( target.node_id, - f"{target_project}.{target.target.dataset}.{target.target.table}", + ".".join(coordinates), target.write_mode.value.upper(), ) console.print(table) diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index 37b6335d..253fb42f 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -91,8 +91,16 @@ def execute_runtime( connectors_dir: Path = typer.Option(Path("connectors"), "--connectors-dir"), # noqa: B008 models_dir: Path = typer.Option(Path("models"), "--models-dir"), # noqa: B008 catalog_output: Path | None = typer.Option(None, "--catalog-output"), # noqa: B008 - project: str | None = typer.Option(None, "--project", help="Override GCP_PROJECT_ID."), - dataset: str | None = typer.Option(None, "--dataset", help="Override BQ_DATASET_RAW."), + project: str | None = typer.Option( + None, + "--project", + help="Override the BigQuery/GCP project for profiles that use GCP.", + ), + dataset: str | None = typer.Option( + None, + "--dataset", + help="Override the raw warehouse dataset or schema.", + ), guarded_free_tier: bool = typer.Option(False, "--guarded-free-tier"), batch_rows: int = typer.Option(10_000, "--batch-rows", min=1, max=100_000), budget_name: str = typer.Option("dander-sbx-cap", "--budget-name", hidden=True), diff --git a/src/dander/executor.py b/src/dander/executor.py index 37a0dc64..6b37c74b 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -13,6 +13,7 @@ from dander.state import LeaseHeartbeat, RunStage, RunStatus, classify_failure from dander.telemetry import RunTelemetry from dander.transform import SqlDialect, TransformProject, TransformRunResult +from dander.warehouse import RelationRef if TYPE_CHECKING: from collections.abc import Iterable @@ -76,7 +77,10 @@ def __init__( source_config: SourceConfig, ingestion: _IngestionRunner, history: RunHistoryStore, - project: str, + catalog: str | None = None, + project: str | None = None, + raw_namespace: str = "raw", + source_relations: dict[str, RelationRef] | None = None, models_dir: Path, selected_models: Iterable[str] | None, build_models: bool, @@ -88,11 +92,26 @@ def __init__( ) -> None: if build_models and transform_runner is None: raise ValueError("A transform runner is required when build_models is enabled") + if catalog is None: + catalog = project + elif project is not None and project != catalog: + raise ValueError("catalog and legacy project must match") + if catalog is None: + raise ValueError("PipelineExecutor requires a warehouse catalog") self._pipeline_id = pipeline_id self._source_config = source_config self._ingestion = ingestion self._history = history - self._project = project + self._catalog = catalog + self._raw_namespace = raw_namespace + self._source_relations = source_relations or { + endpoint.name: RelationRef( + catalog=catalog, + namespace=raw_namespace, + name=f"{source_config.name}_{endpoint.name}", + ) + for endpoint in source_config.endpoints + } self._models_dir = models_dir self._selected_models = tuple(selected_models) if selected_models is not None else None self._build_models = build_models @@ -196,7 +215,8 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: ) transform_project = TransformProject.load( self._models_dir, - project_id=self._project, + catalog=self._catalog, + raw_namespace=self._raw_namespace, target_dialect=getattr( self._transform_runner, "target_dialect", @@ -212,6 +232,7 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: pipeline_id=self._pipeline_id, source=self._source_config, assets=compiled_assets, + source_relations=self._source_relations, ) assets = len(compiled_assets) if self._metadata_store is not None: diff --git a/src/dander/pipeline/compiler.py b/src/dander/pipeline/compiler.py index 2315c739..ed45d206 100644 --- a/src/dander/pipeline/compiler.py +++ b/src/dander/pipeline/compiler.py @@ -29,20 +29,7 @@ ) from dander.transform.model import SqlDialect from dander.warehouse import RelationRef -from dander.writer import ( - BigQueryIncrementalWriter, - BigQueryReplaceWriter, - BigQueryScd1Writer, - BigQueryScd2Writer, - BigQuerySnapshotWriter, - BigQueryStorageIncrementalWriter, - BigQueryStorageScd1Writer, - WriteField, - WriteMode, - WritePattern, - WriteTarget, - WriteTransport, -) +from dander.writer.base import WriteField, WriteMode, WriteTarget if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -55,7 +42,7 @@ PipelineGraph, Transformation, ) - from dander.writer.bigquery import _BigQueryClient + from dander.writer.base import WritePattern _RELATION = re.compile( r"^[A-Za-z][A-Za-z0-9-]{4,61}[A-Za-z0-9]\.[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*$" @@ -137,10 +124,11 @@ def compile_target( graph: PipelineGraph, target_node_id: str, *, - source_relations: Mapping[str, str], + source_relations: Mapping[str, RelationRef | str], + default_catalog: str | None = None, default_project: str | None = None, ) -> CompiledTarget: - """Compile one source dependency subgraph into a BigQuery target SELECT. + """Compile one source dependency subgraph into a provider-neutral target SELECT. Linear mappings and explicit two-input transform joins are executable. Legacy edge joins remain declarative because that shape makes the right input and output the same node. @@ -168,25 +156,28 @@ def compile_target( final_alias = compiler.compile_node(target_node_id) destination = config.writer.destination - project = destination.project or default_project - if project is None: - raise PipelineCompileError( - f"Target node {target_node_id!r} must set destination.project for compilation" - ) + if default_catalog is None: + default_catalog = default_project + elif default_project is not None and default_project != default_catalog: + raise PipelineCompileError("default_catalog and legacy default_project must match") query_ast = exp.Select( expressions=[_column(field.name) for field in target.fields], from_=exp.From(this=_cte_table(final_alias)), ) query_ast.set("with_", exp.With(expressions=compiler.ctes)) query = render_graph_query(query_ast, target_dialect=SqlDialect.BIGQUERY) + try: + target_relation = destination.relation_ref(default_catalog=default_catalog) + except ValueError as error: + raise PipelineCompileError( + f"Target node {target_node_id!r} has an invalid destination" + ) from error return CompiledTarget( node_id=target_node_id, query=query, write_mode=config.writer.write_mode, target=WriteTarget( - project=project, - dataset=destination.dataset, - table=destination.table, + relation=target_relation, business_key=tuple(destination.business_key), schema=tuple( WriteField(name=field.name, data_type=field.cast_to or field.type) @@ -237,7 +228,7 @@ def __init__( *, nodes: Mapping[str, Node], incoming: Mapping[str, list[Edge]], - source_relations: Mapping[str, str], + source_relations: Mapping[str, RelationRef | str], ) -> None: self._nodes = nodes self._incoming = incoming @@ -289,15 +280,18 @@ def _compile_source(self, node: Node, edges: list[Edge]) -> str: relation = self._source_relations.get(node.id) if relation is None: raise PipelineCompileError(f"Missing source relation for node {node.id!r}") - if not _RELATION.fullmatch(relation): - raise PipelineCompileError( - f"Source relation for node {node.id!r} must be project.dataset.table" - ) + if isinstance(relation, str): + if not _RELATION.fullmatch(relation): + raise PipelineCompileError( + f"Source relation for node {node.id!r} must be catalog.namespace.relation" + ) + relation_ref = _parse_relation(relation) + else: + relation_ref = relation columns = [_compile_source_field(field) for field in node.fields] if not columns: raise PipelineCompileError(f"Source node {node.id!r} must declare fields") alias = self._next_alias() - relation_ref = _parse_relation(relation) query = exp.Select(expressions=columns, from_=exp.From(this=_relation_table(relation_ref))) self.ctes.append(_cte(alias, query)) self._aliases[node.id] = alias @@ -472,88 +466,13 @@ def prepare_target_writer( default_project: str, client: object | None = None, ) -> PreparedTargetWriter: - """Resolve target-node configuration to one concrete BigQuery writer.""" - config = target_node.config - if target_node.type != "target" or not isinstance(config, TargetNodeConfig): - raise PipelineCompileError(f"Node {target_node.id!r} is not a configured target") - if config.writer is None: - raise PipelineCompileError(f"Target node {target_node.id!r} has no writer configuration") - writer_config = config.writer - destination = writer_config.destination - project = destination.project or default_project - typed_client = cast("_BigQueryClient | None", client) - match writer_config.write_mode: - case WriteMode.SCD1: - if writer_config.transport is WriteTransport.STORAGE_WRITE: - writer: WritePattern = BigQueryStorageScd1Writer( - project=project, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - schema_evolution=writer_config.schema_evolution, - ) - else: - writer = BigQueryScd1Writer( - project=project, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - schema_evolution=writer_config.schema_evolution, - ) - case WriteMode.SCD2: - writer = BigQueryScd2Writer( - project=project, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - schema_evolution=writer_config.schema_evolution, - ) - case WriteMode.INCREMENTAL: - assert writer_config.cursor_field is not None - if writer_config.transport is WriteTransport.STORAGE_WRITE: - writer = BigQueryStorageIncrementalWriter( - project=project, - cursor_field=writer_config.cursor_field, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - schema_evolution=writer_config.schema_evolution, - ) - else: - writer = BigQueryIncrementalWriter( - project=project, - cursor_field=writer_config.cursor_field, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - schema_evolution=writer_config.schema_evolution, - ) - case WriteMode.SNAPSHOT: - partitioning = writer_config.partitioning - if partitioning is None or partitioning.field is None: - raise PipelineCompileError( - "Snapshot target execution requires field-based partitioning" - ) - writer = BigQuerySnapshotWriter( - project=project, - snapshot_field=partitioning.field, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - schema_evolution=writer_config.schema_evolution, - ) - case WriteMode.REPLACE: - writer = BigQueryReplaceWriter( - project=project, - client=typed_client, - max_batch_rows=writer_config.max_batch_rows, - ) - return PreparedTargetWriter( - writer=writer, - target=WriteTarget( - project=project, - dataset=destination.dataset, - table=destination.table, - business_key=tuple(destination.business_key), - schema=tuple( - WriteField(name=field.name, data_type=field.cast_to or field.type) - for field in target_node.fields - ), - ), + """Compatibility wrapper for BigQuery writer preparation at its provider boundary.""" + from dander.providers.bigquery.graph import prepare_bigquery_target_writer + + return prepare_bigquery_target_writer( + target_node, + default_catalog=default_project, + client=client, ) diff --git a/src/dander/pipeline/node_config.py b/src/dander/pipeline/node_config.py index 25142cc1..66b8a5b6 100644 --- a/src/dander/pipeline/node_config.py +++ b/src/dander/pipeline/node_config.py @@ -26,6 +26,7 @@ # before overriding the rule here. from dander.pipeline.operations import OperationSpec # noqa: TC001 from dander.pipeline.request_spec import RequestSpec # noqa: TC001 +from dander.warehouse.contracts import RelationRef # noqa: TC001 from dander.writer.base import SchemaEvolution, WriteMode, WriteTransport # noqa: TC001 @@ -185,25 +186,10 @@ class PartitioningSpec(BaseModel): class DestinationSpec(BaseModel): - """Declarative BigQuery destination for a target write, mirroring `WriteTarget`. + """Declarative target location retaining the version-1 graph field names. - Structurally mirrors `dander.writer.base.WriteTarget` (`project`/`dataset`/`table`/ - `business_key`) 1:1. `compile_target` and `prepare_target_writer` perform the runtime mapping - after resolving an optional project from deployment context. Unlike `WriteTarget` (a frozen - dataclass — the internal runtime value object), this is a Pydantic model, per - `steering/languages/python.md`'s config/runtime split. Dataset/table values here are ordinary - identifiers, never secrets (`steering/01-security.md`). - - Attributes: - project: GCP project id hosting the destination dataset. `None` means "resolve from - deployment context later" — deliberately not required here since this model only - declares config, never executes a write. - dataset: BigQuery dataset name. Required, non-empty. - table: BigQuery table name. Required, non-empty. - business_key: Ordered column names identifying a logical row for MERGE/versioning writes - (mirrors `WriteTarget.business_key`, which is a `tuple[str, ...]` internally; this is - a `list[str]` at the config boundary). Defaults to empty — required non-empty for the - `WriteMode`s that need one; see `WriterConfig._check_mode_requirements`. + The legacy fields are a configuration boundary only. ``relation_ref`` translates them into + canonical coordinates before provider-neutral compilation and execution. """ model_config = ConfigDict(populate_by_name=True) @@ -213,6 +199,28 @@ class DestinationSpec(BaseModel): table: str = Field(min_length=1) business_key: list[str] = Field(default_factory=list) + def relation_ref(self, *, default_catalog: str | None = None) -> RelationRef: + """Resolve deployment catalog context into one canonical relation.""" + catalog = self.project or default_catalog + if catalog is None: + raise ValueError("Destination requires a catalog") + return RelationRef(catalog=catalog, namespace=self.dataset, name=self.table) + + @classmethod + def from_relation( + cls, + relation: RelationRef, + *, + business_key: list[str] | None = None, + ) -> DestinationSpec: + """Create compatible graph configuration from canonical coordinates.""" + return cls( + project=relation.catalog, + dataset=relation.namespace, + table=relation.name, + business_key=business_key or [], + ) + class WriterConfig(BaseModel): """Declarative target/writer config: how and where a `target` node writes. diff --git a/src/dander/pipeline/runtime.py b/src/dander/pipeline/runtime.py index 5960ac0f..9896a8c1 100644 --- a/src/dander/pipeline/runtime.py +++ b/src/dander/pipeline/runtime.py @@ -1,39 +1,32 @@ -"""Execution bridge from a validated PipelineGraph to configured connectors and BigQuery.""" +"""Provider-neutral graph loading, source binding, and relational planning.""" from __future__ import annotations import json import re from dataclasses import dataclass -from functools import partial -from typing import TYPE_CHECKING, Protocol, cast -from uuid import uuid4 +from typing import TYPE_CHECKING import yaml -from google.cloud import bigquery from pydantic import ValidationError -from dander._bigquery_retry import run_mutation_with_retry -from dander.concurrency import fenced_dml, fencing_job_config from dander.pipeline.compiler import ( CompiledTarget, PipelineCompileError, compile_target, - prepare_target_writer, ) from dander.pipeline.errors import GraphValidationError from dander.pipeline.graph_ops import validate_field_wiring from dander.pipeline.node_config import SourceNodeConfig, TargetNodeConfig -from dander.transform import TransformRunResult -from dander.writer import BigQueryWriteError, WriteMode +from dander.warehouse import RelationRef +from dander.writer.base import WriteMode if TYPE_CHECKING: - from collections.abc import Iterable, Mapping + from collections.abc import Mapping from pathlib import Path - from dander.concurrency import OwnershipGuard from dander.ingestion import SourceConfig - from dander.pipeline.graph import Node, PipelineGraph + from dander.pipeline.graph import PipelineGraph _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _CONNECTOR = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") @@ -56,31 +49,13 @@ def load_graph_for_execution(path: Path) -> PipelineGraph: raise GraphRuntimeError(f"Graph file is invalid: {path.name}") from error -class _QueryJob(Protocol): - def result(self) -> object: - """Wait for BigQuery completion.""" - - -class _BigQueryClient(Protocol): - def query( - self, - query: str, - *, - job_config: bigquery.QueryJobConfig | None = None, - ) -> _QueryJob: - """Submit BigQuery Standard SQL.""" - - def delete_table(self, table: str, *, not_found_ok: bool = False) -> None: - """Delete one run-scoped staging table.""" - - @dataclass(frozen=True) class GraphSourceBindings: """Resolved graph-source nodes for one existing connector configuration.""" connector: str endpoint_names: tuple[str, ...] - source_relations: Mapping[str, str] + source_relations: Mapping[str, RelationRef] @dataclass(frozen=True) @@ -95,18 +70,32 @@ def plan_graph_execution( graph: PipelineGraph, source_config: SourceConfig, *, - project: str, - dataset: str, + endpoint_relations: Mapping[str, RelationRef] | None = None, + project: str | None = None, + dataset: str | None = None, ) -> GraphExecutionPlan: """Bind graph sources to connector endpoints and compile every executable target.""" try: validate_field_wiring(graph) except GraphValidationError as error: raise GraphRuntimeError(str(error)) from error - if not _PROJECT.fullmatch(project): - raise GraphRuntimeError("Graph execution requires a valid GCP project id") - if not _IDENTIFIER.fullmatch(dataset): - raise GraphRuntimeError("Graph execution requires a valid BigQuery raw dataset id") + if endpoint_relations is None: + if project is None or not _PROJECT.fullmatch(project): + raise GraphRuntimeError("Graph execution requires a valid GCP project id") + if dataset is None or not _IDENTIFIER.fullmatch(dataset): + raise GraphRuntimeError("Graph execution requires a valid BigQuery raw dataset id") + endpoint_relations = { + endpoint.name: RelationRef( + catalog=project, + namespace=dataset, + name=f"{source_config.name}_{endpoint.name}", + ) + for endpoint in source_config.endpoints + } + elif project is not None or dataset is not None: + raise GraphRuntimeError( + "Graph execution cannot combine endpoint_relations with project/dataset" + ) unsupported = sorted({node.type for node in graph.nodes} - {"source", "transform", "target"}) if unsupported: raise GraphRuntimeError(f"Graph execution does not support node type {unsupported[0]!r}") @@ -115,7 +104,7 @@ def plan_graph_execution( endpoints = {endpoint.name: endpoint for endpoint in source_config.endpoints} endpoint_names: list[str] = [] - relations: dict[str, str] = {} + relations: dict[str, RelationRef] = {} for node in graph.nodes: if node.type != "source": continue @@ -154,7 +143,12 @@ def plan_graph_execution( ) if config.endpoint not in endpoint_names: endpoint_names.append(config.endpoint) - relations[node.id] = f"{project}.{dataset}.{source_config.name}_{config.endpoint}" + try: + relations[node.id] = endpoint_relations[config.endpoint] + except KeyError as error: + raise GraphRuntimeError( + f"Connector endpoint {config.endpoint!r} has no resolved warehouse relation" + ) from error if not relations: raise GraphRuntimeError("Graph execution requires at least one source node") @@ -171,7 +165,7 @@ def plan_graph_execution( graph, node.id, source_relations=relations, - default_project=project, + default_catalog=next(iter(relations.values())).catalog, ) except PipelineCompileError as error: raise GraphRuntimeError(str(error)) from error @@ -180,10 +174,11 @@ def plan_graph_execution( f"Target node {node.id!r} uses unsupported graph write mode " f"{compiled.write_mode.value!r}; use 'replace'" ) - try: - _validate_target(node, compiled, project) - except (BigQueryWriteError, PipelineCompileError) as error: - raise GraphRuntimeError(str(error)) from error + runtime_catalog = next(iter(relations.values())).catalog + if compiled.target.relation_ref.catalog != runtime_catalog: + raise GraphRuntimeError( + f"Target node {node.id!r} must write inside the runtime catalog" + ) targets.append(compiled) return GraphExecutionPlan( bindings=GraphSourceBindings( @@ -193,101 +188,3 @@ def plan_graph_execution( ), targets=tuple(targets), ) - - -class BigQueryGraphRunner: - """Materialize compiled graph targets inside PipelineExecutor's transform stage.""" - - def __init__( - self, - *, - plan: GraphExecutionPlan, - project: str, - client: _BigQueryClient | None = None, - ) -> None: - self._plan = plan - self._project = project - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) - - def build( - self, - _models_dir: Path, - *, - selected: Iterable[str] | None = None, - ownership: OwnershipGuard | None = None, - ) -> TransformRunResult: - """Stage and transactionally publish selected replace-mode targets.""" - selected_ids = set(selected) if selected is not None else None - known = {target.node_id for target in self._plan.targets} - if selected_ids is not None and (unknown := sorted(selected_ids - known)): - raise GraphRuntimeError(f"Unknown graph target: {unknown[0]!r}") - targets = tuple( - target - for target in self._plan.targets - if selected_ids is None or target.node_id in selected_ids - ) - if not targets: - raise GraphRuntimeError("Graph execution selected no targets") - for target in targets: - self._materialize(target, ownership=ownership) - return TransformRunResult( - models=tuple(target.node_id for target in targets), - assertions=0, - ) - - def _materialize( - self, - compiled: CompiledTarget, - *, - ownership: OwnershipGuard | None, - ) -> None: - target = compiled.target - target_id = f"{target.project}.{target.dataset}.{target.table}" - staging_id = f"{target.project}.{target.dataset}._dander_stage_{target.table}_{uuid4().hex}" - columns = tuple(field.name for field in target.schema) - quoted_columns = ", ".join(f"`{column}`" for column in columns) - try: - if ownership is not None: - ownership.verify() - self._client.query( - f"CREATE TABLE `{staging_id}`\n" - "OPTIONS (expiration_timestamp=" - "TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY))\n" - f"AS\n{compiled.query}" - ).result() - self._client.query( - f"CREATE TABLE IF NOT EXISTS `{target_id}` AS " - f"SELECT {quoted_columns} FROM `{staging_id}` WHERE FALSE" - ).result() - if ownership is not None: - ownership.verify() - replacement = ( - f"DELETE FROM `{target_id}` WHERE TRUE;\n" - f"INSERT INTO `{target_id}` ({quoted_columns})\n" - f"SELECT {quoted_columns} FROM `{staging_id}`" - ) - fence = ownership.fence if ownership is not None else None - if fence is not None: - script = fenced_dml(replacement, fence) - job_config = fencing_job_config(fence) - run_mutation_with_retry(partial(self._client.query, script, job_config=job_config)) - else: - script = f"BEGIN TRANSACTION;\n{replacement};\nCOMMIT TRANSACTION;" - run_mutation_with_retry(partial(self._client.query, script)) - finally: - self._client.delete_table(staging_id, not_found_ok=True) - - -def _validate_target(node: Node, compiled: CompiledTarget, project: str) -> None: - """Reuse writer dispatch validation without constructing credentials or performing I/O.""" - prepare_target_writer(node, default_project=project, client=object()) - target = compiled.target - if target.project != project: - raise GraphRuntimeError(f"Target node {node.id!r} must write inside the runtime project") - if not _IDENTIFIER.fullmatch(target.dataset) or not _IDENTIFIER.fullmatch(target.table): - raise GraphRuntimeError(f"Target node {node.id!r} has an invalid destination") - columns = [field.name for field in target.schema] - if not columns or any(not _IDENTIFIER.fullmatch(column) for column in columns): - raise GraphRuntimeError(f"Target node {node.id!r} must declare valid output fields") - if len(columns) != len(set(columns)): - raise GraphRuntimeError(f"Target node {node.id!r} has duplicate output fields") diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 2c714cec..14a683f8 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -374,7 +374,7 @@ def _resolve( platform_name=selected.platform, deployment_name=selected_name, warehouse_provider=profile.warehouse.provider, - warehouse_config=profile.warehouse.model_dump(mode="json"), + warehouse_config=profile.warehouse.model_dump(mode="json", exclude_none=True), state_provider=profile.state.provider, state_config=profile.state.model_dump(mode="json"), catalog_provider=profile.catalog.provider, diff --git a/src/dander/providers/bigquery/config.py b/src/dander/providers/bigquery/config.py index 14c84da1..192e7e1a 100644 --- a/src/dander/providers/bigquery/config.py +++ b/src/dander/providers/bigquery/config.py @@ -1,8 +1,13 @@ """Dependency-light BigQuery warehouse configuration.""" +import re from typing import Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from dander.warehouse.contracts import RelationRef + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,254}$") class BigQueryWarehouseConfig(BaseModel): @@ -12,6 +17,31 @@ class BigQueryWarehouseConfig(BaseModel): provider: Literal["bigquery"] location: str = Field(default="US", min_length=1) + dataset: str | None = None + + @field_validator("dataset") + @classmethod + def validate_dataset(cls, value: str | None) -> str | None: + if value is not None and not _IDENTIFIER.fullmatch(value): + raise ValueError("dataset must be a valid BigQuery dataset identifier") + return value + + def raw_relation( + self, + name: str, + *, + compatibility_catalog: str | None, + compatibility_namespace: str | None, + default_namespace: str, + ) -> RelationRef: + """Translate legacy project/dataset inputs at the BigQuery boundary.""" + if not compatibility_catalog: + raise ValueError("BigQuery warehouse requires a catalog") + return RelationRef( + catalog=compatibility_catalog, + namespace=compatibility_namespace or self.dataset or default_namespace, + name=name, + ) class BigQueryStateConfig(BaseModel): diff --git a/src/dander/providers/bigquery/graph.py b/src/dander/providers/bigquery/graph.py new file mode 100644 index 00000000..dece0531 --- /dev/null +++ b/src/dander/providers/bigquery/graph.py @@ -0,0 +1,271 @@ +"""BigQuery execution and writer preparation for provider-neutral graph plans.""" + +from __future__ import annotations + +import re +from functools import partial +from typing import TYPE_CHECKING, Protocol, cast +from uuid import uuid4 + +from google.cloud import bigquery + +from dander._bigquery_retry import run_mutation_with_retry +from dander.concurrency import fenced_dml, fencing_job_config +from dander.pipeline.compiler import ( + CompiledTarget, + PipelineCompileError, + PreparedTargetWriter, +) +from dander.pipeline.node_config import TargetNodeConfig +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError +from dander.transform import SqlDialect, TransformRunResult +from dander.warehouse import RelationRef +from dander.writer import ( + BigQueryIncrementalWriter, + BigQueryReplaceWriter, + BigQueryScd1Writer, + BigQueryScd2Writer, + BigQuerySnapshotWriter, + BigQueryStorageIncrementalWriter, + BigQueryStorageScd1Writer, + WriteField, + WriteMode, + WritePattern, + WriteTarget, + WriteTransport, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + from dander.concurrency import OwnershipGuard + from dander.pipeline.graph import Node + from dander.writer.bigquery import _BigQueryClient as _WriterClient + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class _QueryJob(Protocol): + def result(self) -> object: + """Wait for BigQuery completion.""" + + +class _BigQueryClient(Protocol): + def query( + self, + query: str, + *, + job_config: bigquery.QueryJobConfig | None = None, + ) -> _QueryJob: + """Submit BigQuery Standard SQL.""" + + def delete_table(self, table: str, *, not_found_ok: bool = False) -> None: + """Delete one run-scoped staging table.""" + + +class BigQueryGraphRunner: + """Materialize compiled graph targets inside the transform stage.""" + + def __init__( + self, + *, + plan: GraphExecutionPlan, + project: str, + client: _BigQueryClient | None = None, + ) -> None: + for target in plan.targets: + _validate_target(target, project) + self._plan = plan + self._project = project + self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + + def build( + self, + _models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Stage and transactionally publish selected replace-mode targets.""" + selected_ids = set(selected) if selected is not None else None + known = {target.node_id for target in self._plan.targets} + if selected_ids is not None and (unknown := sorted(selected_ids - known)): + raise GraphRuntimeError(f"Unknown graph target: {unknown[0]!r}") + targets = tuple( + target + for target in self._plan.targets + if selected_ids is None or target.node_id in selected_ids + ) + if not targets: + raise GraphRuntimeError("Graph execution selected no targets") + for target in targets: + self._materialize(target, ownership=ownership) + return TransformRunResult( + models=tuple(target.node_id for target in targets), + assertions=0, + ) + + def _materialize( + self, + compiled: CompiledTarget, + *, + ownership: OwnershipGuard | None, + ) -> None: + target = compiled.target + target_id = ".".join(target.relation_ref.coordinates) + staging = RelationRef( + catalog=target.relation_ref.catalog, + namespace=target.relation_ref.namespace, + name=f"_dander_stage_{target.relation_ref.name}_{uuid4().hex}", + ) + staging_id = ".".join(staging.coordinates) + columns = tuple(field.name for field in target.schema) + quoted_columns = ", ".join(f"`{column}`" for column in columns) + query = compiled.render(SqlDialect.BIGQUERY) + try: + if ownership is not None: + ownership.verify() + self._client.query( + f"CREATE TABLE `{staging_id}`\n" + "OPTIONS (expiration_timestamp=" + "TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY))\n" + f"AS\n{query}" + ).result() + self._client.query( + f"CREATE TABLE IF NOT EXISTS `{target_id}` AS " + f"SELECT {quoted_columns} FROM `{staging_id}` WHERE FALSE" + ).result() + if ownership is not None: + ownership.verify() + replacement = ( + f"DELETE FROM `{target_id}` WHERE TRUE;\n" + f"INSERT INTO `{target_id}` ({quoted_columns})\n" + f"SELECT {quoted_columns} FROM `{staging_id}`" + ) + fence = ownership.fence if ownership is not None else None + if fence is not None: + script = fenced_dml(replacement, fence) + job_config = fencing_job_config(fence) + run_mutation_with_retry(partial(self._client.query, script, job_config=job_config)) + else: + script = f"BEGIN TRANSACTION;\n{replacement};\nCOMMIT TRANSACTION;" + run_mutation_with_retry(partial(self._client.query, script)) + finally: + self._client.delete_table(staging_id, not_found_ok=True) + + +def prepare_bigquery_target_writer( + target_node: Node, + *, + default_catalog: str, + client: object | None = None, +) -> PreparedTargetWriter: + """Resolve target-node configuration to one concrete BigQuery writer.""" + config = target_node.config + if target_node.type != "target" or not isinstance(config, TargetNodeConfig): + raise PipelineCompileError(f"Node {target_node.id!r} is not a configured target") + if config.writer is None: + raise PipelineCompileError(f"Target node {target_node.id!r} has no writer configuration") + writer_config = config.writer + destination = writer_config.destination + try: + relation = destination.relation_ref(default_catalog=default_catalog) + except ValueError as error: + raise PipelineCompileError( + f"Target node {target_node.id!r} has an invalid destination" + ) from error + project = relation.catalog + typed_client = cast("_WriterClient | None", client) + match writer_config.write_mode: + case WriteMode.SCD1: + if writer_config.transport is WriteTransport.STORAGE_WRITE: + writer: WritePattern = BigQueryStorageScd1Writer( + project=project, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + schema_evolution=writer_config.schema_evolution, + ) + else: + writer = BigQueryScd1Writer( + project=project, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + schema_evolution=writer_config.schema_evolution, + ) + case WriteMode.SCD2: + writer = BigQueryScd2Writer( + project=project, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + schema_evolution=writer_config.schema_evolution, + ) + case WriteMode.INCREMENTAL: + assert writer_config.cursor_field is not None + if writer_config.transport is WriteTransport.STORAGE_WRITE: + writer = BigQueryStorageIncrementalWriter( + project=project, + cursor_field=writer_config.cursor_field, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + schema_evolution=writer_config.schema_evolution, + ) + else: + writer = BigQueryIncrementalWriter( + project=project, + cursor_field=writer_config.cursor_field, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + schema_evolution=writer_config.schema_evolution, + ) + case WriteMode.SNAPSHOT: + partitioning = writer_config.partitioning + if partitioning is None or partitioning.field is None: + raise PipelineCompileError( + "Snapshot target execution requires field-based partitioning" + ) + writer = BigQuerySnapshotWriter( + project=project, + snapshot_field=partitioning.field, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + schema_evolution=writer_config.schema_evolution, + ) + case WriteMode.REPLACE: + writer = BigQueryReplaceWriter( + project=project, + client=typed_client, + max_batch_rows=writer_config.max_batch_rows, + ) + return PreparedTargetWriter( + writer=writer, + target=WriteTarget( + relation=relation, + business_key=tuple(destination.business_key), + schema=tuple( + WriteField(name=field.name, data_type=field.cast_to or field.type) + for field in target_node.fields + ), + ), + ) + + +def _validate_target(compiled: CompiledTarget, project: str) -> None: + target = compiled.target + if target.relation_ref.catalog != project: + raise GraphRuntimeError( + f"Target node {compiled.node_id!r} must write inside the runtime project" + ) + coordinates = (target.relation_ref.namespace, target.relation_ref.name) + if any(not _IDENTIFIER.fullmatch(value) for value in coordinates): + raise GraphRuntimeError(f"Target node {compiled.node_id!r} has an invalid destination") + columns = [field.name for field in target.schema] + if not columns or any(not _IDENTIFIER.fullmatch(column) for column in columns): + raise GraphRuntimeError( + f"Target node {compiled.node_id!r} must declare valid output fields" + ) + if len(columns) != len(set(columns)): + raise GraphRuntimeError(f"Target node {compiled.node_id!r} has duplicate output fields") + + +__all__ = ["BigQueryGraphRunner", "prepare_bigquery_target_writer"] diff --git a/src/dander/providers/bigquery/runtime.py b/src/dander/providers/bigquery/runtime.py index ea13528c..e9ffbb0f 100644 --- a/src/dander/providers/bigquery/runtime.py +++ b/src/dander/providers/bigquery/runtime.py @@ -5,9 +5,10 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, cast -from dander.pipeline.runtime import BigQueryGraphRunner, GraphExecutionPlan +from dander.pipeline.runtime import GraphExecutionPlan from dander.providers.bigquery.config import BigQueryWarehouseConfig from dander.providers.bigquery.fence import BigQueryTargetFence +from dander.providers.bigquery.graph import BigQueryGraphRunner from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import BigQueryTransformRunner @@ -89,6 +90,7 @@ def build_transform_runner( *, graph_plan: object | None, build_models: bool, + raw_namespace: str = "raw", ) -> WarehouseTransformRunner | None: """Preserve graph precedence and model-build behavior.""" if graph_plan is not None: @@ -96,7 +98,10 @@ def build_transform_runner( raise TypeError("BigQuery graph plan has the wrong type") return BigQueryGraphRunner(plan=graph_plan, project=self.project) if build_models: - return BigQueryTransformRunner(project=self.project) + return BigQueryTransformRunner( + project=self.project, + raw_namespace=raw_namespace, + ) return None @@ -144,9 +149,9 @@ def build_bigquery_warehouse( """Build the selected BigQuery runtime after provider validation.""" if not isinstance(config, BigQueryWarehouseConfig): raise TypeError("BigQuery warehouse factory received the wrong configuration") - project = context.get("project") + project = context.get("catalog", context.get("project")) if not isinstance(project, str) or not project: - raise ValueError("BigQuery warehouse factory requires a project context") + raise ValueError("BigQuery warehouse factory requires a catalog context") return WarehouseRuntime( provider_id="bigquery", relation_codec=BigQueryRelationCodec(), diff --git a/src/dander/providers/bigquery/state.py b/src/dander/providers/bigquery/state.py index 28b17e13..fe291cf6 100644 --- a/src/dander/providers/bigquery/state.py +++ b/src/dander/providers/bigquery/state.py @@ -131,9 +131,13 @@ def _build_bigquery_state( ) -> StateRuntime: if not isinstance(config, BigQueryStateConfig): raise TypeError("BigQuery state factory received the wrong configuration") - project = _required_text(context, "project") - raw_dataset = _required_text(context, "raw_dataset") - metadata_dataset = _required_text(context, "metadata_dataset") + project = _required_text(context, "catalog", legacy="project") + raw_dataset = _required_text(context, "raw_namespace", legacy="raw_dataset") + metadata_dataset = _required_text( + context, + "metadata_namespace", + legacy="metadata_dataset", + ) project_pipeline = _required_bool(context, "project_pipeline") metadata_enabled = _required_bool(context, "metadata_enabled") control_dataset = metadata_dataset if project_pipeline else raw_dataset @@ -177,8 +181,13 @@ def _build_bigquery_state( ) -def _required_text(context: Mapping[str, object], name: str) -> str: - value = context.get(name) +def _required_text( + context: Mapping[str, object], + name: str, + *, + legacy: str | None = None, +) -> str: + value = context.get(name, context.get(legacy) if legacy is not None else None) if not isinstance(value, str) or not value: raise ValueError(f"BigQuery state runtime requires a non-empty {name}") return value diff --git a/src/dander/providers/dataplex/runtime.py b/src/dander/providers/dataplex/runtime.py index 9c948422..519946b0 100644 --- a/src/dander/providers/dataplex/runtime.py +++ b/src/dander/providers/dataplex/runtime.py @@ -23,10 +23,10 @@ def build_dataplex_catalog( """Build Dataplex only after its catalog provider is selected.""" if not isinstance(config, DataplexCatalogConfig): raise TypeError("Dataplex catalog factory received the wrong configuration") - project = context.get("project") + project = context.get("catalog", context.get("project")) location = context.get("location") if not isinstance(project, str) or not project: - raise ValueError("Dataplex catalog factory requires a project context") + raise ValueError("Dataplex catalog factory requires a catalog context") if not isinstance(location, str) or not location: raise ValueError("Dataplex catalog factory requires a location context") client: Any = context.get("client") diff --git a/src/dander/providers/postgresql/config.py b/src/dander/providers/postgresql/config.py index b5b3bd02..77d2405d 100644 --- a/src/dander/providers/postgresql/config.py +++ b/src/dander/providers/postgresql/config.py @@ -7,6 +7,8 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from dander.warehouse.contracts import RelationRef + _ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]{0,126}$") _SCHEMA_NAME = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") _AUTHORITY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{2,255}$") @@ -48,10 +50,16 @@ def validate_settings(self) -> PostgreSQLStateConfig: class PostgreSQLWarehouseConfig(BaseModel): """PostgreSQL warehouse connection, pool, and transaction limits.""" - model_config = ConfigDict(extra="forbid", frozen=True) + model_config = ConfigDict( + extra="forbid", + frozen=True, + populate_by_name=True, + serialize_by_alias=True, + ) provider: Literal["postgresql"] database: str + schema_name: str = Field(default="raw", alias="schema") dsn_env: str = "DANDER_POSTGRES_DSN" pool_min_size: int = Field(default=1, ge=1, le=20) pool_max_size: int = Field(default=5, ge=1, le=50) @@ -64,11 +72,29 @@ class PostgreSQLWarehouseConfig(BaseModel): def validate_settings(self) -> PostgreSQLWarehouseConfig: if not _SCHEMA_NAME.fullmatch(self.database): raise ValueError("database must be a safe lowercase PostgreSQL identifier") + if not _SCHEMA_NAME.fullmatch(self.schema_name): + raise ValueError("schema must be a safe lowercase PostgreSQL identifier") if not _ENVIRONMENT_NAME.fullmatch(self.dsn_env): raise ValueError("dsn_env must be an uppercase environment-variable name") if self.pool_min_size > self.pool_max_size: raise ValueError("pool_min_size must not exceed pool_max_size") return self + def raw_relation( + self, + name: str, + *, + compatibility_catalog: str | None, + compatibility_namespace: str | None, + default_namespace: str, + ) -> RelationRef: + """Translate the legacy namespace option into database/schema coordinates.""" + del compatibility_catalog, default_namespace + return RelationRef( + catalog=self.database, + namespace=compatibility_namespace or self.schema_name, + name=name, + ) + __all__ = ["PostgreSQLStateConfig", "PostgreSQLWarehouseConfig"] diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py index fc685696..6f4fe9e8 100644 --- a/src/dander/providers/postgresql/runtime.py +++ b/src/dander/providers/postgresql/runtime.py @@ -107,6 +107,7 @@ def build_transform_runner( *, graph_plan: object | None, build_models: bool, + raw_namespace: str = "raw", ) -> PostgreSQLTransformRunner | None: if graph_plan is not None: raise ValueError("PostgreSQL graph execution is not available") @@ -117,6 +118,7 @@ def build_transform_runner( pool=self.pool, target_fence=self.target_fence, timeouts=self.timeouts, + raw_namespace=raw_namespace, ) diff --git a/src/dander/providers/postgresql/transform.py b/src/dander/providers/postgresql/transform.py index de2bbbcf..6373ae59 100644 --- a/src/dander/providers/postgresql/transform.py +++ b/src/dander/providers/postgresql/transform.py @@ -19,7 +19,6 @@ TransformRunResult, ) from dander.transform.model import Materialization -from dander.warehouse import RelationRef if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -28,6 +27,7 @@ from dander.concurrency import OwnershipGuard, TargetFence from dander.providers.postgresql.fence import PostgreSQLTargetFence from dander.transform.config import GenericTestMetadata + from dander.warehouse import RelationRef PostgreSQLRow = dict[str, object] PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] @@ -52,11 +52,13 @@ def __init__( pool: PostgreSQLPool, target_fence: PostgreSQLTargetFence, timeouts: PostgreSQLTimeouts, + raw_namespace: str = "raw", ) -> None: self._database = database self._pool = pool self._target_fence = target_fence self._timeouts = timeouts + self._raw_namespace = raw_namespace def build( self, @@ -70,7 +72,8 @@ def build( raise TransformRunError("PostgreSQL hosted transforms require active lease ownership") project = TransformProject.load( models_dir, - project_id=self._database, + catalog=self._database, + raw_namespace=self._raw_namespace, target_dialect=SqlDialect.POSTGRES, ) models = project.ordered(selected) @@ -100,7 +103,8 @@ def test( """Evaluate assertions against already materialized PostgreSQL relations.""" project = TransformProject.load( models_dir, - project_id=self._database, + catalog=self._database, + raw_namespace=self._raw_namespace, target_dialect=SqlDialect.POSTGRES, ) models = project.ordered(selected) @@ -337,32 +341,24 @@ def _assertions_for_test( def _relation_ref(project: TransformProject, model: TransformModel) -> RelationRef: - return RelationRef( - catalog=project.project_id, - namespace=model.metadata.dataset, - name=model.name, - ) + return project.relation_ref_for_model(model) def _relation_identifier(model: TransformModel) -> sql.Identifier: - return sql.Identifier(model.metadata.dataset, model.name) + return sql.Identifier(model.metadata.namespace, model.name) def _relation_identifier_for_ref( project: TransformProject, reference: str, ) -> sql.Identifier: - if reference in project.models: - model = project.models[reference] - return _relation_identifier(model) - if reference.startswith("raw_"): - return sql.Identifier("raw", reference.removeprefix("raw_")) - raise TransformProjectError(f"Unknown model reference: {reference}") + relation = project.relation_ref_for_ref(reference) + return sql.Identifier(relation.namespace, relation.name) def _unique_index_name(model: TransformModel) -> str: digest = hashlib.sha256( - f"{model.metadata.dataset}.{model.name}:{','.join(model.metadata.unique_key)}".encode() + f"{model.metadata.namespace}.{model.name}:{','.join(model.metadata.unique_key)}".encode() ).hexdigest()[:10] return f"dander_uq_{model.name}"[:51] + f"_{digest}" diff --git a/src/dander/providers/postgresql/writer.py b/src/dander/providers/postgresql/writer.py index c346a6b0..3a3de60a 100644 --- a/src/dander/providers/postgresql/writer.py +++ b/src/dander/providers/postgresql/writer.py @@ -118,9 +118,10 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> def _validate_target(target: WriteTarget, *, database: str) -> None: - if target.project != database: + relation = target.relation_ref + if relation.catalog != database: raise PostgreSQLWriteError( - f"Writer database {database!r} does not match target catalog {target.project!r}" + f"Writer database {database!r} does not match target catalog {relation.catalog!r}" ) if not target.business_key: raise PostgreSQLWriteError("PostgreSQL SCD1 writes require a business key") @@ -130,7 +131,7 @@ def _validate_target(target: WriteTarget, *, database: str) -> None: if publication_fence is None: raise PostgreSQLWriteError("PostgreSQL hosted writes require a destination target fence") expected_target = ".".join(target.relation_ref.coordinates) - expected_fence_table = f"{target.dataset}.dander_target_commits" + expected_fence_table = f"{relation.namespace}.dander_target_commits" if ( publication_fence.target_id != expected_target or publication_fence.fence_table != expected_fence_table @@ -158,7 +159,9 @@ def _ensure_target( evolution: SchemaEvolution, ) -> None: connection.execute( - sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format(sql.Identifier(target.dataset)) + sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format( + sql.Identifier(target.relation_ref.namespace) + ) ) definitions = [ sql.SQL("{} {}{}").format( @@ -228,7 +231,7 @@ def _deployed_columns( "JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace " "WHERE namespace.nspname = %s AND relation.relname = %s " "AND attribute.attnum > 0 AND NOT attribute.attisdropped", - (target.dataset, target.table), + (target.relation_ref.namespace, target.relation_ref.name), ).fetchall() deployed: dict[str, tuple[str, bool]] = {} for row in rows: @@ -343,14 +346,17 @@ def _scd1_sql( def _target_identifier(target: WriteTarget) -> sql.Identifier: - return sql.Identifier(target.dataset, target.table) + return sql.Identifier(target.relation_ref.namespace, target.relation_ref.name) def _unique_index_name(target: WriteTarget) -> str: digest = hashlib.sha256( - f"{target.dataset}.{target.table}:{','.join(target.business_key)}".encode() + ( + f"{target.relation_ref.namespace}.{target.relation_ref.name}:" + f"{','.join(target.business_key)}" + ).encode() ).hexdigest()[:10] - prefix = f"dander_uq_{target.table}"[:51] + prefix = f"dander_uq_{target.relation_ref.name}"[:51] return f"{prefix}_{digest}" diff --git a/src/dander/runtime.py b/src/dander/runtime.py index 7c1f37c8..76e46be2 100644 --- a/src/dander/runtime.py +++ b/src/dander/runtime.py @@ -13,6 +13,7 @@ from uuid import uuid4 from dander.state.run_history import RunHistoryStore, RunStatus +from dander.warehouse import RelationRef from dander.writer.base import WriteField, WriteTarget if TYPE_CHECKING: @@ -61,8 +62,9 @@ def __init__( source: Source, writer: WritePattern, watermarks: WatermarkStore, - project: str, - dataset: str, + endpoint_relations: Mapping[str, RelationRef] | None = None, + project: str | None = None, + dataset: str | None = None, resume_from_watermark: bool = True, history: RunHistoryStore | None = None, batch_rows: int = 10_000, @@ -72,8 +74,6 @@ def __init__( self._source = source self._writer = writer self._watermarks = watermarks - self._project = project - self._dataset = dataset self._resume_from_watermark = resume_from_watermark self._history = history self._target_fence = target_fence @@ -81,6 +81,28 @@ def __init__( raise ValueError("batch_rows must be an integer from 1 to 100000") self._batch_rows = batch_rows configured = {endpoint.name for endpoint in source.config.endpoints} + if endpoint_relations is None: + if project is None or dataset is None: + raise ValueError( + "PipelineRunner requires endpoint_relations or legacy project/dataset" + ) + endpoint_relations = { + endpoint: RelationRef( + catalog=project, + namespace=dataset, + name=f"{source.config.name}_{endpoint}", + ) + for endpoint in configured + } + elif project is not None or dataset is not None: + raise ValueError( + "PipelineRunner cannot combine endpoint_relations with project/dataset" + ) + if missing := sorted(configured - endpoint_relations.keys()): + raise ValueError(f"Missing target relation for endpoint: {missing[0]!r}") + if unknown_relations := sorted(endpoint_relations.keys() - configured): + raise ValueError(f"Unknown target relation endpoint: {unknown_relations[0]!r}") + self._endpoint_relations = dict(endpoint_relations) if endpoint_names is None: self._endpoint_names = None else: @@ -168,9 +190,7 @@ def _run_endpoint( if ownership is not None: ownership.verify() target = WriteTarget( - project=self._project, - dataset=self._dataset, - table=f"{source_name}_{endpoint.name}", + relation=self._endpoint_relations[endpoint.name], business_key=tuple(endpoint.primary_key), schema=tuple(_write_field(field) for field in endpoint.raw_schema), fence=ownership.fence if ownership is not None else None, diff --git a/src/dander/transform/__init__.py b/src/dander/transform/__init__.py index 76ebfff7..27454e6d 100644 --- a/src/dander/transform/__init__.py +++ b/src/dander/transform/__init__.py @@ -1,5 +1,9 @@ """Transform engine — our owned dbt-replacement (ref() DAG, materializations, tests).""" +from __future__ import annotations + +from typing import TYPE_CHECKING + from dander.transform.config import ( ColumnMetadata, GenericTestMetadata, @@ -17,7 +21,20 @@ TransformProject, TransformProjectError, ) -from dander.transform.runner import BigQueryTransformRunner, TransformRunError, TransformRunResult +from dander.transform.result import TransformRunError, TransformRunResult + +if TYPE_CHECKING: + from dander.transform.runner import BigQueryTransformRunner + + +def __getattr__(name: str) -> object: + """Load the BigQuery runner only when that compatibility export is requested.""" + if name == "BigQueryTransformRunner": + from dander.transform.runner import BigQueryTransformRunner + + return BigQueryTransformRunner + raise AttributeError(name) + __all__ = [ "BigQueryTransformRunner", diff --git a/src/dander/transform/config.py b/src/dander/transform/config.py index dd347439..b7dda891 100644 --- a/src/dander/transform/config.py +++ b/src/dander/transform/config.py @@ -114,6 +114,11 @@ class ModelMetadata(BaseModel): unique_key: list[str] = Field(default_factory=list) incremental_cursor: str | None = None + @property + def namespace(self) -> str: + """Return the canonical namespace for the legacy model dataset field.""" + return self.dataset + @model_validator(mode="after") def validate_columns_and_tests(self) -> ModelMetadata: """Require unique columns and test references that resolve within the model.""" diff --git a/src/dander/transform/project.py b/src/dander/transform/project.py index a7d88e32..c4ed23e0 100644 --- a/src/dander/transform/project.py +++ b/src/dander/transform/project.py @@ -1,4 +1,4 @@ -"""Model discovery, dependency resolution, and safe BigQuery SQL compilation.""" +"""Model discovery, dependency resolution, and safe warehouse SQL compilation.""" from __future__ import annotations @@ -13,6 +13,7 @@ from dander.transform.config import ModelMetadata, TransformConfigError, load_model_metadata from dander.transform.dialects import PortableSqlError, parse_portable_query, render_portable_query from dander.transform.model import SqlDialect, parse_refs +from dander.warehouse import RelationRef if TYPE_CHECKING: from collections.abc import Iterable @@ -44,27 +45,37 @@ def name(self) -> str: class TransformProject: - """Discovered transform models and their project-scoped relation resolver.""" + """Discovered transform models and their catalog-scoped relation resolver.""" def __init__( self, *, - project_id: str, + catalog: str | None = None, + project_id: str | None = None, + raw_namespace: str = "raw", models: Iterable[TransformModel], target_dialect: SqlDialect | str = SqlDialect.BIGQUERY, ) -> None: + if catalog is None: + catalog = project_id + elif project_id is not None and project_id != catalog: + raise TransformProjectError("catalog and legacy project_id must match") + if catalog is None: + raise TransformProjectError("Warehouse catalog is required") try: self.target_dialect = SqlDialect(target_dialect) except ValueError as error: raise TransformProjectError(f"Unknown target SQL dialect: {target_dialect}") from error if self.target_dialect is SqlDialect.BIGQUERY: - if not _PROJECT_ID.fullmatch(project_id): + if not _PROJECT_ID.fullmatch(catalog): raise TransformProjectError("Invalid GCP project id") elif self.target_dialect is SqlDialect.POSTGRES: - if not _POSTGRESQL_CATALOG.fullmatch(project_id): + if not _POSTGRESQL_CATALOG.fullmatch(catalog): raise TransformProjectError("Invalid PostgreSQL database name") - elif not _IDENTIFIER.fullmatch(project_id): + elif not _IDENTIFIER.fullmatch(catalog): raise TransformProjectError("Invalid warehouse catalog identifier") + if not _IDENTIFIER.fullmatch(raw_namespace): + raise TransformProjectError("Invalid raw namespace identifier") indexed: dict[str, TransformModel] = {} for model in models: if model.name in indexed: @@ -72,23 +83,32 @@ def __init__( indexed[model.name] = model if not indexed: raise TransformProjectError("No SQL models were found") - self.project_id = project_id + self.catalog = catalog + self.raw_namespace = raw_namespace self.models = indexed self._validate_references() + @property + def project_id(self) -> str: + """Return the legacy BigQuery-named alias for the warehouse catalog.""" + return self.catalog + @classmethod def load( cls, models_dir: Path, *, - project_id: str, + catalog: str | None = None, + project_id: str | None = None, + raw_namespace: str = "raw", target_dialect: SqlDialect | str = SqlDialect.BIGQUERY, ) -> TransformProject: """Discover every SQL model beneath a directory and load its YAML sidecar. Args: models_dir: Root containing model SQL and YAML files. - project_id: Warehouse catalog used to qualify or validate compiled relations. + catalog: Warehouse catalog used to qualify or validate compiled relations. + project_id: Deprecated compatibility alias for ``catalog``. Returns: A validated transform project. @@ -121,7 +141,13 @@ def load( sql_path=sql_path, ) ) - return cls(project_id=project_id, models=discovered, target_dialect=target_dialect) + return cls( + catalog=catalog, + project_id=project_id, + raw_namespace=raw_namespace, + models=discovered, + target_dialect=target_dialect, + ) def ordered(self, selected: Iterable[str] | None = None) -> tuple[TransformModel, ...]: """Return selected models plus model dependencies in topological order. @@ -173,9 +199,15 @@ def relation_for_model( ) -> str: """Return a quoted fully-qualified output relation.""" dialect = self._resolved_target(target_dialect) - if dialect is SqlDialect.BIGQUERY: - return self._relation(model.metadata.dataset, model.name) - return self._relation_for_dialect(model.metadata.dataset, model.name, dialect=dialect) + return self._render_relation(self.relation_ref_for_model(model), dialect=dialect) + + def relation_ref_for_model(self, model: TransformModel) -> RelationRef: + """Return canonical coordinates for one materialized model.""" + return RelationRef( + catalog=self.catalog, + namespace=model.metadata.namespace, + name=model.name, + ) def relation_for_ref( self, @@ -185,14 +217,20 @@ def relation_for_ref( ) -> str: """Resolve a model reference or conventional `raw_
` source reference.""" dialect = self._resolved_target(target_dialect) + return self._render_relation(self.relation_ref_for_ref(reference), dialect=dialect) + + def relation_ref_for_ref(self, reference: str) -> RelationRef: + """Resolve a model or raw reference without provider-specific rendering.""" if reference in self.models: - return self.relation_for_model(self.models[reference], target_dialect=dialect) + return self.relation_ref_for_model(self.models[reference]) if reference.startswith(_RAW_PREFIX): table = reference.removeprefix(_RAW_PREFIX) if _IDENTIFIER.fullmatch(table): - if dialect is SqlDialect.BIGQUERY: - return self._relation("raw", table) - return self._relation_for_dialect("raw", table, dialect=dialect) + return RelationRef( + catalog=self.catalog, + namespace=self.raw_namespace, + name=table, + ) raise TransformProjectError(f"Unknown model reference: {reference}") def compile( @@ -272,21 +310,16 @@ def _resolved_target(self, target: SqlDialect | str | None) -> SqlDialect: except ValueError as error: raise TransformProjectError(f"Unknown target SQL dialect: {target}") from error - def _relation(self, dataset: str, table: str) -> str: - if not _IDENTIFIER.fullmatch(dataset) or not _IDENTIFIER.fullmatch(table): - raise TransformProjectError("Unsafe BigQuery relation identifier") - return f"`{self.project_id}.{dataset}.{table}`" - - def _relation_for_dialect(self, dataset: str, table: str, *, dialect: SqlDialect) -> str: - if not _IDENTIFIER.fullmatch(dataset) or not _IDENTIFIER.fullmatch(table): - raise TransformProjectError("Unsafe relation identifier") + def _render_relation(self, relation_ref: RelationRef, *, dialect: SqlDialect) -> str: + if dialect is SqlDialect.BIGQUERY: + return f"`{'.'.join(relation_ref.coordinates)}`" relation = exp.Table( - this=exp.to_identifier(table, quoted=True), - db=exp.to_identifier(dataset, quoted=True), + this=exp.to_identifier(relation_ref.name, quoted=True), + db=exp.to_identifier(relation_ref.namespace, quoted=True), catalog=( None if dialect is SqlDialect.POSTGRES - else exp.to_identifier(self.project_id, quoted=True) + else exp.to_identifier(relation_ref.catalog, quoted=True) ), ) return relation.sql(dialect=dialect.value) @@ -297,15 +330,7 @@ def _relation_coordinates_for_ref( *, dialect: SqlDialect, ) -> tuple[str, ...]: - if reference in self.models: - namespace = self.models[reference].metadata.dataset - name = reference - elif reference.startswith(_RAW_PREFIX) and _IDENTIFIER.fullmatch( - name := reference.removeprefix(_RAW_PREFIX) - ): - namespace = "raw" - else: - raise TransformProjectError(f"Unknown model reference: {reference}") + relation = self.relation_ref_for_ref(reference) if dialect is SqlDialect.POSTGRES: - return (namespace, name) - return (self.project_id, namespace, name) + return (relation.namespace, relation.name) + return relation.coordinates diff --git a/src/dander/transform/result.py b/src/dander/transform/result.py new file mode 100644 index 00000000..01fd085f --- /dev/null +++ b/src/dander/transform/result.py @@ -0,0 +1,18 @@ +"""Provider-neutral transform execution outcomes.""" + +from dataclasses import dataclass + + +class TransformRunError(RuntimeError): + """Raised when model execution or a generic assertion fails.""" + + +@dataclass(frozen=True) +class TransformRunResult: + """Summary of models materialized and assertions evaluated.""" + + models: tuple[str, ...] + assertions: int + + +__all__ = ["TransformRunError", "TransformRunResult"] diff --git a/src/dander/transform/runner.py b/src/dander/transform/runner.py index 5d0faa0b..14a4b126 100644 --- a/src/dander/transform/runner.py +++ b/src/dander/transform/runner.py @@ -17,6 +17,7 @@ TransformProject, TransformProjectError, ) +from dander.transform.result import TransformRunError, TransformRunResult if TYPE_CHECKING: from collections.abc import Iterable @@ -25,10 +26,6 @@ from dander.transform.config import GenericTestMetadata -class TransformRunError(RuntimeError): - """Raised when model execution or a generic assertion fails.""" - - class _QueryRow(Protocol): def __getitem__(self, key: str) -> object: """Return a projected result value by column name.""" @@ -49,14 +46,6 @@ def query( """Submit BigQuery Standard SQL.""" -@dataclass(frozen=True) -class TransformRunResult: - """Summary of models materialized and assertions evaluated.""" - - models: tuple[str, ...] - assertions: int - - @dataclass(frozen=True) class _Assertion: name: str @@ -74,8 +63,15 @@ class BigQueryTransformRunner: target_dialect = SqlDialect.BIGQUERY - def __init__(self, *, project: str, client: _BigQueryClient | None = None) -> None: + def __init__( + self, + *, + project: str, + raw_namespace: str = "raw", + client: _BigQueryClient | None = None, + ) -> None: self._project = project + self._raw_namespace = raw_namespace self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) def build( @@ -86,7 +82,11 @@ def build( ownership: OwnershipGuard | None = None, ) -> TransformRunResult: """Materialize selected models in dependency order, then run their assertions.""" - project = TransformProject.load(models_dir, project_id=self._project) + project = TransformProject.load( + models_dir, + catalog=self._project, + raw_namespace=self._raw_namespace, + ) models = project.ordered(selected) compiled = [(model, project.compile(model)) for model in models] statements = [ @@ -119,7 +119,11 @@ def test( selected: Iterable[str] | None = None, ) -> TransformRunResult: """Run assertions against already-materialized selected model relations.""" - project = TransformProject.load(models_dir, project_id=self._project) + project = TransformProject.load( + models_dir, + catalog=self._project, + raw_namespace=self._raw_namespace, + ) models = project.ordered(selected) assertions = [ assertion for model in models for assertion in _compile_assertions(project, model) diff --git a/src/dander/warehouse/__init__.py b/src/dander/warehouse/__init__.py index 332b5a71..a7e8edc5 100644 --- a/src/dander/warehouse/__init__.py +++ b/src/dander/warehouse/__init__.py @@ -14,6 +14,7 @@ RelationCodec, RelationRef, RelationSchema, + WarehouseCoordinateConfig, ) from dander.warehouse.runtime import ( PreparedWarehouseStatement, @@ -45,6 +46,7 @@ "RelationCodec", "RelationRef", "RelationSchema", + "WarehouseCoordinateConfig", "StagedArtifact", "StagingArtifactError", "StagingManifest", diff --git a/src/dander/warehouse/contracts.py b/src/dander/warehouse/contracts.py index 09b4c9c8..109bc7e7 100644 --- a/src/dander/warehouse/contracts.py +++ b/src/dander/warehouse/contracts.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -_CATALOG = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,254}$") +_CATALOG = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]{0,254}$") _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,254}$") _PROVIDER = re.compile(r"^[a-z][a-z0-9_]{0,62}$") _EXTENSION_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,126}$") @@ -221,6 +221,26 @@ def coordinates(self) -> tuple[str, str, str]: return (self.catalog, self.namespace, self.name) +@runtime_checkable +class WarehouseCoordinateConfig(Protocol): + """Resolve compatibility CLI inputs into canonical warehouse coordinates. + + Provider configuration owns the translation from its native vocabulary. Shared + orchestration only receives the resulting ``RelationRef`` values. + """ + + def raw_relation( + self, + name: str, + *, + compatibility_catalog: str | None, + compatibility_namespace: str | None, + default_namespace: str, + ) -> RelationRef: + """Return one raw relation for a configured endpoint.""" + ... + + @runtime_checkable class RelationCodec(Protocol): """Validate and render canonical coordinates for one warehouse provider.""" diff --git a/src/dander/warehouse/runtime.py b/src/dander/warehouse/runtime.py index 6fb33013..105a4f6c 100644 --- a/src/dander/warehouse/runtime.py +++ b/src/dander/warehouse/runtime.py @@ -97,6 +97,7 @@ def build_transform_runner( *, graph_plan: object | None, build_models: bool, + raw_namespace: str = "raw", ) -> WarehouseTransformRunner | None: """Return the selected provider runner or ``None`` when transforms are disabled.""" ... diff --git a/src/dander/writer/__init__.py b/src/dander/writer/__init__.py index 3ab3a653..25a3e30b 100644 --- a/src/dander/writer/__init__.py +++ b/src/dander/writer/__init__.py @@ -1,4 +1,9 @@ -"""BigQuery writer module: explicit, idempotent write patterns.""" +"""Provider-neutral writer contracts with lazy BigQuery compatibility exports.""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING from dander.writer.base import ( SchemaEvolution, @@ -8,21 +13,57 @@ WriteTarget, WriteTransport, ) -from dander.writer.bigquery import ( - BigQueryIncrementalWriter, - BigQueryReplaceWriter, - BigQueryScd1Writer, - BigQueryScd2Writer, - BigQuerySnapshotWriter, - BigQueryWriteError, + +if TYPE_CHECKING: + from dander.writer.bigquery import ( + BigQueryIncrementalWriter, + BigQueryReplaceWriter, + BigQueryScd1Writer, + BigQueryScd2Writer, + BigQuerySnapshotWriter, + BigQueryWriteError, + ) + from dander.writer.storage_write import ( + BigQueryPendingStreamBackend, + BigQueryStorageIncrementalWriter, + BigQueryStorageScd1Writer, + PendingStreamBackend, + ) + +_BIGQUERY_EXPORTS = frozenset( + { + "BigQueryIncrementalWriter", + "BigQueryReplaceWriter", + "BigQueryScd1Writer", + "BigQueryScd2Writer", + "BigQuerySnapshotWriter", + "BigQueryWriteError", + } ) -from dander.writer.storage_write import ( - BigQueryPendingStreamBackend, - BigQueryStorageIncrementalWriter, - BigQueryStorageScd1Writer, - PendingStreamBackend, +_STORAGE_WRITE_EXPORTS = frozenset( + { + "BigQueryPendingStreamBackend", + "BigQueryStorageIncrementalWriter", + "BigQueryStorageScd1Writer", + "PendingStreamBackend", + } ) + +def __getattr__(name: str) -> object: + """Load concrete BigQuery writers only when a compatibility export is requested.""" + module_name: str | None = None + if name in _BIGQUERY_EXPORTS: + module_name = "dander.writer.bigquery" + elif name in _STORAGE_WRITE_EXPORTS: + module_name = "dander.writer.storage_write" + if module_name is None: + raise AttributeError(name) + value = getattr(import_module(module_name), name) + globals()[name] = value + return value + + __all__ = [ "BigQueryIncrementalWriter", "BigQueryReplaceWriter", diff --git a/src/dander/writer/base.py b/src/dander/writer/base.py index 9d60fe0a..096e6f07 100644 --- a/src/dander/writer/base.py +++ b/src/dander/writer/base.py @@ -1,4 +1,4 @@ -"""BigQuery write patterns — explicit, idempotent load strategies. +"""Provider-neutral contracts for explicit, idempotent load strategies. dlt-sourced data may use dlt's own write dispositions; custom-sourced (enterprise) data and the transform engine's materializations use these patterns. Every pattern is safely re-runnable. See @@ -12,11 +12,13 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any +from dander.warehouse.contracts import RelationRef + if TYPE_CHECKING: from collections.abc import Iterable, Mapping from dander.concurrency import FencingToken, TargetFence - from dander.warehouse import CanonicalField, RelationRef, RelationSchema + from dander.warehouse import CanonicalField, RelationSchema class WriteMode(StrEnum): @@ -60,24 +62,63 @@ def to_canonical(self) -> CanonicalField: return canonical_field_from_bigquery(self) -@dataclass(frozen=True) +@dataclass(frozen=True, init=False) class WriteTarget: - """Fully-qualified BigQuery destination for a write.""" + """Canonical warehouse destination for a write. + + ``project``/``dataset``/``table`` remain accepted and readable for v1 + compatibility, but the stored coordinate is provider-neutral. + """ - project: str - dataset: str - table: str + relation: RelationRef business_key: tuple[str, ...] = field(default_factory=tuple) schema: tuple[WriteField, ...] = field(default_factory=tuple) fence: FencingToken | None = None publication_fence: TargetFence | None = None + def __init__( + self, + project: str | None = None, + dataset: str | None = None, + table: str | None = None, + business_key: tuple[str, ...] = (), + schema: tuple[WriteField, ...] = (), + fence: FencingToken | None = None, + publication_fence: TargetFence | None = None, + *, + relation: RelationRef | None = None, + ) -> None: + if relation is None: + if project is None or dataset is None or table is None: + raise TypeError("WriteTarget requires relation or project/dataset/table") + relation = RelationRef(catalog=project, namespace=dataset, name=table) + elif project is not None or dataset is not None or table is not None: + raise TypeError("WriteTarget cannot combine relation with project/dataset/table") + object.__setattr__(self, "relation", relation) + object.__setattr__(self, "business_key", business_key) + object.__setattr__(self, "schema", schema) + object.__setattr__(self, "fence", fence) + object.__setattr__(self, "publication_fence", publication_fence) + @property - def relation_ref(self) -> RelationRef: - """Return the target coordinates without rendering BigQuery SQL.""" - from dander.warehouse import RelationRef + def project(self) -> str: + """Return the legacy catalog alias used by BigQuery writers.""" + return self.relation.catalog - return RelationRef(catalog=self.project, namespace=self.dataset, name=self.table) + @property + def dataset(self) -> str: + """Return the legacy namespace alias used by BigQuery writers.""" + return self.relation.namespace + + @property + def table(self) -> str: + """Return the legacy relation-name alias used by BigQuery writers.""" + return self.relation.name + + @property + def relation_ref(self) -> RelationRef: + """Return the canonical target coordinates.""" + return self.relation @property def canonical_schema(self) -> RelationSchema: @@ -88,7 +129,7 @@ def canonical_schema(self) -> RelationSchema: class WritePattern(ABC): - """Loads a batch of records into a BigQuery target using one ``WriteMode``.""" + """Loads a batch of records into a warehouse target using one ``WriteMode``.""" mode: WriteMode supports_batched_writes = False diff --git a/tests/catalog/test_dataplex.py b/tests/catalog/test_dataplex.py index aea45345..0a858cf1 100644 --- a/tests/catalog/test_dataplex.py +++ b/tests/catalog/test_dataplex.py @@ -16,6 +16,7 @@ MetadataSpine, ) from dander.transform import TransformProject +from dander.warehouse import RelationRef if TYPE_CHECKING: from google.cloud import dataplex_v1 @@ -157,7 +158,16 @@ def test_request_rejects_asset_from_different_project() -> None: ) with pytest.raises(CatalogPublishError, match="different project"): - publisher.request_for(replace(_asset(), project="other-project-123")) + publisher.request_for( + replace( + _asset(), + relation_ref=RelationRef( + catalog="other-project-123", + namespace="staging", + name="stg_greenhouse__jobs", + ), + ) + ) def test_location_is_validated_before_client_creation() -> None: diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 6c5a189b..db9600e5 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -21,16 +21,14 @@ def test_greenhouse_dry_run_needs_no_credentials_or_gcp() -> None: "run", "greenhouse", "--dry-run", - "--project", - "unit-project", "--connectors-dir", str(_REPO_ROOT / "connectors"), ], ) assert result.exit_code == 0, result.output - assert "greenhouse_candidates" in result.output - assert "greenhouse_jobs" in result.output + assert ".raw.greenhouse_candidates" in result.output + assert ".raw.greenhouse_jobs" in result.output assert "SCD1" in result.output diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 2bb19f53..179b3392 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -4,10 +4,12 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Protocol, cast +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any, Protocol, cast import pytest from click import ClickException +from rich.console import Console from typer.testing import CliRunner import dander.cli.run_command as run_module @@ -16,6 +18,7 @@ from dander.runtime import EndpointRunResult, PipelineRunResult from dander.security import NoAuth from dander.state import StateCapabilities, StateMigration, StateRuntime +from dander.warehouse import RelationRef from dander.writer import SchemaEvolution if TYPE_CHECKING: @@ -47,6 +50,8 @@ class _ResolvedState(Protocol): project: str dataset: str metadata_dataset: str + state_catalog: str | None + state_namespace: str | None class _Migrator: @@ -142,8 +147,10 @@ def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: def build_state(resolved: _ResolvedState) -> StateRuntime: assert resolved.state_provider == "bigquery" assert resolved.project == "unit-project" - assert resolved.dataset == "raw" + assert resolved.dataset == "landing" assert resolved.metadata_dataset == "dander_meta" + assert resolved.state_catalog == "unit-project" + assert resolved.state_namespace == "landing" migrator = _Migrator() captured["migrator"] = migrator for name in ("history", "leases", "metadata", "watermarks"): @@ -186,7 +193,7 @@ def build_secrets(provider_id: str) -> _Built: "--project", "unit-project", "--dataset", - "raw", + "landing", "--config", str(_REPO_ROOT / "dander.yaml"), "--connectors-dir", @@ -200,7 +207,15 @@ def build_secrets(provider_id: str) -> _Built: assert "Dander run run-123" in result.output executor = cast("dict[str, object]", captured["executor"]) assert executor["pipeline_id"] == "greenhouse_jobs" - assert executor["project"] == "unit-project" + assert executor["catalog"] == "unit-project" + assert executor["raw_namespace"] == "landing" + assert executor["source_relations"] == { + "jobs": RelationRef( + catalog="unit-project", + namespace="landing", + name="greenhouse_job_board_jobs", + ) + } assert executor["selected_models"] == ("stg_greenhouse__jobs",) assert executor["build_models"] is True assert executor["history"] == captured["history"] @@ -209,8 +224,7 @@ def build_secrets(provider_id: str) -> _Built: assert executor["transform_runner"] == captured["transform"] ingestion = cast("_Built", executor["ingestion"]) - assert ingestion.kwargs["project"] == "unit-project" - assert ingestion.kwargs["dataset"] == "raw" + assert ingestion.kwargs["endpoint_relations"] == executor["source_relations"] assert ingestion.kwargs["resume_from_watermark"] is True assert ingestion.kwargs["batch_rows"] == 10_000 assert ingestion.kwargs["watermarks"] == captured["watermarks"] @@ -225,6 +239,85 @@ def build_secrets(provider_id: str) -> _Built: assert migrator.calls == 1 +@pytest.mark.parametrize( + ("sandbox", "guarded_free_tier", "message"), + [ + (True, False, "--sandbox is available only with a BigQuery warehouse"), + ( + False, + True, + "--guarded-free-tier is available only with a BigQuery warehouse", + ), + ], +) +def test_postgresql_rejects_bigquery_safety_before_external_clients( + monkeypatch: MonkeyPatch, + *, + sandbox: bool, + guarded_free_tier: bool, + message: str, +) -> None: + options = SimpleNamespace( + sandbox=sandbox, + guarded_free_tier=guarded_free_tier, + dry_run=False, + ) + resolved = SimpleNamespace( + warehouse_provider="postgresql", + publish_dataplex=False, + catalog_provider="none", + ) + monkeypatch.setattr(run_module, "_resolve_run", lambda _options: resolved) + + class _ForbiddenExternalClient: + def __init__(self) -> None: + raise AssertionError("provider guard must run before constructing a GCP client") + + monkeypatch.setattr(run_module, "SandboxDataset", _ForbiddenExternalClient) + monkeypatch.setattr(run_module, "GuardedFreeTierVerifier", _ForbiddenExternalClient) + + with pytest.raises(ClickException, match=message): + run_module.execute_run(cast("Any", options), console=Console()) + + +def test_postgresql_dataplex_publication_fails_before_executor( + monkeypatch: MonkeyPatch, +) -> None: + options = SimpleNamespace(sandbox=False, guarded_free_tier=False, dry_run=False) + resolved = SimpleNamespace( + warehouse_provider="postgresql", + publish_dataplex=True, + catalog_provider="dataplex", + ) + monkeypatch.setattr(run_module, "_resolve_run", lambda _options: resolved) + monkeypatch.setattr( + run_module, + "_build_executor", + lambda *_args, **_kwargs: pytest.fail("executor must not be constructed"), + ) + + with pytest.raises(ClickException, match="requires a BigQuery warehouse"): + run_module.execute_run(cast("Any", options), console=Console()) + + +def test_bigquery_state_location_stays_independent_from_postgresql_warehouse() -> None: + resolved = SimpleNamespace( + state_catalog="gcp-control-project", + state_namespace="dander_state", + metadata_namespace="dander_meta", + project_pipeline=True, + graph_plan=None, + catalog="postgres_database", + raw_namespace="landing", + ) + + context = run_module._state_runtime_context(cast("Any", resolved)) + + assert context["catalog"] == "gcp-control-project" + assert context["raw_namespace"] == "dander_state" + assert context["metadata_namespace"] == "dander_meta" + + def test_only_unsupported_postgresql_state_bigquery_warehouse_pair_fails_closed() -> None: with pytest.raises(ClickException, match="BigQuery write mode"): run_module._require_executable_state_pair( diff --git a/tests/integration/test_postgresql_native_profile.py b/tests/integration/test_postgresql_native_profile.py index b24c130b..5104ab75 100644 --- a/tests/integration/test_postgresql_native_profile.py +++ b/tests/integration/test_postgresql_native_profile.py @@ -17,7 +17,7 @@ from dander.providers import ProviderKind, ProviderRegistry, default_provider_registry from dander.runtime import PipelineRunner from dander.state import RunStatus, StateRuntime -from dander.warehouse import WarehouseRuntime +from dander.warehouse import RelationRef, WarehouseRuntime from dander.writer import SchemaEvolution if TYPE_CHECKING: @@ -85,7 +85,7 @@ def test_postgresql_native_profile_runs_and_replays( assert row is not None database = cast("str", row["database"]) suffix = uuid.uuid4().hex[:12] - raw_schema = "raw" + raw_schema = f"landing_{suffix}" source_name = f"fixture_{suffix}" raw_table = f"{source_name}_widgets" model_schema = f"models_{suffix}" @@ -97,7 +97,6 @@ def test_postgresql_native_profile_runs_and_replays( _write_model( tmp_path, source_name=source_name, - raw_schema=raw_schema, model_schema=model_schema, ) source = _FixtureSource(source_name) @@ -110,14 +109,20 @@ def test_postgresql_native_profile_runs_and_replays( source=source, writer=writer, watermarks=state.watermarks, - project=database, - dataset=raw_schema, + endpoint_relations={ + "widgets": RelationRef( + catalog=database, + namespace=raw_schema, + name=raw_table, + ) + }, batch_rows=1, target_fence=warehouse.target_fence, ) transform = warehouse.transforms.build_transform_runner( graph_plan=None, build_models=True, + raw_namespace=raw_schema, ) assert transform is not None executor = PipelineExecutor( @@ -125,7 +130,15 @@ def test_postgresql_native_profile_runs_and_replays( source_config=source.config, ingestion=ingestion, history=state.history, - project=database, + catalog=database, + raw_namespace=raw_schema, + source_relations={ + "widgets": RelationRef( + catalog=database, + namespace=raw_schema, + name=raw_table, + ) + }, models_dir=tmp_path, selected_models=("stg_fixture__widgets",), build_models=True, @@ -240,11 +253,10 @@ def _write_model( root: Path, *, source_name: str, - raw_schema: str, model_schema: str, ) -> None: (root / "stg_fixture__widgets.sql").write_text( - f"SELECT id, name, updated_at FROM {{{{ ref('{raw_schema}_{source_name}_widgets') }}}}", + f"SELECT id, name, updated_at FROM {{{{ ref('raw_{source_name}_widgets') }}}}", encoding="utf-8", ) (root / "stg_fixture__widgets.yml").write_text( diff --git a/tests/pipeline/test_compiler.py b/tests/pipeline/test_compiler.py index 4fd99e35..fecdb054 100644 --- a/tests/pipeline/test_compiler.py +++ b/tests/pipeline/test_compiler.py @@ -29,6 +29,7 @@ WriterConfig, ) from dander.pipeline.operations import OperationSpec +from dander.warehouse import RelationRef from dander.writer import ( BigQueryIncrementalWriter, BigQueryReplaceWriter, @@ -501,6 +502,44 @@ def test_compile_resolves_default_project() -> None: assert compiled.target.project == "fallback-project" +def test_compile_preserves_canonical_coordinates_without_bigquery_names() -> None: + graph = _linear_graph() + assert isinstance(graph.nodes[-1].config, TargetNodeConfig) + assert graph.nodes[-1].config.writer is not None + graph.nodes[-1].config.writer.destination = DestinationSpec.from_relation( + RelationRef( + catalog="warehouse_db", + namespace="analytics", + name="people", + ), + business_key=["person_id"], + ) + source = RelationRef( + catalog="warehouse_db", + namespace="landing", + name="people", + ) + + compiled = compile_target( + graph, + "target", + source_relations={"source": source}, + default_catalog="warehouse_db", + ) + + assert compiled.target.relation_ref == RelationRef( + catalog="warehouse_db", + namespace="analytics", + name="people", + ) + source_table = next( + table + for table in compiled.query_ast.find_all(exp.Table) + if table.name == "people" and table.db == "landing" + ) + assert source_table.catalog == "warehouse_db" + + @pytest.mark.parametrize( ("mode", "expected_type"), [ diff --git a/tests/pipeline/test_runtime_bridge.py b/tests/pipeline/test_runtime_bridge.py index e65fb5b1..aa9d7402 100644 --- a/tests/pipeline/test_runtime_bridge.py +++ b/tests/pipeline/test_runtime_bridge.py @@ -2,6 +2,8 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path from typing import TYPE_CHECKING @@ -10,11 +12,9 @@ from dander.concurrency import FencingToken from dander.ingestion import Endpoint, RawField, SourceConfig from dander.pipeline.graph import NodeField, PipelineGraph -from dander.pipeline.runtime import ( - BigQueryGraphRunner, - GraphRuntimeError, - plan_graph_execution, -) +from dander.pipeline.runtime import GraphRuntimeError, plan_graph_execution +from dander.providers.bigquery.graph import BigQueryGraphRunner +from dander.warehouse import RelationRef if TYPE_CHECKING: from collections.abc import Callable @@ -54,6 +54,25 @@ def verify(self) -> None: self.verifications += 1 +def test_neutral_graph_planning_does_not_import_bigquery_runtime() -> None: + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import dander.pipeline.runtime, dander.pipeline.compiler; " + "assert 'google.cloud.bigquery' not in sys.modules; " + "assert 'dander.providers.bigquery.graph' not in sys.modules" + ), + ], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + + def _source_config() -> SourceConfig: return SourceConfig( name="greenhouse_job_board", @@ -140,7 +159,13 @@ def test_plan_binds_only_declared_connector_endpoint() -> None: ) assert plan.bindings.endpoint_names == ("jobs",) - assert plan.bindings.source_relations == {"jobs": "unit-project.raw.greenhouse_job_board_jobs"} + assert plan.bindings.source_relations == { + "jobs": RelationRef( + catalog="unit-project", + namespace="raw", + name="greenhouse_job_board_jobs", + ) + } assert plan.targets[0].target.table == "graph_jobs" assert "`unit-project`.`raw`.`greenhouse_job_board_jobs`" in plan.targets[0].query diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index f9b90044..beaa5060 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -202,6 +202,7 @@ def test_version_two_resolves_native_postgresql_profile(tmp_path: Path) -> None: profile["warehouse"] = { "provider": "postgresql", "database": "dander_portable", + "schema": "raw", "dsn_env": "DANDER_POSTGRES_DSN", } profile["state"] = { @@ -234,6 +235,7 @@ def test_version_two_resolves_native_postgresql_profile(tmp_path: Path) -> None: assert resolved.warehouse_config == { "provider": "postgresql", "database": "dander_portable", + "schema": "raw", "dsn_env": "DANDER_POSTGRES_DSN", "pool_min_size": 1, "pool_max_size": 5, diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index 90ec5d4f..91081746 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -11,6 +11,7 @@ from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError from dander.providers import ProviderKind, default_provider_registry +from dander.providers.bigquery.config import BigQueryWarehouseConfig from dander.providers.bigquery.fence import BigQueryTargetFence from dander.telemetry import TelemetryOperation from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime @@ -59,6 +60,21 @@ def test_default_registry_loads_bigquery_runtime_only_after_selection() -> None: ) +def test_bigquery_config_translates_legacy_cli_coordinates() -> None: + relation = BigQueryWarehouseConfig(provider="bigquery").raw_relation( + "accounts", + compatibility_catalog="unit-project", + compatibility_namespace="landing", + default_namespace="raw", + ) + + assert relation == RelationRef( + catalog="unit-project", + namespace="landing", + name="accounts", + ) + + def test_bigquery_runtime_exposes_codec_schema_fence_and_telemetry() -> None: runtime = _runtime() relation = RelationRef(catalog="unit-project", namespace="raw", name="records") diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py index dcc9fc27..3d1ab95d 100644 --- a/tests/providers/test_postgresql_warehouse_runtime.py +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -16,6 +16,7 @@ from dander.concurrency import FencingToken from dander.providers import ProviderKind, default_provider_registry +from dander.providers.postgresql.config import PostgreSQLWarehouseConfig from dander.providers.postgresql.writer import PostgreSQLWriteError from dander.telemetry import TelemetryOperation from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime @@ -46,6 +47,7 @@ def test_postgresql_warehouse_registration_is_lazy_and_explicit() -> None: assert config.model_dump(mode="json") == { "provider": "postgresql", "database": "dander_test", + "schema": "raw", "dsn_env": "DANDER_TEST_POSTGRES_DSN", "pool_min_size": 1, "pool_max_size": 5, @@ -55,6 +57,29 @@ def test_postgresql_warehouse_registration_is_lazy_and_explicit() -> None: "idle_transaction_timeout_ms": 60_000, } + +def test_postgresql_config_uses_database_and_schema_coordinates_directly() -> None: + config = PostgreSQLWarehouseConfig( + provider="postgresql", + database="warehouse_db", + schema_name="landing", + ) + + relation = config.raw_relation( + "accounts", + compatibility_catalog="ignored-gcp-project", + compatibility_namespace=None, + default_namespace="ignored_bigquery_dataset", + ) + + assert relation == RelationRef( + catalog="warehouse_db", + namespace="landing", + name="accounts", + ) + + +def test_postgresql_sslmode_requires_encrypted_connections() -> None: from dander.providers.postgresql.runtime import _required_sslmode assert _required_sslmode("postgresql://example.invalid/db") == "require" diff --git a/tests/test_executor.py b/tests/test_executor.py index ddde0f2a..8cfbfc99 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -20,6 +20,7 @@ RunStatus, ) from dander.transform import TransformRunResult +from dander.warehouse import RelationRef if TYPE_CHECKING: from collections.abc import Iterable @@ -192,6 +193,8 @@ def _executor( metadata: _Metadata, fail_transform: bool = False, leases: LeaseStore | None = None, + raw_namespace: str = "raw", + source_relations: dict[str, RelationRef] | None = None, ) -> PipelineExecutor: source = SourceConfig( name="example", @@ -212,6 +215,8 @@ def _executor( ingestion=_Ingestion(), history=history, project="valid-project-123", + raw_namespace=raw_namespace, + source_relations=source_relations, models_dir=models_dir, selected_models=("stg_widgets",), build_models=True, @@ -258,6 +263,34 @@ def test_executor_preserves_launcher_supplied_run_id(tmp_path: Path) -> None: assert history.started == ("launcher-run-42", "example", "example_pipeline") +def test_executor_keeps_custom_raw_namespace_in_models_and_metadata(tmp_path: Path) -> None: + _models(tmp_path) + metadata = _Metadata() + source_relation = RelationRef( + catalog="valid-project-123", + namespace="landing", + name="example_widgets", + ) + + _executor( + tmp_path, + history=_History(), + metadata=metadata, + raw_namespace="landing", + source_relations={"widgets": source_relation}, + ).execute() + + assert metadata.manifest is not None + source = metadata.manifest["source"] + assert isinstance(source, dict) + endpoints = source["endpoints"] + assert isinstance(endpoints, list) + assert endpoints[0]["relation"] == "valid-project-123.landing.example_widgets" + assets = metadata.manifest["assets"] + assert isinstance(assets, list) + assert assets[0]["upstream_relations"] == ["valid-project-123.landing.example_widgets"] + + def test_executor_marks_transform_failure_without_claiming_ingestion_only_success( tmp_path: Path, caplog: pytest.LogCaptureFixture, diff --git a/tests/test_runtime.py b/tests/test_runtime.py index c0afedd6..5f790c0b 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -13,12 +13,13 @@ from dander.ingestion.source import Endpoint, RawField, Source, SourceConfig from dander.runtime import PipelineRunner, RawSchemaError, WatermarkConflictError from dander.state import LeaseLostError, RunHistoryStore, RunStage, RunStatus, WatermarkStore +from dander.warehouse import RelationRef from dander.writer import WriteMode, WritePattern, WriteTarget if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping - from dander.warehouse import PreparedWarehouseStatement, RelationRef + from dander.warehouse import PreparedWarehouseStatement class _Source(Source): @@ -304,6 +305,32 @@ def test_runner_commits_maximum_cursor_after_write() -> None: assert result.endpoints[0].affected == 2 +def test_runner_preserves_endpoint_relation_without_provider_reinterpretation() -> None: + captured: list[RelationRef] = [] + + class RelationWriter(_Writer): + def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> int: + captured.append(target.relation_ref) + return super().write(records, target) + + events: list[str] = [] + relation = RelationRef( + catalog="warehouse_db", + namespace="landing", + name="example_widgets", + ) + runner = PipelineRunner( + source=_Source(events), + writer=RelationWriter(events), + watermarks=_Watermarks(events), + endpoint_relations={"widgets": relation}, + ) + + runner.run() + + assert captured == [relation] + + def test_runner_claims_required_destination_before_extraction() -> None: events: list[str] = [] runner = PipelineRunner( diff --git a/tests/transform/test_project.py b/tests/transform/test_project.py index bb77a675..d94c5040 100644 --- a/tests/transform/test_project.py +++ b/tests/transform/test_project.py @@ -8,6 +8,7 @@ import pytest from dander.transform import TransformProject, TransformProjectError +from dander.warehouse import RelationRef def _write_model( @@ -126,6 +127,33 @@ def test_postgresql_project_uses_database_local_relations_by_default(tmp_path: P assert '"raw"."fixture"' in project.compile(model) +def test_postgresql_project_preserves_database_and_custom_raw_schema(tmp_path: Path) -> None: + _write_model( + tmp_path, + "portable_model", + "SELECT id FROM {{ ref('raw_fixture') }}", + dialect="portable", + ) + project = TransformProject.load( + tmp_path, + catalog="warehouse_db", + raw_namespace="landing", + target_dialect="postgres", + ) + + assert project.relation_ref_for_ref("raw_fixture") == RelationRef( + catalog="warehouse_db", + namespace="landing", + name="fixture", + ) + assert project.relation_ref_for_model(project.models["portable_model"]) == RelationRef( + catalog="warehouse_db", + namespace="staging", + name="portable_model", + ) + assert '"landing"."fixture"' in project.compile(project.models["portable_model"]) + + def test_unknown_reference_fails_during_project_load(tmp_path: Path) -> None: _write_model(tmp_path, "broken", "SELECT id FROM {{ ref('missing') }}") From c65f53c87224aa130b6de0d245125d911d5df87b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 05:49:53 -0400 Subject: [PATCH 057/163] fix: batch BigQuery run-history migrations --- HANDOFF.md | 39 +++---- src/dander/state/run_history.py | 36 +++++- .../providers/test_bigquery_state_runtime.py | 33 ++++++ tests/state/test_run_history.py | 109 ++++++++++++++++++ 4 files changed, 191 insertions(+), 26 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 3d4b44a0..563a7049 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,36 @@ ## Finished -- Made `RelationRef(catalog, namespace, name)` the dependency-light coordinate passed through run, - graph, transform, metadata, and writer orchestration. -- Moved BigQuery project/dataset and PostgreSQL database/schema translation to provider boundaries. -- Kept legacy CLI and Python compatibility inputs while translating them before neutral execution. -- Fixed non-default raw namespaces so ingestion, transforms, and metadata use the same relation. -- Kept mixed-provider state coordinates independent and rejected BigQuery-only safety/Dataplex - operations before external clients are created. +- Forward-ported the accepted `0.7.1` BigQuery run-history migration fix to current main. +- Read the immediate table schema before considering additive history columns. +- Skip no-op migration DDL when the schema is already current. +- Batch all genuinely missing nullable columns into one race-safe `ALTER TABLE` statement. +- Preserve fail-closed behavior when the single migration statement fails. ## Try It -Run an existing BigQuery command unchanged, or select a PostgreSQL profile. Neutral internals now -receive canonical relations; provider configuration still uses its native vocabulary. +Run `uv run pytest tests/state/test_run_history.py -q`; current schemas emit no ALTER and sparse +legacy schemas emit one bounded additive migration. ## Checks -- All 1,048 tests passed with PostgreSQL 15; Ruff and strict mypy passed. -- Wheel/sdist build, inspection, outside-checkout installs, generation, and validation passed. -- Source-free container conformance, proof assets, dependency audit, and Helm checks passed. -- Repository and generated-project GCP/AWS Terraform validation and module tests passed. +- Full suite passed (`1,038` passed, `13` environment-selected skips); Ruff lint/format and strict + mypy passed. +- Stable `0.7.1` candidate protected CI and retained GCP acceptance passed before this forward-port. +- Current main already locks GitPython `3.1.58`; no dependency change was needed. ## Decisions -- Canonical relations are the neutral contract; provider-shaped names are compatibility aliases. -- Provider configs own native-to-canonical translation; provider runtimes own translation back. -- Serialized v1/GCP catalog output and resource identities remain unchanged. +- Keep the functional fix identical to the accepted maintenance release. +- Do not mix release metadata or provider implementation work into the forward-port. +- Preserve immediate schema failure propagation rather than masking migration errors. ## Remaining -- Run protected Linux CI, image/secret scans, and merge the focused PR if clean. -- Resume Snowflake and Redshift runtime work only after this cleanup merges. +- Run protected Linux CI and merge the focused PR if clean. +- Rebase the provider PR stack on the corrected main before merging it. ## Review First -- `src/dander/cli/run_command.py` -- `src/dander/runtime.py` -- `src/dander/pipeline/runtime.py` +- `src/dander/state/run_history.py` +- `tests/state/test_run_history.py` diff --git a/src/dander/state/run_history.py b/src/dander/state/run_history.py index 8ffb52fe..010f544d 100644 --- a/src/dander/state/run_history.py +++ b/src/dander/state/run_history.py @@ -124,6 +124,21 @@ def query( ) -> _Job: """Run a parameterized statement.""" + def get_table(self, table: str) -> _Table: + """Return the current table schema.""" + + +class _SchemaField(Protocol): + @property + def name(self) -> str: + """Return the warehouse column name.""" + + +class _Table(Protocol): + @property + def schema(self) -> Iterable[_SchemaField]: + """Return the current warehouse schema.""" + class _Row(Protocol): def __getitem__(self, key: str) -> object: @@ -293,7 +308,7 @@ def _ensure_table(self) -> None: "failure_stage STRING, failure_code STRING, failure_summary STRING) " "CLUSTER BY pipeline_id, status" ).result() - for column, data_type in ( + additive_columns = ( ("pipeline_id", "STRING"), ("stage", "STRING"), ("models", "INT64"), @@ -302,10 +317,21 @@ def _ensure_table(self) -> None: ("failure_stage", "STRING"), ("failure_code", "STRING"), ("failure_summary", "STRING"), - ): - self._client.query( - f"ALTER TABLE `{self._table}` ADD COLUMN IF NOT EXISTS {column} {data_type}" - ).result() + ) + existing_columns = { + field.name.casefold() for field in self._client.get_table(self._table).schema + } + missing_columns = tuple( + (column, data_type) + for column, data_type in additive_columns + if column.casefold() not in existing_columns + ) + if missing_columns: + clauses = ", ".join( + f"ADD COLUMN IF NOT EXISTS {column} {data_type}" + for column, data_type in missing_columns + ) + self._client.query(f"ALTER TABLE `{self._table}` {clauses}").result() self._ready = True def recent( diff --git a/tests/providers/test_bigquery_state_runtime.py b/tests/providers/test_bigquery_state_runtime.py index fb144944..3d2358fd 100644 --- a/tests/providers/test_bigquery_state_runtime.py +++ b/tests/providers/test_bigquery_state_runtime.py @@ -30,6 +30,35 @@ def result(self) -> list[dict[str, Any]]: return self._rows +class _Field: + def __init__(self, name: str) -> None: + self.name = name + + +class _Table: + schema = tuple( + _Field(name) + for name in ( + "run_id", + "pipeline_id", + "source_name", + "status", + "stage", + "started_at", + "finished_at", + "endpoints", + "extracted", + "affected", + "models", + "assertions", + "assets", + "failure_stage", + "failure_code", + "failure_summary", + ) + ) + + class _Client: def __init__(self, *, version: int = 0) -> None: self.version = version @@ -44,6 +73,10 @@ def query(self, query: str, *, job_config: object | None = None) -> _Job: self.version = 1 return _Job() + def get_table(self, table: str) -> _Table: + assert table == "unit-project.dander_meta._dander_runs" + return _Table() + def _build(client: _Client, *, metadata_enabled: bool = True) -> StateRuntime: registry = default_provider_registry() diff --git a/tests/state/test_run_history.py b/tests/state/test_run_history.py index c7c3fdd0..18d64d0f 100644 --- a/tests/state/test_run_history.py +++ b/tests/state/test_run_history.py @@ -6,6 +6,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING +import pytest + from dander.state import BigQueryRunHistoryStore, RunStage, RunStatus, SqliteRunHistoryStore if TYPE_CHECKING: @@ -127,7 +129,98 @@ def result(self) -> tuple[object, ...]: @dataclass class _BigQueryClient: queries: list[str] = field(default_factory=list) + existing_columns: set[str] = field( + default_factory=lambda: { + "run_id", + "pipeline_id", + "source_name", + "status", + "stage", + "started_at", + "finished_at", + "endpoints", + "extracted", + "affected", + "models", + "assertions", + "assets", + "failure_stage", + "failure_code", + "failure_summary", + } + ) + + def query( + self, + query: str, + *, + job_config: bigquery.QueryJobConfig | None = None, + ) -> _QueryJob: + del job_config + self.queries.append(query) + return _QueryJob() + + def get_table(self, table: str) -> _BigQueryTable: + assert table == "proof-project.dander_meta._dander_runs" + return _BigQueryTable( + schema=tuple(_BigQuerySchemaField(name=name) for name in self.existing_columns) + ) + + +@dataclass(frozen=True) +class _BigQuerySchemaField: + name: str + + +@dataclass(frozen=True) +class _BigQueryTable: + schema: tuple[_BigQuerySchemaField, ...] + + +def test_bigquery_history_current_schema_emits_no_alter() -> None: + client = _BigQueryClient() + store = BigQueryRunHistoryStore( + project="proof-project", + dataset="dander_meta", + client=client, + ) + + store.start("run-1", "greenhouse", pipeline_id="greenhouse_jobs") + first_queries = tuple(client.queries) + store.start("run-2", "greenhouse", pipeline_id="greenhouse_jobs") + + assert len(first_queries) == 2 + assert first_queries[0].startswith("CREATE TABLE IF NOT EXISTS") + assert first_queries[1].startswith("INSERT INTO") + assert len(client.queries) == len(first_queries) + 1 + assert client.queries[-1].startswith("INSERT INTO") + + +def test_bigquery_history_batches_sparse_legacy_schema_migration() -> None: + client = _BigQueryClient(existing_columns={"RUN_ID", "SOURCE_NAME", "STATUS"}) + store = BigQueryRunHistoryStore( + project="proof-project", + dataset="dander_meta", + client=client, + ) + + store.start("run-1", "greenhouse", pipeline_id="greenhouse_jobs") + + alters = [query for query in client.queries if query.startswith("ALTER TABLE")] + assert len(alters) == 1 + assert "ADD COLUMN IF NOT EXISTS pipeline_id STRING" in alters[0] + assert "ADD COLUMN IF NOT EXISTS stage STRING" in alters[0] + assert "ADD COLUMN IF NOT EXISTS failure_summary STRING" in alters[0] + assert alters[0].count("ADD COLUMN IF NOT EXISTS") == 8 + +class _FailingQueryJob(_QueryJob): + def result(self) -> tuple[object, ...]: + raise RuntimeError("schema migration failed") + + +@dataclass +class _FailingBigQueryClient(_BigQueryClient): def query( self, query: str, @@ -136,9 +229,25 @@ def query( ) -> _QueryJob: del job_config self.queries.append(query) + if query.startswith("ALTER TABLE"): + return _FailingQueryJob() return _QueryJob() +def test_bigquery_history_migration_propagates_schema_failure() -> None: + client = _FailingBigQueryClient(existing_columns={"run_id"}) + store = BigQueryRunHistoryStore( + project="proof-project", + dataset="dander_meta", + client=client, + ) + + with pytest.raises(RuntimeError, match="schema migration failed"): + store.start("run-1", "greenhouse", pipeline_id="greenhouse_jobs") + + assert len([query for query in client.queries if query.startswith("ALTER TABLE")]) == 1 + + def test_bigquery_history_can_read_without_creating_or_altering_tables() -> None: client = _BigQueryClient() store = BigQueryRunHistoryStore( From 6b480050625c4e5c6c6b6dab4beacf1f189741a4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sat, 8 Aug 2026 23:33:37 -0400 Subject: [PATCH 058/163] feat: add experimental Snowflake warehouse --- CHANGELOG.md | 4 + HANDOFF.md | 39 +- docs/compatibility-matrix.md | 2 + docs/decisions.md | 11 + docs/known-limitations.md | 11 +- docs/snowflake.md | 59 ++ pyproject.toml | 1 + src/dander/project/config.py | 4 +- src/dander/project/portable_config.py | 3 +- src/dander/providers/defaults.py | 9 + src/dander/providers/snowflake/__init__.py | 13 + src/dander/providers/snowflake/config.py | 124 ++++ src/dander/providers/snowflake/fence.py | 233 ++++++++ src/dander/providers/snowflake/runtime.py | 294 +++++++++ src/dander/providers/snowflake/session.py | 94 +++ src/dander/providers/snowflake/writer.py | 563 ++++++++++++++++++ src/dander/runtime-compatibility.json | 12 + tests/cli/test_run_command.py | 85 +++ tests/project/test_portable_config.py | 2 +- .../test_snowflake_warehouse_runtime.py | 563 ++++++++++++++++++ tests/test_compatibility.py | 6 +- tickets/DANDER-104-snowflake-warehouse.md | 33 + 22 files changed, 2138 insertions(+), 27 deletions(-) create mode 100644 docs/snowflake.md create mode 100644 src/dander/providers/snowflake/__init__.py create mode 100644 src/dander/providers/snowflake/config.py create mode 100644 src/dander/providers/snowflake/fence.py create mode 100644 src/dander/providers/snowflake/runtime.py create mode 100644 src/dander/providers/snowflake/session.py create mode 100644 src/dander/providers/snowflake/writer.py create mode 100644 tests/providers/test_snowflake_warehouse_runtime.py create mode 100644 tickets/DANDER-104-snowflake-warehouse.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a01cb6b0..4f2352a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ capabilities enter through the next minor release. ### Added +- Add an experimental Snowflake warehouse adapter with native database/schema coordinates, + bounded Parquet staging, scalar SCD1 `COPY`/`MERGE`, additive nullable schema evolution, + destination-side fencing, sanitized telemetry, and process-death-safe temporary staging. This + is local conformance evidence, not a Snowflake support claim. - Add a separately packaged AWS Terraform stack for immutable ECR images, non-root Fargate task definitions, distinct task/execution roles, paused-aware EventBridge schedules, and encrypted failure routing. A Standard Step Functions controller enforces one absolute deadline, retries diff --git a/HANDOFF.md b/HANDOFF.md index 563a7049..de679d6d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,39 @@ ## Finished -- Forward-ported the accepted `0.7.1` BigQuery run-history migration fix to current main. -- Read the immediate table schema before considering additive history columns. -- Skip no-op migration DDL when the schema is already current. -- Batch all genuinely missing nullable columns into one race-safe `ALTER TABLE` statement. -- Preserve fail-closed behavior when the single migration statement fails. +- Added an experimental Snowflake provider with native database/schema coordinates. +- Added bounded Parquet `PUT`/`COPY`, scalar SCD1 merge, schema evolution, and telemetry. +- Explicitly committed fence claims and fenced publication/load history in Snowflake transactions. +- Made remote staging session-temporary and rejected oversized parts before upload. +- Kept Snowflake experimental with transforms, graphs, other modes, and live support excluded. ## Try It -Run `uv run pytest tests/state/test_run_history.py -q`; current schemas emit no ALTER and sparse -legacy schemas emit one bounded additive migration. +Configure a version 2 Snowflake profile from `docs/snowflake.md`, select `build_models: false`, and +use either BigQuery or PostgreSQL state. No live Snowflake proof is claimed. ## Checks -- Full suite passed (`1,038` passed, `13` environment-selected skips); Ruff lint/format and strict - mypy passed. -- Stable `0.7.1` candidate protected CI and retained GCP acceptance passed before this forward-port. -- Current main already locks GitPython `3.1.58`; no dependency change was needed. +- Full suite passed: 1,061 tests with PostgreSQL 15; Ruff and strict mypy also passed. +- Wheel/sdist inspection, source-free installs, generated-project validation, and runtime-all import passed. +- Container build/conformance, Terraform/AWS tests, Helm validation, and dependency audit passed. +- Retained stage-zero plan: no changes; no cloud apply or mutation occurred. +- Retained platform plan matched clean `origin/main`: 2 adds/5 updates already pending, no deletes. ## Decisions -- Keep the functional fix identical to the accepted maintenance release. -- Do not mix release metadata or provider implementation work into the forward-port. -- Preserve immediate schema failure propagation rather than masking migration errors. +- Snowflake translates native database/schema values into canonical `RelationRef` coordinates. +- Explicit Parquet logical/binary settings are part of every effective `COPY` file format. +- Scalar SCD1 is the smallest honest experimental slice; other capabilities remain fail-closed. ## Remaining -- Run protected Linux CI and merge the focused PR if clean. -- Rebase the provider PR stack on the corrected main before merging it. +- Review the known retained-platform baseline drift before any future apply. +- Open the focused draft PR and require protected CI. +- Merge only after review; do not claim live Snowflake qualification. ## Review First -- `src/dander/state/run_history.py` -- `tests/state/test_run_history.py` +- `src/dander/providers/snowflake/writer.py` +- `src/dander/providers/snowflake/fence.py` +- `src/dander/providers/snowflake/config.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index d04e1e5c..2e0047de 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -14,8 +14,10 @@ before provider construction when a pair is absent or marked `unsupported`. |---|---|---|---| | BigQuery | BigQuery | supported | Released GCP profile and BigQuery regression baseline | | BigQuery | PostgreSQL | experimental | Local cross-backend token/fence publication proof | +| BigQuery | Snowflake | experimental | Local staged-SCD1, schema, replay, and fence conformance | | PostgreSQL | BigQuery | unsupported | Not every BigQuery write mode uses destination fencing | | PostgreSQL | PostgreSQL | experimental | Local native profile, state/warehouse conformance, and benchmarks | +| PostgreSQL | Snowflake | experimental | Local staged-SCD1 and destination-fence conformance | `experimental` means the code path is executable and tested locally; it is not a hosted support claim. The packaged `runtime-capabilities.json` remains the supported launcher/profile boundary. diff --git a/docs/decisions.md b/docs/decisions.md index a9a4bd78..f0f6f179 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -866,3 +866,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Correctness:** A selected raw namespace is shared by ingestion, transforms, and metadata; canonical source coordinates are preserved rather than reconstructed from BigQuery defaults. State backends keep their own control catalog and namespace in mixed-provider compositions. + +## 2026-08-08 — Snowflake begins as an explicitly experimental scalar SCD1 slice + +- **Coordinates:** Snowflake configuration translates native `database` and `schema` values into + canonical relations. The neutral runtime never treats them as a GCP project or dataset. +- **Publication:** Each bounded batch becomes checksummed Parquet parts, loads through a temporary + stage and temporary table, and merges behind the exact destination fencing token. Parquet logical + types and binary values use explicit `COPY` settings; oversized singleton parts fail before + remote staging. +- **Boundary:** Only scalar SCD1 ingestion is admitted. Transforms, graphs, semi-structured types, + other write modes, and a support claim remain blocked on their own implementation and live proof. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index fa2be3d4..2e150a78 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -9,11 +9,12 @@ evaluate these limits before using it for an unattended system containing busine verification are implemented but not live-qualified. The runtime bridge copies one temporary ECS task-role session into process memory and therefore caps a Fargate projection at one hour; renewable credentials and live parity remain later gates. -- Provider package extras and the full runtime image contain planned provider SDK dependencies, - but do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. A shared - bounded Parquet staging contract exists for future Snowflake/Redshift bulk loaders, but no remote - upload or warehouse adapter uses it yet. PostgreSQL - state and warehouse execution are implemented and locally conformance-tested. A packaged Helm +- Provider package extras and the full runtime image do not by themselves qualify Snowflake, + Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar SCD1 adapter with + bounded Parquet upload, temporary remote staging, schema checks, and destination fencing. It has + no live Snowflake qualification, transforms, graph execution, semi-structured types, or other + write modes. Redshift has no warehouse adapter yet. PostgreSQL state and warehouse execution are + implemented and locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/snowflake.md b/docs/snowflake.md new file mode 100644 index 00000000..56859ec7 --- /dev/null +++ b/docs/snowflake.md @@ -0,0 +1,59 @@ +# Experimental Snowflake warehouse + +Snowflake is an experimental warehouse adapter, not a supported Dander profile. The current slice +exists to prove the provider boundary with native database/schema coordinates and a real bounded +bulk path before transforms or additional write modes are added. + +## Configuration + +Select Snowflake in a version 2 platform profile. Configuration stores only credential references: + +```yaml +platforms: + snowflake_test: + warehouse: + provider: snowflake + account: myorg-myaccount + user: DANDER_USER + database: DANDER_TEST + schema: RAW + warehouse: DANDER_WH + role: DANDER_ROLE + auth: + method: oauth + token_env: DANDER_SNOWFLAKE_OAUTH_TOKEN + state: + provider: postgresql + authority_id: postgresql:snowflake-test + catalog: + provider: none + secrets: + provider: environment +``` + +Key-pair authentication is also available by setting `method: key_pair` and naming the environment +variable that contains the private-key file path. The optional password is another environment +variable reference. Do not put a token, private key, password, or connection string in YAML. + +## Current contract + +- `database + schema + relation` become one canonical `RelationRef` without GCP aliases. +- Runtime batches are written as checksummed, compressed Parquet parts and uploaded with `PUT`. +- `COPY` preserves Parquet logical and binary types explicitly. +- One session-temporary stage and table contain each batch; normal cleanup is immediate and session + termination removes them after process death. +- SCD1 publication and load-history recording occur in the same transaction as the exact + destination fencing-token touch. +- Only declared nullable columns may be added automatically. Extra columns, required additions, + type drift, nullability drift, malformed rows, and oversized singleton parts fail closed. + +The Snowflake role needs usage on its database and warehouse plus permission to create and operate +schemas, tables, and temporary stages in the selected namespace. Use a dedicated disposable role +until a live least-privilege profile is qualified. + +## Deliberate limits + +Only scalar SCD1 ingestion is implemented. Models, tests, graphs, replace, SCD2, snapshot, +incremental writes, semi-structured fields, live concurrency proof, infrastructure provisioning, +and support promotion remain separate work. Use `build_models: false` and `catalog.provider: none` +for this experimental path. diff --git a/pyproject.toml b/pyproject.toml index 8b043082..b2238db9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,6 +239,7 @@ include = [ "docs/postgresql-benchmarks.md", "docs/oci-artifact-contract.md", "docs/rollback.md", + "docs/snowflake.md", "docs/upgrading.md", "docs/warehouse-staging.md", "examples", diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 32f85331..69ff4d99 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -213,7 +213,9 @@ class DanderProject(BaseModel): pipelines: dict[str, PipelineSpec] = Field(min_length=1) platform_name: str = Field(default="gcp", exclude=True) deployment_name: str = Field(default="gcp_cloud_run", exclude=True) - warehouse_provider: Literal["bigquery", "postgresql"] = Field(default="bigquery", exclude=True) + warehouse_provider: Literal["bigquery", "postgresql", "snowflake"] = Field( + default="bigquery", exclude=True + ) warehouse_config: dict[str, object] = Field( default_factory=_default_warehouse_config, exclude=True, diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 14a683f8..8ff4c61c 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -35,6 +35,7 @@ PostgreSQLStateConfig, PostgreSQLWarehouseConfig, ) +from dander.providers.snowflake import SnowflakeWarehouseConfig # noqa: TC001 if TYPE_CHECKING: from collections.abc import Mapping @@ -146,7 +147,7 @@ def validate_pipeline_ids( WarehouseSpec = Annotated[ - BigQueryWarehouseConfig | PostgreSQLWarehouseConfig, + BigQueryWarehouseConfig | PostgreSQLWarehouseConfig | SnowflakeWarehouseConfig, Field(discriminator="provider"), ] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 4f7cdbce..a976aeb6 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -11,6 +11,7 @@ from dander.providers.no_catalog import NoCatalogConfig from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory +from dander.providers.snowflake import SnowflakeWarehouseConfig def default_provider_registry() -> ProviderRegistry: @@ -32,6 +33,14 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.bigquery.state:BIGQUERY_STATE_FACTORY" ), ) + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="snowflake", + config_model=SnowflakeWarehouseConfig, + load_factory=lazy_provider_factory( + "dander.providers.snowflake.runtime:SNOWFLAKE_WAREHOUSE_FACTORY" + ), + ) registry.register( kind=ProviderKind.WAREHOUSE, provider_id="postgresql", diff --git a/src/dander/providers/snowflake/__init__.py b/src/dander/providers/snowflake/__init__.py new file mode 100644 index 00000000..c57371ec --- /dev/null +++ b/src/dander/providers/snowflake/__init__.py @@ -0,0 +1,13 @@ +"""Dependency-light Snowflake warehouse configuration.""" + +from dander.providers.snowflake.config import ( + SnowflakeKeyPairAuth, + SnowflakeOAuthAuth, + SnowflakeWarehouseConfig, +) + +__all__ = [ + "SnowflakeKeyPairAuth", + "SnowflakeOAuthAuth", + "SnowflakeWarehouseConfig", +] diff --git a/src/dander/providers/snowflake/config.py b/src/dander/providers/snowflake/config.py new file mode 100644 index 00000000..354f70bf --- /dev/null +++ b/src/dander/providers/snowflake/config.py @@ -0,0 +1,124 @@ +"""Validated Snowflake settings containing references, never credentials.""" + +from __future__ import annotations + +import re +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from dander.warehouse.contracts import RelationRef + +_ACCOUNT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,254}$") +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]{0,254}$") +_ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]{0,126}$") + + +class SnowflakeOAuthAuth(BaseModel): + """OAuth access-token reference projected into the runtime environment.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + method: Literal["oauth"] + token_env: str = "DANDER_SNOWFLAKE_OAUTH_TOKEN" + + @field_validator("token_env") + @classmethod + def validate_token_env(cls, value: str) -> str: + return _environment_name(value) + + +class SnowflakeKeyPairAuth(BaseModel): + """Key-pair file references projected into the runtime environment.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + method: Literal["key_pair"] + private_key_file_env: str = "DANDER_SNOWFLAKE_PRIVATE_KEY_FILE" + private_key_password_env: str | None = None + + @field_validator("private_key_file_env", "private_key_password_env") + @classmethod + def validate_key_env(cls, value: str | None) -> str | None: + return None if value is None else _environment_name(value) + + +SnowflakeAuth = Annotated[ + SnowflakeOAuthAuth | SnowflakeKeyPairAuth, + Field(discriminator="method"), +] + + +class SnowflakeWarehouseConfig(BaseModel): + """Snowflake connection, staging, and bounded execution settings.""" + + model_config = ConfigDict( + extra="forbid", + frozen=True, + populate_by_name=True, + serialize_by_alias=True, + ) + + provider: Literal["snowflake"] + account: str + user: str + database: str + schema_name: str = Field(default="raw", alias="schema") + warehouse: str + role: str | None = None + auth: SnowflakeAuth + statement_timeout_seconds: int = Field(default=900, ge=1, le=86_400) + queued_timeout_seconds: int = Field(default=120, ge=1, le=3_600) + login_timeout_seconds: int = Field(default=30, ge=1, le=300) + network_timeout_seconds: int = Field(default=300, ge=1, le=3_600) + max_rows_per_file: int = Field(default=100_000, ge=1, le=1_000_000) + max_logical_bytes_per_file: int = Field( + default=128 * 1_024 * 1_024, + ge=1_024, + le=1_073_741_824, + ) + compression: Literal["snappy", "zstd"] = "zstd" + + @field_validator("account") + @classmethod + def validate_account(cls, value: str) -> str: + if not _ACCOUNT.fullmatch(value): + raise ValueError("account must be a Snowflake account identifier") + return value + + @field_validator("user", "database", "schema_name", "warehouse", "role") + @classmethod + def validate_identifiers(cls, value: str | None) -> str | None: + if value is not None and not _IDENTIFIER.fullmatch(value): + raise ValueError("Snowflake identifiers must use portable identifier syntax") + return value + + def raw_relation( + self, + name: str, + *, + compatibility_catalog: str | None, + compatibility_namespace: str | None, + default_namespace: str, + ) -> RelationRef: + """Translate compatibility inputs into Snowflake database/schema coordinates.""" + del compatibility_catalog, default_namespace + return RelationRef( + catalog=self.database, + namespace=compatibility_namespace or self.schema_name, + name=name, + ) + + +def _environment_name(value: str) -> str: + if not _ENVIRONMENT_NAME.fullmatch(value): + raise ValueError("credential references must be uppercase environment-variable names") + return value + + +__all__ = [ + "SnowflakeAuth", + "SnowflakeKeyPairAuth", + "SnowflakeOAuthAuth", + "SnowflakeWarehouseConfig", +] diff --git a/src/dander/providers/snowflake/fence.py b/src/dander/providers/snowflake/fence.py new file mode 100644 index 00000000..4cffc805 --- /dev/null +++ b/src/dander/providers/snowflake/fence.py @@ -0,0 +1,233 @@ +"""Snowflake destination-side fencing around publication DML.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.providers.snowflake.session import ( + SnowflakeConnection, + SnowflakeConnectionFactory, + SnowflakeStatementResult, + execute, + open_connection, +) +from dander.warehouse.runtime import PreparedWarehouseStatement + +if TYPE_CHECKING: + from collections.abc import Sequence + + from dander.warehouse import RelationRef + + +@dataclass(frozen=True, slots=True) +class SnowflakeFencePlan: + """Parameterized ownership statements surrounding one publication transaction.""" + + touch_sql: str + commit_sql: str + parameters: tuple[object, ...] + + +@dataclass(frozen=True, slots=True) +class SnowflakeTargetFence: + """Claim and DML-touch exact destination ownership before publishing.""" + + connection_factory: SnowflakeConnectionFactory + database: str + + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + """Accept a newer token or an exact idempotent retry in the destination.""" + if target.catalog != self.database: + raise ValueError("Snowflake target fence must use the configured database") + table = _qualified(target.catalog, target.namespace, "dander_target_commits") + target_id = ".".join(target.coordinates) + with open_connection(self.connection_factory) as connection: + execute( + connection, + f"CREATE SCHEMA IF NOT EXISTS {_qualified(target.catalog, target.namespace)}", + ) + execute(connection, _target_fence_table_sql(table)) + claim = execute( + connection, + _target_claim_sql(table), + ( + target_id, + fence.pipeline_id, + fence.resolved_authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + ), + ) + if claim.rowcount == 0: + connection.rollback() + raise TargetFenceLostError( + f"Destination target {target_id!r} rejected stale publication ownership" + ) + # The Python connector inherits the account/user AUTOCOMMIT setting when its + # ``autocommit`` argument is omitted. Persist the accepted claim explicitly so + # accounts configured with AUTOCOMMIT=FALSE cannot lose it on connection close. + connection.commit() + return TargetFence( + fence_table=f"{target.catalog}.{target.namespace}.dander_target_commits", + target_id=target_id, + authority_id=fence.resolved_authority_id, + authority_epoch=fence.authority_epoch, + pipeline_id=fence.pipeline_id, + run_id=fence.run_id, + token=fence.token, + ) + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: + """Return publication DML plus its exact transactional ownership operations.""" + finalizer = statement.strip().removesuffix(";") + return PreparedWarehouseStatement(sql=finalizer, options=self._plan(fence)) + + def execute_dml( + self, + connection: SnowflakeConnection, + statement: str, + fence: TargetFence, + parameters: Sequence[object] = (), + ) -> SnowflakeStatementResult: + """Touch ownership, publish, and commit the fence in one explicit transaction.""" + prepared = self.prepare_dml(statement, fence) + plan = prepared.options + assert isinstance(plan, SnowflakeFencePlan) + execute(connection, "BEGIN TRANSACTION") + try: + touched = execute(connection, plan.touch_sql, plan.parameters) + if touched.rowcount == 0: + raise TargetFenceLostError("Dander destination fence lost before publication") + result = execute(connection, prepared.sql, parameters) + committed = execute(connection, plan.commit_sql, plan.parameters) + if committed.rowcount == 0: + raise TargetFenceLostError("Dander destination fence lost during publication") + connection.commit() + return result + except Exception: + connection.rollback() + raise + + def execute_statements( + self, + connection: SnowflakeConnection, + statements: Sequence[tuple[str, Sequence[object]]], + fence: TargetFence, + ) -> SnowflakeStatementResult: + """Publish an ordered DML group behind one exact target fence.""" + if not statements: + raise ValueError("Snowflake fenced publication requires at least one statement") + plan = self._plan(fence) + execute(connection, "BEGIN TRANSACTION") + try: + touched = execute(connection, plan.touch_sql, plan.parameters) + if touched.rowcount == 0: + raise TargetFenceLostError("Dander destination fence lost before publication") + result: SnowflakeStatementResult | None = None + for statement, parameters in statements: + result = execute(connection, statement, parameters) + committed = execute(connection, plan.commit_sql, plan.parameters) + if committed.rowcount == 0: + raise TargetFenceLostError("Dander destination fence lost during publication") + connection.commit() + assert result is not None + return result + except Exception: + connection.rollback() + raise + + def _plan(self, fence: TargetFence) -> SnowflakeFencePlan: + table = _fence_table(fence.fence_table) + return SnowflakeFencePlan( + touch_sql=_target_touch_sql(table), + commit_sql=_target_commit_sql(table), + parameters=_claim_parameters(fence), + ) + + +def _target_fence_table_sql(table: str) -> str: + return ( + f"CREATE TABLE IF NOT EXISTS {table} (" + '"target_id" VARCHAR NOT NULL, "pipeline_id" VARCHAR NOT NULL, ' + '"authority_id" VARCHAR NOT NULL, "authority_epoch" NUMBER(38,0) NOT NULL, ' + '"run_id" VARCHAR NOT NULL, "fencing_token" NUMBER(38,0) NOT NULL, ' + '"status" VARCHAR NOT NULL, "claimed_at" TIMESTAMP_TZ NOT NULL, ' + '"committed_at" TIMESTAMP_TZ)' + ) + + +def _target_claim_sql(table: str) -> str: + return ( + f"MERGE INTO {table} AS current USING (SELECT ? AS target_id, ? AS pipeline_id, " + "? AS authority_id, ? AS authority_epoch, ? AS run_id, ? AS fencing_token) incoming " + 'ON current."target_id" = incoming.target_id ' + 'AND current."pipeline_id" = incoming.pipeline_id ' + 'WHEN MATCHED AND current."authority_id" = incoming.authority_id ' + 'AND current."authority_epoch" = incoming.authority_epoch ' + 'AND (incoming.fencing_token > current."fencing_token" OR ' + '(incoming.fencing_token = current."fencing_token" ' + 'AND incoming.run_id = current."run_id")) THEN UPDATE SET ' + 'current."run_id" = incoming.run_id, ' + 'current."fencing_token" = incoming.fencing_token, ' + 'current."status" = \'claimed\', current."claimed_at" = CURRENT_TIMESTAMP(), ' + 'current."committed_at" = NULL WHEN NOT MATCHED THEN INSERT ' + '("target_id", "pipeline_id", "authority_id", "authority_epoch", "run_id", ' + '"fencing_token", "status", "claimed_at", "committed_at") VALUES ' + "(incoming.target_id, incoming.pipeline_id, incoming.authority_id, " + "incoming.authority_epoch, incoming.run_id, incoming.fencing_token, " + "'claimed', CURRENT_TIMESTAMP(), NULL)" + ) + + +def _target_touch_sql(table: str) -> str: + return ( + f'UPDATE {table} SET "claimed_at" = "claimed_at" WHERE ' + f"{_target_match_sql()} AND \"status\" IN ('claimed', 'committed')" + ) + + +def _target_commit_sql(table: str) -> str: + return ( + f"UPDATE {table} SET \"status\" = 'committed', " + f'"committed_at" = CURRENT_TIMESTAMP() WHERE {_target_match_sql()} ' + "AND \"status\" IN ('claimed', 'committed')" + ) + + +def _target_match_sql() -> str: + return ( + '"target_id" = ? AND "pipeline_id" = ? AND "authority_id" = ? ' + 'AND "authority_epoch" = ? AND "run_id" = ? AND "fencing_token" = ?' + ) + + +def _claim_parameters(fence: TargetFence) -> tuple[object, ...]: + return ( + fence.target_id, + fence.pipeline_id, + fence.authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + ) + + +def _fence_table(value: str) -> str: + coordinates = value.split(".") + if len(coordinates) != 3 or any(not coordinate for coordinate in coordinates): + raise ValueError("Invalid Snowflake target-fence table") + return _qualified(*coordinates) + + +def _qualified(*coordinates: str) -> str: + return ".".join(_quote(coordinate) for coordinate in coordinates) + + +def _quote(identifier: str) -> str: + return f'"{identifier.replace(chr(34), chr(34) * 2)}"' + + +__all__ = ["SnowflakeFencePlan", "SnowflakeTargetFence"] diff --git a/src/dander/providers/snowflake/runtime.py b/src/dander/providers/snowflake/runtime.py new file mode 100644 index 00000000..d0ffb3d4 --- /dev/null +++ b/src/dander/providers/snowflake/runtime.py @@ -0,0 +1,294 @@ +"""Snowflake implementation of Dander's warehouse capability bundle.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from dander.providers.registry import ( + PROVIDER_API_VERSION, + ProviderFactory, + ProviderFactoryError, + ProviderKind, +) +from dander.providers.snowflake.config import ( + SnowflakeKeyPairAuth, + SnowflakeOAuthAuth, + SnowflakeWarehouseConfig, +) +from dander.providers.snowflake.fence import SnowflakeTargetFence +from dander.providers.snowflake.session import ( + SnowflakeConnectionFactory, + execute, + open_connection, +) +from dander.providers.snowflake.writer import ( + SnowflakeScd1Writer, + SnowflakeStagingSettings, + default_staging_settings, +) +from dander.telemetry import OperationTelemetry, TelemetryOperation +from dander.warehouse import CanonicalField, RelationRef, RelationSchema +from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from pydantic import BaseModel + + +@dataclass(frozen=True, slots=True) +class SnowflakeRelationCodec: + """Render one database-local Snowflake relation coordinate.""" + + database: str + provider_id: str = "snowflake" + + def render(self, relation: RelationRef) -> str: + if relation.catalog != self.database: + raise ValueError("Snowflake relation belongs to another database") + return ".".join(_quote(value) for value in relation.coordinates) + + +@dataclass(frozen=True, slots=True) +class SnowflakeSchemaMapper: + """Map compatibility declarations into canonical schema v1.""" + + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + canonical: list[CanonicalField] = [] + for field in fields: + if isinstance(field, WriteField): + canonical.append(field.to_canonical()) + elif isinstance(field, CanonicalField): + canonical.append(field) + else: + raise TypeError("Snowflake schema mapper received an unsupported field") + return RelationSchema(fields=tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class SnowflakeWriterFactory: + """Construct the bounded Snowflake SCD1 ingestion writer.""" + + database: str + connection_factory: SnowflakeConnectionFactory + target_fence: SnowflakeTargetFence + staging: SnowflakeStagingSettings + + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + ) -> WritePattern: + del batch_rows + if sandbox: + raise ValueError("Snowflake warehouse does not use Dander's BigQuery sandbox mode") + return SnowflakeScd1Writer( + database=self.database, + connection_factory=self.connection_factory, + target_fence=self.target_fence, + schema_evolution=schema_evolution, + staging=self.staging, + ) + + +@dataclass(frozen=True, slots=True) +class SnowflakeTransformFactory: + """Fail clearly until the provider's transform slice is implemented.""" + + def build_transform_runner( + self, + *, + graph_plan: object | None, + build_models: bool, + raw_namespace: str = "raw", + ) -> None: + del raw_namespace + if graph_plan is not None or build_models: + raise ValueError("Snowflake transforms are not available in this experimental slice") + return None + + +@dataclass(frozen=True, slots=True) +class SnowflakeTelemetry: + """Normalize bounded connector counters without retaining SQL or response payloads.""" + + warehouse: str + + def operation( + self, + job: object, + *, + operation: TelemetryOperation, + duration_ms: int = 0, + retry_count: int = 0, + ) -> OperationTelemetry: + rowcount = getattr(job, "rowcount", 0) + query_id = getattr(job, "query_id", None) or getattr(job, "sfqid", None) + affected = rowcount if isinstance(rowcount, int) and rowcount >= 0 else 0 + return OperationTelemetry( + provider="snowflake", + operation=operation, + duration_ms=duration_ms, + retry_count=retry_count, + rows_affected=affected, + query_id=query_id if isinstance(query_id, str) else None, + ) + + +SNOWFLAKE_CAPABILITIES = WarehouseCapabilities( + provider_id="snowflake", + schema_contract_version=1, + write_modes=frozenset({WriteMode.SCD1}), + transports=frozenset({WriteTransport.COPY}), + supports_transforms=False, + supports_graphs=False, + supports_target_fencing=True, +) + + +def build_snowflake_warehouse( + config: BaseModel, + context: Mapping[str, object], +) -> WarehouseRuntime: + """Build a credential-referenced Snowflake runtime or injected conformance session.""" + if not isinstance(config, SnowflakeWarehouseConfig): + raise TypeError("Snowflake warehouse factory received the wrong configuration") + catalog = context.get("catalog") + if not isinstance(catalog, str) or catalog.casefold() != config.database.casefold(): + raise ValueError("Snowflake warehouse factory requires its configured database catalog") + supplied_factory = context.get("connection_factory") + if supplied_factory is None: + connection_factory = _sdk_connection_factory(config) + elif callable(supplied_factory): + connection_factory = cast("SnowflakeConnectionFactory", supplied_factory) + else: + raise TypeError("Snowflake context connection_factory must be callable") + + try: + with open_connection(connection_factory) as connection: + current = execute( + connection, + "SELECT CURRENT_DATABASE(), CURRENT_WAREHOUSE()", + fetch="one", + ).row + except Exception as error: + raise ProviderFactoryError("Snowflake connection validation failed") from error + if not isinstance(current, (tuple, list)) or len(current) < 2: + raise ProviderFactoryError("Snowflake connection validation returned an invalid result") + if str(current[0]).casefold() != config.database.casefold(): + raise ProviderFactoryError("Snowflake connection selected the wrong database") + if str(current[1]).casefold() != config.warehouse.casefold(): + raise ProviderFactoryError("Snowflake connection selected the wrong warehouse") + + target_fence = SnowflakeTargetFence( + connection_factory=connection_factory, + database=config.database, + ) + staging = default_staging_settings( + max_rows_per_file=config.max_rows_per_file, + max_logical_bytes_per_file=config.max_logical_bytes_per_file, + compression=config.compression, + ) + supplied_root = context.get("staging_root") + if supplied_root is not None: + if not isinstance(supplied_root, Path): + raise TypeError("Snowflake context staging_root must be a pathlib.Path") + staging = SnowflakeStagingSettings( + root=supplied_root, + max_rows_per_file=staging.max_rows_per_file, + max_logical_bytes_per_file=staging.max_logical_bytes_per_file, + compression=staging.compression, + ) + return WarehouseRuntime( + provider_id="snowflake", + relation_codec=SnowflakeRelationCodec(config.database), + schema_mapper=SnowflakeSchemaMapper(), + writers=SnowflakeWriterFactory( + config.database, + connection_factory, + target_fence, + staging, + ), + transforms=SnowflakeTransformFactory(), + target_fence=target_fence, + telemetry=SnowflakeTelemetry(config.warehouse), + capabilities=SNOWFLAKE_CAPABILITIES, + ) + + +def _sdk_connection_factory(config: SnowflakeWarehouseConfig) -> SnowflakeConnectionFactory: + parameters: dict[str, object] = { + "account": config.account, + "user": config.user, + "database": config.database, + "warehouse": config.warehouse, + "paramstyle": "qmark", + "login_timeout": config.login_timeout_seconds, + "network_timeout": config.network_timeout_seconds, + "validate_default_parameters": True, + "session_parameters": { + "STATEMENT_TIMEOUT_IN_SECONDS": config.statement_timeout_seconds, + "STATEMENT_QUEUED_TIMEOUT_IN_SECONDS": config.queued_timeout_seconds, + "TIMEZONE": "UTC", + }, + } + if config.role is not None: + parameters["role"] = config.role + if isinstance(config.auth, SnowflakeOAuthAuth): + token = os.environ.get(config.auth.token_env) + if not token: + raise ProviderFactoryError( + f"Snowflake OAuth requires a token in {config.auth.token_env}" + ) + parameters.update({"authenticator": "oauth", "token": token}) + elif isinstance(config.auth, SnowflakeKeyPairAuth): + key_file = os.environ.get(config.auth.private_key_file_env) + if not key_file: + raise ProviderFactoryError( + "Snowflake key-pair authentication requires its configured key-file reference" + ) + parameters["private_key_file"] = key_file + if config.auth.private_key_password_env is not None: + password = os.environ.get(config.auth.private_key_password_env) + if not password: + raise ProviderFactoryError( + "Snowflake key-pair authentication requires its configured password reference" + ) + parameters["private_key_file_pwd"] = password + + def connect() -> object: + try: + import snowflake.connector # type: ignore[import-not-found] + except ModuleNotFoundError as error: + raise ProviderFactoryError( + "Snowflake warehouse requires the dander-platform[snowflake] extra" + ) from error + return snowflake.connector.connect(**parameters) + + return cast("SnowflakeConnectionFactory", connect) + + +SNOWFLAKE_WAREHOUSE_FACTORY = ProviderFactory[WarehouseRuntime]( + kind=ProviderKind.WAREHOUSE, + provider_id="snowflake", + api_version=PROVIDER_API_VERSION, + build=build_snowflake_warehouse, +) + + +def _quote(identifier: str) -> str: + return f'"{identifier.replace(chr(34), chr(34) * 2)}"' + + +__all__ = [ + "SNOWFLAKE_CAPABILITIES", + "SNOWFLAKE_WAREHOUSE_FACTORY", + "SnowflakeRelationCodec", + "SnowflakeSchemaMapper", +] diff --git a/src/dander/providers/snowflake/session.py b/src/dander/providers/snowflake/session.py new file mode 100644 index 00000000..87d61630 --- /dev/null +++ b/src/dander/providers/snowflake/session.py @@ -0,0 +1,94 @@ +"""Small Snowflake connector protocols and sanitized session helpers.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, Self + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + +class SnowflakeCursor(Protocol): + """Connector cursor surface used by Dander and its stateful test double.""" + + rowcount: int + sfqid: str | None + + def execute(self, command: str, params: Sequence[object] | None = None) -> Self: ... + + def fetchone(self) -> object | None: ... + + def fetchall(self) -> list[object]: ... + + def close(self) -> None: ... + + +class SnowflakeConnection(Protocol): + """Connector connection surface kept independent from the optional SDK import.""" + + def cursor(self) -> SnowflakeCursor: ... + + def commit(self) -> None: ... + + def rollback(self) -> None: ... + + def close(self) -> None: ... + + +SnowflakeConnectionFactory = Callable[[], SnowflakeConnection] + + +@dataclass(frozen=True, slots=True) +class SnowflakeStatementResult: + """Non-sensitive statement result retained after its cursor is closed.""" + + rowcount: int + query_id: str | None + row: object | None = None + rows: tuple[object, ...] = () + + +def execute( + connection: SnowflakeConnection, + statement: str, + parameters: Sequence[object] = (), + *, + fetch: str = "none", +) -> SnowflakeStatementResult: + """Execute one statement and detach only the bounded result requested by the caller.""" + cursor = connection.cursor() + try: + cursor.execute(statement, parameters or None) + row = cursor.fetchone() if fetch == "one" else None + rows = tuple(cursor.fetchall()) if fetch == "all" else () + return SnowflakeStatementResult( + rowcount=max(cursor.rowcount, 0), + query_id=cursor.sfqid, + row=row, + rows=rows, + ) + finally: + cursor.close() + + +@contextmanager +def open_connection(factory: SnowflakeConnectionFactory) -> Iterator[SnowflakeConnection]: + """Close one connector session after success or failure.""" + connection = factory() + try: + yield connection + finally: + connection.close() + + +__all__ = [ + "SnowflakeConnection", + "SnowflakeConnectionFactory", + "SnowflakeCursor", + "SnowflakeStatementResult", + "execute", + "open_connection", +] diff --git a/src/dander/providers/snowflake/writer.py b/src/dander/providers/snowflake/writer.py new file mode 100644 index 00000000..bc5b8d16 --- /dev/null +++ b/src/dander/providers/snowflake/writer.py @@ -0,0 +1,563 @@ +"""Bounded Parquet stage/COPY and transactionally fenced Snowflake SCD1 writes.""" + +from __future__ import annotations + +import hashlib +import json +import sys +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING +from uuid import uuid4 + +from dander.concurrency import TargetFenceLostError +from dander.providers.snowflake.session import ( + SnowflakeConnection, + SnowflakeConnectionFactory, + execute, + open_connection, +) +from dander.warehouse import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + ParquetStagingSession, + RelationSchema, + StagingManifest, +) +from dander.writer import SchemaEvolution, WriteMode, WritePattern, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + + from dander.concurrency import TargetFence + from dander.providers.snowflake.fence import SnowflakeTargetFence + +_ORDINAL = "_dander_ordinal" + + +class SnowflakeWriteError(ValueError): + """Raised when staged rows or destination schema violate the Snowflake contract.""" + + +@dataclass(frozen=True, slots=True) +class SnowflakeStagingSettings: + """Bounded local artifact and Snowflake session controls.""" + + root: Path + max_rows_per_file: int + max_logical_bytes_per_file: int + compression: str + + +class SnowflakeScd1Writer(WritePattern): + """Stage one bounded runtime batch as Parquet and MERGE the last row per key.""" + + mode = WriteMode.SCD1 + supports_batched_writes = True + requires_publication_fence = True + + def __init__( + self, + *, + database: str, + connection_factory: SnowflakeConnectionFactory, + target_fence: SnowflakeTargetFence, + schema_evolution: SchemaEvolution, + staging: SnowflakeStagingSettings, + ) -> None: + self._database = database + self._connection_factory = connection_factory + self._target_fence = target_fence + self._schema_evolution = schema_evolution + self._staging = staging + + def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: + """Consume records once, upload checksummed parts, and publish idempotently.""" + _validate_target(target, database=self._database) + publication = target.publication_fence + assert publication is not None + target_schema = target.canonical_schema + if any(field.name == _ORDINAL for field in target_schema.fields): + raise SnowflakeWriteError(f"Declared schema reserves Dander field {_ORDINAL!r}") + _validate_schema(target_schema) + staging_schema = RelationSchema( + fields=( + *target_schema.fields, + CanonicalField( + name=_ORDINAL, + data_type=CanonicalType(kind=LogicalTypeKind.INTEGER, bit_width=64), + cardinality=FieldCardinality.REQUIRED, + ), + ) + ) + local_id = f"snowflake-{uuid4().hex}" + with ParquetStagingSession( + self._staging.root, + run_id=local_id, + max_rows_per_file=self._staging.max_rows_per_file, + max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, + compression=self._staging.compression, + ) as local: + manifest = local.stage(_with_ordinals(records), staging_schema) + return self._load_and_publish( + target, + publication, + target_schema, + staging_schema, + manifest, + ) + + def _load_and_publish( + self, + target: WriteTarget, + publication: TargetFence, + target_schema: RelationSchema, + staging_schema: RelationSchema, + manifest: object, + ) -> int: + if not isinstance(manifest, StagingManifest): + raise TypeError("Snowflake writer received an invalid staging manifest") + _validate_manifest_bounds( + manifest, + max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, + ) + suffix = hashlib.sha256( + f"{publication.target_id}:{publication.run_id}:{uuid4().hex}".encode() + ).hexdigest()[:20] + stage_name = f"dander_files_{suffix}" + staging_table = f"dander_stage_{suffix}" + relation = target.relation_ref + stage = _qualified(relation.catalog, relation.namespace, stage_name) + staging_relation = _qualified(relation.catalog, relation.namespace, staging_table) + cleanup_started = False + with open_connection(self._connection_factory) as connection: + try: + _set_query_tag(connection, publication) + _ensure_target( + connection, + target, + target_schema, + evolution=self._schema_evolution, + ) + _ensure_load_history(connection, target) + if manifest.rows == 0: + result = self._target_fence.execute_dml( + connection, + _no_op_sql(target), + publication, + ) + return result.rowcount + execute(connection, _create_stage_sql(stage)) + execute( + connection, + _create_staging_table_sql( + staging_relation, + staging_schema, + ), + ) + cleanup_started = True + pending = tuple( + artifact + for artifact in manifest.artifacts + if not _artifact_committed(connection, target, publication, artifact.sha256) + ) + if not pending: + result = self._target_fence.execute_dml( + connection, + _no_op_sql(target), + publication, + ) + return result.rowcount + for artifact in pending: + execute(connection, _put_sql(artifact.path, stage)) + execute(connection, _copy_sql(staging_relation, stage, artifact.path.name)) + statements: list[tuple[str, Sequence[object]]] = [] + statements.extend( + ( + _load_history_insert_sql(target), + ( + publication.target_id, + publication.pipeline_id, + publication.run_id, + artifact.sha256, + artifact.rows, + artifact.compressed_bytes, + publication.target_id, + publication.pipeline_id, + publication.run_id, + artifact.sha256, + ), + ) + for artifact in pending + ) + statements.append((_merge_sql(target, staging_relation, target_schema.fields), ())) + result = self._target_fence.execute_statements( + connection, + statements, + publication, + ) + return result.rowcount + except (SnowflakeWriteError, TargetFenceLostError): + raise + except Exception as error: + raise SnowflakeWriteError("Snowflake staged SCD1 write failed") from error + finally: + if cleanup_started: + _cleanup_remote(connection, stage=stage, staging_relation=staging_relation) + + +def default_staging_settings( + *, + max_rows_per_file: int, + max_logical_bytes_per_file: int, + compression: str, +) -> SnowflakeStagingSettings: + """Return container-safe local staging defaults without creating directories.""" + return SnowflakeStagingSettings( + root=Path(tempfile.gettempdir()) / "dander-snowflake-staging", + max_rows_per_file=max_rows_per_file, + max_logical_bytes_per_file=max_logical_bytes_per_file, + compression=compression, + ) + + +def _with_ordinals( + records: Iterable[Mapping[str, object]], +) -> Iterable[Mapping[str, object]]: + for ordinal, record in enumerate(records): + yield {**record, _ORDINAL: ordinal} + + +def _validate_target(target: WriteTarget, *, database: str) -> None: + relation = target.relation_ref + if relation.catalog != database: + raise SnowflakeWriteError( + f"Writer database {database!r} does not match target catalog {relation.catalog!r}" + ) + if not target.business_key: + raise SnowflakeWriteError("Snowflake SCD1 writes require a business key") + if not target.schema: + raise SnowflakeWriteError("Snowflake writes require a declared schema") + publication = target.publication_fence + if publication is None: + raise SnowflakeWriteError("Snowflake hosted writes require a destination target fence") + expected_target = ".".join(target.relation_ref.coordinates) + expected_fence = f"{relation.catalog}.{relation.namespace}.dander_target_commits" + if publication.target_id != expected_target or publication.fence_table != expected_fence: + raise SnowflakeWriteError("Snowflake destination target fence does not match the target") + + +def _validate_schema(schema: RelationSchema) -> None: + for field in schema.fields: + _snowflake_type(field.data_type) + + +def _validate_manifest_bounds( + manifest: StagingManifest, + *, + max_logical_bytes_per_file: int, +) -> None: + if any(artifact.logical_bytes > max_logical_bytes_per_file for artifact in manifest.artifacts): + raise SnowflakeWriteError("Snowflake staged artifact exceeds max_logical_bytes_per_file") + + +def _set_query_tag(connection: SnowflakeConnection, publication: TargetFence) -> None: + tag = json.dumps( + { + "dander": { + "pipeline_id": publication.pipeline_id, + "run_id": publication.run_id, + } + }, + separators=(",", ":"), + sort_keys=True, + ) + execute(connection, f"ALTER SESSION SET QUERY_TAG = {_literal(tag)}") + + +def _ensure_target( + connection: SnowflakeConnection, + target: WriteTarget, + schema: RelationSchema, + *, + evolution: SchemaEvolution, +) -> None: + relation = target.relation_ref + execute( + connection, + f"CREATE SCHEMA IF NOT EXISTS {_qualified(relation.catalog, relation.namespace)}", + ) + definitions = ", ".join( + _column_definition( + field, + required=field.cardinality is FieldCardinality.REQUIRED + or field.name in target.business_key, + ) + for field in schema.fields + ) + execute( + connection, + f"CREATE TABLE IF NOT EXISTS {_target(target)} ({definitions})", + ) + described = execute(connection, f"DESCRIBE TABLE {_target(target)}", fetch="all") + deployed = _described_columns(described.rows) + expected = {field.name: field for field in schema.fields} + extras = sorted(set(deployed) - set(expected)) + if extras: + raise SnowflakeWriteError(f"Deployed target has undeclared column {extras[0]!r}") + for field in schema.fields: + current = deployed.get(field.name) + required = ( + field.cardinality is FieldCardinality.REQUIRED or field.name in target.business_key + ) + expected_type = _normalize_type(_snowflake_type(field.data_type)) + if current is None: + if evolution is not SchemaEvolution.ADDITIVE: + raise SnowflakeWriteError(f"Deployed target is missing column {field.name!r}") + if required: + raise SnowflakeWriteError( + f"Cannot add required Snowflake column {field.name!r} automatically" + ) + execute( + connection, + f"ALTER TABLE {_target(target)} ADD COLUMN {_quote(field.name)} {expected_type}", + ) + continue + deployed_type, deployed_required = current + if _normalize_type(deployed_type) != expected_type: + raise SnowflakeWriteError(f"Snowflake type drift for column {field.name!r}") + if deployed_required != required: + raise SnowflakeWriteError(f"Snowflake nullability drift for column {field.name!r}") + + +def _described_columns(rows: Sequence[object]) -> dict[str, tuple[str, bool]]: + deployed: dict[str, tuple[str, bool]] = {} + for raw in rows: + if isinstance(raw, Mapping): + name = raw.get("name") + data_type = raw.get("type") + nullable = raw.get("null?") + elif isinstance(raw, (tuple, list)) and len(raw) >= 4: + name, data_type, nullable = raw[0], raw[1], raw[3] + else: + raise SnowflakeWriteError("Snowflake DESCRIBE TABLE returned an invalid shape") + if not isinstance(name, str) or not isinstance(data_type, str): + raise SnowflakeWriteError("Snowflake DESCRIBE TABLE returned an invalid column") + if isinstance(nullable, str): + is_required = nullable.upper() == "N" + elif isinstance(nullable, bool): + is_required = not nullable + else: + raise SnowflakeWriteError("Snowflake DESCRIBE TABLE returned invalid nullability") + deployed[name] = (data_type, is_required) + return deployed + + +def _ensure_load_history(connection: SnowflakeConnection, target: WriteTarget) -> None: + execute( + connection, + f"CREATE TABLE IF NOT EXISTS {_load_history(target)} (" + '"target_id" VARCHAR NOT NULL, "pipeline_id" VARCHAR NOT NULL, ' + '"run_id" VARCHAR NOT NULL, "sha256" VARCHAR NOT NULL, ' + '"rows" NUMBER(38,0) NOT NULL, "compressed_bytes" NUMBER(38,0) NOT NULL, ' + '"loaded_at" TIMESTAMP_TZ NOT NULL)', + ) + + +def _artifact_committed( + connection: SnowflakeConnection, + target: WriteTarget, + publication: TargetFence, + checksum: str, +) -> bool: + result = execute( + connection, + f'SELECT 1 FROM {_load_history(target)} WHERE "target_id" = ? ' + 'AND "pipeline_id" = ? AND "run_id" = ? AND "sha256" = ? LIMIT 1', + (publication.target_id, publication.pipeline_id, publication.run_id, checksum), + fetch="one", + ) + return result.row is not None + + +def _load_history_insert_sql(target: WriteTarget) -> str: + return ( + f"INSERT INTO {_load_history(target)} " + '("target_id", "pipeline_id", "run_id", "sha256", "rows", ' + '"compressed_bytes", "loaded_at") SELECT ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP() ' + f"WHERE NOT EXISTS (SELECT 1 FROM {_load_history(target)} WHERE " + '"target_id" = ? AND "pipeline_id" = ? AND "run_id" = ? AND "sha256" = ?)' + ) + + +def _create_stage_sql(stage: str) -> str: + return f"CREATE OR REPLACE TEMPORARY STAGE {stage} FILE_FORMAT = ({_parquet_file_format()})" + + +def _create_staging_table_sql( + relation: str, + schema: RelationSchema, +) -> str: + definitions = ", ".join( + _column_definition( + field, + required=field.cardinality is FieldCardinality.REQUIRED, + ) + for field in schema.fields + ) + return f"CREATE OR REPLACE TEMPORARY TABLE {relation} ({definitions})" + + +def _put_sql(path: Path, stage: str) -> str: + return ( + f"PUT {_literal(path.resolve().as_uri())} @{stage} AUTO_COMPRESS = FALSE OVERWRITE = FALSE" + ) + + +def _copy_sql(staging_relation: str, stage: str, filename: str) -> str: + return ( + f"COPY INTO {staging_relation} FROM @{stage} FILES = ({_literal(filename)}) " + f"FILE_FORMAT = ({_parquet_file_format()}) " + "MATCH_BY_COLUMN_NAME = CASE_SENSITIVE " + "ON_ERROR = ABORT_STATEMENT FORCE = FALSE" + ) + + +def _parquet_file_format() -> str: + return "TYPE = PARQUET USE_LOGICAL_TYPE = TRUE BINARY_AS_TEXT = FALSE" + + +def _merge_sql( + target: WriteTarget, + staging_relation: str, + fields: Sequence[CanonicalField], +) -> str: + names = tuple(field.name for field in fields) + selected = ", ".join(f"incoming.{_quote(name)}" for name in names) + target_names = ", ".join(_quote(name) for name in names) + partition = ", ".join(_quote(key) for key in target.business_key) + match = " AND ".join( + f"target.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + ) + mutable = tuple(name for name in names if name not in target.business_key) + matched = ( + " WHEN MATCHED THEN UPDATE SET " + + ", ".join(f"target.{_quote(name)} = incoming.{_quote(name)}" for name in mutable) + if mutable + else "" + ) + return ( + f"MERGE INTO {_target(target)} AS target USING (SELECT " + f"{', '.join(_quote(name) for name in names)} FROM {staging_relation} " + f"QUALIFY ROW_NUMBER() OVER (PARTITION BY {partition} " + f"ORDER BY {_quote(_ORDINAL)} DESC) = 1) AS incoming ON {match}" + f"{matched} WHEN NOT MATCHED THEN INSERT ({target_names}) VALUES ({selected})" + ) + + +def _no_op_sql(target: WriteTarget) -> str: + key = _quote(target.business_key[0]) + return f"UPDATE {_target(target)} SET {key} = {key} WHERE FALSE" + + +def _cleanup_remote( + connection: SnowflakeConnection, + *, + stage: str, + staging_relation: str, +) -> None: + active_error = sys.exc_info()[0] is not None + failures = 0 + for statement in ( + f"DROP TABLE IF EXISTS {staging_relation}", + f"DROP STAGE IF EXISTS {stage}", + ): + try: + execute(connection, statement) + except Exception: + failures += 1 + if failures and not active_error: + raise SnowflakeWriteError("Snowflake run-scoped staging cleanup failed") + + +def _column_definition( + field: CanonicalField, + *, + required: bool, +) -> str: + data_type = _snowflake_type(field.data_type) + return f"{_quote(field.name)} {data_type}{' NOT NULL' if required else ''}" + + +def _snowflake_type(data_type: CanonicalType) -> str: + match data_type.kind: + case LogicalTypeKind.BOOLEAN: + return "BOOLEAN" + case LogicalTypeKind.INTEGER: + return "NUMBER(38,0)" + case LogicalTypeKind.DECIMAL: + if data_type.precision is None or data_type.scale is None: + raise SnowflakeWriteError("Snowflake decimal requires precision and scale") + if data_type.precision > 38: + raise SnowflakeWriteError("Snowflake decimal precision cannot exceed 38") + return f"NUMBER({data_type.precision},{data_type.scale})" + case LogicalTypeKind.FLOAT: + return "FLOAT" + case LogicalTypeKind.STRING: + return "VARCHAR" + case LogicalTypeKind.BINARY: + return "BINARY" + case LogicalTypeKind.DATE: + return "DATE" + case LogicalTypeKind.TIME: + return f"TIME({min(data_type.fractional_second_precision or 0, 9)})" + case LogicalTypeKind.TIMESTAMP: + flavor = "TZ" if data_type.with_timezone else "NTZ" + return f"TIMESTAMP_{flavor}({min(data_type.fractional_second_precision or 0, 9)})" + case LogicalTypeKind.JSON | LogicalTypeKind.ARRAY | LogicalTypeKind.RECORD: + raise SnowflakeWriteError( + "Snowflake semi-structured fields are not supported in this experimental slice" + ) + raise AssertionError("Unhandled canonical Snowflake type") + + +def _normalize_type(value: str) -> str: + compact = "".join(value.upper().split()) + if compact.startswith("VARCHAR("): + return "VARCHAR" + if compact.startswith("BINARY("): + return "BINARY" + return compact + + +def _target(target: WriteTarget) -> str: + return _qualified(*target.relation_ref.coordinates) + + +def _load_history(target: WriteTarget) -> str: + relation = target.relation_ref + return _qualified(relation.catalog, relation.namespace, "dander_stage_loads") + + +def _qualified(*coordinates: str) -> str: + return ".".join(_quote(coordinate) for coordinate in coordinates) + + +def _quote(identifier: str) -> str: + return f'"{identifier.replace(chr(34), chr(34) * 2)}"' + + +def _literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +__all__ = [ + "SnowflakeScd1Writer", + "SnowflakeStagingSettings", + "SnowflakeWriteError", + "default_staging_settings", +] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 75e840a0..caf16380 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -13,6 +13,12 @@ "status": "experimental", "reason": "Locally conformance-tested with destination-side PostgreSQL fencing; no live hosted profile is qualified." }, + { + "state": "bigquery", + "warehouse": "snowflake", + "status": "experimental", + "reason": "Snowflake staged SCD1 ingestion and destination fencing pass local conformance; no live Snowflake profile is qualified." + }, { "state": "postgresql", "warehouse": "bigquery", @@ -24,6 +30,12 @@ "warehouse": "postgresql", "status": "experimental", "reason": "Local native-profile conformance passed; Kubernetes live qualification remains pending." + }, + { + "state": "postgresql", + "warehouse": "snowflake", + "status": "experimental", + "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged SCD1 writes; live qualification remains pending." } ] } diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 179b3392..ce63c7ff 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -318,6 +318,91 @@ def test_bigquery_state_location_stays_independent_from_postgresql_warehouse() - assert context["metadata_namespace"] == "dander_meta" +def test_snowflake_coordinates_resolve_from_database_and_schema(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text( + """\ +version: 2 +pipelines: + greenhouse_jobs: + source: greenhouse_job_board + models: [stg_greenhouse__jobs] +""", + encoding="utf-8", + ) + platforms_path.write_text( + """\ +version: 1 +platforms: + snowflake: + warehouse: + provider: snowflake + account: org-account + user: dander_user + database: DANDER_DB + schema: LANDING + warehouse: DANDER_WH + auth: + method: oauth + token_env: DANDER_SNOWFLAKE_TOKEN + state: + provider: bigquery + catalog: + provider: none + secrets: + provider: environment +deployments: + snowflake_kubernetes: + platform: snowflake + launcher: + provider: kubernetes + context: test-context + safety: + require_guarded_free_tier: false + pipelines: + greenhouse_jobs: + paused: true +""", + encoding="utf-8", + ) + options = run_module.RunOptions( + pipeline_or_source="greenhouse_jobs", + project="control-project", + dataset=None, + connectors_dir=_REPO_ROOT / "connectors", + project_config=project_path, + platforms_config=platforms_path, + deployment="snowflake_kubernetes", + dry_run=True, + sandbox=False, + guarded_free_tier=False, + batch_rows=10_000, + budget_name="dander-sandbox-budget", + state_path=tmp_path / "state.sqlite3", + build_models=False, + models_dir=_REPO_ROOT / "models", + selected_models=None, + catalog_output=None, + publish_dataplex=False, + dataplex_location="us-central1", + ) + + resolved = run_module._resolve_run(options) + + assert resolved.catalog == "DANDER_DB" + assert resolved.raw_namespace == "LANDING" + assert resolved.endpoint_relations == { + "jobs": RelationRef( + catalog="DANDER_DB", + namespace="LANDING", + name="greenhouse_job_board_jobs", + ) + } + assert resolved.state_catalog == "control-project" + assert resolved.state_namespace == "raw" + + def test_only_unsupported_postgresql_state_bigquery_warehouse_pair_fails_closed() -> None: with pytest.raises(ClickException, match="BigQuery write mode"): run_module._require_executable_state_pair( diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index beaa5060..e5eb44e1 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -121,7 +121,7 @@ def test_version_two_rejects_unknown_provider_and_unknown_pipeline(tmp_path: Pat migration = prepare_version_one_migration(project_path) project_path.write_text(migration.logical_yaml, encoding="utf-8") platforms = yaml.safe_load(migration.platforms_yaml) - platforms["platforms"]["gcp"]["warehouse"]["provider"] = "snowflake" + platforms["platforms"]["gcp"]["warehouse"]["provider"] = "unknown_warehouse" platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") with pytest.raises( diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py new file mode 100644 index 00000000..46a541e7 --- /dev/null +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -0,0 +1,563 @@ +"""Snowflake registration and stateful staged-SCD1 conformance.""" + +from __future__ import annotations + +import re +import sys +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Self + +import pytest + +from dander.concurrency import FencingToken, TargetFenceLostError +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry +from dander.providers.snowflake import SnowflakeWarehouseConfig +from dander.providers.snowflake.writer import SnowflakeWriteError +from dander.telemetry import TelemetryOperation +from dander.warehouse import RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + from pathlib import Path + +_CHECKSUM = re.compile(r"^[0-9a-f]{64}$") + + +@dataclass +class _FakeSnowflake: + """Minimal state shared by independently opened fake connector sessions.""" + + statements: list[tuple[str, tuple[object, ...]]] = field(default_factory=list) + committed_checksums: set[str] = field(default_factory=set) + describe_rows: list[object] = field( + default_factory=lambda: [ + ("id", "VARCHAR(16777216)", "COLUMN", "N"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ] + ) + claim_rowcount: int = 1 + fence_touch_rowcount: int = 1 + merge_rowcount: int = 2 + fail_prefix: str | None = None + commits: int = 0 + rollbacks: int = 0 + closes: int = 0 + committed_claims: int = 0 + discarded_claims: int = 0 + + def connect(self) -> _FakeConnection: + return _FakeConnection(self) + + +class _FakeConnection: + def __init__(self, backend: _FakeSnowflake) -> None: + self.backend = backend + self.pending_claim = False + + def cursor(self) -> _FakeCursor: + return _FakeCursor(self) + + def commit(self) -> None: + self.backend.commits += 1 + if self.pending_claim: + self.backend.committed_claims += 1 + self.pending_claim = False + + def rollback(self) -> None: + self.backend.rollbacks += 1 + self.pending_claim = False + + def close(self) -> None: + if self.pending_claim: + self.backend.discarded_claims += 1 + self.pending_claim = False + self.backend.closes += 1 + + +class _FakeCursor: + def __init__(self, connection: _FakeConnection) -> None: + self.connection = connection + self.backend = connection.backend + self.rowcount = 0 + self.sfqid: str | None = None + self._row: object | None = None + self._rows: list[object] = [] + + def execute(self, command: str, params: Sequence[object] | None = None) -> Self: + parameters = tuple(params or ()) + compact = " ".join(command.split()) + self.backend.statements.append((compact, parameters)) + self.sfqid = f"query-{len(self.backend.statements)}" + if self.backend.fail_prefix and compact.startswith(self.backend.fail_prefix): + raise RuntimeError("private provider response") + if compact == "SELECT CURRENT_DATABASE(), CURRENT_WAREHOUSE()": + self._row = ("DANDER_TEST", "DANDER_WH") + elif compact.startswith("DESCRIBE TABLE"): + self._rows = list(self.backend.describe_rows) + elif compact.startswith("MERGE INTO") and "dander_target_commits" in compact: + self.rowcount = self.backend.claim_rowcount + self.connection.pending_claim = self.rowcount > 0 + elif compact.startswith("UPDATE") and "dander_target_commits" in compact: + self.rowcount = self.backend.fence_touch_rowcount + elif compact.startswith("SELECT 1 FROM") and "dander_stage_loads" in compact: + checksum = str(parameters[3]) + self._row = (1,) if checksum in self.backend.committed_checksums else None + elif compact.startswith("INSERT INTO") and "dander_stage_loads" in compact: + checksum = str(parameters[3]) + assert _CHECKSUM.fullmatch(checksum) + assert len(parameters) == 10 + self.backend.committed_checksums.add(checksum) + self.rowcount = 1 + elif compact.startswith("MERGE INTO"): + self.rowcount = self.backend.merge_rowcount + return self + + def fetchone(self) -> object | None: + return self._row + + def fetchall(self) -> list[object]: + return list(self._rows) + + def close(self) -> None: + return None + + +def _config() -> dict[str, object]: + return { + "provider": "snowflake", + "account": "org-account", + "user": "dander_user", + "database": "DANDER_TEST", + "schema": "raw", + "warehouse": "DANDER_WH", + "role": "DANDER_ROLE", + "auth": { + "method": "key_pair", + "private_key_file_env": "DANDER_TEST_SNOWFLAKE_KEY_FILE", + }, + "max_rows_per_file": 2, + "max_logical_bytes_per_file": 1_048_576, + } + + +@pytest.fixture +def snowflake_runtime( + tmp_path: Path, +) -> tuple[WarehouseRuntime, _FakeSnowflake, Path]: + backend = _FakeSnowflake() + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, _config()) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "DANDER_TEST", + "connection_factory": backend.connect, + "staging_root": tmp_path, + }, + ) + assert isinstance(runtime, WarehouseRuntime) + return runtime, backend, tmp_path + + +def test_snowflake_registration_is_lazy_and_credentials_remain_references() -> None: + module_name = "dander.providers.snowflake.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + + config = registry.parse(ProviderKind.WAREHOUSE, _config()) + + assert module_name not in sys.modules + assert isinstance(config, SnowflakeWarehouseConfig) + dumped = config.model_dump(mode="json") + assert dumped["schema"] == "raw" + assert "schema_name" not in dumped + assert dumped["auth"] == { + "method": "key_pair", + "private_key_file_env": "DANDER_TEST_SNOWFLAKE_KEY_FILE", + "private_key_password_env": None, + } + + relation = config.raw_relation( + "records", + compatibility_catalog="ignored-gcp-project", + compatibility_namespace=None, + default_namespace="ignored_bigquery_default", + ) + assert relation == RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + + +def test_snowflake_runtime_requires_projected_oauth_token(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DANDER_MISSING_SNOWFLAKE_TOKEN", raising=False) + registry = default_provider_registry() + raw = _config() + raw["auth"] = {"method": "oauth", "token_env": "DANDER_MISSING_SNOWFLAKE_TOKEN"} + config = registry.parse(ProviderKind.WAREHOUSE, raw) + + with pytest.raises(ProviderFactoryError, match="requires a token"): + registry.build( + ProviderKind.WAREHOUSE, + config, + context={"catalog": "DANDER_TEST"}, + ) + + +def test_snowflake_claim_commits_when_account_autocommit_is_disabled( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + + runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-claim", + token=3, + authority_id="postgresql:test-state", + ), + ) + + assert backend.committed_claims == 1 + assert backend.discarded_claims == 0 + assert backend.commits == 1 + + +def test_snowflake_stages_bounded_parts_merges_last_record_and_cleans( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, staging_root = snowflake_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=3, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + lease = FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-one", + token=4, + authority_id="postgresql:test-state", + ) + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + publication = runtime.target_fence.claim(relation, lease) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + consumed = 0 + + def records() -> Iterator[dict[str, object]]: + nonlocal consumed + fixture: tuple[dict[str, object], ...] = ( + {"id": "one", "label": "old"}, + {"id": "one", "label": "new"}, + {"id": "two", "label": "second"}, + ) + for record in fixture: + consumed += 1 + yield record + + assert writer.write(records(), target) == 2 + assert consumed == 3 + sql = [statement for statement, _parameters in backend.statements] + assert len([statement for statement in sql if statement.startswith("PUT ")]) == 2 + assert len([statement for statement in sql if statement.startswith("COPY INTO")]) == 2 + assert any(statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") for statement in sql) + copy = next(statement for statement in sql if statement.startswith("COPY INTO")) + assert "USE_LOGICAL_TYPE = TRUE" in copy + assert "BINARY_AS_TEXT = FALSE" in copy + merge = next( + statement + for statement in sql + if statement.startswith("MERGE INTO") and "dander_target_commits" not in statement + ) + assert 'ROW_NUMBER() OVER (PARTITION BY "id" ORDER BY "_dander_ordinal" DESC)' in merge + assert 'WHEN MATCHED THEN UPDATE SET target."label" = incoming."label"' in merge + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + assert any(statement.startswith("DROP STAGE IF EXISTS") for statement in sql) + assert not tuple(staging_root.iterdir()) + assert backend.commits == 2 + assert backend.rollbacks == 0 + assert len(backend.committed_checksums) == 2 + assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) + assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) + assert runtime.capabilities.supports_transforms is False + assert runtime.relation_codec.render(relation) == '"DANDER_TEST"."raw"."records"' + + +def test_snowflake_retry_skips_committed_files_but_still_touches_fence( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + lease = FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-retry", + token=8, + authority_id="postgresql:test-state", + ) + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + publication = runtime.target_fence.claim(relation, lease) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + rows = ({"id": "one", "label": "same"},) + + assert writer.write(rows, target) == 2 + first_puts = sum(statement.startswith("PUT ") for statement, _ in backend.statements) + assert writer.write(rows, target) == 0 + second_puts = sum(statement.startswith("PUT ") for statement, _ in backend.statements) + + assert first_puts == second_puts == 1 + assert backend.commits == 3 + + +def test_snowflake_copy_preserves_binary_and_temporal_parquet_types( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "N"), + ("payload", "BINARY(8388608)", "COLUMN", "Y"), + ("observed_at", "TIMESTAMP_TZ(6)", "COLUMN", "Y"), + ] + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="typed_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_typed_records", + run_id="run-typed", + token=10, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="BYTES"), + WriteField(name="observed_at", data_type="TIMESTAMP"), + ), + publication_fence=publication, + ) + + assert ( + writer.write( + ( + { + "id": "one", + "payload": b"\x00\xff", + "observed_at": datetime(2026, 8, 8, 12, 0, tzinfo=UTC), + }, + ), + target, + ) + == 2 + ) + copy = next(statement for statement, _ in backend.statements if statement.startswith("COPY")) + assert "USE_LOGICAL_TYPE = TRUE" in copy + assert "BINARY_AS_TEXT = FALSE" in copy + + +def test_snowflake_rejects_oversized_singleton_before_remote_staging(tmp_path: Path) -> None: + backend = _FakeSnowflake() + registry = default_provider_registry() + raw = _config() + raw["max_logical_bytes_per_file"] = 1_024 + config = registry.parse(ProviderKind.WAREHOUSE, raw) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "DANDER_TEST", + "connection_factory": backend.connect, + "staging_root": tmp_path, + }, + ) + assert isinstance(runtime, WarehouseRuntime) + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-oversized", + token=11, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + statements_before_write = len(backend.statements) + + with pytest.raises(SnowflakeWriteError, match="exceeds max_logical_bytes_per_file"): + writer.write(({"id": "one", "label": "x" * 3_000},), target) + + assert len(backend.statements) == statements_before_write + assert not tuple(tmp_path.iterdir()) + + +def test_snowflake_fails_closed_on_stale_fence_and_cleans_after_copy_failure( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, staging_root = snowflake_runtime + backend.claim_rowcount = 0 + lease = FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-stale", + token=1, + authority_id="postgresql:test-state", + ) + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + with pytest.raises(TargetFenceLostError, match="rejected stale"): + runtime.target_fence.claim(relation, lease) + + backend.claim_rowcount = 1 + publication = runtime.target_fence.claim(relation, lease) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + backend.fail_prefix = "COPY INTO" + + with pytest.raises(SnowflakeWriteError, match="staged SCD1 write failed") as captured: + writer.write(({"id": "private-record", "label": "secret-value"},), target) + + assert "private-record" not in str(captured.value) + assert "secret-value" not in str(captured.value) + assert not tuple(staging_root.iterdir()) + sql = [statement for statement, _ in backend.statements] + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + assert any(statement.startswith("DROP STAGE IF EXISTS") for statement in sql) + + +def test_snowflake_lost_fence_blocks_publication_and_rolls_back( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, staging_root = snowflake_runtime + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-lost", + token=12, + authority_id="postgresql:test-state", + ), + ) + backend.fence_touch_rowcount = 0 + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + + with pytest.raises(TargetFenceLostError, match="lost before publication"): + writer.write(({"id": "one", "label": "blocked"},), target) + + assert backend.rollbacks == 1 + assert not any( + statement.startswith("MERGE INTO") and "dander_target_commits" not in statement + for statement, _ in backend.statements + ) + assert not tuple(staging_root.iterdir()) + + +def test_snowflake_rejects_semi_structured_fields_and_telemetry_is_bounded( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, _backend, _staging_root = snowflake_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + lease = FencingToken( + lease_table=None, + pipeline_id="snowflake_records", + run_id="run-variant", + token=9, + authority_id="postgresql:test-state", + ) + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") + publication = runtime.target_fence.claim(relation, lease) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON"), + ), + publication_fence=publication, + ) + + with pytest.raises(SnowflakeWriteError, match="semi-structured fields are not supported"): + writer.write(({"id": "one", "payload": {"ready": True}},), target) + + class _Result: + rowcount = 7 + query_id = "query-safe-7" + + telemetry = runtime.telemetry.operation(_Result(), operation=TelemetryOperation.LOAD) + assert telemetry.rows_affected == 7 + assert telemetry.query_id == "query-safe-7" diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 5975a194..313e68ff 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -22,8 +22,10 @@ def test_state_warehouse_matrix_covers_every_current_pair() -> None: assert {(pair.state, pair.warehouse): pair.status for pair in matrix.state_warehouse_pairs} == { ("bigquery", "bigquery"): CompatibilityStatus.SUPPORTED, ("bigquery", "postgresql"): CompatibilityStatus.EXPERIMENTAL, + ("bigquery", "snowflake"): CompatibilityStatus.EXPERIMENTAL, ("postgresql", "bigquery"): CompatibilityStatus.UNSUPPORTED, ("postgresql", "postgresql"): CompatibilityStatus.EXPERIMENTAL, + ("postgresql", "snowflake"): CompatibilityStatus.EXPERIMENTAL, } @@ -32,7 +34,9 @@ def test_state_warehouse_matrix_covers_every_current_pair() -> None: [ ("bigquery", "bigquery"), ("bigquery", "postgresql"), + ("bigquery", "snowflake"), ("postgresql", "postgresql"), + ("postgresql", "snowflake"), ], ) def test_executable_pairs_are_admitted(state: str, warehouse: str) -> None: @@ -56,5 +60,5 @@ def test_runtime_compatibility_cli_prints_one_deterministic_document() -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) assert payload["schema"] == "io.dander.runtime.compatibility/v1" - assert len(payload["state_warehouse_pairs"]) == 4 + assert len(payload["state_warehouse_pairs"]) == 6 assert result.output.strip() == load_runtime_compatibility().to_json() diff --git a/tickets/DANDER-104-snowflake-warehouse.md b/tickets/DANDER-104-snowflake-warehouse.md new file mode 100644 index 00000000..52be129d --- /dev/null +++ b/tickets/DANDER-104-snowflake-warehouse.md @@ -0,0 +1,33 @@ +--- +id: DANDER-104 +title: Add the experimental Snowflake staged-SCD1 warehouse +status: in-review +component: warehouse +epic: cloud-portability +depends_on: [DANDER-103] +created: 2026-08-08 +--- + +## Context + +Canonical relations and bounded Parquet artifacts are merged. The first Snowflake slice should +exercise those contracts without claiming transforms, every write mode, infrastructure, or live +support. + +## Acceptance Criteria + +- [x] Snowflake owns database/schema-to-`RelationRef` translation. +- [x] OAuth and key-pair configuration contain references rather than credentials. +- [x] Bounded Parquet parts load through explicit logical/binary `COPY` settings. +- [x] SCD1 publication, load history, and the exact target fence commit atomically. +- [x] Declared scalar schemas and nullable additive evolution fail closed on drift. +- [x] Temporary remote staging and local artifacts clean up after success and handled failure. +- [x] Lost fencing ownership blocks target publication. +- [x] BigQuery and PostgreSQL compatibility remains unchanged. +- [ ] Full local quality suite and retained GCP no-drift pass. +- [ ] Protected CI passes. + +## Boundary + +This ticket adds no Snowflake transforms, graphs, semi-structured fields, other write modes, +Terraform, live deployment, package publication, or support-status promotion. From 82c10b89819210d6dabb9397003abf2cad3043c5 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 00:36:31 -0400 Subject: [PATCH 059/163] feat: add fenced Snowflake transforms --- HANDOFF.md | 42 +- docs/compatibility-matrix.md | 4 +- docs/decisions.md | 9 + docs/known-limitations.md | 6 +- docs/snowflake.md | 16 +- src/dander/providers/snowflake/runtime.py | 33 +- src/dander/providers/snowflake/transform.py | 404 ++++++++++++++++++ src/dander/transform/project.py | 14 +- .../test_snowflake_warehouse_runtime.py | 305 ++++++++++++- 9 files changed, 789 insertions(+), 44 deletions(-) create mode 100644 src/dander/providers/snowflake/transform.py diff --git a/HANDOFF.md b/HANDOFF.md index de679d6d..c998e33a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Added an experimental Snowflake provider with native database/schema coordinates. -- Added bounded Parquet `PUT`/`COPY`, scalar SCD1 merge, schema evolution, and telemetry. -- Explicitly committed fence claims and fenced publication/load history in Snowflake transactions. -- Made remote staging session-temporary and rejected oversized parts before upload. -- Kept Snowflake experimental with transforms, graphs, other modes, and live support excluded. +- Added experimental Snowflake table and incremental model execution. +- Preflights the full selected DAG before provider mutation and rejects views/graphs fail-closed. +- Publishes through session-temporary staging and exact transactional destination fencing. +- Preserves canonical database/schema/relation coordinates, including hyphenated databases. +- Updated Snowflake capability and limitation documentation without a support promotion. ## Try It -Configure a version 2 Snowflake profile from `docs/snowflake.md`, select `build_models: false`, and -use either BigQuery or PostgreSQL state. No live Snowflake proof is claimed. +Configure the experimental profile in `docs/snowflake.md`; portable table and incremental models +may now run with active lease ownership. No live Snowflake qualification is claimed. ## Checks -- Full suite passed: 1,061 tests with PostgreSQL 15; Ruff and strict mypy also passed. -- Wheel/sdist inspection, source-free installs, generated-project validation, and runtime-all import passed. -- Container build/conformance, Terraform/AWS tests, Helm validation, and dependency audit passed. -- Retained stage-zero plan: no changes; no cloud apply or mutation occurred. -- Retained platform plan matched clean `origin/main`: 2 adds/5 updates already pending, no deletes. +- Full suite passed: 1,065 tests with PostgreSQL 15; Ruff and strict mypy passed. +- Wheel/sdist inspection, source-free installs, generated-project validation, and Terraform validation passed. +- Container build/conformance, Terraform/AWS tests, and dependency audit passed. +- No cloud plan/apply, deployment mutation, or package publication occurred. ## Decisions -- Snowflake translates native database/schema values into canonical `RelationRef` coordinates. -- Explicit Parquet logical/binary settings are part of every effective `COPY` file format. -- Scalar SCD1 is the smallest honest experimental slice; other capabilities remain fail-closed. +- Snowflake view DDL remains excluded because it cannot share the destination-fence transaction. +- Transform targets permit create-if-absent with exact schema; automatic ALTER is not allowed. +- Incremental duplicates resolve by cursor descending, then stable declared-column ordering. ## Remaining -- Review the known retained-platform baseline drift before any future apply. -- Open the focused draft PR and require protected CI. -- Merge only after review; do not claim live Snowflake qualification. +- Complete independent adversarial review. +- Push the stacked branch and open a draft PR against the Snowflake foundation branch. +- Let protected CI repeat Linux, packaging, container, security, and Terraform checks. +- Keep both Snowflake PRs unmerged while retained GCP baseline drift remains unresolved. ## Review First -- `src/dander/providers/snowflake/writer.py` -- `src/dander/providers/snowflake/fence.py` -- `src/dander/providers/snowflake/config.py` +- `src/dander/providers/snowflake/transform.py` +- `tests/providers/test_snowflake_warehouse_runtime.py` +- `src/dander/providers/snowflake/runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 2e0047de..8c9730e7 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -14,10 +14,10 @@ before provider construction when a pair is absent or marked `unsupported`. |---|---|---|---| | BigQuery | BigQuery | supported | Released GCP profile and BigQuery regression baseline | | BigQuery | PostgreSQL | experimental | Local cross-backend token/fence publication proof | -| BigQuery | Snowflake | experimental | Local staged-SCD1, schema, replay, and fence conformance | +| BigQuery | Snowflake | experimental | Local staged-SCD1, fenced transforms, schema, replay, and fence conformance | | PostgreSQL | BigQuery | unsupported | Not every BigQuery write mode uses destination fencing | | PostgreSQL | PostgreSQL | experimental | Local native profile, state/warehouse conformance, and benchmarks | -| PostgreSQL | Snowflake | experimental | Local staged-SCD1 and destination-fence conformance | +| PostgreSQL | Snowflake | experimental | Local staged-SCD1, transforms, and destination-fence conformance | `experimental` means the code path is executable and tested locally; it is not a hosted support claim. The packaged `runtime-capabilities.json` remains the supported launcher/profile boundary. diff --git a/docs/decisions.md b/docs/decisions.md index f0f6f179..fac7273a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -877,3 +877,12 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc remote staging. - **Boundary:** Only scalar SCD1 ingestion is admitted. Transforms, graphs, semi-structured types, other write modes, and a support claim remain blocked on their own implementation and live proof. + +## 2026-08-09 — Snowflake transforms admit only transactionally fenced materializations + +- **Preflight:** The complete selected model DAG, schemas, SQL, materializations, and assertions + compile before any Snowflake session or destination claim is opened. +- **Publication:** Table replacement and deterministic cursor-monotonic incremental merge use + session-temporary staging and DML in the same transaction as the exact destination-fence touch. +- **Boundary:** Permanent view DDL, graph execution, and automatic transform-schema evolution fail + closed. They cannot enter this slice without preserving the same publication guarantee. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 2e150a78..def630d9 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -11,9 +11,9 @@ evaluate these limits before using it for an unattended system containing busine renewable credentials and live parity remain later gates. - Provider package extras and the full runtime image do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar SCD1 adapter with - bounded Parquet upload, temporary remote staging, schema checks, and destination fencing. It has - no live Snowflake qualification, transforms, graph execution, semi-structured types, or other - write modes. Redshift has no warehouse adapter yet. PostgreSQL state and warehouse execution are + bounded Parquet upload, temporary remote staging, schema checks, destination fencing, and fenced + table/incremental transforms. It has no live Snowflake qualification, views, graph execution, + semi-structured types, or other write modes. Redshift has no warehouse adapter yet. PostgreSQL state and warehouse execution are implemented and locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse diff --git a/docs/snowflake.md b/docs/snowflake.md index 56859ec7..83bc31f6 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -1,8 +1,8 @@ # Experimental Snowflake warehouse Snowflake is an experimental warehouse adapter, not a supported Dander profile. The current slice -exists to prove the provider boundary with native database/schema coordinates and a real bounded -bulk path before transforms or additional write modes are added. +exists to prove the provider boundary with native database/schema coordinates, a real bounded +bulk path, and fenced table/incremental models before additional write modes are added. ## Configuration @@ -46,6 +46,9 @@ variable reference. Do not put a token, private key, password, or connection str destination fencing-token touch. - Only declared nullable columns may be added automatically. Extra columns, required additions, type drift, nullability drift, malformed rows, and oversized singleton parts fail closed. +- Portable or Snowflake-authored table models replace rows through fenced `DELETE`/`INSERT` DML. +- Incremental models collapse duplicate keys deterministically and accept only rows whose declared + cursor is at least as new as the stored row. Generic model assertions run after publication. The Snowflake role needs usage on its database and warehouse plus permission to create and operate schemas, tables, and temporary stages in the selected namespace. Use a dedicated disposable role @@ -53,7 +56,8 @@ until a live least-privilege profile is qualified. ## Deliberate limits -Only scalar SCD1 ingestion is implemented. Models, tests, graphs, replace, SCD2, snapshot, -incremental writes, semi-structured fields, live concurrency proof, infrastructure provisioning, -and support promotion remain separate work. Use `build_models: false` and `catalog.provider: none` -for this experimental path. +Only scalar SCD1 ingestion plus table and incremental model materializations are implemented. +Views remain unavailable because Snowflake permanent DDL cannot share the destination-fence +transaction. Graphs, replace, SCD2, snapshot, semi-structured fields, live concurrency proof, +infrastructure provisioning, and support promotion remain separate work. Use +`catalog.provider: none` for this experimental path. diff --git a/src/dander/providers/snowflake/runtime.py b/src/dander/providers/snowflake/runtime.py index d0ffb3d4..6b0096df 100644 --- a/src/dander/providers/snowflake/runtime.py +++ b/src/dander/providers/snowflake/runtime.py @@ -24,6 +24,7 @@ execute, open_connection, ) +from dander.providers.snowflake.transform import SnowflakeTransformRunner from dander.providers.snowflake.writer import ( SnowflakeScd1Writer, SnowflakeStagingSettings, @@ -99,7 +100,11 @@ def build_ingestion_writer( @dataclass(frozen=True, slots=True) class SnowflakeTransformFactory: - """Fail clearly until the provider's transform slice is implemented.""" + """Construct the experimental fenced Snowflake transform runner.""" + + database: str + connection_factory: SnowflakeConnectionFactory + target_fence: SnowflakeTargetFence def build_transform_runner( self, @@ -107,11 +112,19 @@ def build_transform_runner( graph_plan: object | None, build_models: bool, raw_namespace: str = "raw", - ) -> None: - del raw_namespace - if graph_plan is not None or build_models: - raise ValueError("Snowflake transforms are not available in this experimental slice") - return None + ) -> SnowflakeTransformRunner | None: + if graph_plan is not None: + raise ValueError( + "Snowflake graph execution is not available in this experimental slice" + ) + if not build_models: + return None + return SnowflakeTransformRunner( + database=self.database, + connection_factory=self.connection_factory, + target_fence=self.target_fence, + raw_namespace=raw_namespace, + ) @dataclass(frozen=True, slots=True) @@ -146,7 +159,7 @@ def operation( schema_contract_version=1, write_modes=frozenset({WriteMode.SCD1}), transports=frozenset({WriteTransport.COPY}), - supports_transforms=False, + supports_transforms=True, supports_graphs=False, supports_target_fencing=True, ) @@ -215,7 +228,11 @@ def build_snowflake_warehouse( target_fence, staging, ), - transforms=SnowflakeTransformFactory(), + transforms=SnowflakeTransformFactory( + config.database, + connection_factory, + target_fence, + ), target_fence=target_fence, telemetry=SnowflakeTelemetry(config.warehouse), capabilities=SNOWFLAKE_CAPABILITIES, diff --git a/src/dander/providers/snowflake/transform.py b/src/dander/providers/snowflake/transform.py new file mode 100644 index 00000000..5f410582 --- /dev/null +++ b/src/dander/providers/snowflake/transform.py @@ -0,0 +1,404 @@ +"""Fenced Snowflake table and incremental model execution.""" + +from __future__ import annotations + +import hashlib +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander.concurrency import TargetFenceLostError +from dander.providers.snowflake.session import ( + SnowflakeConnection, + SnowflakeConnectionFactory, + execute, + open_connection, +) +from dander.providers.snowflake.writer import ( + SnowflakeWriteError, + _ensure_target, + _qualified, + _quote, + _set_query_tag, + _snowflake_type, +) +from dander.transform import ( + SqlDialect, + TransformModel, + TransformProject, + TransformProjectError, + TransformRunError, + TransformRunResult, +) +from dander.transform.model import Materialization +from dander.writer import SchemaEvolution, WriteField, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from pathlib import Path + + from dander.concurrency import OwnershipGuard, TargetFence + from dander.providers.snowflake.fence import SnowflakeTargetFence + from dander.transform.config import GenericTestMetadata + from dander.warehouse import RelationRef + + +@dataclass(frozen=True, slots=True) +class _SnowflakeModelPlan: + model: TransformModel + target: WriteTarget + query: str + + +@dataclass(frozen=True, slots=True) +class _SnowflakeAssertion: + name: str + statement: str + parameters: tuple[object, ...] = () + + +class SnowflakeTransformRunner: + """Build portable table/incremental models behind destination fencing.""" + + target_dialect = SqlDialect.SNOWFLAKE + + def __init__( + self, + *, + database: str, + connection_factory: SnowflakeConnectionFactory, + target_fence: SnowflakeTargetFence, + raw_namespace: str = "raw", + ) -> None: + self._database = database + self._connection_factory = connection_factory + self._target_fence = target_fence + self._raw_namespace = raw_namespace + + def build( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Preflight the selected DAG, then publish each model through fenced DML.""" + if ownership is None or ownership.fence is None: + raise TransformRunError("Snowflake hosted transforms require active lease ownership") + project, models, plans, assertions = self._preflight(models_dir, selected=selected) + del project + for plan in plans: + ownership.verify() + publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) + ownership.verify() + self._publish(plan, publication) + self._run_assertions(assertions, ownership=ownership) + return TransformRunResult( + models=tuple(model.name for model in models), + assertions=len(assertions), + ) + + def test( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ) -> TransformRunResult: + """Run assertions against already materialized Snowflake relations.""" + _project, models, _plans, assertions = self._preflight(models_dir, selected=selected) + self._run_assertions(assertions) + return TransformRunResult( + models=tuple(model.name for model in models), + assertions=len(assertions), + ) + + def _preflight( + self, + models_dir: Path, + *, + selected: Iterable[str] | None, + ) -> tuple[ + TransformProject, + tuple[TransformModel, ...], + tuple[_SnowflakeModelPlan, ...], + tuple[_SnowflakeAssertion, ...], + ]: + """Compile every selected model and assertion before opening a provider session.""" + project = TransformProject.load( + models_dir, + catalog=self._database, + raw_namespace=self._raw_namespace, + target_dialect=SqlDialect.SNOWFLAKE, + ) + models = project.ordered(selected) + plans = tuple(_model_plan(project, model) for model in models) + assertions = tuple( + assertion for model in models for assertion in _compile_assertions(project, model) + ) + return project, models, plans, assertions + + def _publish(self, plan: _SnowflakeModelPlan, publication: TargetFence) -> None: + target = WriteTarget( + relation=plan.target.relation_ref, + business_key=plan.target.business_key, + schema=plan.target.schema, + publication_fence=publication, + ) + temporary = _temporary_relation(target.relation_ref, publication) + cleanup_started = False + with open_connection(self._connection_factory) as connection: + try: + _set_query_tag(connection, publication) + # Transform targets never evolve automatically: the only permanent DDL permitted + # here is create-if-absent followed by exact declared-schema validation. + _ensure_target( + connection, + target, + target.canonical_schema, + evolution=SchemaEvolution.STRICT, + ) + execute(connection, _create_temporary_sql(plan, temporary)) + cleanup_started = True + self._target_fence.execute_statements( + connection, + _publication_statements(plan, temporary), + publication, + ) + except (TargetFenceLostError, SnowflakeWriteError, TransformRunError): + raise + except Exception as error: + raise TransformRunError("Snowflake transform publication failed") from error + finally: + if cleanup_started: + _drop_temporary( + connection, + temporary, + suppress_failure=sys.exc_info()[0] is not None, + ) + + def _run_assertions( + self, + assertions: Sequence[_SnowflakeAssertion], + *, + ownership: OwnershipGuard | None = None, + ) -> None: + failures: list[str] = [] + with open_connection(self._connection_factory) as connection: + for assertion in assertions: + if ownership is not None: + ownership.verify() + try: + row = execute( + connection, + assertion.statement, + assertion.parameters, + fetch="one", + ).row + except Exception as error: + raise TransformRunError( + f"Snowflake assertion execution failed: {assertion.name}" + ) from error + if _failure_count(row, assertion.name) > 0: + failures.append(assertion.name) + if failures: + raise TransformRunError(f"Data tests failed: {', '.join(failures)}") + + +def _model_plan(project: TransformProject, model: TransformModel) -> _SnowflakeModelPlan: + if model.metadata.materialization is Materialization.VIEW: + raise TransformProjectError( + "Snowflake view materialization is unavailable because permanent DDL cannot satisfy " + "transactional destination fencing" + ) + if model.metadata.materialization not in {Materialization.TABLE, Materialization.INCREMENTAL}: + raise TransformProjectError( + f"Snowflake materialization is unavailable: {model.metadata.materialization.value}" + ) + required = set(model.metadata.unique_key) + if model.metadata.incremental_cursor is not None: + required.add(model.metadata.incremental_cursor) + fields = tuple( + WriteField( + name=column.name, + data_type=column.data_type, + mode="REQUIRED" if column.name in required else "NULLABLE", + ) + for column in model.metadata.columns + ) + target = WriteTarget( + relation=project.relation_ref_for_model(model), + business_key=tuple(model.metadata.unique_key), + schema=fields, + ) + for field in target.canonical_schema.fields: + _snowflake_type(field.data_type) + return _SnowflakeModelPlan( + model=model, + target=target, + query=project.compile(model), + ) + + +def _create_temporary_sql(plan: _SnowflakeModelPlan, temporary: str) -> str: + names = tuple(field.name for field in plan.target.canonical_schema.fields) + selected = ", ".join(f"source.{_quote(name)}" for name in names) + if plan.model.metadata.materialization is Materialization.TABLE: + return ( + f"CREATE OR REPLACE TEMPORARY TABLE {temporary} AS SELECT {selected} " + f"FROM ({plan.query}) AS source" + ) + keys = tuple(plan.model.metadata.unique_key) + cursor = plan.model.metadata.incremental_cursor + assert keys and cursor is not None + partition = ", ".join(f"source.{_quote(key)}" for key in keys) + tie_breakers = tuple(name for name in sorted(names) if name not in {*keys, cursor}) + ordering = [f"source.{_quote(cursor)} DESC NULLS LAST"] + ordering.extend(f"source.{_quote(name)} DESC NULLS LAST" for name in tie_breakers) + projected = ", ".join(f"ranked.{_quote(name)}" for name in names) + return ( + f"CREATE OR REPLACE TEMPORARY TABLE {temporary} AS SELECT {projected} FROM (" + f"SELECT {selected}, ROW_NUMBER() OVER (PARTITION BY {partition} ORDER BY " + f"{', '.join(ordering)}) AS {_quote('_dander_rank')} FROM ({plan.query}) AS source" + f") AS ranked WHERE ranked.{_quote('_dander_rank')} = 1" + ) + + +def _publication_statements( + plan: _SnowflakeModelPlan, + temporary: str, +) -> tuple[tuple[str, Sequence[object]], ...]: + target = _qualified(*plan.target.relation_ref.coordinates) + names = tuple(field.name for field in plan.target.canonical_schema.fields) + columns = ", ".join(_quote(name) for name in names) + incoming = ", ".join(f"incoming.{_quote(name)}" for name in names) + if plan.model.metadata.materialization is Materialization.TABLE: + return ( + (f"DELETE FROM {target}", ()), + (f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}", ()), + ) + keys = tuple(plan.model.metadata.unique_key) + cursor = plan.model.metadata.incremental_cursor + assert keys and cursor is not None + match = " AND ".join(f"target.{_quote(key)} = incoming.{_quote(key)}" for key in keys) + mutable = tuple(name for name in names if name not in keys) + update = ", ".join(f"target.{_quote(name)} = incoming.{_quote(name)}" for name in mutable) + matched = ( + f" WHEN MATCHED AND incoming.{_quote(cursor)} >= target.{_quote(cursor)} " + f"THEN UPDATE SET {update}" + if mutable + else "" + ) + merge = ( + f"MERGE INTO {target} AS target USING {temporary} AS incoming ON {match}{matched} " + f"WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({incoming})" + ) + return ((merge, ()),) + + +def _compile_assertions( + project: TransformProject, + model: TransformModel, +) -> tuple[_SnowflakeAssertion, ...]: + relation = _qualified(*project.relation_ref_for_model(model).coordinates) + assertions: list[_SnowflakeAssertion] = [] + for test in model.metadata.tests: + assertions.extend(_assertions_for_test(project, model.name, relation, test)) + return tuple(assertions) + + +def _assertions_for_test( + project: TransformProject, + model_name: str, + relation: str, + test: GenericTestMetadata, +) -> list[_SnowflakeAssertion]: + column = _quote(test.column) + assertions: list[_SnowflakeAssertion] = [] + if test.not_null: + assertions.append( + _SnowflakeAssertion( + name=f"{model_name}.{test.column}.not_null", + statement=f"SELECT COUNT_IF({column} IS NULL) AS failures FROM {relation}", + ) + ) + if test.unique: + assertions.append( + _SnowflakeAssertion( + name=f"{model_name}.{test.column}.unique", + statement=( + "SELECT COUNT(*) AS failures FROM (" + f"SELECT {column} FROM {relation} WHERE {column} IS NOT NULL " + f"GROUP BY {column} HAVING COUNT(*) > 1) AS duplicates" + ), + ) + ) + if test.accepted_values is not None: + placeholders = ", ".join("?" for _ in test.accepted_values) + assertions.append( + _SnowflakeAssertion( + name=f"{model_name}.{test.column}.accepted_values", + statement=( + f"SELECT COUNT_IF({column} IS NOT NULL AND {column} NOT IN " + f"({placeholders})) AS failures FROM {relation}" + ), + parameters=tuple(test.accepted_values), + ) + ) + if test.relationships is not None: + if test.relationships.to in project.models: + parent_model = project.models[test.relationships.to] + parent_columns = {item.name for item in parent_model.metadata.columns} + if test.relationships.field not in parent_columns: + raise TransformProjectError( + "Relationship test references an undeclared parent column: " + f"{test.relationships.to}.{test.relationships.field}" + ) + parent = _qualified(*project.relation_ref_for_ref(test.relationships.to).coordinates) + parent_field = _quote(test.relationships.field) + assertions.append( + _SnowflakeAssertion( + name=f"{model_name}.{test.column}.relationships", + statement=( + f"SELECT COUNT(*) AS failures FROM {relation} AS child " + f"LEFT JOIN {parent} AS parent ON child.{column} = parent.{parent_field} " + f"WHERE child.{column} IS NOT NULL AND parent.{parent_field} IS NULL" + ), + ) + ) + return assertions + + +def _temporary_relation(relation: RelationRef, publication: TargetFence) -> str: + digest = hashlib.sha256( + f"{'.'.join(relation.coordinates)}:{publication.run_id}:{publication.token}".encode() + ).hexdigest()[:20] + return _qualified(relation.catalog, relation.namespace, f"dander_model_{digest}") + + +def _drop_temporary( + connection: SnowflakeConnection, + temporary: str, + *, + suppress_failure: bool, +) -> None: + try: + execute(connection, f"DROP TABLE IF EXISTS {temporary}") + except Exception as error: + if suppress_failure: + return + raise TransformRunError("Snowflake transform staging cleanup failed") from error + + +def _failure_count(row: object | None, assertion: str) -> int: + if isinstance(row, dict): + value = row.get("FAILURES", row.get("failures")) + elif isinstance(row, (tuple, list)) and row: + value = row[0] + else: + value = None + if isinstance(value, bool) or not isinstance(value, int): + raise TransformRunError(f"Assertion returned an invalid result: {assertion}") + return value + + +__all__ = ["SnowflakeTransformRunner"] diff --git a/src/dander/transform/project.py b/src/dander/transform/project.py index c4ed23e0..01ecb2f9 100644 --- a/src/dander/transform/project.py +++ b/src/dander/transform/project.py @@ -8,6 +8,7 @@ import sqlglot from jinja2 import Environment, StrictUndefined, TemplateError +from pydantic import ValidationError from sqlglot import exp from dander.transform.config import ModelMetadata, TransformConfigError, load_model_metadata @@ -72,9 +73,16 @@ def __init__( elif self.target_dialect is SqlDialect.POSTGRES: if not _POSTGRESQL_CATALOG.fullmatch(catalog): raise TransformProjectError("Invalid PostgreSQL database name") - elif not _IDENTIFIER.fullmatch(catalog): - raise TransformProjectError("Invalid warehouse catalog identifier") - if not _IDENTIFIER.fullmatch(raw_namespace): + else: + try: + RelationRef(catalog=catalog, namespace=raw_namespace, name="dander_validation") + except ValidationError as error: + raise TransformProjectError("Invalid warehouse relation coordinates") from error + legacy_namespace = self.target_dialect in { + SqlDialect.BIGQUERY, + SqlDialect.POSTGRES, + } + if legacy_namespace and not _IDENTIFIER.fullmatch(raw_namespace): raise TransformProjectError("Invalid raw namespace identifier") indexed: dict[str, TransformModel] = {} for model in models: diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index 46a541e7..cbf253ae 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -13,8 +13,10 @@ from dander.concurrency import FencingToken, TargetFenceLostError from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry from dander.providers.snowflake import SnowflakeWarehouseConfig +from dander.providers.snowflake.transform import SnowflakeTransformRunner from dander.providers.snowflake.writer import SnowflakeWriteError from dander.telemetry import TelemetryOperation +from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError from dander.warehouse import RelationRef, WarehouseRuntime from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport @@ -46,6 +48,9 @@ class _FakeSnowflake: closes: int = 0 committed_claims: int = 0 discarded_claims: int = 0 + transform_source_rows: list[dict[str, object]] | None = None + transform_rows: dict[str, dict[str, object]] = field(default_factory=dict) + assertion_failure_count: int = 0 def connect(self) -> _FakeConnection: return _FakeConnection(self) @@ -55,6 +60,7 @@ class _FakeConnection: def __init__(self, backend: _FakeSnowflake) -> None: self.backend = backend self.pending_claim = False + self.temporary_rows: list[dict[str, object]] = [] def cursor(self) -> _FakeCursor: return _FakeCursor(self) @@ -110,8 +116,41 @@ def execute(self, command: str, params: Sequence[object] | None = None) -> Self: assert len(parameters) == 10 self.backend.committed_checksums.add(checksum) self.rowcount = 1 + elif compact.startswith("CREATE OR REPLACE TEMPORARY TABLE"): + rows = self.backend.transform_source_rows + if rows is not None: + self.connection.temporary_rows = [dict(row) for row in rows] + if "ROW_NUMBER() OVER" in compact: + selected: dict[str, dict[str, object]] = {} + for row in sorted( + self.connection.temporary_rows, + key=lambda item: ( + _integer(item, "updated_at"), + str(item.get("label", "")), + ), + reverse=True, + ): + selected.setdefault(str(row["id"]), row) + self.connection.temporary_rows = list(selected.values()) + elif compact.startswith("DELETE FROM") and self.backend.transform_source_rows is not None: + self.backend.transform_rows.clear() + elif compact.startswith("INSERT INTO") and self.backend.transform_source_rows is not None: + self.backend.transform_rows = { + str(row["id"]): dict(row) for row in self.connection.temporary_rows + } elif compact.startswith("MERGE INTO"): self.rowcount = self.backend.merge_rowcount + if self.backend.transform_source_rows is not None: + assert 'incoming."updated_at" >= target."updated_at"' in compact + for row in self.connection.temporary_rows: + key = str(row["id"]) + current = self.backend.transform_rows.get(key) + if current is None or _integer(row, "updated_at") >= _integer( + current, "updated_at" + ): + self.backend.transform_rows[key] = dict(row) + elif compact.startswith("SELECT COUNT"): + self._row = (self.backend.assertion_failure_count,) return self def fetchone(self) -> object | None: @@ -142,6 +181,12 @@ def _config() -> dict[str, object]: } +def _integer(row: dict[str, object], field_name: str) -> int: + value = row[field_name] + assert isinstance(value, int) and not isinstance(value, bool) + return value + + @pytest.fixture def snowflake_runtime( tmp_path: Path, @@ -290,10 +335,268 @@ def records() -> Iterator[dict[str, object]]: assert len(backend.committed_checksums) == 2 assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) - assert runtime.capabilities.supports_transforms is False + assert runtime.capabilities.supports_transforms is True assert runtime.relation_codec.render(relation) == '"DANDER_TEST"."raw"."records"' +@dataclass +class _Ownership: + fence: FencingToken + verifications: int = 0 + + def verify(self) -> None: + self.verifications += 1 + + +def test_snowflake_builds_fenced_table_and_incremental_models( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=None, + build_models=True, + raw_namespace="raw", + ) + assert isinstance(runner, SnowflakeTransformRunner) + _write_snowflake_model( + tmp_path, + name="table_model", + materialization="table", + with_tests=True, + ) + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "Y"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ("updated_at", "NUMBER(38,0)", "COLUMN", "Y"), + ] + backend.transform_rows = {"stale": {"id": "stale", "label": "old-target", "updated_at": 0}} + backend.transform_source_rows = [{"id": "fresh", "label": "new-target", "updated_at": 1}] + table_ownership = _ownership("run-table", 1) + + result = runner.build(tmp_path, selected=["table_model"], ownership=table_ownership) + + assert result.models == ("table_model",) + assert result.assertions == 4 + sql = [statement for statement, _parameters in backend.statements] + assert any( + statement.startswith("DELETE FROM") and "table_model" in statement for statement in sql + ) + assert any( + statement.startswith("INSERT INTO") and "table_model" in statement for statement in sql + ) + assert not any(statement.startswith("ALTER TABLE") for statement in sql) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + assert backend.transform_rows == { + "fresh": {"id": "fresh", "label": "new-target", "updated_at": 1} + } + backend.assertion_failure_count = 1 + with pytest.raises(TransformRunError, match=r"table_model\.id\.not_null") as error: + runner.test(tmp_path, selected=["table_model"]) + assert "older" not in str(error.value) + backend.assertion_failure_count = 0 + backend.transform_rows.clear() + + before_incremental = len(backend.statements) + _write_snowflake_model(tmp_path, name="incremental_model", materialization="incremental") + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "N"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ("updated_at", "NUMBER(38,0)", "COLUMN", "N"), + ] + backend.transform_source_rows = [ + {"id": "one", "label": "older", "updated_at": 1}, + {"id": "one", "label": "aaa-equal", "updated_at": 2}, + {"id": "one", "label": "zzz-equal", "updated_at": 2}, + {"id": "two", "label": "second", "updated_at": 1}, + ] + incremental_ownership = _ownership("run-incremental", 2) + result = runner.build( + tmp_path, + selected=["incremental_model"], + ownership=incremental_ownership, + ) + + assert result.models == ("incremental_model",) + incremental_sql = [ + statement for statement, _parameters in backend.statements[before_incremental:] + ] + temporary = next( + statement + for statement in incremental_sql + if statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") + ) + assert ( + 'ROW_NUMBER() OVER (PARTITION BY source."id" ORDER BY ' + 'source."updated_at" DESC NULLS LAST, source."label" DESC NULLS LAST)' in temporary + ) + merge = next( + statement + for statement in incremental_sql + if statement.startswith("MERGE INTO") and "dander_target_commits" not in statement + ) + assert 'incoming."updated_at" >= target."updated_at"' in merge + assert 'target."label" = incoming."label"' in merge + assert backend.transform_rows == { + "one": {"id": "one", "label": "zzz-equal", "updated_at": 2}, + "two": {"id": "two", "label": "second", "updated_at": 1}, + } + backend.transform_source_rows = [{"id": "one", "label": "stale-replay", "updated_at": 1}] + runner.build( + tmp_path, + selected=["incremental_model"], + ownership=_ownership("run-replay", 3), + ) + assert backend.transform_rows["one"] == { + "id": "one", + "label": "zzz-equal", + "updated_at": 2, + } + assert backend.rollbacks == 0 + + +def test_snowflake_preflights_whole_dag_before_provider_mutation( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + runner = runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + assert isinstance(runner, SnowflakeTransformRunner) + _write_snowflake_model(tmp_path, name="good_model", materialization="table") + _write_snowflake_model(tmp_path, name="unsupported_view", materialization="view") + before = list(backend.statements) + + with pytest.raises(TransformProjectError, match="view materialization is unavailable"): + runner.build( + tmp_path, + ownership=_ownership("run-preflight", 3), + ) + + assert backend.statements == before + + +def test_snowflake_transform_coordinates_are_canonical_and_graphs_fail_closed( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, _backend, _staging_root = snowflake_runtime + _write_snowflake_model(tmp_path, name="coordinate_model", materialization="table") + project = TransformProject.load( + tmp_path, + catalog="DANDER-TEST", + raw_namespace="RAW_DATA", + target_dialect=SqlDialect.SNOWFLAKE, + ) + model = project.ordered(["coordinate_model"])[0] + + assert project.relation_ref_for_model(model).coordinates == ( + "DANDER-TEST", + "analytics", + "coordinate_model", + ) + assert '"DANDER-TEST"."RAW_DATA"."records"' in project.compile(model) + with pytest.raises(ValueError, match="graph execution is not available"): + runtime.transforms.build_transform_runner(graph_plan=object(), build_models=False) + assert runtime.transforms.build_transform_runner(graph_plan=None, build_models=False) is None + + +def test_snowflake_transform_requires_ownership_and_fails_closed_after_staging( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + runner = runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + assert isinstance(runner, SnowflakeTransformRunner) + _write_snowflake_model(tmp_path, name="owned_model", materialization="table") + with pytest.raises(TransformRunError, match="require active lease ownership"): + runner.build(tmp_path) + backend.transform_source_rows = [ + {"id": "blocked", "label": "must-not-publish", "updated_at": 1} + ] + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "Y"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ("updated_at", "NUMBER(38,0)", "COLUMN", "Y"), + ] + backend.fence_touch_rowcount = 0 + before_publication = len(backend.statements) + rollbacks = backend.rollbacks + + with pytest.raises(TargetFenceLostError, match="lost before publication"): + runner.build(tmp_path, ownership=_ownership("run-stale", 1)) + + attempted = [statement for statement, _ in backend.statements[before_publication:]] + assert any(statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") for statement in attempted) + assert not any(statement.startswith(("DELETE FROM", "INSERT INTO")) for statement in attempted) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in attempted) + assert backend.rollbacks == rollbacks + 1 + assert backend.transform_rows == {} + + +def _ownership(run_id: str, token: int) -> _Ownership: + return _Ownership( + FencingToken( + lease_table=None, + pipeline_id="snowflake_transforms", + run_id=run_id, + token=token, + authority_id="postgresql:test-state", + ) + ) + + +def _write_snowflake_model( + root: Path, + *, + name: str, + materialization: str, + with_tests: bool = False, +) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / f"{name}.sql").write_text("SELECT id, label, updated_at FROM {{ ref('raw_records') }}") + incremental = ( + "unique_key: [id]\nincremental_cursor: updated_at\n" + if materialization == "incremental" + else "" + ) + tests = ( + "tests:\n" + " - column: id\n" + " not_null: true\n" + " unique: true\n" + " - column: label\n" + " accepted_values: [older, newer, second]\n" + " - column: id\n" + " relationships:\n" + " to: raw_parent_records\n" + " field: id\n" + if with_tests + else "tests: []\n" + ) + (root / f"{name}.yml").write_text( + f"model: {name}\n" + "description: Portable Snowflake transform fixture.\n" + "owner: data-eng\n" + "dialect: portable\n" + f"materialization: {materialization}\n" + "dataset: analytics\n" + "source_system: fixture\n" + "sensitivity: public\n" + f"{incremental}" + "columns:\n" + " - name: id\n" + " type: STRING\n" + " description: Stable fixture identifier.\n" + " - name: label\n" + " type: STRING\n" + " description: Deterministic tie breaker.\n" + " - name: updated_at\n" + " type: INT64\n" + " description: Monotonic fixture cursor.\n" + f"{tests}" + ) + + def test_snowflake_retry_skips_committed_files_but_still_touches_fence( snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], ) -> None: From 377ea78ada71b6fe11aa6af2dd2a6e9164ab3a25 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 00:40:14 -0400 Subject: [PATCH 060/163] docs: finalize Snowflake transform handoff --- HANDOFF.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index c998e33a..4c8a7648 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -18,6 +18,7 @@ may now run with active lease ownership. No live Snowflake qualification is clai - Full suite passed: 1,065 tests with PostgreSQL 15; Ruff and strict mypy passed. - Wheel/sdist inspection, source-free installs, generated-project validation, and Terraform validation passed. - Container build/conformance, Terraform/AWS tests, and dependency audit passed. +- Protected CI passed Python/PostgreSQL, Terraform/static, distribution, container, and secret jobs. - No cloud plan/apply, deployment mutation, or package publication occurred. ## Decisions @@ -28,9 +29,7 @@ may now run with active lease ownership. No live Snowflake qualification is clai ## Remaining -- Complete independent adversarial review. -- Push the stacked branch and open a draft PR against the Snowflake foundation branch. -- Let protected CI repeat Linux, packaging, container, security, and Terraform checks. +- Review stacked draft PR #137 after its Snowflake foundation PR #136. - Keep both Snowflake PRs unmerged while retained GCP baseline drift remains unresolved. ## Review First From 472dc3cd5b62a7054100becd969aa0eb7a75e0ac Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 01:30:37 -0400 Subject: [PATCH 061/163] feat: add experimental Redshift SCD1 warehouse --- HANDOFF.md | 44 +- docs/compatibility-matrix.md | 2 + docs/decisions.md | 12 + docs/known-limitations.md | 8 +- docs/warehouse-staging.md | 8 +- src/dander/project/config.py | 5 +- src/dander/project/portable_config.py | 6 +- src/dander/providers/defaults.py | 9 + src/dander/providers/redshift/__init__.py | 5 + src/dander/providers/redshift/config.py | 124 ++++ src/dander/providers/redshift/fence.py | 237 +++++++ src/dander/providers/redshift/runtime.py | 308 +++++++++ src/dander/providers/redshift/session.py | 75 ++ src/dander/providers/redshift/writer.py | 639 ++++++++++++++++++ src/dander/runtime-compatibility.json | 12 + tests/cli/test_run_command.py | 8 + tests/project/test_portable_config.py | 34 + .../test_redshift_warehouse_runtime.py | 518 ++++++++++++++ tests/test_compatibility.py | 4 + 19 files changed, 2030 insertions(+), 28 deletions(-) create mode 100644 src/dander/providers/redshift/__init__.py create mode 100644 src/dander/providers/redshift/config.py create mode 100644 src/dander/providers/redshift/fence.py create mode 100644 src/dander/providers/redshift/runtime.py create mode 100644 src/dander/providers/redshift/session.py create mode 100644 src/dander/providers/redshift/writer.py create mode 100644 tests/providers/test_redshift_warehouse_runtime.py diff --git a/HANDOFF.md b/HANDOFF.md index 4c8a7648..91b17d55 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,42 @@ ## Finished -- Added experimental Snowflake table and incremental model execution. -- Preflights the full selected DAG before provider mutation and rejects views/graphs fail-closed. -- Publishes through session-temporary staging and exact transactional destination fencing. -- Preserves canonical database/schema/relation coordinates, including hyphenated databases. -- Updated Snowflake capability and limitation documentation without a support promotion. +- Added an experimental Redshift warehouse provider using native database/schema coordinates. +- Implemented bounded Parquet staging, same-region S3 manifest `COPY`, and IAM-only authentication. +- Added deterministic SCD1 merge/replay, additive schema checks, transactional destination fencing, + and exact owned-object cleanup. +- Registered only BigQuery-state/Redshift and PostgreSQL-state/Redshift as experimental pairs. +- Updated existing compatibility, staging, limitations, and architecture decision records. ## Try It -Configure the experimental profile in `docs/snowflake.md`; portable table and incremental models -may now run with active lease ownership. No live Snowflake qualification is claimed. +Configure a v2 `warehouse.provider: redshift` profile with provisioned-cluster or Serverless IAM +settings, then use the normal run path with SCD1. No AWS live profile was created or contacted. ## Checks -- Full suite passed: 1,065 tests with PostgreSQL 15; Ruff and strict mypy passed. -- Wheel/sdist inspection, source-free installs, generated-project validation, and Terraform validation passed. -- Container build/conformance, Terraform/AWS tests, and dependency audit passed. -- Protected CI passed Python/PostgreSQL, Terraform/static, distribution, container, and secret jobs. -- No cloud plan/apply, deployment mutation, or package publication occurred. +- Ruff, formatting, and strict mypy passed; all 1,063 tests passed with PostgreSQL 15 integration. +- Wheel/sdist build, inspection, outside-checkout installs, source-free generation, and full runtime + dependency loading passed; strict dependency audit found no known vulnerabilities. +- The exact final container built and passed non-root, read-only, runtime, and bundled-asset checks. +- GCP/AWS Terraform roots and module tests, cross-cloud feasibility roots, and Helm checks passed. ## Decisions -- Snowflake view DDL remains excluded because it cannot share the destination-fence transaction. -- Transform targets permit create-if-absent with exact schema; automatic ALTER is not allowed. -- Incremental duplicates resolve by cursor descending, then stable declared-column ordering. +- Redshift accepts ambient AWS credentials and uses a configured IAM role for `COPY`; no static + access keys or database passwords enter configuration or SQL. +- Publication is SCD1-only and transactionally fenced; replay identity includes target, pipeline, + run, schema fingerprint, and deterministic manifest digest. +- Unsupported transforms, graphs, write modes, and semi-structured mappings fail closed. ## Remaining -- Review stacked draft PR #137 after its Snowflake foundation PR #136. -- Keep both Snowflake PRs unmerged while retained GCP baseline drift remains unresolved. +- Let protected Linux CI repeat image/config/secret scans before merge consideration. +- Resolve the pre-existing retained GCP no-drift baseline before merging provider work. +- Qualify Redshift transforms, other write modes, and one live disposable AWS profile separately. ## Review First -- `src/dander/providers/snowflake/transform.py` -- `tests/providers/test_snowflake_warehouse_runtime.py` -- `src/dander/providers/snowflake/runtime.py` +- `src/dander/providers/redshift/fence.py` +- `src/dander/providers/redshift/writer.py` +- `tests/providers/test_redshift_warehouse_runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 8c9730e7..bc4b8c40 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -15,9 +15,11 @@ before provider construction when a pair is absent or marked `unsupported`. | BigQuery | BigQuery | supported | Released GCP profile and BigQuery regression baseline | | BigQuery | PostgreSQL | experimental | Local cross-backend token/fence publication proof | | BigQuery | Snowflake | experimental | Local staged-SCD1, fenced transforms, schema, replay, and fence conformance | +| BigQuery | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, and destination-fence conformance | | PostgreSQL | BigQuery | unsupported | Not every BigQuery write mode uses destination fencing | | PostgreSQL | PostgreSQL | experimental | Local native profile, state/warehouse conformance, and benchmarks | | PostgreSQL | Snowflake | experimental | Local staged-SCD1, transforms, and destination-fence conformance | +| PostgreSQL | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, and destination-fence conformance | `experimental` means the code path is executable and tested locally; it is not a hosted support claim. The packaged `runtime-capabilities.json` remains the supported launcher/profile boundary. diff --git a/docs/decisions.md b/docs/decisions.md index fac7273a..5cafb78f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -886,3 +886,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc session-temporary staging and DML in the same transaction as the exact destination-fence touch. - **Boundary:** Permanent view DDL, graph execution, and automatic transform-schema evolution fail closed. They cannot enter this slice without preserving the same publication guarantee. + +## 2026-08-09 — Redshift begins with IAM-only bounded SCD1 publication + +- **Boundary:** Provisioned and Serverless profiles map native database/schema coordinates into + canonical relations. The first slice supports scalar-schema SCD1 only; transforms, graphs, + other write modes, SUPER fallback, infrastructure, and live qualification remain unavailable. +- **Loading:** Bounded Parquet parts use a mandatory, content-length S3 manifest in the warehouse + region. Ambient AWS credentials upload and clean owned objects; SQL receives only a validated + cluster/workgroup COPY-role ARN and never an AWS access key. +- **Correctness:** Claims serialize on the destination ledger. One publication transaction locks + and DML-touches the exact authority/run/token, evolves declared nullable columns, performs the + deterministic ordinal MERGE, records replay identity, and commits the fence or rolls back all. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index def630d9..7be70c16 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -13,8 +13,12 @@ evaluate these limits before using it for an unattended system containing busine Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar SCD1 adapter with bounded Parquet upload, temporary remote staging, schema checks, destination fencing, and fenced table/incremental transforms. It has no live Snowflake qualification, views, graph execution, - semi-structured types, or other write modes. Redshift has no warehouse adapter yet. PostgreSQL state and warehouse execution are - implemented and locally conformance-tested. A packaged Helm + semi-structured types, or other write modes. + Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded + Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, and destination fencing; + it has no live qualification, transforms, executable graphs, non-SCD1 modes, or semi-structured + `SUPER` mapping. PostgreSQL state and warehouse execution are implemented and locally + conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/warehouse-staging.md b/docs/warehouse-staging.md index 07aee237..8f8ed9ce 100644 --- a/docs/warehouse-staging.md +++ b/docs/warehouse-staging.md @@ -25,6 +25,8 @@ or string representation. Decimal precision above Parquet/Arrow's 76-digit limit file is published. Missing/extra fields, required nulls, malformed JSON, and Arrow conversion errors produce row-indexed messages without echoing record contents. -This is a prerequisite, not a support claim. Snowflake and Redshift remain unavailable until their -upload, idempotency, target fencing, schema, transform, telemetry, cleanup, and live-profile gates -pass independently. +Redshift's experimental SCD1 adapter consumes this contract through same-region S3 manifest `COPY`, +with IAM-role authentication, deterministic replay history, target fencing, and owned-object +cleanup. Transforms, executable graphs, other write modes, semi-structured `SUPER` mapping, and a +live Redshift profile remain unavailable. Snowflake remains unavailable until its independent +upload, idempotency, fencing, schema, transform, telemetry, cleanup, and live-profile gates pass. diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 69ff4d99..065618cc 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -213,8 +213,9 @@ class DanderProject(BaseModel): pipelines: dict[str, PipelineSpec] = Field(min_length=1) platform_name: str = Field(default="gcp", exclude=True) deployment_name: str = Field(default="gcp_cloud_run", exclude=True) - warehouse_provider: Literal["bigquery", "postgresql", "snowflake"] = Field( - default="bigquery", exclude=True + warehouse_provider: Literal["bigquery", "postgresql", "snowflake", "redshift"] = Field( + default="bigquery", + exclude=True, ) warehouse_config: dict[str, object] = Field( default_factory=_default_warehouse_config, diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 8ff4c61c..2fa87a87 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -35,6 +35,7 @@ PostgreSQLStateConfig, PostgreSQLWarehouseConfig, ) +from dander.providers.redshift import RedshiftWarehouseConfig # noqa: TC001 from dander.providers.snowflake import SnowflakeWarehouseConfig # noqa: TC001 if TYPE_CHECKING: @@ -147,7 +148,10 @@ def validate_pipeline_ids( WarehouseSpec = Annotated[ - BigQueryWarehouseConfig | PostgreSQLWarehouseConfig | SnowflakeWarehouseConfig, + BigQueryWarehouseConfig + | PostgreSQLWarehouseConfig + | SnowflakeWarehouseConfig + | RedshiftWarehouseConfig, Field(discriminator="provider"), ] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index a976aeb6..aa71a0b2 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -10,6 +10,7 @@ from dander.providers.kubernetes import KubernetesLauncherConfig from dander.providers.no_catalog import NoCatalogConfig from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig +from dander.providers.redshift import RedshiftWarehouseConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory from dander.providers.snowflake import SnowflakeWarehouseConfig @@ -41,6 +42,14 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.snowflake.runtime:SNOWFLAKE_WAREHOUSE_FACTORY" ), ) + registry.register( + kind=ProviderKind.WAREHOUSE, + provider_id="redshift", + config_model=RedshiftWarehouseConfig, + load_factory=lazy_provider_factory( + "dander.providers.redshift.runtime:REDSHIFT_WAREHOUSE_FACTORY" + ), + ) registry.register( kind=ProviderKind.WAREHOUSE, provider_id="postgresql", diff --git a/src/dander/providers/redshift/__init__.py b/src/dander/providers/redshift/__init__.py new file mode 100644 index 00000000..95df942f --- /dev/null +++ b/src/dander/providers/redshift/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light Redshift provider registration surface.""" + +from dander.providers.redshift.config import RedshiftWarehouseConfig + +__all__ = ["RedshiftWarehouseConfig"] diff --git a/src/dander/providers/redshift/config.py b/src/dander/providers/redshift/config.py new file mode 100644 index 00000000..59d2f7b9 --- /dev/null +++ b/src/dander/providers/redshift/config.py @@ -0,0 +1,124 @@ +"""Validated Redshift warehouse settings containing no AWS credentials.""" + +from __future__ import annotations + +import re +from typing import Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from dander.warehouse import RelationRef + +_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]{0,126}$") +_AWS_REGION = re.compile(r"^[a-z]{2}(?:-gov)?-[a-z]+-\d$") +_HOST = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}[A-Za-z0-9]$") +_S3_BUCKET = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$") +_IAM_ROLE = re.compile(r"^arn:(?:aws|aws-us-gov):iam::\d{12}:role/[A-Za-z0-9+=,.@_/-]+$") +_RESOURCE_NAME = re.compile(r"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$") + + +class RedshiftWarehouseConfig(BaseModel): + """One provisioned or Serverless Redshift data-plane profile.""" + + model_config = ConfigDict( + extra="forbid", + frozen=True, + populate_by_name=True, + serialize_by_alias=True, + ) + + provider: Literal["redshift"] + deployment: Literal["provisioned", "serverless"] + host: str + port: int = Field(default=5439, ge=1, le=65_535) + database: str + schema_name: str = Field(default="raw", alias="schema") + db_user: str | None = None + region: str + cluster_identifier: str | None = None + workgroup_name: str | None = None + copy_role_arn: str + staging_bucket: str + staging_prefix: str = "dander/staging" + connect_timeout_seconds: int = Field(default=30, ge=1, le=300) + statement_timeout_ms: int = Field(default=900_000, ge=1_000, le=3_600_000) + max_rows_per_file: int = Field(default=100_000, ge=1, le=1_000_000) + max_logical_bytes_per_file: int = Field( + default=128 * 1_024 * 1_024, + ge=1_024, + le=1_073_741_824, + ) + compression: Literal["snappy", "zstd"] = "zstd" + + @model_validator(mode="after") + def validate_settings(self) -> Self: + for field_name, value in ( + ("database", self.database), + ("schema", self.schema_name), + ): + if not _IDENTIFIER.fullmatch(value): + raise ValueError(f"{field_name} must use portable lowercase identifier syntax") + if self.db_user is not None and not _IDENTIFIER.fullmatch(self.db_user): + raise ValueError("db_user must use portable lowercase identifier syntax") + if not _HOST.fullmatch(self.host): + raise ValueError("host must be a DNS name") + if not _AWS_REGION.fullmatch(self.region): + raise ValueError("region must be an AWS region") + if not _IAM_ROLE.fullmatch(self.copy_role_arn): + raise ValueError("copy_role_arn must reference one AWS IAM role") + if not _S3_BUCKET.fullmatch(self.staging_bucket): + raise ValueError("staging_bucket must be a valid S3 bucket name") + prefix = self.staging_prefix.strip("/") + if not prefix or ".." in prefix.split("/") or any(ord(char) < 32 for char in prefix): + raise ValueError("staging_prefix must be a safe non-empty S3 key prefix") + object.__setattr__(self, "staging_prefix", prefix) + if self.deployment == "provisioned": + if ( + not self.cluster_identifier + or self.workgroup_name is not None + or self.db_user is None + ): + raise ValueError( + "provisioned Redshift requires cluster_identifier and db_user only" + ) + assert self.cluster_identifier is not None + if not _RESOURCE_NAME.fullmatch(self.cluster_identifier): + raise ValueError("cluster_identifier must use lowercase AWS resource syntax") + elif ( + not self.workgroup_name + or self.cluster_identifier is not None + or self.db_user is not None + ): + raise ValueError("Redshift Serverless requires workgroup_name and an AWS-derived user") + else: + assert self.workgroup_name is not None + if not _RESOURCE_NAME.fullmatch(self.workgroup_name): + raise ValueError("workgroup_name must use lowercase AWS resource syntax") + return self + + def raw_relation( + self, + name: str, + *, + compatibility_catalog: str | None, + compatibility_namespace: str | None, + default_namespace: str, + ) -> RelationRef: + """Translate compatibility inputs into Redshift database/schema coordinates.""" + del compatibility_catalog, default_namespace + relation = RelationRef( + catalog=self.database, + namespace=compatibility_namespace or self.schema_name, + name=name, + ) + validate_redshift_relation(relation) + return relation + + +def validate_redshift_relation(relation: RelationRef) -> None: + """Reject identifiers Redshift would silently truncate.""" + if any(len(value.encode()) > 127 for value in (relation.namespace, relation.name)): + raise ValueError("Redshift namespace and relation identifiers cannot exceed 127 bytes") + + +__all__ = ["RedshiftWarehouseConfig", "validate_redshift_relation"] diff --git a/src/dander/providers/redshift/fence.py b/src/dander/providers/redshift/fence.py new file mode 100644 index 00000000..450be4ce --- /dev/null +++ b/src/dander/providers/redshift/fence.py @@ -0,0 +1,237 @@ +"""Serialized Redshift destination claims and transactional publication fencing.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.providers.redshift.config import validate_redshift_relation +from dander.providers.redshift.session import ( + RedshiftConnection, + RedshiftConnectionFactory, + RedshiftStatementResult, + execute, + open_connection, +) +from dander.warehouse.runtime import PreparedWarehouseStatement + +if TYPE_CHECKING: + from collections.abc import Sequence + + from dander.warehouse import RelationRef + + +@dataclass(frozen=True, slots=True) +class RedshiftFencePlan: + touch_sql: str + commit_sql: str + parameters: tuple[object, ...] + + +@dataclass(frozen=True, slots=True) +class RedshiftTargetFence: + connection_factory: RedshiftConnectionFactory + database: str + + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + validate_redshift_relation(target) + if target.catalog != self.database: + raise ValueError("Redshift target fence must use the configured database") + table = _qualified(target.namespace, "dander_target_commits") + target_id = ".".join(target.coordinates) + with open_connection(self.connection_factory) as connection: + try: + execute(connection, f"CREATE SCHEMA IF NOT EXISTS {_quote(target.namespace)}") + execute(connection, _fence_table_sql(table)) + connection.commit() + execute(connection, "BEGIN") + execute(connection, f"LOCK {table}") + current = execute( + connection, + _select_claim_sql(table), + (target_id, fence.pipeline_id), + fetch="one", + ).row + if current is None: + execute(connection, _insert_claim_sql(table), _claim_values(target_id, fence)) + elif _claim_is_allowed(current, fence): + execute(connection, _update_claim_sql(table), _update_values(target_id, fence)) + else: + raise TargetFenceLostError( + f"Destination target {target_id!r} rejected stale publication ownership" + ) + connection.commit() + except Exception: + connection.rollback() + raise + return TargetFence( + fence_table=f"{target.catalog}.{target.namespace}.dander_target_commits", + target_id=target_id, + authority_id=fence.resolved_authority_id, + authority_epoch=fence.authority_epoch, + pipeline_id=fence.pipeline_id, + run_id=fence.run_id, + token=fence.token, + ) + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: + return PreparedWarehouseStatement( + sql=statement.strip().removesuffix(";"), + options=self._plan(fence), + ) + + def execute_dml( + self, + connection: RedshiftConnection, + statement: str, + fence: TargetFence, + parameters: Sequence[object] = (), + ) -> RedshiftStatementResult: + return self.execute_statements(connection, ((statement, parameters),), fence)[0] + + def execute_statements( + self, + connection: RedshiftConnection, + statements: Sequence[tuple[str, Sequence[object]]], + fence: TargetFence, + ) -> tuple[RedshiftStatementResult, ...]: + if not statements: + raise ValueError("Redshift fenced publication requires at least one statement") + plan = self._plan(fence) + table = _fence_table(fence.fence_table) + execute(connection, "BEGIN") + try: + execute(connection, f"LOCK {table}") + touched = execute(connection, plan.touch_sql, plan.parameters) + if touched.rowcount != 1: + raise TargetFenceLostError("Dander destination fence lost before publication") + results: list[RedshiftStatementResult] = [] + for statement, parameters in statements: + results.append(execute(connection, statement, parameters)) + committed = execute(connection, plan.commit_sql, plan.parameters) + if committed.rowcount != 1: + raise TargetFenceLostError("Dander destination fence lost during publication") + connection.commit() + return tuple(results) + except Exception: + connection.rollback() + raise + + def _plan(self, fence: TargetFence) -> RedshiftFencePlan: + table = _fence_table(fence.fence_table) + parameters = ( + fence.target_id, + fence.pipeline_id, + fence.authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + ) + match = ( + '"target_id" = %s AND "pipeline_id" = %s AND "authority_id" = %s ' + 'AND "authority_epoch" = %s AND "run_id" = %s AND "fencing_token" = %s' + ) + return RedshiftFencePlan( + touch_sql=f'UPDATE {table} SET "claimed_at" = "claimed_at" WHERE {match} ' + "AND \"status\" IN ('claimed', 'committed')", + commit_sql=f"UPDATE {table} SET \"status\" = 'committed', " + f'"committed_at" = GETDATE() WHERE {match} ' + "AND \"status\" IN ('claimed', 'committed')", + parameters=parameters, + ) + + +def _claim_is_allowed(current: object, fence: FencingToken) -> bool: + if not isinstance(current, (tuple, list)) or len(current) != 4: + raise ValueError("Redshift fence claim returned an invalid row") + authority_id, authority_epoch, run_id, token = current + if ( + not isinstance(authority_id, str) + or not isinstance(authority_epoch, int) + or not isinstance(run_id, str) + or not isinstance(token, int) + ): + raise ValueError("Redshift fence claim returned invalid values") + exact_authority = ( + authority_id == fence.resolved_authority_id and authority_epoch == fence.authority_epoch + ) + return exact_authority and ( + token < fence.token or (token == fence.token and run_id == fence.run_id) + ) + + +def _claim_values(target_id: str, fence: FencingToken) -> tuple[object, ...]: + return ( + target_id, + fence.pipeline_id, + fence.resolved_authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + ) + + +def _update_values(target_id: str, fence: FencingToken) -> tuple[object, ...]: + return ( + fence.resolved_authority_id, + fence.authority_epoch, + fence.run_id, + fence.token, + target_id, + fence.pipeline_id, + ) + + +def _fence_table_sql(table: str) -> str: + return ( + f"CREATE TABLE IF NOT EXISTS {table} (" + '"target_id" VARCHAR(1024) NOT NULL, "pipeline_id" VARCHAR(127) NOT NULL, ' + '"authority_id" VARCHAR(256) NOT NULL, "authority_epoch" BIGINT NOT NULL, ' + '"run_id" VARCHAR(127) NOT NULL, "fencing_token" BIGINT NOT NULL, ' + '"status" VARCHAR(16) NOT NULL, "claimed_at" TIMESTAMPTZ NOT NULL, ' + '"committed_at" TIMESTAMPTZ)' + ) + + +def _select_claim_sql(table: str) -> str: + return ( + f'SELECT "authority_id", "authority_epoch", "run_id", "fencing_token" ' + f'FROM {table} WHERE "target_id" = %s AND "pipeline_id" = %s' + ) + + +def _insert_claim_sql(table: str) -> str: + return ( + f'INSERT INTO {table} ("target_id", "pipeline_id", "authority_id", ' + '"authority_epoch", "run_id", "fencing_token", "status", "claimed_at", ' + '"committed_at") VALUES (%s, %s, %s, %s, %s, %s, ' + "'claimed', GETDATE(), NULL)" + ) + + +def _update_claim_sql(table: str) -> str: + return ( + f'UPDATE {table} SET "authority_id" = %s, "authority_epoch" = %s, ' + '"run_id" = %s, "fencing_token" = %s, "status" = \'claimed\', ' + '"claimed_at" = GETDATE(), "committed_at" = NULL ' + 'WHERE "target_id" = %s AND "pipeline_id" = %s' + ) + + +def _fence_table(value: str) -> str: + parts = value.split(".") + if len(parts) != 3: + raise ValueError("Invalid Redshift target-fence table") + return _qualified(parts[1], parts[2]) + + +def _qualified(*parts: str) -> str: + return ".".join(_quote(part) for part in parts) + + +def _quote(value: str) -> str: + return f'"{value.replace(chr(34), chr(34) * 2)}"' + + +__all__ = ["RedshiftFencePlan", "RedshiftTargetFence"] diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py new file mode 100644 index 00000000..aed51ed1 --- /dev/null +++ b/src/dander/providers/redshift/runtime.py @@ -0,0 +1,308 @@ +"""Amazon Redshift implementation of Dander's warehouse capability bundle.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from dander.providers.redshift.config import RedshiftWarehouseConfig, validate_redshift_relation +from dander.providers.redshift.fence import RedshiftTargetFence +from dander.providers.redshift.session import ( + RedshiftConnectionFactory, + execute, + open_connection, +) +from dander.providers.redshift.writer import ( + RedshiftS3Client, + RedshiftScd1Writer, + RedshiftStagingSettings, + default_staging_settings, +) +from dander.providers.registry import ( + PROVIDER_API_VERSION, + ProviderFactory, + ProviderFactoryError, + ProviderKind, +) +from dander.telemetry import OperationTelemetry, TelemetryOperation +from dander.warehouse import CanonicalField, RelationRef, RelationSchema +from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + + from pydantic import BaseModel + + +@dataclass(frozen=True, slots=True) +class RedshiftRelationCodec: + """Render canonical database/schema/relation coordinates for Redshift.""" + + database: str + provider_id: str = "redshift" + + def render(self, relation: RelationRef) -> str: + validate_redshift_relation(relation) + if relation.catalog != self.database: + raise ValueError("Redshift relation belongs to another database") + return ".".join(_quote(value) for value in relation.coordinates) + + +@dataclass(frozen=True, slots=True) +class RedshiftSchemaMapper: + """Map compatibility declarations into canonical schema v1.""" + + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + canonical: list[CanonicalField] = [] + for field in fields: + if isinstance(field, WriteField): + canonical.append(field.to_canonical()) + elif isinstance(field, CanonicalField): + canonical.append(field) + else: + raise TypeError("Redshift schema mapper received an unsupported field") + return RelationSchema(fields=tuple(canonical)) + + +@dataclass(frozen=True, slots=True) +class RedshiftWriterFactory: + """Construct Redshift's bounded Parquet/COPY SCD1 writer.""" + + database: str + connection_factory: RedshiftConnectionFactory + s3_client: RedshiftS3Client + target_fence: RedshiftTargetFence + staging: RedshiftStagingSettings + + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + ) -> WritePattern: + del batch_rows + if sandbox: + raise ValueError("Redshift warehouse does not use Dander's BigQuery sandbox mode") + return RedshiftScd1Writer( + database=self.database, + connection_factory=self.connection_factory, + s3_client=self.s3_client, + target_fence=self.target_fence, + schema_evolution=schema_evolution, + staging=self.staging, + ) + + +@dataclass(frozen=True, slots=True) +class RedshiftTransformFactory: + """Fail clearly until the Redshift transform slice is implemented.""" + + def build_transform_runner( + self, + *, + graph_plan: object | None, + build_models: bool, + raw_namespace: str = "raw", + ) -> None: + del raw_namespace + if graph_plan is not None or build_models: + raise ValueError("Redshift transforms are not available in this experimental slice") + return None + + +@dataclass(frozen=True, slots=True) +class RedshiftTelemetry: + """Normalize DB-API counters without retaining SQL or provider payloads.""" + + def operation( + self, + job: object, + *, + operation: TelemetryOperation, + duration_ms: int = 0, + retry_count: int = 0, + ) -> OperationTelemetry: + rowcount = getattr(job, "rowcount", 0) + query_id = getattr(job, "query_id", None) + affected = rowcount if isinstance(rowcount, int) and rowcount >= 0 else 0 + return OperationTelemetry( + provider="redshift", + operation=operation, + duration_ms=duration_ms, + retry_count=retry_count, + rows_affected=affected, + query_id=query_id if isinstance(query_id, str) else None, + ) + + +REDSHIFT_CAPABILITIES = WarehouseCapabilities( + provider_id="redshift", + schema_contract_version=1, + write_modes=frozenset({WriteMode.SCD1}), + transports=frozenset({WriteTransport.COPY}), + supports_transforms=False, + supports_graphs=False, + supports_target_fencing=True, +) + + +def build_redshift_warehouse( + config: BaseModel, + context: Mapping[str, object], +) -> WarehouseRuntime: + """Build an IAM-authenticated Redshift runtime or injected conformance session.""" + if not isinstance(config, RedshiftWarehouseConfig): + raise TypeError("Redshift warehouse factory received the wrong configuration") + catalog = context.get("catalog") + if catalog != config.database: + raise ValueError("Redshift warehouse factory requires its configured database catalog") + supplied_factory = context.get("connection_factory") + if supplied_factory is None: + connection_factory = _sdk_connection_factory(config) + elif callable(supplied_factory): + connection_factory = cast("RedshiftConnectionFactory", supplied_factory) + else: + raise TypeError("Redshift context connection_factory must be callable") + + supplied_s3 = context.get("s3_client") + if supplied_s3 is None: + s3_client = _sdk_s3_client(config.region) + else: + s3_client = cast("RedshiftS3Client", supplied_s3) + + try: + with open_connection(connection_factory) as connection: + current = execute( + connection, + "SELECT current_database(), current_user", + fetch="one", + ).row + except Exception as error: + raise ProviderFactoryError("Redshift connection validation failed") from error + if not isinstance(current, (tuple, list)) or len(current) < 2: + raise ProviderFactoryError("Redshift connection validation returned an invalid result") + if current[0] != config.database: + raise ProviderFactoryError("Redshift connection selected the wrong database") + if not isinstance(current[1], str) or not current[1]: + raise ProviderFactoryError("Redshift connection returned an invalid database user") + if config.db_user is not None and not _matches_iam_user(current[1], config.db_user): + raise ProviderFactoryError("Redshift connection selected the wrong database user") + + target_fence = RedshiftTargetFence( + connection_factory=connection_factory, + database=config.database, + ) + staging = default_staging_settings( + bucket=config.staging_bucket, + prefix=config.staging_prefix, + region=config.region, + copy_role_arn=config.copy_role_arn, + max_rows_per_file=config.max_rows_per_file, + max_logical_bytes_per_file=config.max_logical_bytes_per_file, + compression=config.compression, + statement_timeout_ms=config.statement_timeout_ms, + ) + supplied_root = context.get("staging_root") + if supplied_root is not None: + if not isinstance(supplied_root, Path): + raise TypeError("Redshift context staging_root must be a pathlib.Path") + staging = RedshiftStagingSettings( + root=supplied_root, + bucket=staging.bucket, + prefix=staging.prefix, + region=staging.region, + copy_role_arn=staging.copy_role_arn, + max_rows_per_file=staging.max_rows_per_file, + max_logical_bytes_per_file=staging.max_logical_bytes_per_file, + compression=staging.compression, + statement_timeout_ms=staging.statement_timeout_ms, + ) + return WarehouseRuntime( + provider_id="redshift", + relation_codec=RedshiftRelationCodec(config.database), + schema_mapper=RedshiftSchemaMapper(), + writers=RedshiftWriterFactory( + config.database, + connection_factory, + s3_client, + target_fence, + staging, + ), + transforms=RedshiftTransformFactory(), + target_fence=target_fence, + telemetry=RedshiftTelemetry(), + capabilities=REDSHIFT_CAPABILITIES, + ) + + +def _sdk_connection_factory(config: RedshiftWarehouseConfig) -> RedshiftConnectionFactory: + def connect() -> object: + try: + import redshift_connector + except ModuleNotFoundError as error: + raise ProviderFactoryError( + "Redshift warehouse requires the dander-platform[redshift] extra" + ) from error + connector = cast("Callable[..., object]", redshift_connector.connect) + common: dict[str, object] = { + "iam": True, + "ssl": True, + "sslmode": "verify-full", + "host": config.host, + "port": config.port, + "database": config.database, + "region": config.region, + "timeout": config.connect_timeout_seconds, + "application_name": "dander", + } + if config.deployment == "provisioned": + assert config.db_user is not None + return connector( + **common, + cluster_identifier=config.cluster_identifier, + db_user=config.db_user, + ) + return connector( + **common, + is_serverless=True, + serverless_work_group=config.workgroup_name, + ) + + return cast("RedshiftConnectionFactory", connect) + + +def _sdk_s3_client(region: str) -> RedshiftS3Client: + try: + import boto3 # type: ignore[import-untyped] + except ModuleNotFoundError as error: + raise ProviderFactoryError( + "Redshift warehouse requires the dander-platform[redshift] extra" + ) from error + return cast("RedshiftS3Client", boto3.client("s3", region_name=region)) + + +REDSHIFT_WAREHOUSE_FACTORY = ProviderFactory[WarehouseRuntime]( + kind=ProviderKind.WAREHOUSE, + provider_id="redshift", + api_version=PROVIDER_API_VERSION, + build=build_redshift_warehouse, +) + + +def _quote(identifier: str) -> str: + return f'"{identifier.replace(chr(34), chr(34) * 2)}"' + + +def _matches_iam_user(actual: str, configured: str) -> bool: + return actual in {configured, f"IAM:{configured}"} or actual.startswith(f"IAMA:{configured}:") + + +__all__ = [ + "REDSHIFT_CAPABILITIES", + "REDSHIFT_WAREHOUSE_FACTORY", + "RedshiftRelationCodec", + "RedshiftSchemaMapper", +] diff --git a/src/dander/providers/redshift/session.py b/src/dander/providers/redshift/session.py new file mode 100644 index 00000000..a3e2e4c4 --- /dev/null +++ b/src/dander/providers/redshift/session.py @@ -0,0 +1,75 @@ +"""Small DB-API boundary for injected and SDK Redshift sessions.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Protocol + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + +class RedshiftCursor(Protocol): + rowcount: int + + def execute(self, operation: str, args: Sequence[object] | None = None) -> object: ... + def fetchone(self) -> object | None: ... + def fetchall(self) -> list[object]: ... + def close(self) -> None: ... + + +class RedshiftConnection(Protocol): + autocommit: bool + + def cursor(self) -> RedshiftCursor: ... + def commit(self) -> None: ... + def rollback(self) -> None: ... + def close(self) -> None: ... + + +class RedshiftConnectionFactory(Protocol): + def __call__(self) -> RedshiftConnection: ... + + +@dataclass(frozen=True, slots=True) +class RedshiftStatementResult: + rowcount: int + row: object | None = None + rows: tuple[object, ...] = () + + +@contextmanager +def open_connection(factory: RedshiftConnectionFactory) -> Iterator[RedshiftConnection]: + connection = factory() + connection.autocommit = False + try: + yield connection + finally: + connection.close() + + +def execute( + connection: RedshiftConnection, + statement: str, + parameters: Sequence[object] = (), + *, + fetch: Literal["one", "all"] | None = None, +) -> RedshiftStatementResult: + cursor = connection.cursor() + try: + cursor.execute(statement, parameters or None) + row = cursor.fetchone() if fetch == "one" else None + rows = tuple(cursor.fetchall()) if fetch == "all" else () + return RedshiftStatementResult(rowcount=cursor.rowcount, row=row, rows=rows) + finally: + cursor.close() + + +__all__ = [ + "RedshiftConnection", + "RedshiftConnectionFactory", + "RedshiftStatementResult", + "execute", + "open_connection", +] diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py new file mode 100644 index 00000000..04a23482 --- /dev/null +++ b/src/dander/providers/redshift/writer.py @@ -0,0 +1,639 @@ +"""Bounded S3/Parquet COPY and transactionally fenced Redshift SCD1 writes.""" + +# ruff: noqa: N803 -- boto3's public S3 API uses capitalized parameter names. + +from __future__ import annotations + +import hashlib +import json +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Protocol +from uuid import uuid4 + +from dander.concurrency import TargetFenceLostError +from dander.providers.redshift.config import validate_redshift_relation +from dander.providers.redshift.session import ( + RedshiftConnection, + RedshiftConnectionFactory, + execute, + open_connection, +) +from dander.warehouse import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + ParquetStagingSession, + RelationSchema, + StagingManifest, +) +from dander.writer import SchemaEvolution, WriteMode, WritePattern, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + + from dander.concurrency import TargetFence + from dander.providers.redshift.fence import RedshiftTargetFence + +_ORDINAL = "_dander_ordinal" +_MAX_PARQUET_ROW_BYTES = 4 * 1_024 * 1_024 + + +class RedshiftWriteError(ValueError): + """Raised when Redshift staging, schema, or publication fails closed.""" + + +class RedshiftS3Client(Protocol): + def get_bucket_location( # noqa: N803 + self, *, Bucket: str + ) -> Mapping[str, object]: ... + def upload_file( # noqa: N803 + self, Filename: str, Bucket: str, Key: str + ) -> object: ... + def put_object( # noqa: N803 + self, *, Bucket: str, Key: str, Body: bytes + ) -> object: ... + def delete_objects( # noqa: N803 + self, *, Bucket: str, Delete: Mapping[str, object] + ) -> object: ... + + +@dataclass(frozen=True, slots=True) +class RedshiftStagingSettings: + root: Path + bucket: str + prefix: str + region: str + copy_role_arn: str + max_rows_per_file: int + max_logical_bytes_per_file: int + compression: str + statement_timeout_ms: int + + +@dataclass(frozen=True, slots=True) +class RedshiftPublication: + statements: tuple[tuple[str, Sequence[object]], ...] + affected_statement_index: int + + +class RedshiftScd1Writer(WritePattern): + """Upload bounded Parquet parts and merge deterministic last-record-wins rows.""" + + mode = WriteMode.SCD1 + supports_batched_writes = True + requires_publication_fence = True + + def __init__( + self, + *, + database: str, + connection_factory: RedshiftConnectionFactory, + s3_client: RedshiftS3Client, + target_fence: RedshiftTargetFence, + schema_evolution: SchemaEvolution, + staging: RedshiftStagingSettings, + ) -> None: + self._database = database + self._connection_factory = connection_factory + self._s3 = s3_client + self._target_fence = target_fence + self._evolution = schema_evolution + self._staging = staging + + def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: + _validate_target(target, self._database) + publication = target.publication_fence + assert publication is not None + schema = target.canonical_schema + _validate_schema(schema) + if any(field.name == _ORDINAL for field in schema.fields): + raise RedshiftWriteError(f"Declared schema reserves Dander field {_ORDINAL!r}") + staged_schema = RelationSchema( + fields=( + *schema.fields, + CanonicalField( + name=_ORDINAL, + data_type=CanonicalType(kind=LogicalTypeKind.INTEGER, bit_width=64), + cardinality=FieldCardinality.REQUIRED, + ), + ) + ) + local_id = f"redshift-{uuid4().hex}" + with ParquetStagingSession( + self._staging.root, + run_id=local_id, + max_rows_per_file=self._staging.max_rows_per_file, + max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, + compression=self._staging.compression, + ) as local: + manifest = local.stage(_with_ordinals(records), staged_schema) + _validate_manifest(manifest) + return self._publish(target, publication, schema, staged_schema, manifest) + + def _publish( + self, + target: WriteTarget, + publication: TargetFence, + target_schema: RelationSchema, + staged_schema: RelationSchema, + manifest: StagingManifest, + ) -> int: + _validate_bucket_region(self._s3, self._staging.bucket, self._staging.region) + remote_id = hashlib.sha256( + f"{publication.target_id}:{publication.run_id}:{uuid4().hex}".encode() + ).hexdigest()[:24] + remote_prefix = f"{self._staging.prefix}/{remote_id}" + temp_table = f"dander_stage_{remote_id}" + uploaded: list[str] = [] + with open_connection(self._connection_factory) as connection: + publication_failed = True + try: + _set_query_group( + connection, + publication, + statement_timeout_ms=self._staging.statement_timeout_ms, + ) + if manifest.rows: + for artifact in manifest.artifacts: + key = f"{remote_prefix}/{artifact.path.name}" + uploaded.append(key) + self._s3.upload_file(str(artifact.path), self._staging.bucket, key) + manifest_key = f"{remote_prefix}/manifest.json" + uploaded.append(manifest_key) + self._s3.put_object( + Bucket=self._staging.bucket, + Key=manifest_key, + Body=_copy_manifest(self._staging.bucket, remote_prefix, manifest), + ) + execute(connection, _temporary_table_sql(temp_table, staged_schema)) + execute( + connection, + _copy_sql( + temp_table, + bucket=self._staging.bucket, + manifest_key=manifest_key, + role_arn=self._staging.copy_role_arn, + ), + ) + # COPY and SET open an implicit transaction. Close it while retaining the + # session-scoped temp table so the fenced publication owns one transaction. + connection.commit() + plan = _publication_plan( + connection, + target, + temp_table, + target_schema, + manifest, + evolution=self._evolution, + has_rows=bool(manifest.rows), + ) + # The schema read also opens an implicit transaction. End that read-only + # snapshot before beginning the one fenced publication transaction. + connection.commit() + results = self._target_fence.execute_statements( + connection, + plan.statements, + publication, + ) + affected_rows = results[plan.affected_statement_index].rowcount + if affected_rows < 0: + raise RedshiftWriteError("Redshift MERGE did not report affected rows") + publication_failed = False + return affected_rows + except (RedshiftWriteError, TargetFenceLostError): + raise + except Exception as error: + raise RedshiftWriteError("Redshift staged SCD1 write failed") from error + finally: + cleanup_error: Exception | None = None + try: + _drop_temporary(connection, temp_table) + except Exception as error: + cleanup_error = error + try: + _delete_owned(self._s3, self._staging.bucket, uploaded) + except Exception as error: + cleanup_error = cleanup_error or error + if cleanup_error is not None and not publication_failed: + raise cleanup_error + + +def default_staging_settings( + *, + bucket: str, + prefix: str, + region: str, + copy_role_arn: str, + max_rows_per_file: int, + max_logical_bytes_per_file: int, + compression: str, + statement_timeout_ms: int, +) -> RedshiftStagingSettings: + return RedshiftStagingSettings( + root=Path(tempfile.gettempdir()) / "dander-redshift-staging", + bucket=bucket, + prefix=prefix, + region=region, + copy_role_arn=copy_role_arn, + max_rows_per_file=max_rows_per_file, + max_logical_bytes_per_file=max_logical_bytes_per_file, + compression=compression, + statement_timeout_ms=statement_timeout_ms, + ) + + +def _with_ordinals(records: Iterable[Mapping[str, object]]) -> Iterable[Mapping[str, object]]: + for ordinal, record in enumerate(records): + yield {**record, _ORDINAL: ordinal} + + +def _validate_target(target: WriteTarget, database: str) -> None: + try: + validate_redshift_relation(target.relation_ref) + except ValueError as error: + raise RedshiftWriteError(str(error)) from error + if target.relation_ref.catalog != database: + raise RedshiftWriteError("Redshift target belongs to another database") + if not target.business_key or not target.schema: + raise RedshiftWriteError("Redshift SCD1 requires a business key and declared schema") + declared = {field.name for field in target.canonical_schema.fields} + if any(key not in declared for key in target.business_key): + raise RedshiftWriteError("Redshift business keys must be declared schema fields") + publication = target.publication_fence + if publication is None: + raise RedshiftWriteError("Redshift hosted writes require a destination target fence") + expected = ".".join(target.relation_ref.coordinates) + expected_table = f"{database}.{target.relation_ref.namespace}.dander_target_commits" + if publication.target_id != expected or publication.fence_table != expected_table: + raise RedshiftWriteError("Redshift destination target fence does not match the target") + + +def _validate_schema(schema: RelationSchema) -> None: + for field in schema.fields: + if len(field.name.encode()) > 127: + raise RedshiftWriteError("Redshift column identifiers cannot exceed 127 bytes") + _redshift_type(field.data_type) + + +def _validate_manifest(manifest: StagingManifest) -> None: + if any( + artifact.rows == 1 and artifact.logical_bytes > _MAX_PARQUET_ROW_BYTES + for artifact in manifest.artifacts + ): + raise RedshiftWriteError("Redshift Parquet row exceeds the 4 MB COPY limit") + + +def _validate_bucket_region(client: RedshiftS3Client, bucket: str, expected: str) -> None: + response = client.get_bucket_location(Bucket=bucket) + location = response.get("LocationConstraint") + actual = "us-east-1" if location in {None, ""} else location + if actual == "EU": + actual = "eu-west-1" + if actual != expected: + raise RedshiftWriteError("Redshift staging bucket must be in the warehouse AWS region") + + +def _copy_manifest(bucket: str, prefix: str, manifest: StagingManifest) -> bytes: + return json.dumps( + { + "entries": [ + { + "url": f"s3://{bucket}/{prefix}/{artifact.path.name}", + "mandatory": True, + "meta": {"content_length": artifact.compressed_bytes}, + } + for artifact in manifest.artifacts + ] + }, + separators=(",", ":"), + sort_keys=True, + ).encode() + + +def _publication_plan( + connection: RedshiftConnection, + target: WriteTarget, + temp_table: str, + schema: RelationSchema, + manifest: StagingManifest, + *, + evolution: SchemaEvolution, + has_rows: bool, +) -> RedshiftPublication: + statements: list[tuple[str, Sequence[object]]] = [ + (_create_target_sql(target, schema), ()), + (_history_table_sql(target), ()), + ] + statements.extend(_schema_changes(connection, target, schema, evolution=evolution)) + digest = _manifest_digest(manifest) + history_values: tuple[object, ...] = ( + ".".join(target.relation_ref.coordinates), + target.publication_fence.pipeline_id if target.publication_fence else "", + target.publication_fence.run_id if target.publication_fence else "", + manifest.schema_fingerprint, + digest, + ) + if has_rows: + statements.append((_merge_sql(target, temp_table, schema.fields), history_values)) + else: + statements.append((_no_op_sql(target), ())) + affected_statement_index = len(statements) - 1 + statements.append((_history_insert_sql(target), (*history_values, *history_values))) + return RedshiftPublication( + statements=tuple(statements), + affected_statement_index=affected_statement_index, + ) + + +def _schema_changes( + connection: RedshiftConnection, + target: WriteTarget, + schema: RelationSchema, + *, + evolution: SchemaEvolution, +) -> tuple[tuple[str, Sequence[object]], ...]: + relation = target.relation_ref + rows = execute( + connection, + "SELECT column_name, data_type, character_maximum_length, numeric_precision, " + "numeric_scale, is_nullable FROM svv_columns " + "WHERE database_name = %s AND schema_name = %s AND table_name = %s", + relation.coordinates, + fetch="all", + ).rows + deployed = _deployed_columns(rows) + expected = {field.name: field for field in schema.fields} + if extras := sorted(set(deployed) - set(expected)): + raise RedshiftWriteError(f"Deployed target has undeclared column {extras[0]!r}") + changes: list[tuple[str, Sequence[object]]] = [] + for field in schema.fields: + current = deployed.get(field.name) + required = ( + field.cardinality is FieldCardinality.REQUIRED or field.name in target.business_key + ) + if current is None: + if not deployed: + continue + if evolution is not SchemaEvolution.ADDITIVE or required: + raise RedshiftWriteError(f"Cannot add Redshift column {field.name!r}") + changes.append( + ( + f"ALTER TABLE {_target(target)} ADD COLUMN {_quote(field.name)} " + f"{_redshift_type(field.data_type)}", + (), + ) + ) + continue + deployed_type, nullable = current + if deployed_type.casefold() != _redshift_type(field.data_type).casefold(): + raise RedshiftWriteError(f"Redshift type drift for column {field.name!r}") + if nullable == required: + raise RedshiftWriteError(f"Redshift nullability drift for column {field.name!r}") + return tuple(changes) + + +def _deployed_columns(rows: Sequence[object]) -> dict[str, tuple[str, bool]]: + result: dict[str, tuple[str, bool]] = {} + for row in rows: + if not isinstance(row, (tuple, list)) or len(row) < 6: + raise RedshiftWriteError("Redshift schema inspection returned an invalid row") + name, data_type, length, precision, scale, nullable = row[:6] + if not all(isinstance(value, str) for value in (name, data_type, nullable)): + raise RedshiftWriteError("Redshift schema inspection returned invalid values") + result[name] = ( + _normalize_deployed_type(data_type, length, precision, scale), + nullable.casefold() == "yes", + ) + return result + + +def _create_target_sql(target: WriteTarget, schema: RelationSchema) -> str: + fields = ", ".join(_target_column(field, target) for field in schema.fields) + return f"CREATE TABLE IF NOT EXISTS {_target(target)} ({fields})" + + +def _target_column(field: CanonicalField, target: WriteTarget) -> str: + required = field.cardinality is FieldCardinality.REQUIRED or field.name in target.business_key + return ( + f"{_quote(field.name)} {_redshift_type(field.data_type)}{' NOT NULL' if required else ''}" + ) + + +def _temporary_table_sql(name: str, schema: RelationSchema) -> str: + fields = ", ".join( + f"{_quote(field.name)} {_redshift_type(field.data_type)}" + f"{' NOT NULL' if field.cardinality is FieldCardinality.REQUIRED else ''}" + for field in schema.fields + ) + return f"CREATE TEMP TABLE {_quote(name)} ({fields})" + + +def _copy_sql(name: str, *, bucket: str, manifest_key: str, role_arn: str) -> str: + return ( + f"COPY {_quote(name)} FROM {_literal(f's3://{bucket}/{manifest_key}')} " + f"IAM_ROLE {_literal(role_arn)} FORMAT AS PARQUET MANIFEST" + ) + + +def _merge_sql(target: WriteTarget, temporary: str, fields: Sequence[CanonicalField]) -> str: + names = tuple(field.name for field in fields) + columns = ", ".join(_quote(name) for name in names) + keys = ", ".join(_quote(key) for key in target.business_key) + match = " AND ".join( + f"target.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + ) + mutable = tuple(name for name in names if name not in target.business_key) + updates = ", ".join(f"{_quote(name)} = incoming.{_quote(name)}" for name in mutable) + selected = ", ".join(f"incoming.{_quote(name)}" for name in names) + history = _history(target) + matched = f" WHEN MATCHED THEN UPDATE SET {updates}" if updates else "" + return ( + f"MERGE INTO {_target(target)} AS target USING (SELECT {columns} FROM " + f"(SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {keys} ORDER BY " + f"{_quote(_ORDINAL)} DESC) AS {_quote('_dander_rank')} FROM {_quote(temporary)}) ranked " + f"WHERE {_quote('_dander_rank')} = 1 AND NOT EXISTS (SELECT 1 FROM {history} WHERE " + '"target_id" = %s AND "pipeline_id" = %s AND "run_id" = %s ' + 'AND "schema_fingerprint" = %s AND "manifest_digest" = %s)) AS incoming ' + f"ON {match}{matched} WHEN NOT MATCHED THEN " + f"INSERT ({columns}) VALUES ({selected})" + ) + + +def _history_table_sql(target: WriteTarget) -> str: + return ( + f"CREATE TABLE IF NOT EXISTS {_history(target)} (" + '"target_id" VARCHAR(1024) NOT NULL, "pipeline_id" VARCHAR(127) NOT NULL, ' + '"run_id" VARCHAR(127) NOT NULL, "schema_fingerprint" CHAR(64) NOT NULL, ' + '"manifest_digest" CHAR(64) NOT NULL, "loaded_at" TIMESTAMPTZ NOT NULL)' + ) + + +def _history_insert_sql(target: WriteTarget) -> str: + return ( + f'INSERT INTO {_history(target)} ("target_id", "pipeline_id", "run_id", ' + '"schema_fingerprint", "manifest_digest", "loaded_at") SELECT %s, %s, %s, %s, %s, ' + f"GETDATE() WHERE NOT EXISTS (SELECT 1 FROM {_history(target)} WHERE " + '"target_id" = %s AND "pipeline_id" = %s AND "run_id" = %s ' + 'AND "schema_fingerprint" = %s AND "manifest_digest" = %s)' + ) + + +def _history(target: WriteTarget) -> str: + return _qualified(target.relation_ref.namespace, "dander_stage_loads") + + +def _no_op_sql(target: WriteTarget) -> str: + key = _quote(target.business_key[0]) + return f"UPDATE {_target(target)} SET {key} = {key} WHERE FALSE" + + +def _set_query_group( + connection: RedshiftConnection, + publication: TargetFence, + *, + statement_timeout_ms: int, +) -> None: + digest = hashlib.sha256(f"{publication.pipeline_id}:{publication.run_id}".encode()).hexdigest()[ + :20 + ] + execute(connection, f"SET query_group TO {_literal(f'dander_{digest}')}") + execute(connection, f"SET statement_timeout TO {statement_timeout_ms}") + + +def _drop_temporary(connection: RedshiftConnection, name: str) -> None: + try: + execute(connection, f"DROP TABLE IF EXISTS {_quote(name)}") + connection.commit() + except Exception as error: + connection.rollback() + raise RedshiftWriteError("Redshift temporary staging cleanup failed") from error + + +def _delete_owned(client: RedshiftS3Client, bucket: str, keys: Sequence[str]) -> None: + if not keys: + return + try: + for start in range(0, len(keys), 1_000): + response = client.delete_objects( + Bucket=bucket, + Delete={ + "Objects": [{"Key": key} for key in keys[start : start + 1_000]], + "Quiet": True, + }, + ) + if isinstance(response, Mapping) and response.get("Errors"): + raise RedshiftWriteError("Redshift S3 staging cleanup reported object errors") + except Exception as error: + if isinstance(error, RedshiftWriteError): + raise + raise RedshiftWriteError("Redshift S3 staging cleanup failed") from error + + +def _redshift_type(data_type: CanonicalType) -> str: + match data_type.kind: + case LogicalTypeKind.BOOLEAN: + return "BOOLEAN" + case LogicalTypeKind.INTEGER: + return "BIGINT" + case LogicalTypeKind.DECIMAL: + if data_type.precision is None or data_type.scale is None or data_type.precision > 38: + raise RedshiftWriteError("Redshift decimal precision cannot exceed 38") + return f"DECIMAL({data_type.precision},{data_type.scale})" + case LogicalTypeKind.FLOAT: + return "DOUBLE PRECISION" + case LogicalTypeKind.STRING: + return "VARCHAR(65535)" + case LogicalTypeKind.BINARY: + return "VARBYTE" + case LogicalTypeKind.DATE: + return "DATE" + case LogicalTypeKind.TIME: + if (data_type.fractional_second_precision or 0) > 6: + raise RedshiftWriteError("Redshift time precision cannot exceed 6") + return "TIME" + case LogicalTypeKind.TIMESTAMP: + if (data_type.fractional_second_precision or 0) > 6: + raise RedshiftWriteError("Redshift timestamp precision cannot exceed 6") + return "TIMESTAMPTZ" if data_type.with_timezone else "TIMESTAMP" + case LogicalTypeKind.JSON | LogicalTypeKind.ARRAY | LogicalTypeKind.RECORD: + raise RedshiftWriteError( + "Redshift semi-structured types require an explicit SUPER fallback" + ) + raise AssertionError("Unhandled Redshift canonical type") + + +def _target(target: WriteTarget) -> str: + return _qualified(target.relation_ref.namespace, target.relation_ref.name) + + +def _qualified(*parts: str) -> str: + return ".".join(_quote(part) for part in parts) + + +def _quote(value: str) -> str: + return f'"{value.replace(chr(34), chr(34) * 2)}"' + + +def _literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _manifest_digest(manifest: StagingManifest) -> str: + payload = { + "schema_fingerprint": manifest.schema_fingerprint, + "artifacts": [ + { + "rows": artifact.rows, + "logical_bytes": artifact.logical_bytes, + "compressed_bytes": artifact.compressed_bytes, + "sha256": artifact.sha256, + } + for artifact in manifest.artifacts + ], + } + return hashlib.sha256( + json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + + +def _normalize_deployed_type( + data_type: str, + length: object, + precision: object, + scale: object, +) -> str: + normalized = " ".join(data_type.casefold().split()) + if normalized in {"character varying", "varchar"}: + if not isinstance(length, int): + raise RedshiftWriteError("Redshift VARCHAR metadata has no length") + return f"VARCHAR({length})" + if normalized in {"numeric", "decimal"}: + if not isinstance(precision, int) or not isinstance(scale, int): + raise RedshiftWriteError("Redshift DECIMAL metadata has no precision or scale") + return f"DECIMAL({precision},{scale})" + aliases = { + "int8": "BIGINT", + "bigint": "BIGINT", + "float8": "DOUBLE PRECISION", + "double precision": "DOUBLE PRECISION", + "bool": "BOOLEAN", + "boolean": "BOOLEAN", + "timestamp with time zone": "TIMESTAMPTZ", + "timestamp without time zone": "TIMESTAMP", + "time without time zone": "TIME", + "binary varying": "VARBYTE", + } + return aliases.get(normalized, normalized.upper()) + + +__all__ = [ + "RedshiftS3Client", + "RedshiftScd1Writer", + "RedshiftStagingSettings", + "RedshiftWriteError", + "default_staging_settings", +] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index caf16380..4725dfd6 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -19,6 +19,12 @@ "status": "experimental", "reason": "Snowflake staged SCD1 ingestion and destination fencing pass local conformance; no live Snowflake profile is qualified." }, + { + "state": "bigquery", + "warehouse": "redshift", + "status": "experimental", + "reason": "Locally conformance-tested with destination-side Redshift fencing; no live hosted profile is qualified." + }, { "state": "postgresql", "warehouse": "bigquery", @@ -36,6 +42,12 @@ "warehouse": "snowflake", "status": "experimental", "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged SCD1 writes; live qualification remains pending." + }, + { + "state": "postgresql", + "warehouse": "redshift", + "status": "experimental", + "reason": "Locally conformance-tested with destination-side Redshift fencing; no live hosted profile is qualified." } ] } diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index ce63c7ff..7f94de94 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -422,3 +422,11 @@ def test_only_unsupported_postgresql_state_bigquery_warehouse_pair_fails_closed( state_provider="bigquery", warehouse_provider="postgresql", ) + run_module._require_executable_state_pair( + state_provider="bigquery", + warehouse_provider="redshift", + ) + run_module._require_executable_state_pair( + state_provider="postgresql", + warehouse_provider="redshift", + ) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index e5eb44e1..137ba067 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -249,6 +249,40 @@ def test_version_two_resolves_native_postgresql_profile(tmp_path: Path) -> None: assert resolved.secret_provider == "environment" +def test_version_two_resolves_native_redshift_warehouse_coordinates(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + profile = platforms["platforms"]["gcp"] + profile["warehouse"] = { + "provider": "redshift", + "deployment": "provisioned", + "host": "example.abc123.us-east-1.redshift.amazonaws.com", + "database": "analytics", + "schema": "landing", + "db_user": "dander_user", + "region": "us-east-1", + "cluster_identifier": "dander-test", + "copy_role_arn": "arn:aws:iam::123456789012:role/DanderRedshiftCopy", + "staging_bucket": "dander-redshift-staging", + } + profile["catalog"] = {"provider": "none"} + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path) + + assert resolved.warehouse_provider == "redshift" + assert resolved.warehouse_config["database"] == "analytics" + assert resolved.warehouse_config["schema"] == "landing" + assert resolved.warehouse_config["copy_role_arn"] == ( + "arn:aws:iam::123456789012:role/DanderRedshiftCopy" + ) + assert "password" not in resolved.warehouse_config + + def test_cloud_run_rejects_environment_only_secrets(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py new file mode 100644 index 00000000..9f61891e --- /dev/null +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -0,0 +1,518 @@ +"""Redshift registration and stateful Parquet/COPY SCD1 conformance.""" + +# ruff: noqa: N803 -- boto3's public S3 API uses capitalized parameter names. + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Self + +import pytest + +from dander.concurrency import FencingToken, TargetFenceLostError +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry +from dander.providers.redshift import RedshiftWarehouseConfig +from dander.providers.redshift.writer import RedshiftWriteError, _delete_owned +from dander.warehouse import RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + +@dataclass +class _FakeRedshift: + statements: list[tuple[int, str, tuple[object, ...]]] = field(default_factory=list) + schema_rows: list[object] = field( + default_factory=lambda: [ + ("id", "character varying", 65_535, None, None, "NO"), + ("label", "character varying", 65_535, None, None, "YES"), + ] + ) + claim: tuple[str, int, str, int] | None = None + fence_touch_rowcount: int = 1 + merge_rowcount: int = 1 + history: set[tuple[object, ...]] = field(default_factory=set) + commits: int = 0 + rollbacks: int = 0 + closes: int = 0 + connections: int = 0 + + def connect(self) -> _FakeConnection: + self.connections += 1 + return _FakeConnection(self, self.connections) + + +class _FakeConnection: + def __init__(self, backend: _FakeRedshift, connection_id: int) -> None: + self.backend = backend + self.connection_id = connection_id + self.autocommit = True + self.in_transaction = False + + def cursor(self) -> _FakeCursor: + return _FakeCursor(self) + + def commit(self) -> None: + self.in_transaction = False + self.backend.commits += 1 + + def rollback(self) -> None: + self.in_transaction = False + self.backend.rollbacks += 1 + + def close(self) -> None: + self.backend.closes += 1 + + +class _FakeCursor: + def __init__(self, connection: _FakeConnection) -> None: + self.connection = connection + self.backend = connection.backend + self.rowcount = 0 + self._row: object | None = None + self._rows: list[object] = [] + + def execute(self, command: str, args: Sequence[object] | None = None) -> Self: + parameters = tuple(args or ()) + compact = " ".join(command.split()) + if compact == "BEGIN" and self.connection.in_transaction: + raise AssertionError("Redshift writer attempted to nest a transaction") + self.connection.in_transaction = True + self.backend.statements.append((self.connection.connection_id, compact, parameters)) + if compact == "SELECT current_database(), current_user": + self._row = ("analytics", "dander_user") + elif compact.startswith('SELECT "authority_id"'): + self._row = self.backend.claim + elif compact.startswith("INSERT INTO") and "dander_target_commits" in compact: + self.backend.claim = ( + str(parameters[2]), + _required_int(parameters[3]), + str(parameters[4]), + _required_int(parameters[5]), + ) + self.rowcount = 1 + elif ( + compact.startswith("UPDATE") + and "dander_target_commits" in compact + and 'SET "authority_id"' in compact + ): + self.backend.claim = ( + str(parameters[0]), + _required_int(parameters[1]), + str(parameters[2]), + _required_int(parameters[3]), + ) + self.rowcount = 1 + elif compact.startswith("UPDATE") and "dander_target_commits" in compact: + self.rowcount = self.backend.fence_touch_rowcount + elif compact.startswith("SELECT column_name") and "svv_columns" in compact: + self._rows = list(self.backend.schema_rows) + elif compact.startswith("INSERT INTO") and "dander_stage_loads" in compact: + self.rowcount = 0 if parameters[:5] in self.backend.history else 1 + self.backend.history.add(parameters[:5]) + elif compact.startswith("MERGE INTO"): + self.rowcount = ( + 0 if parameters[:5] in self.backend.history else self.backend.merge_rowcount + ) + return self + + def fetchone(self) -> object | None: + return self._row + + def fetchall(self) -> list[object]: + return list(self._rows) + + def close(self) -> None: + return None + + +@dataclass +class _FakeS3: + region: str = "us-east-1" + uploads: list[tuple[str, str, bytes]] = field(default_factory=list) + puts: list[tuple[str, str, bytes]] = field(default_factory=list) + deleted: list[tuple[str, ...]] = field(default_factory=list) + region_checks: int = 0 + fail_upload_at: int | None = None + + def get_bucket_location(self, *, Bucket: str) -> Mapping[str, object]: + del Bucket + self.region_checks += 1 + return {"LocationConstraint": None if self.region == "us-east-1" else self.region} + + def upload_file(self, Filename: str, Bucket: str, Key: str) -> None: + self.uploads.append((Bucket, Key, Path(Filename).read_bytes())) + if self.fail_upload_at == len(self.uploads): + raise RuntimeError("ambiguous S3 upload failure") + + def put_object(self, *, Bucket: str, Key: str, Body: bytes) -> None: + self.puts.append((Bucket, Key, Body)) + + def delete_objects( + self, + *, + Bucket: str, + Delete: Mapping[str, object], + ) -> Mapping[str, object]: + del Bucket + raw = Delete["Objects"] + assert isinstance(raw, list) + keys = tuple(str(item["Key"]) for item in raw if isinstance(item, dict)) + self.deleted.append(keys) + return {} + + +def _required_int(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise AssertionError("fake Redshift received a non-integer fence value") + return value + + +def _config() -> dict[str, object]: + return { + "provider": "redshift", + "deployment": "provisioned", + "host": "example.abc123.us-east-1.redshift.amazonaws.com", + "database": "analytics", + "schema": "raw", + "db_user": "dander_user", + "region": "us-east-1", + "cluster_identifier": "dander-test", + "copy_role_arn": "arn:aws:iam::123456789012:role/DanderRedshiftCopy", + "staging_bucket": "dander-redshift-staging", + "max_rows_per_file": 2, + "max_logical_bytes_per_file": 1_048_576, + } + + +@pytest.fixture +def redshift_runtime( + tmp_path: Path, +) -> tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path]: + backend = _FakeRedshift() + s3 = _FakeS3() + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, _config()) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "analytics", + "connection_factory": backend.connect, + "s3_client": s3, + "staging_root": tmp_path, + }, + ) + assert isinstance(runtime, WarehouseRuntime) + return runtime, backend, s3, tmp_path + + +def _target(runtime: WarehouseRuntime, *, run_id: str = "run-one") -> WriteTarget: + relation = RelationRef(catalog="analytics", namespace="raw", name="records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="redshift_records", + run_id=run_id, + token=4, + authority_id="bigquery:gcp-control.dander_state", + ), + ) + return WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + + +def test_redshift_registration_is_lazy_and_uses_native_coordinates() -> None: + module_name = "dander.providers.redshift.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + + config = registry.parse(ProviderKind.WAREHOUSE, _config()) + + assert module_name not in sys.modules + assert isinstance(config, RedshiftWarehouseConfig) + relation = config.raw_relation( + "records", + compatibility_catalog="ignored-gcp-project", + compatibility_namespace=None, + default_namespace="ignored_bigquery_default", + ) + assert relation == RelationRef(catalog="analytics", namespace="raw", name="records") + assert "password" not in config.model_dump_json() + + +def test_redshift_serverless_uses_an_aws_derived_database_user() -> None: + registry = default_provider_registry() + raw = _config() + raw.update( + { + "deployment": "serverless", + "workgroup_name": "dander-test", + } + ) + raw.pop("cluster_identifier") + raw.pop("db_user") + + config = registry.parse(ProviderKind.WAREHOUSE, raw) + + assert isinstance(config, RedshiftWarehouseConfig) + assert config.db_user is None + with pytest.raises(ProviderFactoryError, match="Invalid warehouse provider"): + registry.parse(ProviderKind.WAREHOUSE, {**raw, "db_user": "dander_user"}) + + +def test_redshift_runtime_validates_connection_and_fails_closed_for_transforms( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, _backend, _s3, _root = redshift_runtime + relation = RelationRef(catalog="analytics", namespace="raw", name="records") + + assert runtime.relation_codec.render(relation) == '"analytics"."raw"."records"' + assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) + assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) + assert runtime.capabilities.supports_transforms is False + with pytest.raises(ValueError, match="transforms are not available"): + runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + + +def test_redshift_claim_serializes_and_rejects_a_stale_token( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + relation = RelationRef(catalog="analytics", namespace="raw", name="records") + newer = FencingToken( + lease_table=None, + pipeline_id="redshift_records", + run_id="new-run", + token=9, + authority_id="postgresql:test-state", + ) + runtime.target_fence.claim(relation, newer) + rollback_before = backend.rollbacks + + with pytest.raises(TargetFenceLostError, match="rejected stale"): + runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="redshift_records", + run_id="old-run", + token=8, + authority_id="postgresql:test-state", + ), + ) + + assert backend.rollbacks == rollback_before + 1 + claim_sql = [statement for _, statement, _ in backend.statements] + assert "BEGIN" in claim_sql + assert any( + statement.startswith('LOCK "raw"."dander_target_commits"') for statement in claim_sql + ) + + +def test_redshift_rejects_truncated_relation_identifiers_before_connecting( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + relation = RelationRef(catalog="analytics", namespace="n" * 128, name="records") + connections_before = backend.connections + + with pytest.raises(ValueError, match="cannot exceed 127 bytes"): + runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="redshift_records", + run_id="too-long", + token=1, + authority_id="postgresql:test-state", + ), + ) + + assert backend.connections == connections_before + + +def test_redshift_stages_bounded_parts_merges_deterministically_and_cleans( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, s3, staging_root = redshift_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=3, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = _target(runtime) + backend.merge_rowcount = 2 + + assert ( + writer.write( + ( + {"id": "one", "label": "old"}, + {"id": "one", "label": "new"}, + {"id": "two", "label": "second"}, + ), + target, + ) + == 2 + ) + + assert len(s3.uploads) == 2 + assert len(s3.puts) == 1 + manifest = json.loads(s3.puts[0][2]) + assert all(entry["mandatory"] is True for entry in manifest["entries"]) + assert all(entry["meta"]["content_length"] > 0 for entry in manifest["entries"]) + assert s3.deleted and len(s3.deleted[-1]) == 3 + assert not tuple(staging_root.iterdir()) + + writer_connection = max(connection_id for connection_id, _, _ in backend.statements) + sql = [ + statement + for connection_id, statement, _ in backend.statements + if connection_id == writer_connection + ] + assert any(statement.startswith("CREATE TEMP TABLE") for statement in sql) + copy = next(statement for statement in sql if statement.startswith("COPY ")) + assert "IAM_ROLE 'arn:aws:iam::123456789012:role/DanderRedshiftCopy'" in copy + assert "ACCESS_KEY_ID" not in copy and "SECRET_ACCESS_KEY" not in copy + merge = next(statement for statement in sql if statement.startswith("MERGE INTO")) + assert 'ROW_NUMBER() OVER (PARTITION BY "id" ORDER BY "_dander_ordinal" DESC)' in merge + touch_indexes = [ + index + for index, statement in enumerate(sql) + if statement.startswith("UPDATE") and "dander_target_commits" in statement + ] + assert len(touch_indexes) == 2 + assert touch_indexes[0] < sql.index(merge) < touch_indexes[1] + assert sql[touch_indexes[0] - 1].startswith("LOCK ") + + +def test_redshift_retry_identity_excludes_random_local_staging_id( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = _target(runtime, run_id="run-retry") + row = ({"id": "one", "label": "same"},) + + assert writer.write(row, target) == 1 + assert writer.write(row, target) == 0 + + history_parameters = [ + parameters[:5] + for _, statement, parameters in backend.statements + if statement.startswith("INSERT INTO") and "dander_stage_loads" in statement + ] + assert len(history_parameters) == 2 + assert history_parameters[0] == history_parameters[1] + assert len(backend.history) == 1 + + +def test_redshift_lost_publication_rolls_back_and_removes_owned_staging( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, s3, staging_root = redshift_runtime + target = _target(runtime, run_id="run-stale") + backend.fence_touch_rowcount = 0 + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + rollback_before = backend.rollbacks + + with pytest.raises(TargetFenceLostError, match="lost before publication"): + writer.write(({"id": "one", "label": "blocked"},), target) + + assert backend.rollbacks == rollback_before + 1 + assert s3.deleted and len(s3.deleted[-1]) == 2 + assert not tuple(staging_root.iterdir()) + assert not any(statement.startswith("MERGE INTO") for _, statement, _ in backend.statements) + + +def test_redshift_ambiguous_upload_failure_still_deletes_the_owned_key( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, _backend, s3, staging_root = redshift_runtime + target = _target(runtime, run_id="run-upload-failure") + s3.fail_upload_at = 1 + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + with pytest.raises(RedshiftWriteError, match="staged SCD1 write failed"): + writer.write(({"id": "one", "label": "ambiguous"},), target) + + assert len(s3.deleted) == 1 + assert s3.deleted[0] == (s3.uploads[0][1],) + assert not tuple(staging_root.iterdir()) + + +def test_redshift_s3_cleanup_batches_the_service_limit() -> None: + s3 = _FakeS3() + keys = tuple(f"dander/staging/owned-{index}" for index in range(1_001)) + + _delete_owned(s3, "dander-redshift-staging", keys) + + assert tuple(map(len, s3.deleted)) == (1_000, 1) + assert s3.deleted[0][0] == keys[0] + assert s3.deleted[1][0] == keys[-1] + + +def test_redshift_rejects_oversized_single_row_before_s3_or_database_mutation( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, s3, _root = redshift_runtime + target = _target(runtime, run_id="run-large") + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + connections_before = backend.connections + + with pytest.raises(RedshiftWriteError, match="4 MB COPY limit"): + writer.write(({"id": "one", "label": "x" * 4_300_000},), target) + + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts + assert backend.connections == connections_before + + +def test_redshift_connection_validation_errors_are_sanitized(tmp_path: Path) -> None: + class _BrokenConnectionFactory: + def __call__(self) -> object: + raise RuntimeError("private AWS response") + + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, _config()) + + with pytest.raises(ProviderFactoryError, match="connection validation failed") as raised: + registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "analytics", + "connection_factory": _BrokenConnectionFactory(), + "s3_client": _FakeS3(), + "staging_root": tmp_path, + }, + ) + assert "private AWS response" not in str(raised.value) diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 313e68ff..54e46df2 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -23,9 +23,11 @@ def test_state_warehouse_matrix_covers_every_current_pair() -> None: ("bigquery", "bigquery"): CompatibilityStatus.SUPPORTED, ("bigquery", "postgresql"): CompatibilityStatus.EXPERIMENTAL, ("bigquery", "snowflake"): CompatibilityStatus.EXPERIMENTAL, + ("bigquery", "redshift"): CompatibilityStatus.EXPERIMENTAL, ("postgresql", "bigquery"): CompatibilityStatus.UNSUPPORTED, ("postgresql", "postgresql"): CompatibilityStatus.EXPERIMENTAL, ("postgresql", "snowflake"): CompatibilityStatus.EXPERIMENTAL, + ("postgresql", "redshift"): CompatibilityStatus.EXPERIMENTAL, } @@ -35,8 +37,10 @@ def test_state_warehouse_matrix_covers_every_current_pair() -> None: ("bigquery", "bigquery"), ("bigquery", "postgresql"), ("bigquery", "snowflake"), + ("bigquery", "redshift"), ("postgresql", "postgresql"), ("postgresql", "snowflake"), + ("postgresql", "redshift"), ], ) def test_executable_pairs_are_admitted(state: str, warehouse: str) -> None: From 121c7f7c606f979ca849c81f07a0e66dc9a730b4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 01:34:25 -0400 Subject: [PATCH 062/163] fix: keep Redshift SDK imports optional --- src/dander/providers/redshift/runtime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index aed51ed1..e3ba40a0 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, cast @@ -241,7 +242,7 @@ def build_redshift_warehouse( def _sdk_connection_factory(config: RedshiftWarehouseConfig) -> RedshiftConnectionFactory: def connect() -> object: try: - import redshift_connector + redshift_connector = importlib.import_module("redshift_connector") except ModuleNotFoundError as error: raise ProviderFactoryError( "Redshift warehouse requires the dander-platform[redshift] extra" @@ -276,7 +277,7 @@ def connect() -> object: def _sdk_s3_client(region: str) -> RedshiftS3Client: try: - import boto3 # type: ignore[import-untyped] + boto3 = importlib.import_module("boto3") except ModuleNotFoundError as error: raise ProviderFactoryError( "Redshift warehouse requires the dander-platform[redshift] extra" From a733e56f7260d2b55e499899edc1163312355dde Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 01:37:41 -0400 Subject: [PATCH 063/163] docs: record Redshift CI evidence --- HANDOFF.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HANDOFF.md b/HANDOFF.md index 91b17d55..bf3f10e7 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -21,6 +21,7 @@ settings, then use the normal run path with SCD1. No AWS live profile was create dependency loading passed; strict dependency audit found no known vulnerabilities. - The exact final container built and passed non-root, read-only, runtime, and bundled-asset checks. - GCP/AWS Terraform roots and module tests, cross-cloud feasibility roots, and Helm checks passed. +- Protected Linux CI passed Python, Terraform, distribution, container/security, and secret scans. ## Decisions @@ -32,7 +33,6 @@ settings, then use the normal run path with SCD1. No AWS live profile was create ## Remaining -- Let protected Linux CI repeat image/config/secret scans before merge consideration. - Resolve the pre-existing retained GCP no-drift baseline before merging provider work. - Qualify Redshift transforms, other write modes, and one live disposable AWS profile separately. From c0f9c7b30a417803b777553adf5f532b4b6f6fc1 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 06:07:26 -0400 Subject: [PATCH 064/163] fix: reconcile warehouse compatibility matrix --- src/dander/runtime-compatibility.json | 16 ++++++++-------- tests/test_compatibility.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 4725dfd6..d06d3ff3 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -15,15 +15,15 @@ }, { "state": "bigquery", - "warehouse": "snowflake", + "warehouse": "redshift", "status": "experimental", - "reason": "Snowflake staged SCD1 ingestion and destination fencing pass local conformance; no live Snowflake profile is qualified." + "reason": "Locally conformance-tested with destination-side Redshift fencing; no live hosted profile is qualified." }, { "state": "bigquery", - "warehouse": "redshift", + "warehouse": "snowflake", "status": "experimental", - "reason": "Locally conformance-tested with destination-side Redshift fencing; no live hosted profile is qualified." + "reason": "Snowflake staged SCD1 ingestion and destination fencing pass local conformance; no live Snowflake profile is qualified." }, { "state": "postgresql", @@ -39,15 +39,15 @@ }, { "state": "postgresql", - "warehouse": "snowflake", + "warehouse": "redshift", "status": "experimental", - "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged SCD1 writes; live qualification remains pending." + "reason": "Locally conformance-tested with destination-side Redshift fencing; no live hosted profile is qualified." }, { "state": "postgresql", - "warehouse": "redshift", + "warehouse": "snowflake", "status": "experimental", - "reason": "Locally conformance-tested with destination-side Redshift fencing; no live hosted profile is qualified." + "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged SCD1 writes; live qualification remains pending." } ] } diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 54e46df2..8d441432 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -64,5 +64,5 @@ def test_runtime_compatibility_cli_prints_one_deterministic_document() -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) assert payload["schema"] == "io.dander.runtime.compatibility/v1" - assert len(payload["state_warehouse_pairs"]) == 6 + assert len(payload["state_warehouse_pairs"]) == 8 assert result.output.strip() == load_runtime_compatibility().to_json() From db5d79dc3e20377bd22de7cdcba12b2bda75b38a Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 06:26:45 -0400 Subject: [PATCH 065/163] docs: record clean Redshift merge gate --- HANDOFF.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index bf3f10e7..dd4caca4 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -16,12 +16,13 @@ settings, then use the normal run path with SCD1. No AWS live profile was create ## Checks -- Ruff, formatting, and strict mypy passed; all 1,063 tests passed with PostgreSQL 15 integration. +- Ruff, formatting, and strict mypy passed; protected CI ran 1,083 tests with PostgreSQL 15. - Wheel/sdist build, inspection, outside-checkout installs, source-free generation, and full runtime dependency loading passed; strict dependency audit found no known vulnerabilities. - The exact final container built and passed non-root, read-only, runtime, and bundled-asset checks. - GCP/AWS Terraform roots and module tests, cross-cloud feasibility roots, and Helm checks passed. - Protected Linux CI passed Python, Terraform, distribution, container/security, and secret scans. +- A refreshed retained-project plan from current `main` reported exactly `No changes.` ## Decisions @@ -33,7 +34,7 @@ settings, then use the normal run path with SCD1. No AWS live profile was create ## Remaining -- Resolve the pre-existing retained GCP no-drift baseline before merging provider work. +- Merge the focused PR only through the protected-main process. - Qualify Redshift transforms, other write modes, and one live disposable AWS profile separately. ## Review First From 1fa1aacc3616bf21f22098096b1fbd3e2fe5e774 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 06:56:19 -0400 Subject: [PATCH 066/163] Add fenced Redshift transforms --- CHANGELOG.md | 4 + HANDOFF.md | 49 +- docs/compatibility-matrix.md | 4 +- docs/decisions.md | 11 + docs/known-limitations.md | 8 +- docs/redshift.md | 55 +++ pyproject.toml | 1 + src/dander/providers/redshift/runtime.py | 34 +- src/dander/providers/redshift/transform.py | 425 ++++++++++++++++++ .../test_redshift_warehouse_runtime.py | 312 ++++++++++++- 10 files changed, 861 insertions(+), 42 deletions(-) create mode 100644 docs/redshift.md create mode 100644 src/dander/providers/redshift/transform.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f2352a9..ffe0a32f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ capabilities enter through the next minor release. ### Added +- Extend the experimental Redshift warehouse adapter with whole-DAG-preflighted portable and + Redshift-authored table/incremental models, strict declared schemas, generic assertions, and + transactionally fenced DML publication. Views, graphs, other write modes, and support promotion + remain unavailable. - Add an experimental Snowflake warehouse adapter with native database/schema coordinates, bounded Parquet staging, scalar SCD1 `COPY`/`MERGE`, additive nullable schema evolution, destination-side fencing, sanitized telemetry, and process-death-safe temporary staging. This diff --git a/HANDOFF.md b/HANDOFF.md index dd4caca4..91de8e09 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,43 +2,44 @@ ## Finished -- Added an experimental Redshift warehouse provider using native database/schema coordinates. -- Implemented bounded Parquet staging, same-region S3 manifest `COPY`, and IAM-only authentication. -- Added deterministic SCD1 merge/replay, additive schema checks, transactional destination fencing, - and exact owned-object cleanup. -- Registered only BigQuery-state/Redshift and PostgreSQL-state/Redshift as experimental pairs. -- Updated existing compatibility, staging, limitations, and architecture decision records. +- Added experimental Redshift portable/provider-exact table and incremental model execution. +- Preflighted the complete selected DAG, schemas, SQL, materializations, and assertions before any + provider connection, claim, or mutation. +- Published through session-temporary CTAS plus fenced table replacement or cursor-monotonic + `UPDATE`/`INSERT`; target DML and the exact fence touch share one transaction. +- Added generic assertions, stale-fence/cleanup/replay coverage, and a packaged Redshift guide. +- Kept views, graphs, schema evolution, `SUPER`, other write modes, and support promotion blocked. ## Try It -Configure a v2 `warehouse.provider: redshift` profile with provisioned-cluster or Serverless IAM -settings, then use the normal run path with SCD1. No AWS live profile was created or contacted. +Configure a v2 `warehouse.provider: redshift` profile as shown in `docs/redshift.md`, author a +portable or Redshift-exact table/incremental model, and use the normal hosted run path. This slice +was conformance-tested locally; no AWS warehouse was created or contacted. ## Checks -- Ruff, formatting, and strict mypy passed; protected CI ran 1,083 tests with PostgreSQL 15. -- Wheel/sdist build, inspection, outside-checkout installs, source-free generation, and full runtime - dependency loading passed; strict dependency audit found no known vulnerabilities. -- The exact final container built and passed non-root, read-only, runtime, and bundled-asset checks. -- GCP/AWS Terraform roots and module tests, cross-cloud feasibility roots, and Helm checks passed. -- Protected Linux CI passed Python, Terraform, distribution, container/security, and secret scans. -- A refreshed retained-project plan from current `main` reported exactly `No changes.` +- Ruff and strict mypy passed; focused Redshift/transform tests passed. +- Full suite passed with 1,086 tests against an ephemeral PostgreSQL 15 service. +- Wheel/sdist inspection, outside-checkout generation/install, full-runtime install, dependency + audit, Terraform/AWS/Helm validation, and generated-project Terraform validation passed. +- The final source-free container passed non-root, read-only, runtime, and bundled-asset checks. ## Decisions -- Redshift accepts ambient AWS credentials and uses a configured IAM role for `COPY`; no static - access keys or database passwords enter configuration or SQL. -- Publication is SCD1-only and transactionally fenced; replay identity includes target, pipeline, - run, schema fingerprint, and deterministic manifest digest. -- Unsupported transforms, graphs, write modes, and semi-structured mappings fail closed. +- Redshift incremental publication uses documented `UPDATE ... FROM` plus `INSERT ... WHERE NOT + EXISTS`; Redshift does not support Snowflake-style conditional `WHEN MATCHED` clauses. +- Canonical database/schema/relation coordinates survive compilation; Redshift renders local + schema/table target DML only at its provider boundary. +- Existing destination fencing and strict transform schemas are reused without a parallel runtime. ## Remaining -- Merge the focused PR only through the protected-main process. -- Qualify Redshift transforms, other write modes, and one live disposable AWS profile separately. +- Run the retained GCP no-drift plan; never apply it. +- Push the focused branch, let protected CI validate it, and merge only if every check is green. +- Continue the cloud-portability roadmap with a separate next slice. ## Review First -- `src/dander/providers/redshift/fence.py` -- `src/dander/providers/redshift/writer.py` +- `src/dander/providers/redshift/transform.py` - `tests/providers/test_redshift_warehouse_runtime.py` +- `docs/redshift.md` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index bc4b8c40..7556e1b2 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -15,11 +15,11 @@ before provider construction when a pair is absent or marked `unsupported`. | BigQuery | BigQuery | supported | Released GCP profile and BigQuery regression baseline | | BigQuery | PostgreSQL | experimental | Local cross-backend token/fence publication proof | | BigQuery | Snowflake | experimental | Local staged-SCD1, fenced transforms, schema, replay, and fence conformance | -| BigQuery | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, and destination-fence conformance | +| BigQuery | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, fenced transforms, and destination-fence conformance | | PostgreSQL | BigQuery | unsupported | Not every BigQuery write mode uses destination fencing | | PostgreSQL | PostgreSQL | experimental | Local native profile, state/warehouse conformance, and benchmarks | | PostgreSQL | Snowflake | experimental | Local staged-SCD1, transforms, and destination-fence conformance | -| PostgreSQL | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, and destination-fence conformance | +| PostgreSQL | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, fenced transforms, and destination-fence conformance | `experimental` means the code path is executable and tested locally; it is not a hosted support claim. The packaged `runtime-capabilities.json` remains the supported launcher/profile boundary. diff --git a/docs/decisions.md b/docs/decisions.md index 5cafb78f..13b61bf7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -898,3 +898,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Correctness:** Claims serialize on the destination ledger. One publication transaction locks and DML-touches the exact authority/run/token, evolves declared nullable columns, performs the deterministic ordinal MERGE, records replay identity, and commits the fence or rolls back all. + +## 2026-08-09 — Redshift transforms reuse native transactions and canonical relations + +- **Preflight:** The complete selected model DAG, declared scalar schemas, SQL, materializations, + and assertions compile before Redshift receives a connection, destination claim, or mutation. +- **Publication:** Session-temporary CTAS staging feeds table replacement or deterministic, + cursor-monotonic incremental `UPDATE`/`INSERT`; target DML and the exact fence touch commit + together. +- **Boundary:** Canonical database/schema/relation coordinates survive compilation, while the + provider renders database-local target DML. Views, graphs, automatic transform-schema evolution, + `SUPER`, and support promotion remain separate gates. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 7be70c16..4abc062d 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -15,10 +15,10 @@ evaluate these limits before using it for an unattended system containing busine table/incremental transforms. It has no live Snowflake qualification, views, graph execution, semi-structured types, or other write modes. Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded - Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, and destination fencing; - it has no live qualification, transforms, executable graphs, non-SCD1 modes, or semi-structured - `SUPER` mapping. PostgreSQL state and warehouse execution are implemented and locally - conformance-tested. A packaged Helm + Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, + and fenced table/incremental transforms. It has no live qualification, views, executable graphs, + non-SCD1 modes, or semi-structured `SUPER` mapping. PostgreSQL state and warehouse execution are + implemented and locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/redshift.md b/docs/redshift.md new file mode 100644 index 00000000..3aa59f7d --- /dev/null +++ b/docs/redshift.md @@ -0,0 +1,55 @@ +# Experimental Amazon Redshift warehouse + +Redshift is an experimental warehouse adapter, not a supported Dander profile. The current slice +proves native database/schema coordinates, bounded IAM-only bulk loading, and fenced table and +incremental models before broader write modes or a live support claim. + +## Configuration + +Select Redshift in a version 2 platform profile. This provisioned-cluster example stores resource +identifiers but no AWS access keys or database password: + +```yaml +platforms: + redshift_test: + warehouse: + provider: redshift + deployment: provisioned + host: example.abc123.us-east-1.redshift.amazonaws.com + database: analytics + schema: raw + db_user: dander_user + region: us-east-1 + cluster_identifier: dander-test + copy_role_arn: arn:aws:iam::123456789012:role/DanderRedshiftCopy + staging_bucket: dander-redshift-staging + state: + provider: postgresql + authority_id: postgresql:redshift-test + catalog: + provider: none + secrets: + provider: environment +``` + +Serverless uses `deployment: serverless` and `workgroup_name` instead of `cluster_identifier` and +`db_user`. Dander obtains temporary IAM credentials from the ordinary AWS SDK chain. The configured +COPY role must read the dedicated same-region staging prefix. + +## Current contract + +- `database + schema + relation` become one canonical `RelationRef`; GCP aliases are not used. +- Bounded, checksummed Parquet parts load through a mandatory same-region S3 manifest and `COPY`. +- SCD1 publication, replay history, and the exact destination fencing-token touch commit together. +- Only declared nullable ingestion columns evolve automatically; drift fails closed. +- Portable or Redshift-authored table models use fenced `DELETE`/`INSERT` replacement. +- Incremental models collapse duplicate keys deterministically and reject cursor regression. +- Generic not-null, uniqueness, accepted-value, and relationship assertions run after publication. +- The complete selected transform DAG preflights before any provider connection or mutation. + +## Deliberate limits + +Only scalar SCD1 ingestion plus table and incremental model materializations are implemented. +Views, graphs, replace, SCD2, snapshot, `SUPER`, live concurrency proof, infrastructure +provisioning, and support promotion remain separate work. Use `catalog.provider: none` for this +experimental path. diff --git a/pyproject.toml b/pyproject.toml index b2238db9..5d03799d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,6 +239,7 @@ include = [ "docs/postgresql-benchmarks.md", "docs/oci-artifact-contract.md", "docs/rollback.md", + "docs/redshift.md", "docs/snowflake.md", "docs/upgrading.md", "docs/warehouse-staging.md", diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index e3ba40a0..9896ee9b 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -14,6 +14,7 @@ execute, open_connection, ) +from dander.providers.redshift.transform import RedshiftTransformRunner from dander.providers.redshift.writer import ( RedshiftS3Client, RedshiftScd1Writer, @@ -99,7 +100,12 @@ def build_ingestion_writer( @dataclass(frozen=True, slots=True) class RedshiftTransformFactory: - """Fail clearly until the Redshift transform slice is implemented.""" + """Construct fenced Redshift model execution while rejecting graph plans.""" + + database: str + connection_factory: RedshiftConnectionFactory + target_fence: RedshiftTargetFence + statement_timeout_ms: int def build_transform_runner( self, @@ -107,11 +113,18 @@ def build_transform_runner( graph_plan: object | None, build_models: bool, raw_namespace: str = "raw", - ) -> None: - del raw_namespace - if graph_plan is not None or build_models: - raise ValueError("Redshift transforms are not available in this experimental slice") - return None + ) -> RedshiftTransformRunner | None: + if graph_plan is not None: + raise ValueError("Redshift graph execution is not available in this experimental slice") + if not build_models: + return None + return RedshiftTransformRunner( + database=self.database, + connection_factory=self.connection_factory, + target_fence=self.target_fence, + statement_timeout_ms=self.statement_timeout_ms, + raw_namespace=raw_namespace, + ) @dataclass(frozen=True, slots=True) @@ -144,7 +157,7 @@ def operation( schema_contract_version=1, write_modes=frozenset({WriteMode.SCD1}), transports=frozenset({WriteTransport.COPY}), - supports_transforms=False, + supports_transforms=True, supports_graphs=False, supports_target_fencing=True, ) @@ -232,7 +245,12 @@ def build_redshift_warehouse( target_fence, staging, ), - transforms=RedshiftTransformFactory(), + transforms=RedshiftTransformFactory( + config.database, + connection_factory, + target_fence, + config.statement_timeout_ms, + ), target_fence=target_fence, telemetry=RedshiftTelemetry(), capabilities=REDSHIFT_CAPABILITIES, diff --git a/src/dander/providers/redshift/transform.py b/src/dander/providers/redshift/transform.py new file mode 100644 index 00000000..26b92176 --- /dev/null +++ b/src/dander/providers/redshift/transform.py @@ -0,0 +1,425 @@ +"""Fenced Amazon Redshift table and incremental model execution.""" + +from __future__ import annotations + +import hashlib +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander.concurrency import TargetFenceLostError +from dander.providers.redshift.config import validate_redshift_relation +from dander.providers.redshift.session import ( + RedshiftConnection, + RedshiftConnectionFactory, + execute, + open_connection, +) +from dander.providers.redshift.writer import ( + RedshiftWriteError, + _create_target_sql, + _qualified, + _quote, + _redshift_type, + _schema_changes, + _set_query_group, +) +from dander.transform import ( + SqlDialect, + TransformModel, + TransformProject, + TransformProjectError, + TransformRunError, + TransformRunResult, +) +from dander.transform.model import Materialization +from dander.writer import SchemaEvolution, WriteField, WriteTarget + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from pathlib import Path + + from dander.concurrency import OwnershipGuard, TargetFence + from dander.providers.redshift.fence import RedshiftTargetFence + from dander.transform.config import GenericTestMetadata + from dander.warehouse import RelationRef + + +@dataclass(frozen=True, slots=True) +class _RedshiftModelPlan: + model: TransformModel + target: WriteTarget + query: str + + +@dataclass(frozen=True, slots=True) +class _RedshiftAssertion: + name: str + statement: str + parameters: tuple[object, ...] = () + + +class RedshiftTransformRunner: + """Build portable table/incremental models behind destination fencing.""" + + target_dialect = SqlDialect.REDSHIFT + + def __init__( + self, + *, + database: str, + connection_factory: RedshiftConnectionFactory, + target_fence: RedshiftTargetFence, + statement_timeout_ms: int, + raw_namespace: str = "raw", + ) -> None: + self._database = database + self._connection_factory = connection_factory + self._target_fence = target_fence + self._statement_timeout_ms = statement_timeout_ms + self._raw_namespace = raw_namespace + + def build( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Preflight the selected DAG, then publish every model through fenced DML.""" + if ownership is None or ownership.fence is None: + raise TransformRunError("Redshift hosted transforms require active lease ownership") + _project, models, plans, assertions = self._preflight(models_dir, selected=selected) + for plan in plans: + ownership.verify() + publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) + ownership.verify() + self._publish(plan, publication) + self._run_assertions(assertions, ownership=ownership) + return TransformRunResult( + models=tuple(model.name for model in models), + assertions=len(assertions), + ) + + def test( + self, + models_dir: Path, + *, + selected: Iterable[str] | None = None, + ) -> TransformRunResult: + """Run assertions against already materialized Redshift relations.""" + _project, models, _plans, assertions = self._preflight(models_dir, selected=selected) + self._run_assertions(assertions) + return TransformRunResult( + models=tuple(model.name for model in models), + assertions=len(assertions), + ) + + def _preflight( + self, + models_dir: Path, + *, + selected: Iterable[str] | None, + ) -> tuple[ + TransformProject, + tuple[TransformModel, ...], + tuple[_RedshiftModelPlan, ...], + tuple[_RedshiftAssertion, ...], + ]: + """Compile every selected model and assertion before opening a provider session.""" + project = TransformProject.load( + models_dir, + catalog=self._database, + raw_namespace=self._raw_namespace, + target_dialect=SqlDialect.REDSHIFT, + ) + models = project.ordered(selected) + plans = tuple(_model_plan(project, model) for model in models) + assertions = tuple( + assertion for model in models for assertion in _compile_assertions(project, model) + ) + return project, models, plans, assertions + + def _publish(self, plan: _RedshiftModelPlan, publication: TargetFence) -> None: + target = WriteTarget( + relation=plan.target.relation_ref, + business_key=plan.target.business_key, + schema=plan.target.schema, + publication_fence=publication, + ) + temporary = _temporary_name(target.relation_ref, publication) + cleanup_started = False + with open_connection(self._connection_factory) as connection: + try: + _set_query_group( + connection, + publication, + statement_timeout_ms=self._statement_timeout_ms, + ) + cleanup_started = True + execute(connection, _create_temporary_sql(plan, temporary)) + # Redshift temp tables survive commit. End CTAS and schema-inspection snapshots + # before the single destination-fenced publication transaction begins. + connection.commit() + schema_changes = _schema_changes( + connection, + target, + target.canonical_schema, + evolution=SchemaEvolution.STRICT, + ) + connection.commit() + statements = ( + (_create_target_sql(target, target.canonical_schema), ()), + *schema_changes, + *_publication_statements(plan, temporary), + ) + self._target_fence.execute_statements(connection, statements, publication) + except (TargetFenceLostError, RedshiftWriteError, TransformRunError): + raise + except Exception as error: + raise TransformRunError("Redshift transform publication failed") from error + finally: + if cleanup_started: + _drop_temporary( + connection, + temporary, + suppress_failure=sys.exc_info()[0] is not None, + ) + + def _run_assertions( + self, + assertions: Sequence[_RedshiftAssertion], + *, + ownership: OwnershipGuard | None = None, + ) -> None: + failures: list[str] = [] + with open_connection(self._connection_factory) as connection: + for assertion in assertions: + if ownership is not None: + ownership.verify() + try: + row = execute( + connection, + assertion.statement, + assertion.parameters, + fetch="one", + ).row + except Exception as error: + raise TransformRunError( + f"Redshift assertion execution failed: {assertion.name}" + ) from error + if _failure_count(row, assertion.name) > 0: + failures.append(assertion.name) + connection.commit() + if failures: + raise TransformRunError(f"Data tests failed: {', '.join(failures)}") + + +def _model_plan(project: TransformProject, model: TransformModel) -> _RedshiftModelPlan: + if model.metadata.materialization is Materialization.VIEW: + raise TransformProjectError( + "Redshift view materialization is unavailable in the fenced transform slice" + ) + if model.metadata.materialization not in {Materialization.TABLE, Materialization.INCREMENTAL}: + raise TransformProjectError( + f"Redshift materialization is unavailable: {model.metadata.materialization.value}" + ) + required = set(model.metadata.unique_key) + if model.metadata.incremental_cursor is not None: + required.add(model.metadata.incremental_cursor) + fields = tuple( + WriteField( + name=column.name, + data_type=column.data_type, + mode="REQUIRED" if column.name in required else "NULLABLE", + ) + for column in model.metadata.columns + ) + target = WriteTarget( + relation=project.relation_ref_for_model(model), + business_key=tuple(model.metadata.unique_key), + schema=fields, + ) + validate_redshift_relation(target.relation_ref) + for field in target.canonical_schema.fields: + _redshift_type(field.data_type) + return _RedshiftModelPlan(model=model, target=target, query=project.compile(model)) + + +def _create_temporary_sql(plan: _RedshiftModelPlan, temporary: str) -> str: + names = tuple(field.name for field in plan.target.canonical_schema.fields) + selected = ", ".join(f"source.{_quote(name)}" for name in names) + if plan.model.metadata.materialization is Materialization.TABLE: + return ( + f"CREATE TEMP TABLE {_quote(temporary)} AS SELECT {selected} " + f"FROM ({plan.query}) AS source" + ) + keys = tuple(plan.model.metadata.unique_key) + cursor = plan.model.metadata.incremental_cursor + assert keys and cursor is not None + partition = ", ".join(f"source.{_quote(key)}" for key in keys) + tie_breakers = tuple(name for name in sorted(names) if name not in {*keys, cursor}) + ordering = [f"source.{_quote(cursor)} DESC NULLS LAST"] + ordering.extend(f"source.{_quote(name)} DESC NULLS LAST" for name in tie_breakers) + projected = ", ".join(f"ranked.{_quote(name)}" for name in names) + return ( + f"CREATE TEMP TABLE {_quote(temporary)} AS SELECT {projected} FROM (" + f"SELECT {selected}, ROW_NUMBER() OVER (PARTITION BY {partition} ORDER BY " + f"{', '.join(ordering)}) AS {_quote('_dander_rank')} FROM ({plan.query}) AS source" + f") AS ranked WHERE ranked.{_quote('_dander_rank')} = 1" + ) + + +def _publication_statements( + plan: _RedshiftModelPlan, + temporary: str, +) -> tuple[tuple[str, Sequence[object]], ...]: + target = _qualified(plan.target.relation_ref.namespace, plan.target.relation_ref.name) + staged = _quote(temporary) + names = tuple(field.name for field in plan.target.canonical_schema.fields) + columns = ", ".join(_quote(name) for name in names) + incoming = ", ".join(f"incoming.{_quote(name)}" for name in names) + if plan.model.metadata.materialization is Materialization.TABLE: + return ( + (f"DELETE FROM {target}", ()), + (f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {staged}", ()), + ) + keys = tuple(plan.model.metadata.unique_key) + cursor = plan.model.metadata.incremental_cursor + assert keys and cursor is not None + match = " AND ".join(f"target.{_quote(key)} = incoming.{_quote(key)}" for key in keys) + mutable = tuple(name for name in names if name not in keys) + updates = ", ".join(f"{_quote(name)} = incoming.{_quote(name)}" for name in mutable) + statements: list[tuple[str, Sequence[object]]] = [] + if mutable: + statements.append( + ( + f"UPDATE {target} AS target SET {updates} FROM {staged} AS incoming " + f"WHERE {match} AND incoming.{_quote(cursor)} >= target.{_quote(cursor)}", + (), + ) + ) + missing = " AND ".join(f"existing.{_quote(key)} = incoming.{_quote(key)}" for key in keys) + statements.append( + ( + f"INSERT INTO {target} ({columns}) SELECT {incoming} FROM {staged} AS incoming " + f"WHERE NOT EXISTS (SELECT 1 FROM {target} AS existing WHERE {missing})", + (), + ) + ) + return tuple(statements) + + +def _compile_assertions( + project: TransformProject, + model: TransformModel, +) -> tuple[_RedshiftAssertion, ...]: + relation = _local_relation(project.relation_ref_for_model(model)) + assertions: list[_RedshiftAssertion] = [] + for test in model.metadata.tests: + assertions.extend(_assertions_for_test(project, model.name, relation, test)) + return tuple(assertions) + + +def _assertions_for_test( + project: TransformProject, + model_name: str, + relation: str, + test: GenericTestMetadata, +) -> list[_RedshiftAssertion]: + column = _quote(test.column) + assertions: list[_RedshiftAssertion] = [] + if test.not_null: + assertions.append( + _RedshiftAssertion( + name=f"{model_name}.{test.column}.not_null", + statement=f"SELECT COUNT(*) AS failures FROM {relation} WHERE {column} IS NULL", + ) + ) + if test.unique: + assertions.append( + _RedshiftAssertion( + name=f"{model_name}.{test.column}.unique", + statement=( + "SELECT COUNT(*) AS failures FROM (" + f"SELECT {column} FROM {relation} WHERE {column} IS NOT NULL " + f"GROUP BY {column} HAVING COUNT(*) > 1) AS duplicates" + ), + ) + ) + if test.accepted_values is not None: + placeholders = ", ".join("%s" for _ in test.accepted_values) + assertions.append( + _RedshiftAssertion( + name=f"{model_name}.{test.column}.accepted_values", + statement=( + f"SELECT COUNT(*) AS failures FROM {relation} WHERE {column} IS NOT NULL " + f"AND {column} NOT IN ({placeholders})" + ), + parameters=tuple(test.accepted_values), + ) + ) + if test.relationships is not None: + if test.relationships.to in project.models: + parent_model = project.models[test.relationships.to] + parent_columns = {item.name for item in parent_model.metadata.columns} + if test.relationships.field not in parent_columns: + raise TransformProjectError( + "Relationship test references an undeclared parent column: " + f"{test.relationships.to}.{test.relationships.field}" + ) + parent = _local_relation(project.relation_ref_for_ref(test.relationships.to)) + parent_field = _quote(test.relationships.field) + assertions.append( + _RedshiftAssertion( + name=f"{model_name}.{test.column}.relationships", + statement=( + f"SELECT COUNT(*) AS failures FROM {relation} AS child " + f"LEFT JOIN {parent} AS parent ON child.{column} = parent.{parent_field} " + f"WHERE child.{column} IS NOT NULL AND parent.{parent_field} IS NULL" + ), + ) + ) + return assertions + + +def _local_relation(relation: RelationRef) -> str: + return _qualified(relation.namespace, relation.name) + + +def _temporary_name(relation: RelationRef, publication: TargetFence) -> str: + digest = hashlib.sha256( + f"{'.'.join(relation.coordinates)}:{publication.run_id}:{publication.token}".encode() + ).hexdigest()[:20] + return f"dander_model_{digest}" + + +def _drop_temporary( + connection: RedshiftConnection, + temporary: str, + *, + suppress_failure: bool, +) -> None: + try: + execute(connection, f"DROP TABLE IF EXISTS {_quote(temporary)}") + connection.commit() + except Exception as error: + connection.rollback() + if suppress_failure: + return + raise TransformRunError("Redshift transform staging cleanup failed") from error + + +def _failure_count(row: object | None, assertion: str) -> int: + if isinstance(row, dict): + value = row.get("failures") + elif isinstance(row, (tuple, list)) and row: + value = row[0] + else: + value = None + if isinstance(value, bool) or not isinstance(value, int): + raise TransformRunError(f"Assertion returned an invalid result: {assertion}") + return value + + +__all__ = ["RedshiftTransformRunner"] diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index 9f61891e..a3f07e34 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -15,7 +15,9 @@ from dander.concurrency import FencingToken, TargetFenceLostError from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry from dander.providers.redshift import RedshiftWarehouseConfig +from dander.providers.redshift.transform import RedshiftTransformRunner from dander.providers.redshift.writer import RedshiftWriteError, _delete_owned +from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError from dander.warehouse import RelationRef, WarehouseRuntime from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport @@ -40,6 +42,9 @@ class _FakeRedshift: rollbacks: int = 0 closes: int = 0 connections: int = 0 + transform_source_rows: list[dict[str, object]] | None = None + transform_rows: dict[str, dict[str, object]] = field(default_factory=dict) + assertion_failure_count: int = 0 def connect(self) -> _FakeConnection: self.connections += 1 @@ -52,6 +57,7 @@ def __init__(self, backend: _FakeRedshift, connection_id: int) -> None: self.connection_id = connection_id self.autocommit = True self.in_transaction = False + self.temporary_rows: list[dict[str, object]] = [] def cursor(self) -> _FakeCursor: return _FakeCursor(self) @@ -111,6 +117,50 @@ def execute(self, command: str, args: Sequence[object] | None = None) -> Self: self.rowcount = self.backend.fence_touch_rowcount elif compact.startswith("SELECT column_name") and "svv_columns" in compact: self._rows = list(self.backend.schema_rows) + elif compact.startswith("CREATE TEMP TABLE") and " AS SELECT" in compact: + rows = self.backend.transform_source_rows + if rows is not None: + self.connection.temporary_rows = [dict(row) for row in rows] + if "ROW_NUMBER() OVER" in compact: + selected: dict[str, dict[str, object]] = {} + for row in sorted( + self.connection.temporary_rows, + key=lambda item: ( + _integer(item, "updated_at"), + str(item.get("label", "")), + ), + reverse=True, + ): + selected.setdefault(str(row["id"]), row) + self.connection.temporary_rows = list(selected.values()) + elif compact.startswith("DELETE FROM") and self.backend.transform_source_rows is not None: + self.backend.transform_rows.clear() + elif ( + compact.startswith("UPDATE") + and "dander_target_commits" not in compact + and self.backend.transform_source_rows is not None + ): + assert 'incoming."updated_at" >= target."updated_at"' in compact + for row in self.connection.temporary_rows: + key = str(row["id"]) + current = self.backend.transform_rows.get(key) + if current is not None and _integer(row, "updated_at") >= _integer( + current, "updated_at" + ): + self.backend.transform_rows[key] = dict(row) + elif ( + compact.startswith("INSERT INTO") + and "dander_target_commits" not in compact + and "dander_stage_loads" not in compact + and self.backend.transform_source_rows is not None + ): + if "WHERE NOT EXISTS" in compact: + for row in self.connection.temporary_rows: + self.backend.transform_rows.setdefault(str(row["id"]), dict(row)) + else: + self.backend.transform_rows = { + str(row["id"]): dict(row) for row in self.connection.temporary_rows + } elif compact.startswith("INSERT INTO") and "dander_stage_loads" in compact: self.rowcount = 0 if parameters[:5] in self.backend.history else 1 self.backend.history.add(parameters[:5]) @@ -118,6 +168,8 @@ def execute(self, command: str, args: Sequence[object] | None = None) -> Self: self.rowcount = ( 0 if parameters[:5] in self.backend.history else self.backend.merge_rowcount ) + elif compact.startswith("SELECT COUNT"): + self._row = (self.backend.assertion_failure_count,) return self def fetchone(self) -> object | None: @@ -172,6 +224,12 @@ def _required_int(value: object) -> int: return value +def _integer(row: dict[str, object], field_name: str) -> int: + value = row[field_name] + assert isinstance(value, int) and not isinstance(value, bool) + return value + + def _config() -> dict[str, object]: return { "provider": "redshift", @@ -273,7 +331,7 @@ def test_redshift_serverless_uses_an_aws_derived_database_user() -> None: registry.parse(ProviderKind.WAREHOUSE, {**raw, "db_user": "dander_user"}) -def test_redshift_runtime_validates_connection_and_fails_closed_for_transforms( +def test_redshift_runtime_validates_connection_and_exposes_fenced_transforms( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], ) -> None: runtime, _backend, _s3, _root = redshift_runtime @@ -282,9 +340,255 @@ def test_redshift_runtime_validates_connection_and_fails_closed_for_transforms( assert runtime.relation_codec.render(relation) == '"analytics"."raw"."records"' assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) - assert runtime.capabilities.supports_transforms is False - with pytest.raises(ValueError, match="transforms are not available"): - runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + assert runtime.capabilities.supports_transforms is True + assert isinstance( + runtime.transforms.build_transform_runner(graph_plan=None, build_models=True), + RedshiftTransformRunner, + ) + + +@dataclass +class _Ownership: + fence: FencingToken + verifications: int = 0 + + def verify(self) -> None: + self.verifications += 1 + + +def test_redshift_builds_fenced_table_and_incremental_models( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=None, + build_models=True, + raw_namespace="raw", + ) + assert isinstance(runner, RedshiftTransformRunner) + _write_redshift_model(tmp_path, name="table_model", materialization="table", with_tests=True) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "YES"), + ("label", "character varying", 65_535, None, None, "YES"), + ("updated_at", "bigint", None, None, None, "YES"), + ] + backend.transform_rows = {"stale": {"id": "stale", "label": "old", "updated_at": 0}} + backend.transform_source_rows = [{"id": "fresh", "label": "newer", "updated_at": 1}] + + result = runner.build( + tmp_path, + selected=["table_model"], + ownership=_ownership("run-table", 1), + ) + + assert result.models == ("table_model",) + assert result.assertions == 4 + assert backend.transform_rows == {"fresh": {"id": "fresh", "label": "newer", "updated_at": 1}} + sql = [statement for _, statement, _ in backend.statements] + assert any( + statement.startswith("DELETE FROM") and "table_model" in statement for statement in sql + ) + assert any( + statement.startswith("INSERT INTO") and "table_model" in statement for statement in sql + ) + assert not any(statement.startswith("ALTER TABLE") for statement in sql) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + + _write_redshift_model(tmp_path, name="incremental_model", materialization="incremental") + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "NO"), + ("label", "character varying", 65_535, None, None, "YES"), + ("updated_at", "bigint", None, None, None, "NO"), + ] + backend.transform_rows.clear() + backend.transform_source_rows = [ + {"id": "one", "label": "older", "updated_at": 1}, + {"id": "one", "label": "aaa-equal", "updated_at": 2}, + {"id": "one", "label": "zzz-equal", "updated_at": 2}, + {"id": "two", "label": "second", "updated_at": 1}, + ] + before_incremental = len(backend.statements) + + result = runner.build( + tmp_path, + selected=["incremental_model"], + ownership=_ownership("run-incremental", 2), + ) + + assert result.models == ("incremental_model",) + incremental_sql = [statement for _, statement, _ in backend.statements[before_incremental:]] + temporary = next( + statement + for statement in incremental_sql + if statement.startswith("CREATE TEMP TABLE") and " AS SELECT" in statement + ) + assert ( + 'ROW_NUMBER() OVER (PARTITION BY source."id" ORDER BY ' + 'source."updated_at" DESC NULLS LAST, source."label" DESC NULLS LAST)' in temporary + ) + update = next( + statement + for statement in incremental_sql + if statement.startswith("UPDATE ") and "incremental_model" in statement + ) + insert = next( + statement + for statement in incremental_sql + if statement.startswith("INSERT INTO") and "incremental_model" in statement + ) + assert 'incoming."updated_at" >= target."updated_at"' in update + assert '"label" = incoming."label"' in update + assert "WHERE NOT EXISTS" in insert + assert backend.transform_rows == { + "one": {"id": "one", "label": "zzz-equal", "updated_at": 2}, + "two": {"id": "two", "label": "second", "updated_at": 1}, + } + backend.transform_source_rows = [{"id": "one", "label": "stale", "updated_at": 1}] + runner.build( + tmp_path, + selected=["incremental_model"], + ownership=_ownership("run-replay", 3), + ) + assert backend.transform_rows["one"] == { + "id": "one", + "label": "zzz-equal", + "updated_at": 2, + } + + +def test_redshift_preflights_the_dag_and_preserves_canonical_coordinates( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + runner = runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + assert isinstance(runner, RedshiftTransformRunner) + _write_redshift_model(tmp_path, name="good_model", materialization="table") + _write_redshift_model(tmp_path, name="unsupported_view", materialization="view") + before = list(backend.statements) + + with pytest.raises(TransformProjectError, match="view materialization is unavailable"): + runner.build(tmp_path, ownership=_ownership("run-preflight", 1)) + + assert backend.statements == before + project = TransformProject.load( + tmp_path, + catalog="analytics", + raw_namespace="raw_data", + target_dialect=SqlDialect.REDSHIFT, + ) + model = project.ordered(["good_model"])[0] + assert project.relation_ref_for_model(model).coordinates == ( + "analytics", + "analytics", + "good_model", + ) + assert '"analytics"."raw_data"."records"' in project.compile(model) + with pytest.raises(ValueError, match="graph execution is not available"): + runtime.transforms.build_transform_runner(graph_plan=object(), build_models=False) + assert runtime.transforms.build_transform_runner(graph_plan=None, build_models=False) is None + + +def test_redshift_transform_requires_ownership_and_fails_closed_after_staging( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + runner = runtime.transforms.build_transform_runner(graph_plan=None, build_models=True) + assert isinstance(runner, RedshiftTransformRunner) + _write_redshift_model(tmp_path, name="owned_model", materialization="table") + with pytest.raises(TransformRunError, match="require active lease ownership"): + runner.build(tmp_path) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "YES"), + ("label", "character varying", 65_535, None, None, "YES"), + ("updated_at", "bigint", None, None, None, "YES"), + ] + backend.transform_source_rows = [ + {"id": "blocked", "label": "must-not-publish", "updated_at": 1} + ] + backend.fence_touch_rowcount = 0 + before_publication = len(backend.statements) + rollbacks = backend.rollbacks + + with pytest.raises(TargetFenceLostError, match="lost before publication"): + runner.build(tmp_path, ownership=_ownership("run-stale", 1)) + + attempted = [statement for _, statement, _ in backend.statements[before_publication:]] + assert any(statement.startswith("CREATE TEMP TABLE") for statement in attempted) + assert not any(statement.startswith("DELETE FROM") for statement in attempted) + assert not any( + statement.startswith("INSERT INTO") and "owned_model" in statement + for statement in attempted + ) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in attempted) + assert backend.rollbacks == rollbacks + 1 + assert backend.transform_rows == {} + + +def _ownership(run_id: str, token: int) -> _Ownership: + return _Ownership( + FencingToken( + lease_table=None, + pipeline_id="redshift_transforms", + run_id=run_id, + token=token, + authority_id="postgresql:test-state", + ) + ) + + +def _write_redshift_model( + root: Path, + *, + name: str, + materialization: str, + with_tests: bool = False, +) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / f"{name}.sql").write_text("SELECT id, label, updated_at FROM {{ ref('raw_records') }}") + incremental = ( + "unique_key: [id]\nincremental_cursor: updated_at\n" + if materialization == "incremental" + else "" + ) + tests = ( + "tests:\n" + " - column: id\n" + " not_null: true\n" + " unique: true\n" + " - column: label\n" + " accepted_values: [older, newer, second]\n" + " - column: id\n" + " relationships:\n" + " to: raw_parent_records\n" + " field: id\n" + if with_tests + else "tests: []\n" + ) + (root / f"{name}.yml").write_text( + f"model: {name}\n" + "description: Portable Redshift transform fixture.\n" + "owner: data-eng\n" + "dialect: portable\n" + f"materialization: {materialization}\n" + "dataset: analytics\n" + "source_system: fixture\n" + "sensitivity: public\n" + f"{incremental}" + "columns:\n" + " - name: id\n" + " type: STRING\n" + " description: Stable fixture identifier.\n" + " - name: label\n" + " type: STRING\n" + " description: Deterministic tie breaker.\n" + " - name: updated_at\n" + " type: INT64\n" + " description: Monotonic fixture cursor.\n" + f"{tests}" + ) def test_redshift_claim_serializes_and_rejects_a_stale_token( From 9c6dd4c7276b443577f74a29fbbcf3ebcc70c5b3 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 07:39:58 -0400 Subject: [PATCH 067/163] feat: add AWS Glue catalog provider --- CHANGELOG.md | 3 + HANDOFF.md | 57 +-- docs/aws-glue.md | 61 ++++ docs/compatibility-matrix.md | 4 + docs/decisions.md | 11 + docs/known-limitations.md | 3 + docs/platform-profiles.md | 7 +- pyproject.toml | 1 + src/dander/catalog/glue.py | 443 ++++++++++++++++++++++++ src/dander/cli/provider_runtime.py | 15 +- src/dander/cli/run_command.py | 32 +- src/dander/executor.py | 22 +- src/dander/project/config.py | 17 +- src/dander/project/portable_config.py | 12 +- src/dander/providers/defaults.py | 7 + src/dander/providers/glue/__init__.py | 5 + src/dander/providers/glue/config.py | 22 ++ src/dander/providers/glue/runtime.py | 57 +++ tests/catalog/test_glue.py | 238 +++++++++++++ tests/cli/test_run_command.py | 4 +- tests/project/test_portable_config.py | 25 ++ tests/providers/test_catalog_runtime.py | 44 +++ tests/test_executor.py | 29 ++ 23 files changed, 1062 insertions(+), 57 deletions(-) create mode 100644 docs/aws-glue.md create mode 100644 src/dander/catalog/glue.py create mode 100644 src/dander/providers/glue/__init__.py create mode 100644 src/dander/providers/glue/config.py create mode 100644 src/dander/providers/glue/runtime.py create mode 100644 tests/catalog/test_glue.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ffe0a32f..cf95f4ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Add an experimental AWS Glue Data Catalog provider with canonical relation mapping, direct + database/table API publication, Dander-owned parameters, unrelated-field preservation, and + normalized readback. No crawler, deletion, IAM provisioning, or live-support claim is included. - Extend the experimental Redshift warehouse adapter with whole-DAG-preflighted portable and Redshift-authored table/incremental models, strict declared schemas, generic assertions, and transactionally fenced DML publication. Views, graphs, other write modes, and support promotion diff --git a/HANDOFF.md b/HANDOFF.md index 91de8e09..f55bd221 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,44 +2,49 @@ ## Finished -- Added experimental Redshift portable/provider-exact table and incremental model execution. -- Preflighted the complete selected DAG, schemas, SQL, materializations, and assertions before any - provider connection, claim, or mutation. -- Published through session-temporary CTAS plus fenced table replacement or cursor-monotonic - `UPDATE`/`INSERT`; target DML and the exact fence touch share one transaction. -- Added generic assertions, stale-fence/cleanup/replay coverage, and a packaged Redshift guide. -- Kept views, graphs, schema evolution, `SUPER`, other write modes, and support promotion blocked. +- Added a lazy, explicitly configured AWS Glue Data Catalog provider. +- Published canonical relation, schema, lineage, tests, metrics, ownership, and sensitivity through + direct database/table APIs with deterministic readback. +- Preserved unrelated database, table, storage-descriptor, partition, and column metadata. +- Replaced provider-neutral executor/runtime use of “Dataplex publisher” with a generic catalog + publisher while retaining the legacy constructor and CLI compatibility inputs. +- Added focused configuration, lazy-loading, create/update/readback, preservation, and sanitization + coverage plus operator documentation. ## Try It -Configure a v2 `warehouse.provider: redshift` profile as shown in `docs/redshift.md`, author a -portable or Redshift-exact table/incremental model, and use the normal hosted run path. This slice -was conformance-tested locally; no AWS warehouse was created or contacted. +Select `catalog.provider: glue` in a version 2 platform profile as shown in `docs/aws-glue.md`, set +`publish_catalog: true` on a model pipeline, and run with an ambient AWS identity. This slice has +not contacted AWS and is not a support claim. ## Checks -- Ruff and strict mypy passed; focused Redshift/transform tests passed. -- Full suite passed with 1,086 tests against an ephemeral PostgreSQL 15 service. -- Wheel/sdist inspection, outside-checkout generation/install, full-runtime install, dependency - audit, Terraform/AWS/Helm validation, and generated-project Terraform validation passed. -- The final source-free container passed non-root, read-only, runtime, and bundled-asset checks. +- Ruff, 34 focused tests, strict mypy across 303 files, and the full 1,094-test suite passed; + the full suite included ephemeral PostgreSQL 15 integration. +- Wheel, sdist, source-free installation, runtime-all installation, dependency audit, and the final + non-root/read-only container conformance checks passed. +- GCP/AWS Terraform validation and tests plus Helm lint/template passed. The retained GCP project + produced exactly `No changes.` using its reviewed 600-second runtime override; no apply ran. ## Decisions -- Redshift incremental publication uses documented `UPDATE ... FROM` plus `INSERT ... WHERE NOT - EXISTS`; Redshift does not support Snowflake-style conditional `WHEN MATCHED` clauses. -- Canonical database/schema/relation coordinates survive compilation; Redshift renders local - schema/table target DML only at its provider boundary. -- Existing destination fencing and strict transform schemas are reused without a parallel runtime. +- Glue databases encode canonical catalog and namespace coordinates; a digest is added only when + lowercase normalization would be lossy. +- Dander owns descriptions, columns, classification, and `dander.*` parameters; unrelated metadata + is retained and catalog objects are never deleted. +- Glue uses direct APIs and ambient AWS identity; crawlers, IAM provisioning, Lake Formation, tags, + live proof, and support promotion remain separate gates. ## Remaining -- Run the retained GCP no-drift plan; never apply it. -- Push the focused branch, let protected CI validate it, and merge only if every check is green. -- Continue the cloud-portability roadmap with a separate next slice. +- Let protected CI repeat Linux tests, packaging, container scanning, and secret scanning. +- Reconcile the tracked 300-second job default and retained 600-second operator override in a + separate change; do not mix it into the Glue catalog slice. +- Continue the cloud-portability plan in a separate branch after merge; do not deploy or mutate + AWS/GCP resources in this slice. ## Review First -- `src/dander/providers/redshift/transform.py` -- `tests/providers/test_redshift_warehouse_runtime.py` -- `docs/redshift.md` +- `src/dander/catalog/glue.py` +- `src/dander/providers/glue/` +- `tests/catalog/test_glue.py` diff --git a/docs/aws-glue.md b/docs/aws-glue.md new file mode 100644 index 00000000..2c96be82 --- /dev/null +++ b/docs/aws-glue.md @@ -0,0 +1,61 @@ +# AWS Glue Data Catalog + +AWS Glue is an experimental external catalog provider for version 2 platform profiles. It consumes +the same canonical `CatalogAsset` records as the local semantic registry and Dataplex; it does not +change warehouse execution or make an AWS profile supported. + +```yaml +# dander.yaml +version: 2 +pipelines: + salesforce_crm: + source: salesforce + models: [stg_salesforce__accounts] + publish_catalog: true +``` + +```yaml +# dander.platforms.yaml +version: 1 +platforms: + aws: + warehouse: + provider: redshift + # Redshift settings omitted here. + state: + provider: postgresql + # PostgreSQL settings omitted here. + catalog: + provider: glue + region: us-east-1 + catalog_id: "123456789012" + database_prefix: dander + connection_name: analytics-redshift # optional metadata only + secrets: + provider: environment +``` + +The runtime uses ambient boto3 credentials. Do not put AWS access keys in either manifest. The +runtime principal needs only these actions for the selected catalog and Dander databases/tables: + +- `glue:GetDatabase`, `glue:CreateDatabase`, and `glue:UpdateDatabase` +- `glue:GetTable`, `glue:CreateTable`, and `glue:UpdateTable` + +Dander never calls a Glue crawler and never deletes a database or table. It maps canonical +`catalog + namespace` coordinates to one lowercase Glue database and the canonical relation name +to one lowercase table. A short digest is added only when normalization could otherwise be lossy +or ambiguous. + +Publication owns the table description, owner, columns, `classification=dander`, and parameters +whose names begin with `dander.`. It preserves unrelated table parameters, database parameters, +storage-descriptor fields, partition metadata, and non-Dander column parameters. Readback returns +only the normalized Dander-owned projection for deterministic comparison. + +Glue columns retain each canonical Dander type in `dander.logical_type`. Types without a direct +Hive representation, including JSON and semi-structured values, use a `string` presentation type +without changing the warehouse schema. Warehouse-backed relations do not imply an S3 location; +the optional connection name is catalog metadata and does not create or validate a Glue +connection. + +This adapter is locally stateful-fake/conformance tested. IAM provisioning, a live Glue proof, +cross-cloud identity, tags/Lake Formation behavior, and support promotion remain separate gates. diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 7556e1b2..61d8c3d6 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -25,3 +25,7 @@ before provider construction when a pair is absent or marked `unsupported`. claim. The packaged `runtime-capabilities.json` remains the supported launcher/profile boundary. Promotion requires the named live profile, identity, failure, rollback, and no-drift evidence from the cloud-portability plan. Adding an adapter does not silently add a matrix row. + +Catalog selection is independent from this state/warehouse matrix. Dataplex is supported only in +the released GCP profile, `none` is a complete no-cloud-mutation provider, and AWS Glue is an +experimental direct API projection with local create/update/readback conformance only. diff --git a/docs/decisions.md b/docs/decisions.md index 13b61bf7..91dda5f5 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -909,3 +909,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Canonical database/schema/relation coordinates survive compilation, while the provider renders database-local target DML. Views, graphs, automatic transform-schema evolution, `SUPER`, and support promotion remain separate gates. + +## 2026-08-09 — Glue projects the canonical catalog directly without crawlers + +- **Coordinates:** One deterministic lowercase Glue database represents canonical + `catalog + namespace`; one Glue table represents the relation. Lossy name normalization receives + a stable digest instead of collapsing distinct warehouse objects. +- **Ownership:** Dander updates descriptions, columns, `classification`, and `dander.*` parameters + while preserving unrelated database/table/storage/column metadata. It never deletes catalog + objects or makes crawlers authoritative. +- **Boundary:** Ambient AWS identity and direct API readback are implemented. IAM infrastructure, + Glue connections, Lake Formation, tags, live proof, and support promotion remain separate work. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 4abc062d..91be0929 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -77,6 +77,9 @@ evaluate these limits before using it for an unattended system containing busine authenticated administrator. - Dataplex and Storage Write remain optional paths and are not exercised by the retained operator trial. +- AWS Glue publication is experimental and locally conformance-tested only. It does not provision + IAM, Glue connections, Lake Formation permissions, crawlers, or tags, and it has not completed a + live AWS catalog proof. Warehouse-backed entries intentionally have no inferred S3 location. ## Support boundary diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 7533ec66..69c92f5d 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -10,7 +10,8 @@ The first supported hosted profile remains BigQuery, BigQuery state, Dataplex (o catalog), GCP Secret Manager, and Cloud Run. Version 2 may select PostgreSQL 15+ for both durable state and the warehouse, but that composition remains locally qualified until the Kubernetes launcher and live-profile gates pass. A provider name outside the installed contract fails -validation; adapter registration alone does not imply hosted support. +validation; adapter registration alone does not imply hosted support. AWS Glue is available as an +experimental catalog projection only. One logical project may have multiple named platforms and deployments. `dander validate` and `dander run` accept `--deployment`; the OCI runtime's `--platform` value selects the named @@ -47,7 +48,9 @@ The runtime composes the selected warehouse and state through one API-v1 provide Registration loads only a small configuration model; selecting and building a provider loads its implementation and SDK dependencies. BigQuery behavior and names remain unchanged. Dataplex keeps aspect-only updates and normalized readback; selecting `none` loads no Dataplex implementation or -credentials. GCP Secret Manager preserves its existing environment indirection and audit behavior; +credentials. Glue uses exact region/account configuration, ambient AWS identity, direct +database/table APIs, and normalized readback; see [AWS Glue](aws-glue.md). GCP Secret Manager +preserves its existing environment indirection and audit behavior; environment-only resolution remains local or launcher-injected because Cloud Run requires managed secret references. The registry is a construction contract, not a support claim. diff --git a/pyproject.toml b/pyproject.toml index 5d03799d..0604a67a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,7 @@ include = [ "connectors", "dander.yaml", "docs/getting-started.md", + "docs/aws-glue.md", "docs/connector-plugins.md", "docs/compatibility-matrix.md", "docs/execution-projection.md", diff --git a/src/dander/catalog/glue.py b/src/dander/catalog/glue.py new file mode 100644 index 00000000..58874fa7 --- /dev/null +++ b/src/dander/catalog/glue.py @@ -0,0 +1,443 @@ +"""AWS Glue Data Catalog publication from canonical Dander metadata.""" + +# boto3 method keyword names are part of AWS's public request contract. +# ruff: noqa: N803 + +from __future__ import annotations + +import hashlib +import json +import re +from importlib import import_module +from typing import TYPE_CHECKING, Any, Protocol, cast + +from dander.catalog.publisher import CatalogPublishError + +if TYPE_CHECKING: + from collections.abc import Mapping + + from dander.catalog.spine import CatalogAsset, CatalogColumn + +_DANDER_PARAMETER_PREFIX = "dander." +_INPUT_TABLE_FIELDS = frozenset( + { + "LastAccessTime", + "LastAnalyzedTime", + "Retention", + "PartitionKeys", + "ViewOriginalText", + "ViewExpandedText", + "TargetTable", + "ViewDefinition", + } +) +_INPUT_STORAGE_FIELDS = frozenset( + { + "Location", + "AdditionalLocations", + "InputFormat", + "OutputFormat", + "Compressed", + "NumberOfBuckets", + "SerdeInfo", + "BucketColumns", + "SortColumns", + "Parameters", + "SkewedInfo", + "StoredAsSubDirectories", + "SchemaReference", + } +) +_NON_NAME = re.compile(r"[^a-z0-9_]+") + + +class GlueClient(Protocol): + """Small subset of the boto3 Glue client used by Dander.""" + + def get_database(self, *, CatalogId: str, Name: str) -> Mapping[str, object]: # noqa: N803 + """Read one database.""" + + def create_database( # noqa: N803 + self, *, CatalogId: str, DatabaseInput: Mapping[str, object] + ) -> Mapping[str, object]: + """Create one database.""" + + def update_database( # noqa: N803 + self, *, CatalogId: str, Name: str, DatabaseInput: Mapping[str, object] + ) -> Mapping[str, object]: + """Update one database.""" + + def get_table( # noqa: N803 + self, *, CatalogId: str, DatabaseName: str, Name: str + ) -> Mapping[str, object]: + """Read one table.""" + + def create_table( # noqa: N803 + self, + *, + CatalogId: str, + DatabaseName: str, + TableInput: Mapping[str, object], + ) -> Mapping[str, object]: + """Create one table.""" + + def update_table( # noqa: N803 + self, + *, + CatalogId: str, + DatabaseName: str, + Name: str, + TableInput: Mapping[str, object], + SkipArchive: bool, + ) -> Mapping[str, object]: + """Update one table.""" + + +class GlueCatalogPublisher: + """Create or update Dander-owned Glue fields without deleting unrelated metadata.""" + + def __init__( + self, + *, + region: str, + catalog_id: str, + database_prefix: str, + warehouse_provider: str, + connection_name: str | None = None, + client: GlueClient | None = None, + ) -> None: + self._region = region + self._catalog_id = catalog_id + self._database_prefix = database_prefix + self._warehouse_provider = warehouse_provider + self._connection_name = connection_name + self._client = client + + def publish(self, asset: CatalogAsset) -> str: + """Upsert one canonical asset and return its stable Glue ARN.""" + database_name = self.database_name(asset) + table_name = self.table_name(asset) + self._ensure_database(asset, database_name) + try: + response = self._glue_client().get_table( + CatalogId=self._catalog_id, + DatabaseName=database_name, + Name=table_name, + ) + existing = _mapping(response.get("Table")) + except Exception as error: + if not _is_not_found(error): + raise CatalogPublishError( + f"Cannot read Glue catalog asset: {asset.name}" + ) from error + existing = None + + table_input = self.table_input_for(asset, existing=existing) + try: + if existing is None: + self._glue_client().create_table( + CatalogId=self._catalog_id, + DatabaseName=database_name, + TableInput=table_input, + ) + else: + self._glue_client().update_table( + CatalogId=self._catalog_id, + DatabaseName=database_name, + Name=table_name, + TableInput=table_input, + SkipArchive=False, + ) + except Exception as error: + raise CatalogPublishError(f"Cannot publish Glue catalog asset: {asset.name}") from error + return f"arn:aws:glue:{self._region}:{self._catalog_id}:table/{database_name}/{table_name}" + + def read(self, asset: CatalogAsset) -> Mapping[str, object]: + """Read back one published Glue table.""" + try: + response = self._glue_client().get_table( + CatalogId=self._catalog_id, + DatabaseName=self.database_name(asset), + Name=self.table_name(asset), + ) + except Exception as error: + raise CatalogPublishError(f"Cannot read Glue catalog asset: {asset.name}") from error + table = _mapping(response.get("Table")) + if table is None: + raise CatalogPublishError(f"Glue catalog asset is malformed: {asset.name}") + return table + + def normalized_table(self, asset: CatalogAsset) -> dict[str, object]: + """Return Dander-owned readback fields in deterministic provider-neutral form.""" + table = self.read(asset) + storage = _mapping(table.get("StorageDescriptor")) or {} + raw_columns = storage.get("Columns") + columns = raw_columns if isinstance(raw_columns, list) else [] + parameters = _string_mapping(table.get("Parameters")) + return { + "database": self.database_name(asset), + "table": str(table.get("Name", "")), + "description": str(table.get("Description", "")), + "owner": str(table.get("Owner", "")), + "columns": [_normalize_column(item) for item in columns if isinstance(item, dict)], + "parameters": { + key: value + for key, value in sorted(parameters.items()) + if key == "classification" or key.startswith(_DANDER_PARAMETER_PREFIX) + }, + } + + def database_name(self, asset: CatalogAsset) -> str: + """Map catalog and namespace into one collision-safe Glue database name.""" + coordinate = f"{asset.relation_ref.catalog}.{asset.relation_ref.namespace}" + raw = f"{self._database_prefix}_{asset.relation_ref.catalog}_{asset.relation_ref.namespace}" + return _glue_name(raw, identity=coordinate) + + def table_name(self, asset: CatalogAsset) -> str: + """Map one relation name into Hive-compatible lowercase form.""" + return _glue_name(asset.relation_ref.name, identity=asset.relation) + + def table_input_for( + self, + asset: CatalogAsset, + *, + existing: Mapping[str, object] | None = None, + ) -> dict[str, object]: + """Build an update-safe TableInput while retaining unrelated provider fields.""" + existing = existing or {} + table_input = {key: value for key, value in existing.items() if key in _INPUT_TABLE_FIELDS} + existing_storage = _mapping(existing.get("StorageDescriptor")) or {} + storage = { + key: value for key, value in existing_storage.items() if key in _INPUT_STORAGE_FIELDS + } + existing_columns = _columns_by_name(existing_storage.get("Columns")) + storage["Columns"] = [ + _column_input(column, existing=existing_columns.get(column.name)) + for column in asset.columns + ] + + parameters = _string_mapping(existing.get("Parameters")) + parameters.update(self._owned_parameters(asset)) + table_input.update( + { + "Name": self.table_name(asset), + "Description": _utf8_prefix(asset.description, 2048), + "Owner": _utf8_prefix(asset.owner, 255), + "StorageDescriptor": storage, + "TableType": "EXTERNAL_TABLE", + "Parameters": parameters, + } + ) + return table_input + + def _ensure_database(self, asset: CatalogAsset, database_name: str) -> None: + try: + response = self._glue_client().get_database( + CatalogId=self._catalog_id, + Name=database_name, + ) + existing = _mapping(response.get("Database")) + except Exception as error: + if not _is_not_found(error): + raise CatalogPublishError( + f"Cannot read Glue database for catalog asset: {asset.name}" + ) from error + existing = None + + parameters = _string_mapping(existing.get("Parameters") if existing else None) + desired_parameters = parameters | { + "dander.catalog": asset.relation_ref.catalog, + "dander.namespace": asset.relation_ref.namespace, + "dander.managed": "true", + } + database_input: dict[str, object] = { + "Name": database_name, + "Parameters": desired_parameters, + } + try: + if existing is None: + database_input["Description"] = ( + "Dander catalog projection for " + f"{asset.relation_ref.catalog}.{asset.relation_ref.namespace}" + ) + self._glue_client().create_database( + CatalogId=self._catalog_id, + DatabaseInput=database_input, + ) + elif desired_parameters != parameters: + for key in ( + "Description", + "LocationUri", + "CreateTableDefaultPermissions", + "TargetDatabase", + "FederatedDatabase", + ): + if key in existing: + database_input[key] = existing[key] + self._glue_client().update_database( + CatalogId=self._catalog_id, + Name=database_name, + DatabaseInput=database_input, + ) + except Exception as error: + raise CatalogPublishError( + f"Cannot prepare Glue database for catalog asset: {asset.name}" + ) from error + + def _owned_parameters(self, asset: CatalogAsset) -> dict[str, str]: + manifest = asset.to_manifest() + parameters = { + "classification": "dander", + "dander.catalog": asset.relation_ref.catalog, + "dander.namespace": asset.relation_ref.namespace, + "dander.relation": asset.relation, + "dander.warehouse_provider": self._warehouse_provider, + "dander.materialization": asset.materialization, + "dander.source_system": asset.source_system, + "dander.sensitivity": asset.sensitivity, + "dander.owner": asset.owner, + "dander.lineage": _json(list(asset.upstream_relations)), + "dander.tests": _json([test.to_manifest() for test in asset.tests]), + "dander.metrics": _json([metric.to_manifest() for metric in asset.metrics]), + "dander.manifest_sha256": hashlib.sha256(_json(manifest).encode("utf-8")).hexdigest(), + } + if self._connection_name is not None: + parameters["dander.connection_name"] = self._connection_name + return {key: _utf8_prefix(value, 512_000) for key, value in parameters.items()} + + def _glue_client(self) -> GlueClient: + if self._client is None: + boto3 = cast("Any", import_module("boto3")) + self._client = cast( + "GlueClient", + boto3.client("glue", region_name=self._region), + ) + return self._client + + +def _column_input( + column: CatalogColumn, + *, + existing: Mapping[str, object] | None, +) -> dict[str, object]: + parameters = _string_mapping(existing.get("Parameters") if existing else None) + parameters.update( + { + "dander.logical_type": column.data_type, + "dander.nullable": str(column.nullable).lower(), + } + ) + return { + "Name": column.name, + "Type": _glue_type(column.data_type), + "Comment": _utf8_prefix(column.description, 255), + "Parameters": parameters, + } + + +def _glue_type(data_type: str) -> str: + normalized = data_type.strip().upper() + if normalized.startswith("DECIMAL("): + return normalized.lower() + return { + "BOOLEAN": "boolean", + "BOOL": "boolean", + "SMALLINT": "smallint", + "INTEGER": "bigint", + "INT64": "bigint", + "BIGINT": "bigint", + "FLOAT": "double", + "FLOAT64": "double", + "DOUBLE": "double", + "NUMERIC": "decimal(38,9)", + "BIGNUMERIC": "decimal(38,9)", + "DECIMAL": "decimal(38,9)", + "STRING": "string", + "TEXT": "string", + "VARCHAR": "string", + "BYTES": "binary", + "BINARY": "binary", + "DATE": "date", + "DATETIME": "timestamp", + "TIMESTAMP": "timestamp", + "TIMESTAMPTZ": "timestamp", + "TIME": "string", + "JSON": "string", + "JSONB": "string", + "VARIANT": "string", + "SUPER": "string", + "RECORD": "string", + "STRUCT": "string", + }.get(normalized, "string") + + +def _glue_name(value: str, *, identity: str) -> str: + lowered = value.lower() + normalized = _NON_NAME.sub("_", lowered).strip("_") or "relation" + if normalized != value or len(normalized.encode("utf-8")) > 255: + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:10] + normalized = f"{normalized[:244].rstrip('_')}_{digest}" + return normalized + + +def _columns_by_name(value: object) -> dict[str, Mapping[str, object]]: + if not isinstance(value, list): + return {} + return { + str(item["Name"]): item + for item in value + if isinstance(item, dict) and isinstance(item.get("Name"), str) + } + + +def _mapping(value: object) -> Mapping[str, object] | None: + if not isinstance(value, dict): + return None + return {str(key): item for key, item in value.items()} + + +def _string_mapping(value: object) -> dict[str, str]: + if not isinstance(value, dict): + return {} + return { + str(key): str(item) + for key, item in value.items() + if isinstance(key, str) and isinstance(item, str) + } + + +def _normalize_column(value: Mapping[str, object]) -> dict[str, object]: + parameters = _string_mapping(value.get("Parameters")) + return { + "name": str(value.get("Name", "")), + "type": str(value.get("Type", "")), + "comment": str(value.get("Comment", "")), + "parameters": { + key: item + for key, item in sorted(parameters.items()) + if key.startswith(_DANDER_PARAMETER_PREFIX) + }, + } + + +def _is_not_found(error: Exception) -> bool: + response = getattr(error, "response", None) + if isinstance(response, dict): + detail = response.get("Error") + if isinstance(detail, dict) and detail.get("Code") == "EntityNotFoundException": + return True + return type(error).__name__ == "EntityNotFoundException" + + +def _json(value: object) -> str: + return json.dumps(value, separators=(",", ":"), sort_keys=True) + + +def _utf8_prefix(value: str, maximum_bytes: int) -> str: + encoded = value.encode("utf-8") + if len(encoded) <= maximum_bytes: + return value + return encoded[:maximum_bytes].decode("utf-8", errors="ignore") + + +__all__ = ["GlueCatalogPublisher", "GlueClient"] diff --git a/src/dander/cli/provider_runtime.py b/src/dander/cli/provider_runtime.py index 6ec710df..09658e04 100644 --- a/src/dander/cli/provider_runtime.py +++ b/src/dander/cli/provider_runtime.py @@ -11,6 +11,8 @@ def build_catalog_publisher( *, provider_id: str, + provider_config: dict[str, object] | None = None, + warehouse_provider: str = "bigquery", catalog: str | None = None, project: str | None = None, location: str, @@ -22,13 +24,22 @@ def build_catalog_publisher( raise ClickException("catalog and legacy project must match") if catalog is None: raise ClickException("Catalog provider requires a warehouse catalog") + if provider_config is not None and provider_config.get("provider") != provider_id: + raise ClickException("Catalog provider id and configuration must match") registry = default_provider_registry() try: - config = registry.parse(ProviderKind.CATALOG, {"provider": provider_id}) + config = registry.parse( + ProviderKind.CATALOG, + provider_config or {"provider": provider_id}, + ) runtime = registry.build( ProviderKind.CATALOG, config, - context={"catalog": catalog, "location": location}, + context={ + "catalog": catalog, + "location": location, + "warehouse_provider": warehouse_provider, + }, ) except ProviderFactoryError as error: raise ClickException(str(error)) from error diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index 0c2fefc5..f5557561 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -121,12 +121,13 @@ class _ResolvedRun: state_namespace: str | None selected_models: tuple[str, ...] | None build_models: bool - publish_dataplex: bool + publish_catalog: bool warehouse_provider: str warehouse_config: dict[str, object] state_provider: str state_config: dict[str, object] catalog_provider: str + catalog_config: dict[str, object] secret_provider: str @property @@ -199,12 +200,13 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: tuple(options.selected_models) if options.selected_models is not None else None ) build_models = options.build_models - publish_dataplex = options.publish_dataplex + publish_catalog = options.publish_dataplex warehouse_provider = "bigquery" warehouse_config: dict[str, object] = {"provider": "bigquery", "location": "US"} state_provider = "bigquery" state_config: dict[str, object] = {"provider": "bigquery"} catalog_provider = "dataplex" + catalog_config: dict[str, object] = {"provider": "dataplex"} secret_provider = "gcp_secret_manager" plugin_registry: ConnectorPluginRegistry | None = None @@ -231,7 +233,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: elif selected_models is None: selected_models = tuple(pipeline.models) build_models = build_models or pipeline.build_models - publish_dataplex = publish_dataplex or pipeline.publish_dataplex + publish_catalog = publish_catalog or pipeline.publish_catalog warehouse_provider = manifest.warehouse_provider warehouse_config = manifest.warehouse_config state_provider = manifest.state_provider @@ -239,6 +241,11 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: catalog_provider = ( "dataplex" if options.publish_dataplex else manifest.catalog_provider ) + catalog_config = ( + {"provider": "dataplex"} + if options.publish_dataplex + else manifest.catalog_config + ) secret_provider = manifest.secret_provider except (ConnectorPluginError, ProjectConfigError) as error: raise ClickException(str(error)) from error @@ -293,7 +300,7 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: build_models=build_models, selected_models=selected_models, catalog_output=options.catalog_output, - publish_dataplex=publish_dataplex, + publish_catalog=publish_catalog, ) return _ResolvedRun( pipeline_id=options.pipeline_or_source, @@ -311,12 +318,13 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: state_namespace=state_namespace, selected_models=selected_models, build_models=build_models, - publish_dataplex=publish_dataplex, + publish_catalog=publish_catalog, warehouse_provider=warehouse_provider, warehouse_config=warehouse_config, state_provider=state_provider, state_config=state_config, catalog_provider=catalog_provider, + catalog_config=catalog_config, secret_provider=secret_provider, ) @@ -329,13 +337,13 @@ def _resolve_graph_plan( build_models: bool, selected_models: Sequence[str] | None, catalog_output: Path | None, - publish_dataplex: bool, + publish_catalog: bool, ) -> GraphExecutionPlan | None: if graph_file is None: return None if build_models or selected_models is not None: raise ClickException("Graph pipelines do not accept --build-models or --select-model") - if catalog_output is not None or publish_dataplex: + if catalog_output is not None or publish_catalog: raise ClickException("Graph metadata publication is not supported yet") try: graph = load_graph_for_execution(graph_file) @@ -371,7 +379,7 @@ def _require_provider_compatible_options(options: RunOptions, resolved: _Resolve raise ClickException("--sandbox is available only with a BigQuery warehouse") if options.guarded_free_tier: raise ClickException("--guarded-free-tier is available only with a BigQuery warehouse") - if resolved.publish_dataplex and resolved.catalog_provider == "dataplex": + if resolved.publish_catalog and resolved.catalog_provider == "dataplex": raise ClickException("Dataplex publication currently requires a BigQuery warehouse") @@ -388,13 +396,15 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec raise ClickException(str(error)) from error stores = _build_control_stores(options, resolved) - dataplex_publisher = ( + catalog_publisher = ( build_catalog_publisher( provider_id=resolved.catalog_provider, + provider_config=resolved.catalog_config, + warehouse_provider=resolved.warehouse_provider, catalog=resolved.catalog, location=options.dataplex_location, ) - if resolved.publish_dataplex + if resolved.publish_catalog else None ) warehouse = _build_warehouse_runtime(resolved) @@ -416,7 +426,7 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec transform_runner=transform_runner, metadata_store=stores.metadata, registry_output=options.catalog_output, - dataplex_publisher=dataplex_publisher, + catalog_publisher=catalog_publisher, leases=stores.leases, ) diff --git a/src/dander/executor.py b/src/dander/executor.py index 6b37c74b..4f7aed88 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -19,7 +19,7 @@ from collections.abc import Iterable from pathlib import Path - from dander.catalog import CatalogAsset, MetadataStore + from dander.catalog import CatalogAsset, CatalogPublisher, MetadataStore from dander.concurrency import OwnershipGuard from dander.ingestion import SourceConfig from dander.state import LeaseStore, RunHistoryStore @@ -48,11 +48,6 @@ def run( """Run configured endpoints and return a non-sensitive summary.""" -class _DataplexPublisher(Protocol): - def publish(self, asset: CatalogAsset) -> str: - """Project one canonical asset into Dataplex.""" - - @dataclass(frozen=True) class PipelineExecutionResult: """Non-sensitive summary for the complete Dander pipeline lifecycle.""" @@ -87,7 +82,8 @@ def __init__( transform_runner: _TransformRunner | None = None, metadata_store: MetadataStore | None = None, registry_output: Path | None = None, - dataplex_publisher: _DataplexPublisher | None = None, + catalog_publisher: CatalogPublisher | None = None, + dataplex_publisher: CatalogPublisher | None = None, leases: LeaseStore | None = None, ) -> None: if build_models and transform_runner is None: @@ -98,6 +94,10 @@ def __init__( raise ValueError("catalog and legacy project must match") if catalog is None: raise ValueError("PipelineExecutor requires a warehouse catalog") + if catalog_publisher is not None and dataplex_publisher is not None: + raise ValueError( + "catalog_publisher and legacy dataplex_publisher are mutually exclusive" + ) self._pipeline_id = pipeline_id self._source_config = source_config self._ingestion = ingestion @@ -118,7 +118,7 @@ def __init__( self._transform_runner = transform_runner self._metadata_store = metadata_store self._registry_output = registry_output - self._dataplex_publisher = dataplex_publisher + self._catalog_publisher = catalog_publisher or dataplex_publisher self._leases = leases def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: @@ -247,11 +247,11 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: if heartbeat is not None: heartbeat.verify() SemanticRegistryPublisher().publish(manifest, self._registry_output) - if self._dataplex_publisher is not None: + if self._catalog_publisher is not None: for asset in compiled_assets: if heartbeat is not None: heartbeat.verify() - self._dataplex_publisher.publish(asset) + self._catalog_publisher.publish(asset) if heartbeat is not None: heartbeat.verify() @@ -332,7 +332,7 @@ def _publishes_metadata(self) -> bool: for value in ( self._metadata_store, self._registry_output, - self._dataplex_publisher, + self._catalog_publisher, ) ) diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 065618cc..1b35a5ab 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -30,6 +30,10 @@ def _default_warehouse_config() -> dict[str, object]: return {"provider": "bigquery", "location": "US"} +def _default_catalog_config() -> dict[str, object]: + return {"provider": "dataplex"} + + _DISTRIBUTION = re.compile(r"^[A-Za-z0-9]+(?:[-_.][A-Za-z0-9]+)*$") if TYPE_CHECKING: @@ -105,6 +109,11 @@ class PipelineSpec(BaseModel): secrets: dict[str, str] = Field(default_factory=dict) resources: PipelineResourceNames = Field(default_factory=PipelineResourceNames) + @property + def publish_catalog(self) -> bool: + """Return the provider-neutral meaning of the legacy v1 publication flag.""" + return self.publish_dataplex + @field_validator("models") @classmethod def validate_models(cls, values: list[str]) -> list[str]: @@ -226,7 +235,11 @@ class DanderProject(BaseModel): default_factory=_default_state_config, exclude=True, ) - catalog_provider: Literal["dataplex", "none"] = Field(default="dataplex", exclude=True) + catalog_provider: Literal["dataplex", "glue", "none"] = Field(default="dataplex", exclude=True) + catalog_config: dict[str, object] = Field( + default_factory=_default_catalog_config, + exclude=True, + ) secret_provider: Literal["gcp_secret_manager", "environment"] = Field( default="gcp_secret_manager", exclude=True, @@ -265,6 +278,8 @@ def validate_deployed_pipeline_ids(self) -> DanderProject: raise ValueError("warehouse config provider must match warehouse_provider") if self.state_config.get("provider") != self.state_provider: raise ValueError("state config provider must match state_provider") + if self.catalog_config.get("provider") != self.catalog_provider: + raise ValueError("catalog config provider must match catalog_provider") if self.launcher_config: configured_provider = self.launcher_config.get("provider") if configured_provider != self.launcher_provider: diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 2fa87a87..00d65161 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -29,6 +29,7 @@ from dander.providers.environment_secrets import EnvironmentSecretConfig # noqa: TC001 from dander.providers.fargate import FargateLauncherConfig # noqa: TC001 from dander.providers.gcp_secret_manager import GcpSecretManagerConfig # noqa: TC001 +from dander.providers.glue import GlueCatalogConfig # noqa: TC001 from dander.providers.kubernetes import KubernetesLauncherConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 from dander.providers.postgresql import ( # noqa: TC001 @@ -126,7 +127,10 @@ def validate_pipeline_ids( return values -CatalogSpec = Annotated[DataplexCatalogConfig | NoCatalogConfig, Field(discriminator="provider")] +CatalogSpec = Annotated[ + DataplexCatalogConfig | GlueCatalogConfig | NoCatalogConfig, + Field(discriminator="provider"), +] SecretProviderSpec = Annotated[ @@ -354,7 +358,9 @@ def _resolve( graph=pipeline.graph, models=pipeline.models, build_models=pipeline.build_models, - publish_dataplex=(pipeline.publish_catalog and profile.catalog.provider == "dataplex"), + # ``publish_dataplex`` is the version-1 compatibility field. In a resolved + # version-2 project it means publication through the selected catalog provider. + publish_dataplex=(pipeline.publish_catalog and profile.catalog.provider != "none"), schedule=deployment_pipeline.schedule, time_zone=deployment_pipeline.time_zone, paused=deployment_pipeline.paused, @@ -383,6 +389,7 @@ def _resolve( state_provider=profile.state.provider, state_config=profile.state.model_dump(mode="json"), catalog_provider=profile.catalog.provider, + catalog_config=profile.catalog.model_dump(mode="json", exclude_none=True), secret_provider=profile.secrets.provider, launcher_provider=selected.launcher.provider, launcher_config=selected.launcher.model_dump(mode="json"), @@ -490,6 +497,7 @@ def _equivalent(legacy: DanderProject, migrated: DanderProject) -> bool: and legacy.state_provider == migrated.state_provider and legacy.state_config == migrated.state_config and legacy.catalog_provider == migrated.catalog_provider + and legacy.catalog_config == migrated.catalog_config and legacy.secret_provider == migrated.secret_provider and legacy.launcher_provider == migrated.launcher_provider and legacy.resolved_launcher_config() == migrated.resolved_launcher_config() diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index aa71a0b2..84f236a1 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -7,6 +7,7 @@ from dander.providers.environment_secrets import EnvironmentSecretConfig from dander.providers.fargate import FargateLauncherConfig from dander.providers.gcp_secret_manager import GcpSecretManagerConfig +from dander.providers.glue import GlueCatalogConfig from dander.providers.kubernetes import KubernetesLauncherConfig from dander.providers.no_catalog import NoCatalogConfig from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig @@ -82,6 +83,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.no_catalog.runtime:NO_CATALOG_FACTORY" ), ) + registry.register( + kind=ProviderKind.CATALOG, + provider_id="glue", + config_model=GlueCatalogConfig, + load_factory=lazy_provider_factory("dander.providers.glue.runtime:GLUE_CATALOG_FACTORY"), + ) registry.register( kind=ProviderKind.SECRETS, provider_id="gcp_secret_manager", diff --git a/src/dander/providers/glue/__init__.py b/src/dander/providers/glue/__init__.py new file mode 100644 index 00000000..17906234 --- /dev/null +++ b/src/dander/providers/glue/__init__.py @@ -0,0 +1,5 @@ +"""Dependency-light AWS Glue Data Catalog configuration.""" + +from dander.providers.glue.config import GlueCatalogConfig + +__all__ = ["GlueCatalogConfig"] diff --git a/src/dander/providers/glue/config.py b/src/dander/providers/glue/config.py new file mode 100644 index 00000000..fa351c03 --- /dev/null +++ b/src/dander/providers/glue/config.py @@ -0,0 +1,22 @@ +"""Validated AWS Glue Data Catalog provider configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class GlueCatalogConfig(BaseModel): + """Select one account-scoped Glue Data Catalog.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["glue"] + region: str = Field(pattern=r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") + catalog_id: str = Field(pattern=r"^[0-9]{12}$") + database_prefix: str = Field( + default="dander", + min_length=1, + max_length=32, + pattern=r"^[a-z][a-z0-9_]*$", + ) + connection_name: str | None = Field(default=None, min_length=1, max_length=255) diff --git a/src/dander/providers/glue/runtime.py b/src/dander/providers/glue/runtime.py new file mode 100644 index 00000000..98269295 --- /dev/null +++ b/src/dander/providers/glue/runtime.py @@ -0,0 +1,57 @@ +"""AWS Glue Data Catalog runtime composition.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from dander.catalog.glue import GlueCatalogPublisher +from dander.catalog.runtime import CatalogCapabilities, CatalogRuntime +from dander.providers.glue.config import GlueCatalogConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind + +if TYPE_CHECKING: + from collections.abc import Mapping + + from pydantic import BaseModel + + from dander.catalog.glue import GlueClient + + +def build_glue_catalog( + config: BaseModel, + context: Mapping[str, object], +) -> CatalogRuntime: + """Build Glue only after its catalog provider is selected.""" + if not isinstance(config, GlueCatalogConfig): + raise TypeError("Glue catalog factory received the wrong configuration") + warehouse_provider = context.get("warehouse_provider") + if not isinstance(warehouse_provider, str) or not warehouse_provider: + raise ValueError("Glue catalog factory requires a warehouse provider context") + client = context.get("client") + return CatalogRuntime( + provider_id="glue", + publisher=GlueCatalogPublisher( + region=config.region, + catalog_id=config.catalog_id, + database_prefix=config.database_prefix, + warehouse_provider=warehouse_provider, + connection_name=config.connection_name, + client=cast("GlueClient", client) if client is not None else None, + ), + capabilities=CatalogCapabilities( + provider_id="glue", + readback=True, + preserves_unrelated_fields=True, + first_party_entries=False, + ), + ) + + +GLUE_CATALOG_FACTORY: ProviderFactory[CatalogRuntime] = ProviderFactory( + kind=ProviderKind.CATALOG, + provider_id="glue", + api_version=PROVIDER_API_VERSION, + build=build_glue_catalog, +) + +__all__ = ["GLUE_CATALOG_FACTORY"] diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py new file mode 100644 index 00000000..77c1b8e1 --- /dev/null +++ b/tests/catalog/test_glue.py @@ -0,0 +1,238 @@ +"""Stateful AWS Glue catalog publication and preservation tests.""" + +# The fake intentionally matches boto3's public keyword names. +# ruff: noqa: N803 + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from dander.catalog import CatalogPublishError, MetadataSpine +from dander.catalog.glue import GlueCatalogPublisher +from dander.transform import TransformProject +from dander.warehouse import RelationRef + +if TYPE_CHECKING: + from collections.abc import Mapping + + from dander.catalog import CatalogAsset + +_MODELS_DIR = Path(__file__).resolve().parents[2] / "models" + + +class _EntityNotFoundError(RuntimeError): + """Minimal botocore-shaped not-found response.""" + + response = {"Error": {"Code": "EntityNotFoundException"}} + + +class _FakeGlue: + def __init__(self) -> None: + self.databases: dict[str, dict[str, object]] = {} + self.tables: dict[tuple[str, str], dict[str, object]] = {} + self.calls: list[str] = [] + + def get_database(self, *, CatalogId: str, Name: str) -> Mapping[str, object]: # noqa: N803 + assert CatalogId == "123456789012" + self.calls.append("get_database") + try: + return {"Database": deepcopy(self.databases[Name])} + except KeyError as error: + raise _EntityNotFoundError from error + + def create_database( # noqa: N803 + self, *, CatalogId: str, DatabaseInput: Mapping[str, object] + ) -> Mapping[str, object]: + assert CatalogId == "123456789012" + self.calls.append("create_database") + self.databases[str(DatabaseInput["Name"])] = dict(deepcopy(DatabaseInput)) + return {} + + def update_database( # noqa: N803 + self, *, CatalogId: str, Name: str, DatabaseInput: Mapping[str, object] + ) -> Mapping[str, object]: + assert CatalogId == "123456789012" + self.calls.append("update_database") + self.databases[Name] = dict(deepcopy(DatabaseInput)) + return {} + + def get_table( # noqa: N803 + self, *, CatalogId: str, DatabaseName: str, Name: str + ) -> Mapping[str, object]: + assert CatalogId == "123456789012" + self.calls.append("get_table") + try: + return {"Table": deepcopy(self.tables[(DatabaseName, Name)])} + except KeyError as error: + raise _EntityNotFoundError from error + + def create_table( # noqa: N803 + self, + *, + CatalogId: str, + DatabaseName: str, + TableInput: Mapping[str, object], + ) -> Mapping[str, object]: + assert CatalogId == "123456789012" + self.calls.append("create_table") + self.tables[(DatabaseName, str(TableInput["Name"]))] = dict(deepcopy(TableInput)) + return {} + + def update_table( # noqa: N803 + self, + *, + CatalogId: str, + DatabaseName: str, + Name: str, + TableInput: Mapping[str, object], + SkipArchive: bool, + ) -> Mapping[str, object]: + assert CatalogId == "123456789012" + assert SkipArchive is False + self.calls.append("update_table") + self.tables[(DatabaseName, Name)] = dict(deepcopy(TableInput)) + return {} + + +def _asset() -> CatalogAsset: + project = TransformProject.load(_MODELS_DIR, project_id="analytics") + (asset,) = MetadataSpine().compile(project, selected=["stg_greenhouse__jobs"]) + return asset + + +def _publisher(client: _FakeGlue) -> GlueCatalogPublisher: + return GlueCatalogPublisher( + region="us-east-1", + catalog_id="123456789012", + database_prefix="dander", + warehouse_provider="redshift", + connection_name="analytics-redshift", + client=client, + ) + + +def test_publish_creates_database_table_and_normalized_readback() -> None: + client = _FakeGlue() + publisher = _publisher(client) + asset = _asset() + + resource = publisher.publish(asset) + normalized = publisher.normalized_table(asset) + + assert resource == ( + "arn:aws:glue:us-east-1:123456789012:table/dander_analytics_staging/stg_greenhouse__jobs" + ) + assert client.calls == [ + "get_database", + "create_database", + "get_table", + "create_table", + "get_table", + ] + assert normalized["database"] == "dander_analytics_staging" + assert normalized["table"] == "stg_greenhouse__jobs" + parameters = normalized["parameters"] + assert isinstance(parameters, dict) + assert parameters["classification"] == "dander" + assert parameters["dander.relation"] == "analytics.staging.stg_greenhouse__jobs" + assert parameters["dander.warehouse_provider"] == "redshift" + assert parameters["dander.connection_name"] == "analytics-redshift" + columns = normalized["columns"] + assert isinstance(columns, list) + assert columns[0]["name"] == "job_id" + assert columns[0]["type"] == "string" + assert columns[0]["parameters"] == { + "dander.logical_type": "STRING", + "dander.nullable": "false", + } + + +def test_update_preserves_unrelated_table_database_storage_and_column_fields() -> None: + client = _FakeGlue() + publisher = _publisher(client) + asset = _asset() + database = publisher.database_name(asset) + table = publisher.table_name(asset) + client.databases[database] = { + "Name": database, + "Description": "operator description", + "LocationUri": "s3://operator-owned/catalog/", + "Parameters": {"operator.note": "keep"}, + } + client.tables[(database, table)] = { + "Name": table, + "Description": "old", + "Retention": 17, + "Parameters": {"operator.note": "keep", "dander.owner": "old"}, + "StorageDescriptor": { + "Location": "s3://operator-owned/table/", + "SerdeInfo": {"SerializationLibrary": "operator.serde"}, + "Columns": [ + { + "Name": "job_id", + "Type": "string", + "Parameters": {"operator.column": "keep"}, + } + ], + }, + } + + publisher.publish(asset) + + assert client.databases[database]["Description"] == "operator description" + assert client.databases[database]["LocationUri"] == "s3://operator-owned/catalog/" + assert client.databases[database]["Parameters"] == { + "operator.note": "keep", + "dander.catalog": "analytics", + "dander.namespace": "staging", + "dander.managed": "true", + } + updated = client.tables[(database, table)] + assert updated["Retention"] == 17 + updated_parameters = updated["Parameters"] + assert isinstance(updated_parameters, dict) + assert updated_parameters["operator.note"] == "keep" + storage = updated["StorageDescriptor"] + assert isinstance(storage, dict) + assert storage["Location"] == "s3://operator-owned/table/" + assert storage["SerdeInfo"] == {"SerializationLibrary": "operator.serde"} + assert storage["Columns"][0]["Parameters"]["operator.column"] == "keep" + assert client.calls[-1] == "update_table" + + +def test_provider_errors_are_sanitized() -> None: + class _BrokenGlue(_FakeGlue): + def create_table( # noqa: N803 + self, + *, + CatalogId: str, + DatabaseName: str, + TableInput: Mapping[str, object], + ) -> Mapping[str, object]: + del CatalogId, DatabaseName, TableInput + raise RuntimeError("secret-bearing provider detail") + + with pytest.raises(CatalogPublishError, match="Cannot publish Glue catalog asset") as raised: + _publisher(_BrokenGlue()).publish(_asset()) + + assert "secret-bearing" not in str(raised.value) + + +def test_lossy_hive_name_normalization_gets_stable_coordinate_digest() -> None: + publisher = _publisher(_FakeGlue()) + asset = replace( + _asset(), + relation_ref=RelationRef( + catalog="Analytics-Prod", + namespace="SalesOps", + name="DailyRevenue", + ), + ) + + assert publisher.database_name(asset) == "dander_analytics_prod_salesops_5e931afe4c" + assert publisher.table_name(asset) == "dailyrevenue_52c237dcdf" diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index 7f94de94..daf42346 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -264,7 +264,7 @@ def test_postgresql_rejects_bigquery_safety_before_external_clients( ) resolved = SimpleNamespace( warehouse_provider="postgresql", - publish_dataplex=False, + publish_catalog=False, catalog_provider="none", ) monkeypatch.setattr(run_module, "_resolve_run", lambda _options: resolved) @@ -286,7 +286,7 @@ def test_postgresql_dataplex_publication_fails_before_executor( options = SimpleNamespace(sandbox=False, guarded_free_tier=False, dry_run=False) resolved = SimpleNamespace( warehouse_provider="postgresql", - publish_dataplex=True, + publish_catalog=True, catalog_provider="dataplex", ) monkeypatch.setattr(run_module, "_resolve_run", lambda _options: resolved) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index 137ba067..e3d54ea4 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -283,6 +283,31 @@ def test_version_two_resolves_native_redshift_warehouse_coordinates(tmp_path: Pa assert "password" not in resolved.warehouse_config +def test_version_two_resolves_glue_catalog_and_enables_generic_publication( + tmp_path: Path, +) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["platforms"]["gcp"]["catalog"] = { + "provider": "glue", + "region": "us-east-1", + "catalog_id": "123456789012", + "database_prefix": "dander", + "connection_name": "analytics-redshift", + } + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path) + + assert resolved.catalog_provider == "glue" + assert resolved.catalog_config == platforms["platforms"]["gcp"]["catalog"] + assert resolved.pipelines["example_records"].publish_dataplex is True + + def test_cloud_run_rejects_environment_only_secrets(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_catalog_runtime.py b/tests/providers/test_catalog_runtime.py index 0bea1e63..ddc2265d 100644 --- a/tests/providers/test_catalog_runtime.py +++ b/tests/providers/test_catalog_runtime.py @@ -73,6 +73,40 @@ def test_no_catalog_runtime_has_no_publisher_or_dataplex_import() -> None: assert implementation_module not in sys.modules +def test_default_registry_loads_glue_only_after_selection() -> None: + module_name = "dander.providers.glue.runtime" + implementation_module = "dander.catalog.glue" + sys.modules.pop(module_name, None) + sys.modules.pop(implementation_module, None) + registry = default_provider_registry() + config = registry.parse( + ProviderKind.CATALOG, + { + "provider": "glue", + "region": "us-east-1", + "catalog_id": "123456789012", + }, + ) + + assert module_name not in sys.modules + assert implementation_module not in sys.modules + runtime = registry.build( + ProviderKind.CATALOG, + config, + context={"warehouse_provider": "redshift", "client": object()}, + ) + + assert isinstance(runtime, CatalogRuntime) + assert runtime.provider_id == "glue" + assert module_name in sys.modules + assert implementation_module in sys.modules + assert runtime.publisher is not None + assert type(runtime.publisher).__name__ == "GlueCatalogPublisher" + assert runtime.capabilities.readback is True + assert runtime.capabilities.preserves_unrelated_fields is True + assert runtime.capabilities.first_party_entries is False + + def test_cli_rejects_publication_when_catalog_is_disabled() -> None: with pytest.raises(ClickException, match="does not publish assets"): build_catalog_publisher( @@ -80,3 +114,13 @@ def test_cli_rejects_publication_when_catalog_is_disabled() -> None: project="unit-project", location="us", ) + + +def test_cli_rejects_mismatched_catalog_provider_configuration() -> None: + with pytest.raises(ClickException, match="id and configuration must match"): + build_catalog_publisher( + provider_id="dataplex", + provider_config={"provider": "none"}, + project="unit-project", + location="us", + ) diff --git a/tests/test_executor.py b/tests/test_executor.py index 8cfbfc99..f75f9107 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -26,6 +26,7 @@ from collections.abc import Iterable from pathlib import Path + from dander.catalog import CatalogAsset, CatalogPublisher from dander.concurrency import OwnershipGuard @@ -132,6 +133,15 @@ def snapshots(self, *, pipeline_id: str | None = None) -> tuple[MetadataSnapshot return () +class _CatalogPublisher: + def __init__(self) -> None: + self.assets: list[CatalogAsset] = [] + + def publish(self, asset: CatalogAsset) -> str: + self.assets.append(asset) + return f"catalog://{asset.relation}" + + class _Leases(LeaseStore): def __init__(self, *, available: bool, heartbeat: bool = True) -> None: self._available = available @@ -195,6 +205,7 @@ def _executor( leases: LeaseStore | None = None, raw_namespace: str = "raw", source_relations: dict[str, RelationRef] | None = None, + catalog_publisher: CatalogPublisher | None = None, ) -> PipelineExecutor: source = SourceConfig( name="example", @@ -222,6 +233,7 @@ def _executor( build_models=True, transform_runner=_Transform(fail=fail_transform), metadata_store=metadata, + catalog_publisher=catalog_publisher, leases=leases, ) @@ -263,6 +275,23 @@ def test_executor_preserves_launcher_supplied_run_id(tmp_path: Path) -> None: assert history.started == ("launcher-run-42", "example", "example_pipeline") +def test_executor_publishes_through_provider_neutral_catalog_boundary(tmp_path: Path) -> None: + _models(tmp_path) + publisher = _CatalogPublisher() + + result = _executor( + tmp_path, + history=_History(), + metadata=_Metadata(), + catalog_publisher=publisher, + ).execute() + + assert result.assets == 1 + assert [asset.relation for asset in publisher.assets] == [ + "valid-project-123.staging.stg_widgets" + ] + + def test_executor_keeps_custom_raw_namespace_in_models_and_metadata(tmp_path: Path) -> None: _models(tmp_path) metadata = _Metadata() From 1c64bab7bde31f5d14c29d2037fc250bdaa8c4f1 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 08:44:40 -0400 Subject: [PATCH 068/163] refactor: finish provider-neutral relation coordinates --- HANDOFF.md | 58 +++++----- src/dander/cli/run_command.py | 85 +++++++++------ src/dander/executor.py | 50 +++++++-- src/dander/providers/bigquery/config.py | 3 +- src/dander/providers/postgresql/config.py | 3 +- src/dander/providers/redshift/config.py | 3 +- src/dander/providers/snowflake/config.py | 3 +- src/dander/warehouse/contracts.py | 1 - tests/cli/test_run_command.py | 42 +++++-- tests/providers/test_bigquery_runtime.py | 13 ++- .../test_postgresql_warehouse_runtime.py | 1 - .../test_redshift_warehouse_runtime.py | 1 - .../test_snowflake_warehouse_runtime.py | 1 - tests/test_executor.py | 103 +++++++++++++++++- 14 files changed, 267 insertions(+), 100 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index f55bd221..b238b251 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,49 +2,45 @@ ## Finished -- Added a lazy, explicitly configured AWS Glue Data Catalog provider. -- Published canonical relation, schema, lineage, tests, metrics, ownership, and sensitivity through - direct database/table APIs with deterministic readback. -- Preserved unrelated database, table, storage-descriptor, partition, and column metadata. -- Replaced provider-neutral executor/runtime use of “Dataplex publisher” with a generic catalog - publisher while retaining the legacy constructor and CLI compatibility inputs. -- Added focused configuration, lazy-loading, create/update/readback, preservation, and sanitization - coverage plus operator documentation. +- Removed the BigQuery-derived namespace fallback from the provider-neutral warehouse coordinate + protocol; each provider now accepts only canonical compatibility inputs. +- Kept BigQuery namespace precedence exactly `--dataset`, profile `warehouse.dataset`, then + `BQ_DATASET_RAW`, resolved before neutral orchestration. +- Made the endpoint `RelationRef` map authoritative through CLI composition and + `PipelineExecutor`, including complete-map and shared catalog/namespace validation. +- Removed unused internal `project`, `dataset`, `metadata_dataset`, and `warehouse_catalog` aliases + while retaining public v1/CLI compatibility entry points. ## Try It -Select `catalog.provider: glue` in a version 2 platform profile as shown in `docs/aws-glue.md`, set -`publish_catalog: true` on a model pipeline, and run with an ambient AWS identity. This slice has -not contacted AWS and is not a support claim. +Run `uv run pytest -q tests/cli/test_run_command.py tests/test_executor.py` to exercise compatibility +translation and canonical relation flow without contacting a warehouse. ## Checks -- Ruff, 34 focused tests, strict mypy across 303 files, and the full 1,094-test suite passed; - the full suite included ephemeral PostgreSQL 15 integration. -- Wheel, sdist, source-free installation, runtime-all installation, dependency audit, and the final - non-root/read-only container conformance checks passed. -- GCP/AWS Terraform validation and tests plus Helm lint/template passed. The retained GCP project - produced exactly `No changes.` using its reviewed 600-second runtime override; no apply ran. +- Ruff passed; strict mypy passed across 303 source files. +- The full suite passed: 1,100 tests with an ephemeral pinned PostgreSQL 15 service. +- Wheel, sdist, outside-checkout source-free installs, runtime-all install, generated-project + validation, dependency audit, and non-root/read-only container conformance passed. +- GCP/AWS Terraform validation and tests plus Helm lint/template passed. ## Decisions -- Glue databases encode canonical catalog and namespace coordinates; a digest is added only when - lowercase normalization would be lossy. -- Dander owns descriptions, columns, classification, and `dander.*` parameters; unrelated metadata - is retained and catalog objects are never deleted. -- Glue uses direct APIs and ambient AWS identity; crawlers, IAM provisioning, Lake Formation, tags, - live proof, and support promotion remain separate gates. +- Canonical endpoint relations are the single warehouse-location authority once CLI compatibility + inputs have been translated. +- Mixed catalogs or raw namespaces fail before execution instead of selecting one relation + implicitly. +- Provider registry, capabilities, fencing, schema contracts, v1 resources, and Terraform remain + unchanged. ## Remaining -- Let protected CI repeat Linux tests, packaging, container scanning, and secret scanning. -- Reconcile the tracked 300-second job default and retained 600-second operator override in a - separate change; do not mix it into the Glue catalog slice. -- Continue the cloud-portability plan in a separate branch after merge; do not deploy or mutate - AWS/GCP resources in this slice. +- Let protected CI repeat Linux packaging, image scanning, and secret scanning before merge. +- Keep the separate warehouse-capability worktree and future provider work out of this slice. +- Do not deploy, apply Terraform, publish packages, or expand provider support from this branch. ## Review First -- `src/dander/catalog/glue.py` -- `src/dander/providers/glue/` -- `tests/catalog/test_glue.py` +- `src/dander/cli/run_command.py` +- `src/dander/executor.py` +- `src/dander/warehouse/contracts.py` diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index f5557561..d4aa4664 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -113,12 +113,10 @@ class _ResolvedRun: graph_file: Path | None graph_plan: GraphExecutionPlan | None gcp_project: str - catalog: str - raw_namespace: str - metadata_namespace: str endpoint_relations: dict[str, RelationRef] state_catalog: str | None state_namespace: str | None + metadata_namespace: str | None selected_models: tuple[str, ...] | None build_models: bool publish_catalog: bool @@ -131,24 +129,19 @@ class _ResolvedRun: secret_provider: str @property - def project(self) -> str: - """Return the v1 CLI compatibility project value.""" - return self.gcp_project + def raw_relation(self) -> RelationRef: + """Return one canonical relation carrying the shared raw location.""" + return next(iter(self.endpoint_relations.values())) @property - def dataset(self) -> str: - """Return the v1 CLI compatibility dataset value.""" - return self.raw_namespace + def catalog(self) -> str: + """Return the provider-neutral warehouse catalog.""" + return self.raw_relation.catalog @property - def metadata_dataset(self) -> str: - """Return the v1 BigQuery metadata namespace alias.""" - return self.metadata_namespace - - @property - def warehouse_catalog(self) -> str: - """Return the former internal warehouse-catalog shim.""" - return self.catalog + def raw_namespace(self) -> str: + """Return the provider-neutral raw namespace.""" + return self.raw_relation.namespace @dataclass(frozen=True, slots=True) @@ -262,33 +255,37 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: raise ClickException(str(error)) from error settings = Settings() - resolved_project = options.project or settings.gcp_project_id - state_catalog = (resolved_project or None) if state_provider == "bigquery" else None + gcp_project = options.project or settings.gcp_project_id + state_catalog = (gcp_project or None) if state_provider == "bigquery" else None state_namespace = ( (options.dataset or settings.bq_dataset_raw) if state_provider == "bigquery" else None ) + metadata_namespace = settings.bq_dataset_metadata if state_provider == "bigquery" else None registry = default_provider_registry() try: - coordinate_config = registry.parse(ProviderKind.WAREHOUSE, warehouse_config) + coordinate_values = _warehouse_coordinate_values( + provider=warehouse_provider, + config=warehouse_config, + compatibility_namespace=options.dataset, + bigquery_default_namespace=settings.bq_dataset_raw, + ) + coordinate_config = registry.parse(ProviderKind.WAREHOUSE, coordinate_values) if not isinstance(coordinate_config, WarehouseCoordinateConfig): raise ProviderFactoryError( f"Warehouse provider {warehouse_provider!r} cannot resolve relation coordinates" ) - coordinate = coordinate_config.raw_relation( - "_dander_coordinate", - compatibility_catalog=resolved_project or "dander-dry-run", - compatibility_namespace=options.dataset, - default_namespace=settings.bq_dataset_raw, + compatibility_catalog = ( + gcp_project or "dander-dry-run" if warehouse_provider == "bigquery" else None ) endpoint_relations = { endpoint.name: coordinate_config.raw_relation( f"{config.name}_{endpoint.name}", - compatibility_catalog=resolved_project or "dander-dry-run", + compatibility_catalog=compatibility_catalog, compatibility_namespace=options.dataset, - default_namespace=settings.bq_dataset_raw, ) for endpoint in config.endpoints } + _require_common_raw_location(endpoint_relations) except (ProviderFactoryError, ValueError) as error: raise ClickException(str(error)) from error if options.sandbox and options.guarded_free_tier: @@ -309,13 +306,11 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: project_pipeline=project_pipeline, graph_file=graph_file, graph_plan=graph_plan, - gcp_project=resolved_project, - catalog=coordinate.catalog, - raw_namespace=coordinate.namespace, - metadata_namespace=settings.bq_dataset_metadata, + gcp_project=gcp_project, endpoint_relations=endpoint_relations, state_catalog=state_catalog, state_namespace=state_namespace, + metadata_namespace=metadata_namespace, selected_models=selected_models, build_models=build_models, publish_catalog=publish_catalog, @@ -329,6 +324,32 @@ def _resolve_run(options: RunOptions) -> _ResolvedRun: ) +def _require_common_raw_location(relations: dict[str, RelationRef]) -> None: + """Require one catalog/namespace pair for a pipeline's raw relations.""" + if not relations: + raise ValueError("Warehouse coordinate resolution returned no endpoint relations") + first = next(iter(relations.values())) + if any( + (relation.catalog, relation.namespace) != (first.catalog, first.namespace) + for relation in relations.values() + ): + raise ValueError("Pipeline endpoint relations must share one catalog and raw namespace") + + +def _warehouse_coordinate_values( + *, + provider: str, + config: dict[str, object], + compatibility_namespace: str | None, + bigquery_default_namespace: str, +) -> dict[str, object]: + """Resolve the legacy BigQuery namespace default before neutral orchestration.""" + values = dict(config) + if provider == "bigquery" and compatibility_namespace is None and values.get("dataset") is None: + values["dataset"] = bigquery_default_namespace + return values + + def _resolve_graph_plan( graph_file: Path | None, config: SourceConfig, @@ -415,8 +436,6 @@ def _build_executor(options: RunOptions, resolved: _ResolvedRun) -> PipelineExec source_config=resolved.source_config, ingestion=ingestion, history=stores.history, - catalog=resolved.catalog, - raw_namespace=resolved.raw_namespace, source_relations=resolved.endpoint_relations, models_dir=( resolved.graph_file.parent if resolved.graph_file is not None else options.models_dir diff --git a/src/dander/executor.py b/src/dander/executor.py index 4f7aed88..6ae2b6f0 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -74,7 +74,7 @@ def __init__( history: RunHistoryStore, catalog: str | None = None, project: str | None = None, - raw_namespace: str = "raw", + raw_namespace: str | None = None, source_relations: dict[str, RelationRef] | None = None, models_dir: Path, selected_models: Iterable[str] | None, @@ -92,26 +92,54 @@ def __init__( catalog = project elif project is not None and project != catalog: raise ValueError("catalog and legacy project must match") - if catalog is None: - raise ValueError("PipelineExecutor requires a warehouse catalog") if catalog_publisher is not None and dataplex_publisher is not None: raise ValueError( "catalog_publisher and legacy dataplex_publisher are mutually exclusive" ) + configured_endpoints = {endpoint.name for endpoint in source_config.endpoints} + if source_relations is not None: + if missing := sorted(configured_endpoints - source_relations.keys()): + raise ValueError(f"Missing source relation for endpoint: {missing[0]!r}") + if unknown := sorted(source_relations.keys() - configured_endpoints): + raise ValueError(f"Unknown source relation endpoint: {unknown[0]!r}") + if not source_relations: + raise ValueError("PipelineExecutor requires at least one source relation") + canonical = next(iter(source_relations.values())) + if any( + (relation.catalog, relation.namespace) != (canonical.catalog, canonical.namespace) + for relation in source_relations.values() + ): + raise ValueError( + "Pipeline source relations must share one catalog and raw namespace" + ) + if catalog is not None and catalog != canonical.catalog: + raise ValueError("catalog conflicts with canonical source relations") + if raw_namespace is not None and raw_namespace != canonical.namespace: + raise ValueError("raw_namespace conflicts with canonical source relations") + catalog = canonical.catalog + raw_namespace = canonical.namespace + resolved_source_relations = dict(source_relations) + else: + if catalog is None: + raise ValueError("PipelineExecutor requires a warehouse catalog") + raw_namespace = raw_namespace or "raw" + resolved_source_relations = { + endpoint.name: RelationRef( + catalog=catalog, + namespace=raw_namespace, + name=f"{source_config.name}_{endpoint.name}", + ) + for endpoint in source_config.endpoints + } + assert catalog is not None + assert raw_namespace is not None self._pipeline_id = pipeline_id self._source_config = source_config self._ingestion = ingestion self._history = history self._catalog = catalog self._raw_namespace = raw_namespace - self._source_relations = source_relations or { - endpoint.name: RelationRef( - catalog=catalog, - namespace=raw_namespace, - name=f"{source_config.name}_{endpoint.name}", - ) - for endpoint in source_config.endpoints - } + self._source_relations = resolved_source_relations self._models_dir = models_dir self._selected_models = tuple(selected_models) if selected_models is not None else None self._build_models = build_models diff --git a/src/dander/providers/bigquery/config.py b/src/dander/providers/bigquery/config.py index 192e7e1a..71d42bb3 100644 --- a/src/dander/providers/bigquery/config.py +++ b/src/dander/providers/bigquery/config.py @@ -32,14 +32,13 @@ def raw_relation( *, compatibility_catalog: str | None, compatibility_namespace: str | None, - default_namespace: str, ) -> RelationRef: """Translate legacy project/dataset inputs at the BigQuery boundary.""" if not compatibility_catalog: raise ValueError("BigQuery warehouse requires a catalog") return RelationRef( catalog=compatibility_catalog, - namespace=compatibility_namespace or self.dataset or default_namespace, + namespace=compatibility_namespace or self.dataset or "raw", name=name, ) diff --git a/src/dander/providers/postgresql/config.py b/src/dander/providers/postgresql/config.py index 77d2405d..044bebe3 100644 --- a/src/dander/providers/postgresql/config.py +++ b/src/dander/providers/postgresql/config.py @@ -86,10 +86,9 @@ def raw_relation( *, compatibility_catalog: str | None, compatibility_namespace: str | None, - default_namespace: str, ) -> RelationRef: """Translate the legacy namespace option into database/schema coordinates.""" - del compatibility_catalog, default_namespace + del compatibility_catalog return RelationRef( catalog=self.database, namespace=compatibility_namespace or self.schema_name, diff --git a/src/dander/providers/redshift/config.py b/src/dander/providers/redshift/config.py index 59d2f7b9..0056dbf4 100644 --- a/src/dander/providers/redshift/config.py +++ b/src/dander/providers/redshift/config.py @@ -102,10 +102,9 @@ def raw_relation( *, compatibility_catalog: str | None, compatibility_namespace: str | None, - default_namespace: str, ) -> RelationRef: """Translate compatibility inputs into Redshift database/schema coordinates.""" - del compatibility_catalog, default_namespace + del compatibility_catalog relation = RelationRef( catalog=self.database, namespace=compatibility_namespace or self.schema_name, diff --git a/src/dander/providers/snowflake/config.py b/src/dander/providers/snowflake/config.py index 354f70bf..33d113bb 100644 --- a/src/dander/providers/snowflake/config.py +++ b/src/dander/providers/snowflake/config.py @@ -99,10 +99,9 @@ def raw_relation( *, compatibility_catalog: str | None, compatibility_namespace: str | None, - default_namespace: str, ) -> RelationRef: """Translate compatibility inputs into Snowflake database/schema coordinates.""" - del compatibility_catalog, default_namespace + del compatibility_catalog return RelationRef( catalog=self.database, namespace=compatibility_namespace or self.schema_name, diff --git a/src/dander/warehouse/contracts.py b/src/dander/warehouse/contracts.py index 109bc7e7..75334b27 100644 --- a/src/dander/warehouse/contracts.py +++ b/src/dander/warehouse/contracts.py @@ -235,7 +235,6 @@ def raw_relation( *, compatibility_catalog: str | None, compatibility_namespace: str | None, - default_namespace: str, ) -> RelationRef: """Return one raw relation for a configured endpoint.""" ... diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index daf42346..d920bebf 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -47,11 +47,10 @@ class _ResolvedWarehouse(Protocol): class _ResolvedState(Protocol): state_provider: str - project: str - dataset: str - metadata_dataset: str + gcp_project: str state_catalog: str | None state_namespace: str | None + metadata_namespace: str | None class _Migrator: @@ -146,11 +145,10 @@ def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: def build_state(resolved: _ResolvedState) -> StateRuntime: assert resolved.state_provider == "bigquery" - assert resolved.project == "unit-project" - assert resolved.dataset == "landing" - assert resolved.metadata_dataset == "dander_meta" + assert resolved.gcp_project == "unit-project" assert resolved.state_catalog == "unit-project" assert resolved.state_namespace == "landing" + assert resolved.metadata_namespace == "dander_meta" migrator = _Migrator() captured["migrator"] = migrator for name in ("history", "leases", "metadata", "watermarks"): @@ -207,8 +205,8 @@ def build_secrets(provider_id: str) -> _Built: assert "Dander run run-123" in result.output executor = cast("dict[str, object]", captured["executor"]) assert executor["pipeline_id"] == "greenhouse_jobs" - assert executor["catalog"] == "unit-project" - assert executor["raw_namespace"] == "landing" + assert "catalog" not in executor + assert "raw_namespace" not in executor assert executor["source_relations"] == { "jobs": RelationRef( catalog="unit-project", @@ -318,6 +316,34 @@ def test_bigquery_state_location_stays_independent_from_postgresql_warehouse() - assert context["metadata_namespace"] == "dander_meta" +def test_bigquery_default_namespace_is_resolved_only_for_bigquery() -> None: + assert run_module._warehouse_coordinate_values( + provider="bigquery", + config={"provider": "bigquery", "location": "US"}, + compatibility_namespace=None, + bigquery_default_namespace="environment_raw", + ) == { + "provider": "bigquery", + "location": "US", + "dataset": "environment_raw", + } + assert ( + run_module._warehouse_coordinate_values( + provider="bigquery", + config={"provider": "bigquery", "location": "US", "dataset": "profile_raw"}, + compatibility_namespace=None, + bigquery_default_namespace="environment_raw", + )["dataset"] + == "profile_raw" + ) + assert run_module._warehouse_coordinate_values( + provider="postgresql", + config={"provider": "postgresql", "database": "warehouse"}, + compatibility_namespace=None, + bigquery_default_namespace="environment_raw", + ) == {"provider": "postgresql", "database": "warehouse"} + + def test_snowflake_coordinates_resolve_from_database_and_schema(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index 91081746..9c80c1ed 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -61,11 +61,10 @@ def test_default_registry_loads_bigquery_runtime_only_after_selection() -> None: def test_bigquery_config_translates_legacy_cli_coordinates() -> None: - relation = BigQueryWarehouseConfig(provider="bigquery").raw_relation( + relation = BigQueryWarehouseConfig(provider="bigquery", dataset="profile_raw").raw_relation( "accounts", compatibility_catalog="unit-project", compatibility_namespace="landing", - default_namespace="raw", ) assert relation == RelationRef( @@ -75,6 +74,16 @@ def test_bigquery_config_translates_legacy_cli_coordinates() -> None: ) +def test_bigquery_config_uses_its_native_namespace_when_cli_alias_is_absent() -> None: + relation = BigQueryWarehouseConfig(provider="bigquery", dataset="profile_raw").raw_relation( + "accounts", + compatibility_catalog="unit-project", + compatibility_namespace=None, + ) + + assert relation.namespace == "profile_raw" + + def test_bigquery_runtime_exposes_codec_schema_fence_and_telemetry() -> None: runtime = _runtime() relation = RelationRef(catalog="unit-project", namespace="raw", name="records") diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py index 3d1ab95d..1e1e1982 100644 --- a/tests/providers/test_postgresql_warehouse_runtime.py +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -69,7 +69,6 @@ def test_postgresql_config_uses_database_and_schema_coordinates_directly() -> No "accounts", compatibility_catalog="ignored-gcp-project", compatibility_namespace=None, - default_namespace="ignored_bigquery_dataset", ) assert relation == RelationRef( diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index a3f07e34..783cada5 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -305,7 +305,6 @@ def test_redshift_registration_is_lazy_and_uses_native_coordinates() -> None: "records", compatibility_catalog="ignored-gcp-project", compatibility_namespace=None, - default_namespace="ignored_bigquery_default", ) assert relation == RelationRef(catalog="analytics", namespace="raw", name="records") assert "password" not in config.model_dump_json() diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index cbf253ae..208d62c8 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -229,7 +229,6 @@ def test_snowflake_registration_is_lazy_and_credentials_remain_references() -> N "records", compatibility_catalog="ignored-gcp-project", compatibility_namespace=None, - default_namespace="ignored_bigquery_default", ) assert relation == RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") diff --git a/tests/test_executor.py b/tests/test_executor.py index f75f9107..c75995d1 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -203,7 +203,9 @@ def _executor( metadata: _Metadata, fail_transform: bool = False, leases: LeaseStore | None = None, - raw_namespace: str = "raw", + catalog: str | None = None, + project: str | None = "valid-project-123", + raw_namespace: str | None = "raw", source_relations: dict[str, RelationRef] | None = None, catalog_publisher: CatalogPublisher | None = None, ) -> PipelineExecutor: @@ -225,7 +227,8 @@ def _executor( source_config=source, ingestion=_Ingestion(), history=history, - project="valid-project-123", + catalog=catalog, + project=project, raw_namespace=raw_namespace, source_relations=source_relations, models_dir=models_dir, @@ -305,7 +308,9 @@ def test_executor_keeps_custom_raw_namespace_in_models_and_metadata(tmp_path: Pa tmp_path, history=_History(), metadata=metadata, - raw_namespace="landing", + catalog=None, + project=None, + raw_namespace=None, source_relations={"widgets": source_relation}, ).execute() @@ -320,6 +325,98 @@ def test_executor_keeps_custom_raw_namespace_in_models_and_metadata(tmp_path: Pa assert assets[0]["upstream_relations"] == ["valid-project-123.landing.example_widgets"] +@pytest.mark.parametrize( + ("catalog", "raw_namespace", "message"), + [ + ("other-catalog", None, "catalog conflicts with canonical source relations"), + (None, "other_namespace", "raw_namespace conflicts with canonical source relations"), + ], +) +def test_executor_rejects_legacy_coordinates_that_conflict_with_source_relations( + tmp_path: Path, + *, + catalog: str | None, + raw_namespace: str | None, + message: str, +) -> None: + _models(tmp_path) + source_relation = RelationRef( + catalog="canonical_catalog", + namespace="canonical_namespace", + name="example_widgets", + ) + + with pytest.raises(ValueError, match=message): + _executor( + tmp_path, + history=_History(), + metadata=_Metadata(), + catalog=catalog, + project=None, + raw_namespace=raw_namespace, + source_relations={"widgets": source_relation}, + ) + + +@pytest.mark.parametrize( + ("source_relations", "message"), + [ + ( + { + "widgets": RelationRef( + catalog="warehouse", + namespace="landing", + name="example_widgets", + ) + }, + "Missing source relation for endpoint: 'gadgets'", + ), + ( + { + "widgets": RelationRef( + catalog="warehouse", + namespace="landing", + name="example_widgets", + ), + "gadgets": RelationRef( + catalog="warehouse", + namespace="other_namespace", + name="example_gadgets", + ), + }, + "must share one catalog and raw namespace", + ), + ], +) +def test_executor_requires_one_complete_canonical_source_location( + tmp_path: Path, + *, + source_relations: dict[str, RelationRef], + message: str, +) -> None: + source = SourceConfig( + name="example", + base_url="https://example.test", + auth_strategy="none", + endpoints=[ + Endpoint(name="widgets", path="/widgets", primary_key=["id"]), + Endpoint(name="gadgets", path="/gadgets", primary_key=["id"]), + ], + ) + + with pytest.raises(ValueError, match=message): + PipelineExecutor( + pipeline_id="example_pipeline", + source_config=source, + ingestion=_Ingestion(), + history=_History(), + source_relations=source_relations, + models_dir=tmp_path, + selected_models=None, + build_models=False, + ) + + def test_executor_marks_transform_failure_without_claiming_ingestion_only_success( tmp_path: Path, caplog: pytest.LogCaptureFixture, From ec16c32427ead8ff8e10d364029de7115cb0f802 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 08:51:44 -0400 Subject: [PATCH 069/163] Add warehouse capability diagnostics --- CHANGELOG.md | 4 + HANDOFF.md | 51 ++++---- docs/compatibility-matrix.md | 19 +++ docs/decisions.md | 14 +++ docs/known-limitations.md | 7 +- src/dander/cli/run_command.py | 1 + src/dander/compatibility.py | 103 +++++++++++++++- src/dander/providers/bigquery/runtime.py | 15 ++- src/dander/providers/postgresql/runtime.py | 24 +++- src/dander/providers/redshift/runtime.py | 36 +++++- src/dander/providers/snowflake/runtime.py | 36 +++++- src/dander/runtime-compatibility.json | 50 ++++++++ src/dander/runtime.py | 18 +-- src/dander/state/failure.py | 5 +- src/dander/warehouse/__init__.py | 4 + src/dander/warehouse/runtime.py | 75 +++++++++++- tests/cli/test_run_command.py | 3 + tests/providers/test_bigquery_runtime.py | 9 ++ .../test_postgresql_warehouse_runtime.py | 1 + .../test_redshift_warehouse_runtime.py | 1 + .../test_snowflake_warehouse_runtime.py | 1 + tests/state/test_failure.py | 16 +++ tests/test_compatibility.py | 37 ++++++ tests/test_runtime.py | 65 +++++++++- tests/warehouse/test_schema_support.py | 115 ++++++++++++++++++ 25 files changed, 654 insertions(+), 56 deletions(-) create mode 100644 tests/warehouse/test_schema_support.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cf95f4ae..49423cfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ capabilities enter through the next minor release. ### Added +- Publish exact write-mode, transport, transform, graph, fencing, logical-type, decimal, and + temporal limits for every packaged warehouse through `dander runtime compatibility`. Selected + portable warehouse schema mappers now reject unsupported types and precision before extraction + or destination mutation without changing BigQuery's provider-native v1 schema behavior. - Add an experimental AWS Glue Data Catalog provider with canonical relation mapping, direct database/table API publication, Dander-owned parameters, unrelated-field preservation, and normalized readback. No crawler, deletion, IAM provisioning, or live-support claim is included. diff --git a/HANDOFF.md b/HANDOFF.md index b238b251..6118ca0a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,45 +2,42 @@ ## Finished -- Removed the BigQuery-derived namespace fallback from the provider-neutral warehouse coordinate - protocol; each provider now accepts only canonical compatibility inputs. -- Kept BigQuery namespace precedence exactly `--dataset`, profile `warehouse.dataset`, then - `BQ_DATASET_RAW`, resolved before neutral orchestration. -- Made the endpoint `RelationRef` map authoritative through CLI composition and - `PipelineExecutor`, including complete-map and shared catalog/namespace validation. -- Removed unused internal `project`, `dataset`, `metadata_dataset`, and `warehouse_catalog` aliases - while retaining public v1/CLI compatibility entry points. +- Added exact packaged capability reports for BigQuery, PostgreSQL, Redshift, and Snowflake. +- Added fail-closed canonical schema validation before extraction and destination mutation. +- Added field-path diagnostics for unsupported types, precision, and nested arrays. +- Kept the compatibility schema additive and the supported-profile manifest unchanged. +- Documented the implemented capability surface without promoting experimental providers. ## Try It -Run `uv run pytest -q tests/cli/test_run_command.py tests/test_executor.py` to exercise compatibility -translation and canonical relation flow without contacting a warehouse. +Run `dander runtime compatibility` and inspect the `warehouses` array. Portable-provider schemas +now fail through the selected provider's schema mapper before extraction or destination mutation. ## Checks -- Ruff passed; strict mypy passed across 303 source files. -- The full suite passed: 1,100 tests with an ephemeral pinned PostgreSQL 15 service. -- Wheel, sdist, outside-checkout source-free installs, runtime-all install, generated-project - validation, dependency audit, and non-root/read-only container conformance passed. -- GCP/AWS Terraform validation and tests plus Helm lint/template passed. +- Ruff and strict mypy across 304 source files passed; 1,110 tests passed with PostgreSQL 15. +- Wheel, sdist, source-free installs, runtime-all install, generated-project validation, dependency + audit, and non-root/read-only container conformance passed. +- GCP/AWS Terraform validation/tests and Helm lint/template passed. The retained GCP read-only plan + reported exactly `No changes.` with its reviewed 600-second override; no apply ran. ## Decisions -- Canonical endpoint relations are the single warehouse-location authority once CLI compatibility - inputs have been translated. -- Mixed catalogs or raw namespaces fail before execution instead of selecting one relation - implicitly. -- Provider registry, capabilities, fencing, schema contracts, v1 resources, and Terraform remain - unchanged. +- Runtime capability declarations are authoritative; packaged JSON is checked against them. +- Unsupported semi-structured mappings remain fail-closed instead of silently selecting VARIANT + or SUPER. +- This slice reports current behavior only; it adds no provider feature or support claim. ## Remaining -- Let protected CI repeat Linux packaging, image scanning, and secret scanning before merge. -- Keep the separate warehouse-capability worktree and future provider work out of this slice. -- Do not deploy, apply Terraform, publish packages, or expand provider support from this branch. +- Repeat the protected-quality-equivalent suite after the provider-neutral coordinate rebase. +- Reconcile the tracked 300-second job default and retained 600-second operator override in a + separate change rather than mixing it into this capability slice. +- Continue provider write modes and live qualification separately; do not deploy or publish a + package from this branch. ## Review First -- `src/dander/cli/run_command.py` -- `src/dander/executor.py` -- `src/dander/warehouse/contracts.py` +- `src/dander/warehouse/runtime.py` +- `src/dander/compatibility.py` +- `tests/warehouse/test_schema_support.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 61d8c3d6..8f0de804 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -29,3 +29,22 @@ the cloud-portability plan. Adding an adapter does not silently add a matrix row Catalog selection is independent from this state/warehouse matrix. Dataplex is supported only in the released GCP profile, `none` is a complete no-cloud-mutation provider, and AWS Glue is an experimental direct API projection with local create/update/readback conformance only. + +## Warehouse capabilities + +The same command publishes the exact implemented warehouse surface. These declarations are +capabilities, not support promotion: Snowflake, Redshift, and PostgreSQL remain experimental until +their named live profiles pass. + +| Warehouse | Ingestion modes | Transport | Canonical schema | Models | Graphs | Fence | +|---|---|---|---|---|---|---| +| BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | +| PostgreSQL | SCD1 | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | +| Redshift | SCD1 | COPY | scalar types; decimal 38, time 6; no SUPER fallback | yes | no | yes | +| Snowflake | SCD1 | COPY | scalar types; decimal 38, time 9; no VARIANT fallback | yes | no | yes | + +Portable-provider schema validation uses these declarations before source extraction, staging +upload, or destination mutation. BigQuery retains its existing provider-native v1 schema path so +types without lossless canonical mappings remain compatible. An unsupported portable field reports +the provider, field path, type or precision, and supported limit. The packaged JSON and runtime +capability constants are checked for drift in the test suite. diff --git a/docs/decisions.md b/docs/decisions.md index 91dda5f5..02705d86 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -920,3 +920,17 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc objects or makes crawlers authoritative. - **Boundary:** Ambient AWS identity and direct API readback are implemented. IAM infrastructure, Glue connections, Lake Formation, tags, live proof, and support promotion remain separate work. + +## 2026-08-09 — Warehouse capabilities are exact and schema validation is fail-early + +- **Contract:** Every warehouse runtime declares its implemented write modes, transports, model, + graph, fencing, logical-type, decimal, and temporal support. The credential-free compatibility + report mirrors those declarations and tests prevent drift. +- **Failure boundary:** Provider schema mappers reject unsupported types, precision, and nested + arrays before source extraction, staging, or destination mutation. BigQuery retains its native + v1 schema path because types such as GEOGRAPHY have no lossless canonical mapping. Diagnostics + name the provider and + exact field path without including provider responses or record values. +- **Scope:** The declaration reports implemented behavior only. It does not promote experimental + providers, add write modes, introduce semi-structured fallbacks, or widen the supported profile + manifest. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 91be0929..c2098422 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -22,9 +22,10 @@ evaluate these limits before using it for an unattended system containing busine chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The - package publishes this pair matrix through `dander runtime compatibility`, while the packaged - capability manifest remains the support boundary. Local PostgreSQL benchmark results are - regression evidence, not a paid or controlled-memory scale qualification. + package publishes this pair matrix and each warehouse's exact implemented modes, transports, + schema limits, transforms, graphs, and fencing through `dander runtime compatibility`, while the + packaged capability manifest remains the support boundary. Local PostgreSQL benchmark results + are regression evidence, not a paid or controlled-memory scale qualification. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/src/dander/cli/run_command.py b/src/dander/cli/run_command.py index d4aa4664..f5086827 100644 --- a/src/dander/cli/run_command.py +++ b/src/dander/cli/run_command.py @@ -497,6 +497,7 @@ def _build_ingestion_runner( resolved.graph_plan.bindings.endpoint_names if resolved.graph_plan is not None else None ), target_fence=warehouse.target_fence, + schema_mapper=warehouse.ingestion_schema_mapper, ) diff --git a/src/dander/compatibility.py b/src/dander/compatibility.py index 095e6f93..682588d7 100644 --- a/src/dander/compatibility.py +++ b/src/dander/compatibility.py @@ -40,18 +40,50 @@ def as_dict(self) -> dict[str, str]: } +@dataclass(frozen=True, slots=True) +class WarehouseCapabilityReport: + """Credential-free capability declaration for one packaged warehouse adapter.""" + + provider: str + write_modes: tuple[str, ...] + transports: tuple[str, ...] + logical_types: tuple[str, ...] + max_decimal_precision: int + max_temporal_precision: int + supports_transforms: bool + supports_graphs: bool + supports_target_fencing: bool + limitations: tuple[str, ...] + + def as_dict(self) -> dict[str, object]: + return { + "provider": self.provider, + "write_modes": list(self.write_modes), + "transports": list(self.transports), + "logical_types": list(self.logical_types), + "max_decimal_precision": self.max_decimal_precision, + "max_temporal_precision": self.max_temporal_precision, + "supports_transforms": self.supports_transforms, + "supports_graphs": self.supports_graphs, + "supports_target_fencing": self.supports_target_fencing, + "limitations": list(self.limitations), + } + + @dataclass(frozen=True, slots=True) class RuntimeCompatibility: """Validated, deterministic compatibility report for the installed runtime.""" schema: str state_warehouse_pairs: tuple[StateWarehousePair, ...] + warehouses: tuple[WarehouseCapabilityReport, ...] def to_json(self) -> str: return json.dumps( { "schema": self.schema, "state_warehouse_pairs": [pair.as_dict() for pair in self.state_warehouse_pairs], + "warehouses": [warehouse.as_dict() for warehouse in self.warehouses], }, separators=(",", ":"), sort_keys=True, @@ -110,7 +142,37 @@ def load_runtime_compatibility() -> RuntimeCompatibility: raise CompatibilityError( "runtime compatibility pairs must be unique and sorted by state/warehouse" ) - return RuntimeCompatibility(schema=_SCHEMA, state_warehouse_pairs=tuple(pairs)) + raw_warehouses = payload.get("warehouses") + if not isinstance(raw_warehouses, list) or not raw_warehouses: + raise CompatibilityError("runtime compatibility matrix has no warehouse capabilities") + warehouses = tuple(_warehouse_capability(item) for item in raw_warehouses) + providers = [warehouse.provider for warehouse in warehouses] + if len(providers) != len(set(providers)) or providers != sorted(providers): + raise CompatibilityError( + "runtime warehouse capabilities must be unique and sorted by provider" + ) + return RuntimeCompatibility( + schema=_SCHEMA, + state_warehouse_pairs=tuple(pairs), + warehouses=warehouses, + ) + + +def _warehouse_capability(raw: object) -> WarehouseCapabilityReport: + if not isinstance(raw, dict): + raise CompatibilityError("runtime compatibility matrix contains an invalid warehouse") + return WarehouseCapabilityReport( + provider=_required_text(raw, "provider"), + write_modes=_required_sorted_texts(raw, "write_modes"), + transports=_required_sorted_texts(raw, "transports"), + logical_types=_required_sorted_texts(raw, "logical_types"), + max_decimal_precision=_required_positive_int(raw, "max_decimal_precision"), + max_temporal_precision=_required_nonnegative_int(raw, "max_temporal_precision"), + supports_transforms=_required_bool(raw, "supports_transforms"), + supports_graphs=_required_bool(raw, "supports_graphs"), + supports_target_fencing=_required_bool(raw, "supports_target_fencing"), + limitations=_required_sorted_texts(raw, "limitations", allow_empty=True), + ) def _required_text(payload: dict[object, object], name: str) -> str: @@ -120,10 +182,49 @@ def _required_text(payload: dict[object, object], name: str) -> str: return value +def _required_sorted_texts( + payload: dict[object, object], + name: str, + *, + allow_empty: bool = False, +) -> tuple[str, ...]: + value = payload.get(name) + if ( + not isinstance(value, list) + or (not value and not allow_empty) + or any(not isinstance(item, str) or not item.strip() for item in value) + or value != sorted(set(value)) + ): + raise CompatibilityError(f"runtime warehouse capability has invalid {name}") + return tuple(value) + + +def _required_positive_int(payload: dict[object, object], name: str) -> int: + value = payload.get(name) + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise CompatibilityError(f"runtime warehouse capability has invalid {name}") + return value + + +def _required_nonnegative_int(payload: dict[object, object], name: str) -> int: + value = payload.get(name) + if not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= 9: + raise CompatibilityError(f"runtime warehouse capability has invalid {name}") + return value + + +def _required_bool(payload: dict[object, object], name: str) -> bool: + value = payload.get(name) + if not isinstance(value, bool): + raise CompatibilityError(f"runtime warehouse capability has invalid {name}") + return value + + __all__ = [ "CompatibilityError", "CompatibilityStatus", "RuntimeCompatibility", "StateWarehousePair", + "WarehouseCapabilityReport", "load_runtime_compatibility", ] diff --git a/src/dander/providers/bigquery/runtime.py b/src/dander/providers/bigquery/runtime.py index e9ffbb0f..04c1cf73 100644 --- a/src/dander/providers/bigquery/runtime.py +++ b/src/dander/providers/bigquery/runtime.py @@ -13,9 +13,11 @@ from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import BigQueryTransformRunner from dander.warehouse.bigquery_compat import canonical_schema_from_bigquery +from dander.warehouse.contracts import LogicalTypeKind from dander.warehouse.runtime import ( WarehouseCapabilities, WarehouseRuntime, + WarehouseSchemaSupport, WarehouseTransformRunner, ) from dander.writer import ( @@ -36,6 +38,15 @@ from dander.warehouse.contracts import RelationRef, RelationSchema +BIGQUERY_SCHEMA_SUPPORT = WarehouseSchemaSupport( + provider_id="bigquery", + logical_types=frozenset(LogicalTypeKind), + max_decimal_precision=38, + max_temporal_precision=6, + supports_nested_arrays=False, +) + + @dataclass(frozen=True, slots=True) class BigQueryRelationCodec: """Render validated canonical coordinates as quoted BigQuery identifiers.""" @@ -53,7 +64,8 @@ class BigQuerySchemaMapper: def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: """Map a complete legacy BigQuery declaration to canonical schema v1.""" - return canonical_schema_from_bigquery(cast("Sequence[BigQueryFieldLike]", fields)) + schema = canonical_schema_from_bigquery(cast("Sequence[BigQueryFieldLike]", fields)) + return BIGQUERY_SCHEMA_SUPPORT.require(schema) @dataclass(frozen=True, slots=True) @@ -139,6 +151,7 @@ def operation( supports_transforms=True, supports_graphs=True, supports_target_fencing=True, + schema_support=BIGQUERY_SCHEMA_SUPPORT, ) diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py index 6f4fe9e8..df805fc7 100644 --- a/src/dander/providers/postgresql/runtime.py +++ b/src/dander/providers/postgresql/runtime.py @@ -17,8 +17,12 @@ from dander.providers.postgresql.writer import PostgreSQLScd1Writer, PostgreSQLTimeouts from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.telemetry import OperationTelemetry, TelemetryOperation -from dander.warehouse import CanonicalField, RelationRef, RelationSchema -from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.warehouse import CanonicalField, LogicalTypeKind, RelationRef, RelationSchema +from dander.warehouse.runtime import ( + WarehouseCapabilities, + WarehouseRuntime, + WarehouseSchemaSupport, +) from dander.writer import ( SchemaEvolution, WriteField, @@ -35,6 +39,14 @@ PostgreSQLRow = dict[str, object] PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] +POSTGRESQL_SCHEMA_SUPPORT = WarehouseSchemaSupport( + provider_id="postgresql", + logical_types=frozenset(LogicalTypeKind), + max_decimal_precision=1_000, + max_temporal_precision=6, + supports_nested_arrays=True, +) + @dataclass(frozen=True, slots=True) class PostgreSQLRelationCodec: @@ -62,7 +74,7 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: canonical.append(field) else: raise TypeError("PostgreSQL schema mapper received an unsupported field") - return RelationSchema(fields=tuple(canonical)) + return POSTGRESQL_SCHEMA_SUPPORT.require(RelationSchema(fields=tuple(canonical))) @dataclass(frozen=True, slots=True) @@ -153,6 +165,7 @@ def operation( supports_transforms=True, supports_graphs=False, supports_target_fencing=True, + schema_support=POSTGRESQL_SCHEMA_SUPPORT, ) @@ -198,15 +211,17 @@ def build_postgresql_warehouse( lock_ms=config.lock_timeout_ms, idle_transaction_ms=config.idle_transaction_timeout_ms, ) + schema_mapper = PostgreSQLSchemaMapper() return WarehouseRuntime( provider_id="postgresql", relation_codec=PostgreSQLRelationCodec(config.database), - schema_mapper=PostgreSQLSchemaMapper(), + schema_mapper=schema_mapper, writers=PostgreSQLWriterFactory(config.database, pool, target_fence, timeouts), transforms=PostgreSQLTransformFactory(config.database, pool, target_fence, timeouts), target_fence=target_fence, telemetry=PostgreSQLTelemetry(), capabilities=POSTGRESQL_CAPABILITIES, + ingestion_schema_mapper=schema_mapper, ) @@ -232,6 +247,7 @@ def _required_sslmode(dsn: str) -> str: __all__ = [ "POSTGRESQL_CAPABILITIES", + "POSTGRESQL_SCHEMA_SUPPORT", "POSTGRESQL_WAREHOUSE_FACTORY", "PostgreSQLRelationCodec", "PostgreSQLSchemaMapper", diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index 9896ee9b..5e66925e 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -28,8 +28,12 @@ ProviderKind, ) from dander.telemetry import OperationTelemetry, TelemetryOperation -from dander.warehouse import CanonicalField, RelationRef, RelationSchema -from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.warehouse import CanonicalField, LogicalTypeKind, RelationRef, RelationSchema +from dander.warehouse.runtime import ( + WarehouseCapabilities, + WarehouseRuntime, + WarehouseSchemaSupport, +) from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport if TYPE_CHECKING: @@ -38,6 +42,26 @@ from pydantic import BaseModel +REDSHIFT_SCHEMA_SUPPORT = WarehouseSchemaSupport( + provider_id="redshift", + logical_types=frozenset( + { + LogicalTypeKind.BOOLEAN, + LogicalTypeKind.INTEGER, + LogicalTypeKind.DECIMAL, + LogicalTypeKind.FLOAT, + LogicalTypeKind.STRING, + LogicalTypeKind.BINARY, + LogicalTypeKind.DATE, + LogicalTypeKind.TIME, + LogicalTypeKind.TIMESTAMP, + } + ), + max_decimal_precision=38, + max_temporal_precision=6, +) + + @dataclass(frozen=True, slots=True) class RedshiftRelationCodec: """Render canonical database/schema/relation coordinates for Redshift.""" @@ -65,7 +89,7 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: canonical.append(field) else: raise TypeError("Redshift schema mapper received an unsupported field") - return RelationSchema(fields=tuple(canonical)) + return REDSHIFT_SCHEMA_SUPPORT.require(RelationSchema(fields=tuple(canonical))) @dataclass(frozen=True, slots=True) @@ -160,6 +184,7 @@ def operation( supports_transforms=True, supports_graphs=False, supports_target_fencing=True, + schema_support=REDSHIFT_SCHEMA_SUPPORT, ) @@ -234,10 +259,11 @@ def build_redshift_warehouse( compression=staging.compression, statement_timeout_ms=staging.statement_timeout_ms, ) + schema_mapper = RedshiftSchemaMapper() return WarehouseRuntime( provider_id="redshift", relation_codec=RedshiftRelationCodec(config.database), - schema_mapper=RedshiftSchemaMapper(), + schema_mapper=schema_mapper, writers=RedshiftWriterFactory( config.database, connection_factory, @@ -254,6 +280,7 @@ def build_redshift_warehouse( target_fence=target_fence, telemetry=RedshiftTelemetry(), capabilities=REDSHIFT_CAPABILITIES, + ingestion_schema_mapper=schema_mapper, ) @@ -321,6 +348,7 @@ def _matches_iam_user(actual: str, configured: str) -> bool: __all__ = [ "REDSHIFT_CAPABILITIES", + "REDSHIFT_SCHEMA_SUPPORT", "REDSHIFT_WAREHOUSE_FACTORY", "RedshiftRelationCodec", "RedshiftSchemaMapper", diff --git a/src/dander/providers/snowflake/runtime.py b/src/dander/providers/snowflake/runtime.py index 6b0096df..eef82bf5 100644 --- a/src/dander/providers/snowflake/runtime.py +++ b/src/dander/providers/snowflake/runtime.py @@ -31,8 +31,12 @@ default_staging_settings, ) from dander.telemetry import OperationTelemetry, TelemetryOperation -from dander.warehouse import CanonicalField, RelationRef, RelationSchema -from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.warehouse import CanonicalField, LogicalTypeKind, RelationRef, RelationSchema +from dander.warehouse.runtime import ( + WarehouseCapabilities, + WarehouseRuntime, + WarehouseSchemaSupport, +) from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport if TYPE_CHECKING: @@ -41,6 +45,26 @@ from pydantic import BaseModel +SNOWFLAKE_SCHEMA_SUPPORT = WarehouseSchemaSupport( + provider_id="snowflake", + logical_types=frozenset( + { + LogicalTypeKind.BOOLEAN, + LogicalTypeKind.INTEGER, + LogicalTypeKind.DECIMAL, + LogicalTypeKind.FLOAT, + LogicalTypeKind.STRING, + LogicalTypeKind.BINARY, + LogicalTypeKind.DATE, + LogicalTypeKind.TIME, + LogicalTypeKind.TIMESTAMP, + } + ), + max_decimal_precision=38, + max_temporal_precision=9, +) + + @dataclass(frozen=True, slots=True) class SnowflakeRelationCodec: """Render one database-local Snowflake relation coordinate.""" @@ -67,7 +91,7 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: canonical.append(field) else: raise TypeError("Snowflake schema mapper received an unsupported field") - return RelationSchema(fields=tuple(canonical)) + return SNOWFLAKE_SCHEMA_SUPPORT.require(RelationSchema(fields=tuple(canonical))) @dataclass(frozen=True, slots=True) @@ -162,6 +186,7 @@ def operation( supports_transforms=True, supports_graphs=False, supports_target_fencing=True, + schema_support=SNOWFLAKE_SCHEMA_SUPPORT, ) @@ -218,10 +243,11 @@ def build_snowflake_warehouse( max_logical_bytes_per_file=staging.max_logical_bytes_per_file, compression=staging.compression, ) + schema_mapper = SnowflakeSchemaMapper() return WarehouseRuntime( provider_id="snowflake", relation_codec=SnowflakeRelationCodec(config.database), - schema_mapper=SnowflakeSchemaMapper(), + schema_mapper=schema_mapper, writers=SnowflakeWriterFactory( config.database, connection_factory, @@ -236,6 +262,7 @@ def build_snowflake_warehouse( target_fence=target_fence, telemetry=SnowflakeTelemetry(config.warehouse), capabilities=SNOWFLAKE_CAPABILITIES, + ingestion_schema_mapper=schema_mapper, ) @@ -305,6 +332,7 @@ def _quote(identifier: str) -> str: __all__ = [ "SNOWFLAKE_CAPABILITIES", + "SNOWFLAKE_SCHEMA_SUPPORT", "SNOWFLAKE_WAREHOUSE_FACTORY", "SnowflakeRelationCodec", "SnowflakeSchemaMapper", diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index d06d3ff3..766e0c13 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -49,5 +49,55 @@ "status": "experimental", "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged SCD1 writes; live qualification remains pending." } + ], + "warehouses": [ + { + "provider": "bigquery", + "write_modes": ["incremental", "replace", "scd1", "scd2", "snapshot"], + "transports": ["load_job", "storage_write"], + "logical_types": ["array", "binary", "boolean", "date", "decimal", "float", "integer", "json", "record", "string", "time", "timestamp"], + "max_decimal_precision": 38, + "max_temporal_precision": 6, + "supports_transforms": true, + "supports_graphs": true, + "supports_target_fencing": true, + "limitations": ["nested arrays are not supported"] + }, + { + "provider": "postgresql", + "write_modes": ["scd1"], + "transports": ["copy"], + "logical_types": ["array", "binary", "boolean", "date", "decimal", "float", "integer", "json", "record", "string", "time", "timestamp"], + "max_decimal_precision": 1000, + "max_temporal_precision": 6, + "supports_transforms": true, + "supports_graphs": false, + "supports_target_fencing": true, + "limitations": ["graph execution is not available", "only scd1 ingestion is implemented"] + }, + { + "provider": "redshift", + "write_modes": ["scd1"], + "transports": ["copy"], + "logical_types": ["binary", "boolean", "date", "decimal", "float", "integer", "string", "time", "timestamp"], + "max_decimal_precision": 38, + "max_temporal_precision": 6, + "supports_transforms": true, + "supports_graphs": false, + "supports_target_fencing": true, + "limitations": ["graph execution is not available", "only scd1 ingestion is implemented", "semi-structured types require a future explicit SUPER mapping"] + }, + { + "provider": "snowflake", + "write_modes": ["scd1"], + "transports": ["copy"], + "logical_types": ["binary", "boolean", "date", "decimal", "float", "integer", "string", "time", "timestamp"], + "max_decimal_precision": 38, + "max_temporal_precision": 9, + "supports_transforms": true, + "supports_graphs": false, + "supports_target_fencing": true, + "limitations": ["graph execution is not available", "only scd1 ingestion is implemented", "semi-structured types require a future explicit VARIANT mapping"] + } ] } diff --git a/src/dander/runtime.py b/src/dander/runtime.py index 76e46be2..e37cf8ea 100644 --- a/src/dander/runtime.py +++ b/src/dander/runtime.py @@ -20,7 +20,7 @@ from dander.concurrency import OwnershipGuard from dander.ingestion.source import Endpoint, RawField, Source from dander.state.watermark import WatermarkStore - from dander.warehouse import WarehouseTargetFence + from dander.warehouse import WarehouseSchemaMapper, WarehouseTargetFence from dander.writer.base import WritePattern _LOGGER = logging.getLogger(__name__) @@ -70,6 +70,7 @@ def __init__( batch_rows: int = 10_000, endpoint_names: Iterable[str] | None = None, target_fence: WarehouseTargetFence | None = None, + schema_mapper: WarehouseSchemaMapper | None = None, ) -> None: self._source = source self._writer = writer @@ -77,6 +78,7 @@ def __init__( self._resume_from_watermark = resume_from_watermark self._history = history self._target_fence = target_fence + self._schema_mapper = schema_mapper if isinstance(batch_rows, bool) or not 1 <= batch_rows <= 100_000: raise ValueError("batch_rows must be an integer from 1 to 100000") self._batch_rows = batch_rows @@ -181,6 +183,14 @@ def _run_endpoint( ownership: OwnershipGuard | None, ) -> EndpointRunResult: source_name = self._source.config.name + target = WriteTarget( + relation=self._endpoint_relations[endpoint.name], + business_key=tuple(endpoint.primary_key), + schema=tuple(_write_field(field) for field in endpoint.raw_schema), + fence=ownership.fence if ownership is not None else None, + ) + if target.schema and self._schema_mapper is not None: + self._schema_mapper.canonical_schema(target.schema) stored_cursor = ( self._watermarks.get(source_name, endpoint.name) if endpoint.incremental_cursor @@ -189,12 +199,6 @@ def _run_endpoint( cursor = stored_cursor if self._resume_from_watermark else None if ownership is not None: ownership.verify() - target = WriteTarget( - relation=self._endpoint_relations[endpoint.name], - business_key=tuple(endpoint.primary_key), - schema=tuple(_write_field(field) for field in endpoint.raw_schema), - fence=ownership.fence if ownership is not None else None, - ) if self._writer.requires_publication_fence: if ownership is None or ownership.fence is None or self._target_fence is None: raise RuntimeError("Hosted destination writes require target-fence ownership") diff --git a/src/dander/state/failure.py b/src/dander/state/failure.py index 58462295..a82342c4 100644 --- a/src/dander/state/failure.py +++ b/src/dander/state/failure.py @@ -57,10 +57,11 @@ def classify_failure(error: Exception, *, stage: RunStage, run_id: str) -> Failu "lease_failed", "Pipeline ownership or cursor fencing was lost. A fresh run can retry safely.", ) - if names & {"RawSchemaError", "CursorValueError"}: + if names & {"RawSchemaError", "CursorValueError", "WarehouseSchemaSupportError"}: return _details( "source_schema_failed", - f"A source record did not match the declared schema. Inspect logs for run {run_id}.", + f"The declared source schema is unsupported or a record did not match it. " + f"Inspect logs for run {run_id}.", ) if names & {"BigQueryWriteError", "DestinationWriteError", "WarehouseWriteError"}: return _details( diff --git a/src/dander/warehouse/__init__.py b/src/dander/warehouse/__init__.py index a7e8edc5..6fd8e0b7 100644 --- a/src/dander/warehouse/__init__.py +++ b/src/dander/warehouse/__init__.py @@ -21,6 +21,8 @@ WarehouseCapabilities, WarehouseRuntime, WarehouseSchemaMapper, + WarehouseSchemaSupport, + WarehouseSchemaSupportError, WarehouseTargetFence, WarehouseTelemetry, WarehouseTransformFactory, @@ -53,6 +55,8 @@ "WarehouseCapabilities", "WarehouseRuntime", "WarehouseSchemaMapper", + "WarehouseSchemaSupport", + "WarehouseSchemaSupportError", "WarehouseTargetFence", "WarehouseTelemetry", "WarehouseTransformFactory", diff --git a/src/dander/warehouse/runtime.py b/src/dander/warehouse/runtime.py index 105a4f6c..ba40b9be 100644 --- a/src/dander/warehouse/runtime.py +++ b/src/dander/warehouse/runtime.py @@ -5,6 +5,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol, runtime_checkable +from dander.warehouse.contracts import CanonicalType, LogicalTypeKind, RelationSchema + if TYPE_CHECKING: from collections.abc import Iterable, Sequence from pathlib import Path @@ -12,7 +14,7 @@ from dander.concurrency import FencingToken, OwnershipGuard, TargetFence from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import TransformRunResult - from dander.warehouse.contracts import RelationCodec, RelationRef, RelationSchema + from dander.warehouse.contracts import RelationCodec, RelationRef from dander.writer.base import SchemaEvolution, WriteMode, WritePattern, WriteTransport @@ -28,6 +30,73 @@ def __post_init__(self) -> None: raise ValueError("prepared warehouse statement must not be blank") +class WarehouseSchemaSupportError(ValueError): + """A canonical schema cannot be represented by the selected warehouse.""" + + +@dataclass(frozen=True, slots=True) +class WarehouseSchemaSupport: + """Fail-closed canonical type support advertised by one warehouse.""" + + provider_id: str + logical_types: frozenset[LogicalTypeKind] + max_decimal_precision: int + max_temporal_precision: int + supports_nested_arrays: bool = False + + def __post_init__(self) -> None: + if not self.provider_id: + raise ValueError("warehouse schema support provider_id must not be blank") + if not self.logical_types: + raise ValueError("warehouse schema support must declare logical types") + if self.max_decimal_precision < 1: + raise ValueError("warehouse decimal precision limit must be positive") + if not 0 <= self.max_temporal_precision <= 9: + raise ValueError("warehouse temporal precision limit must be between 0 and 9") + + def require(self, schema: RelationSchema) -> RelationSchema: + """Return ``schema`` after validating every field before extraction or mutation.""" + for field in schema.fields: + self._require_type(field.data_type, path=field.name) + return schema + + def _require_type(self, data_type: CanonicalType, *, path: str) -> None: + if data_type.kind not in self.logical_types: + raise WarehouseSchemaSupportError( + f"{self.provider_id} warehouse does not support canonical type " + f"{data_type.kind.value!r} at field {path!r}" + ) + if ( + data_type.kind is LogicalTypeKind.DECIMAL + and data_type.precision is not None + and data_type.precision > self.max_decimal_precision + ): + raise WarehouseSchemaSupportError( + f"{self.provider_id} warehouse supports decimal precision up to " + f"{self.max_decimal_precision}; field {path!r} declares {data_type.precision}" + ) + if ( + data_type.kind in {LogicalTypeKind.TIME, LogicalTypeKind.TIMESTAMP} + and data_type.fractional_second_precision is not None + and data_type.fractional_second_precision > self.max_temporal_precision + ): + raise WarehouseSchemaSupportError( + f"{self.provider_id} warehouse supports temporal precision up to " + f"{self.max_temporal_precision}; field {path!r} declares " + f"{data_type.fractional_second_precision}" + ) + if data_type.kind is LogicalTypeKind.ARRAY: + assert data_type.element is not None + if data_type.element.kind is LogicalTypeKind.ARRAY and not self.supports_nested_arrays: + raise WarehouseSchemaSupportError( + f"{self.provider_id} warehouse does not support nested arrays at field {path!r}" + ) + self._require_type(data_type.element, path=f"{path}[]") + if data_type.kind is LogicalTypeKind.RECORD: + for field in data_type.fields: + self._require_type(field.data_type, path=f"{path}.{field.name}") + + @dataclass(frozen=True, slots=True) class WarehouseCapabilities: """Closed support declaration for one concrete warehouse implementation.""" @@ -39,6 +108,7 @@ class WarehouseCapabilities: supports_transforms: bool supports_graphs: bool supports_target_fencing: bool + schema_support: WarehouseSchemaSupport def __post_init__(self) -> None: if not self.provider_id: @@ -47,6 +117,8 @@ def __post_init__(self) -> None: raise ValueError("warehouse schema contract version must be 1") if not self.write_modes: raise ValueError("warehouse capabilities must declare at least one write mode") + if self.provider_id != self.schema_support.provider_id: + raise ValueError("warehouse schema support provider does not match capabilities") @runtime_checkable @@ -144,6 +216,7 @@ class WarehouseRuntime: target_fence: WarehouseTargetFence telemetry: WarehouseTelemetry capabilities: WarehouseCapabilities + ingestion_schema_mapper: WarehouseSchemaMapper | None = None def __post_init__(self) -> None: if self.provider_id != self.relation_codec.provider_id: diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index d920bebf..e747bc50 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -137,6 +137,8 @@ class _Warehouse: writers = _WriterFactory() transforms = _TransformFactory() target_fence = _Built(name="target-fence", args=(), kwargs={}) + schema_mapper = _Built(name="schema-mapper", args=(), kwargs={}) + ingestion_schema_mapper = None def build_warehouse(resolved: _ResolvedWarehouse) -> _Warehouse: assert resolved.warehouse_provider == "bigquery" @@ -227,6 +229,7 @@ def build_secrets(provider_id: str) -> _Built: assert ingestion.kwargs["batch_rows"] == 10_000 assert ingestion.kwargs["watermarks"] == captured["watermarks"] assert ingestion.kwargs["target_fence"] == _Warehouse.target_fence + assert ingestion.kwargs["schema_mapper"] is None writer = cast("_Built", ingestion.kwargs["writer"]) assert writer.kwargs == { "sandbox": False, diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index 9c80c1ed..d3cbcd98 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -118,6 +118,7 @@ class _Job: ) assert runtime.relation_codec.render(relation) == "`unit-project`.`raw`.`records`" + assert runtime.ingestion_schema_mapper is None assert schema.fields[0].data_type.kind is LogicalTypeKind.STRING assert prepared.sql.count("ASSERT @@row_count = 1") == 2 assert "status = 'committed'" in prepared.sql @@ -138,6 +139,14 @@ class _Job: } +def test_bigquery_keeps_provider_native_types_out_of_portable_ingestion_preflight() -> None: + runtime = _runtime() + + assert runtime.ingestion_schema_mapper is None + with pytest.raises(ValueError, match="explicit canonical fallback"): + runtime.schema_mapper.canonical_schema([WriteField(name="location", data_type="GEOGRAPHY")]) + + class _FenceJob: def __init__(self, *, affected: int | None = None) -> None: self.num_dml_affected_rows = affected diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py index 1e1e1982..1f384892 100644 --- a/tests/providers/test_postgresql_warehouse_runtime.py +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -124,6 +124,7 @@ def postgresql_warehouse() -> Iterator[tuple[WarehouseRuntime, PostgreSQLPool, s ) runtime = registry.build(ProviderKind.WAREHOUSE, config, context={"pool": pool}) assert isinstance(runtime, WarehouseRuntime) + assert runtime.ingestion_schema_mapper is runtime.schema_mapper try: yield runtime, pool, database, schema_name finally: diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index 783cada5..c02c957a 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -266,6 +266,7 @@ def redshift_runtime( }, ) assert isinstance(runtime, WarehouseRuntime) + assert runtime.ingestion_schema_mapper is runtime.schema_mapper return runtime, backend, s3, tmp_path diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index 208d62c8..de54dcb6 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -204,6 +204,7 @@ def snowflake_runtime( }, ) assert isinstance(runtime, WarehouseRuntime) + assert runtime.ingestion_schema_mapper is runtime.schema_mapper return runtime, backend, tmp_path diff --git a/tests/state/test_failure.py b/tests/state/test_failure.py index 75fd173a..d04e4585 100644 --- a/tests/state/test_failure.py +++ b/tests/state/test_failure.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from dander.state import RunStage, classify_failure +from dander.warehouse import WarehouseSchemaSupportError @dataclass @@ -93,6 +94,21 @@ class CatalogPublishError(RuntimeError): assert "private" not in catalog.summary +def test_warehouse_schema_support_failures_keep_the_source_schema_code() -> None: + failure = classify_failure( + WarehouseSchemaSupportError("private provider schema detail"), + stage=RunStage.INGEST, + run_id="run-123", + ) + + assert failure.code == "source_schema_failed" + assert failure.summary == ( + "The declared source schema is unsupported or a record did not match it. " + "Inspect logs for run run-123." + ) + assert "private" not in failure.summary + + def test_provider_specific_failures_use_cloud_neutral_summaries() -> None: class PermissionDenied(RuntimeError): # noqa: N818 - matches provider SDK class name pass diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 8d441432..f643d8fa 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -13,6 +13,10 @@ CompatibilityStatus, load_runtime_compatibility, ) +from dander.providers.bigquery.runtime import BIGQUERY_CAPABILITIES +from dander.providers.postgresql.runtime import POSTGRESQL_CAPABILITIES +from dander.providers.redshift.runtime import REDSHIFT_CAPABILITIES +from dander.providers.snowflake.runtime import SNOWFLAKE_CAPABILITIES def test_state_warehouse_matrix_covers_every_current_pair() -> None: @@ -65,4 +69,37 @@ def test_runtime_compatibility_cli_prints_one_deterministic_document() -> None: payload = json.loads(result.output) assert payload["schema"] == "io.dander.runtime.compatibility/v1" assert len(payload["state_warehouse_pairs"]) == 8 + assert [warehouse["provider"] for warehouse in payload["warehouses"]] == [ + "bigquery", + "postgresql", + "redshift", + "snowflake", + ] assert result.output.strip() == load_runtime_compatibility().to_json() + + +def test_published_warehouse_capabilities_match_runtime_declarations() -> None: + runtime_capabilities = { + capability.provider_id: capability + for capability in ( + BIGQUERY_CAPABILITIES, + POSTGRESQL_CAPABILITIES, + REDSHIFT_CAPABILITIES, + SNOWFLAKE_CAPABILITIES, + ) + } + + for report in load_runtime_compatibility().warehouses: + capability = runtime_capabilities[report.provider] + assert report.write_modes == tuple(sorted(mode.value for mode in capability.write_modes)) + assert report.transports == tuple( + sorted(transport.value for transport in capability.transports) + ) + assert report.logical_types == tuple( + sorted(kind.value for kind in capability.schema_support.logical_types) + ) + assert report.max_decimal_precision == capability.schema_support.max_decimal_precision + assert report.max_temporal_precision == capability.schema_support.max_temporal_precision + assert report.supports_transforms is capability.supports_transforms + assert report.supports_graphs is capability.supports_graphs + assert report.supports_target_fencing is capability.supports_target_fencing diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 5f790c0b..2949bd5f 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -17,9 +17,9 @@ from dander.writer import WriteMode, WritePattern, WriteTarget if TYPE_CHECKING: - from collections.abc import Iterable, Iterator, Mapping + from collections.abc import Iterable, Iterator, Mapping, Sequence - from dander.warehouse import PreparedWarehouseStatement + from dander.warehouse import PreparedWarehouseStatement, RelationSchema class _Source(Source): @@ -839,6 +839,67 @@ def compare_and_set( assert watermarks.committed is None +def test_runner_validates_declared_schema_before_state_source_or_writer_io() -> None: + events: list[str] = [] + + class _DeclaredIncrementalSource(Source): + def __init__(self) -> None: + super().__init__( + SourceConfig( + name="example", + base_url="https://example.test", + auth_strategy="none", + endpoints=[ + Endpoint( + name="widgets", + path="/widgets", + incremental_cursor="updated_at", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING", mode="REQUIRED"), + RawField(name="updated_at", data_type="TIMESTAMP"), + ], + ) + ], + ) + ) + + def discover(self) -> Mapping[str, Any]: + return {} + + def extract( + self, + endpoint: str, + *, + since: str | None = None, + ) -> Iterator[Mapping[str, Any]]: + del endpoint, since + events.append("extract") + yield {"id": "one"} + + class _RejectingSchemaMapper: + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + assert fields + events.append("schema") + raise ValueError("synthetic unsupported schema") + + writer = _BatchedWriter() + runner = PipelineRunner( + source=_DeclaredIncrementalSource(), + writer=writer, + watermarks=_Watermarks(events), + project="unit-project", + dataset="raw", + schema_mapper=_RejectingSchemaMapper(), + ) + + with pytest.raises(ValueError, match="synthetic unsupported schema"): + runner.run() + + assert events == ["schema"] + assert writer.batches == [] + + @pytest.mark.parametrize("batch_rows", [0, 100_001, True]) def test_runner_rejects_invalid_batch_rows(batch_rows: int) -> None: with pytest.raises(ValueError, match="batch_rows"): diff --git a/tests/warehouse/test_schema_support.py b/tests/warehouse/test_schema_support.py new file mode 100644 index 00000000..8528bcf0 --- /dev/null +++ b/tests/warehouse/test_schema_support.py @@ -0,0 +1,115 @@ +"""Warehouse capability declarations reject unsupported schemas before provider I/O.""" + +from __future__ import annotations + +import pytest + +from dander.providers.bigquery.runtime import BIGQUERY_SCHEMA_SUPPORT +from dander.providers.postgresql.runtime import ( + POSTGRESQL_SCHEMA_SUPPORT, + PostgreSQLSchemaMapper, +) +from dander.providers.redshift.runtime import REDSHIFT_SCHEMA_SUPPORT, RedshiftSchemaMapper +from dander.providers.snowflake.runtime import ( + SNOWFLAKE_SCHEMA_SUPPORT, + SnowflakeSchemaMapper, +) +from dander.warehouse import ( + CanonicalField, + CanonicalType, + LogicalTypeKind, + RelationSchema, + WarehouseSchemaSupportError, +) + + +def _schema(name: str, data_type: CanonicalType) -> RelationSchema: + return RelationSchema(fields=(CanonicalField(name=name, data_type=data_type),)) + + +@pytest.mark.parametrize( + "mapper", + [SnowflakeSchemaMapper(), RedshiftSchemaMapper()], +) +def test_experimental_mappers_reject_semistructured_fields_before_provider_io( + mapper: SnowflakeSchemaMapper | RedshiftSchemaMapper, +) -> None: + field = CanonicalField( + name="payload", + data_type=CanonicalType(kind=LogicalTypeKind.JSON), + ) + + with pytest.raises( + WarehouseSchemaSupportError, + match="does not support canonical type 'json' at field 'payload'", + ): + mapper.canonical_schema([field]) + + +def test_decimal_and_temporal_limits_name_provider_field_and_limit() -> None: + decimal = CanonicalType(kind=LogicalTypeKind.DECIMAL, precision=39, scale=2) + timestamp = CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=True, + fractional_second_precision=7, + ) + + with pytest.raises( + WarehouseSchemaSupportError, + match="redshift warehouse supports decimal precision up to 38; field 'amount' declares 39", + ): + REDSHIFT_SCHEMA_SUPPORT.require(_schema("amount", decimal)) + with pytest.raises( + WarehouseSchemaSupportError, + match=( + "postgresql warehouse supports temporal precision up to 6; " + "field 'observed_at' declares 7" + ), + ): + POSTGRESQL_SCHEMA_SUPPORT.require(_schema("observed_at", timestamp)) + + +def test_bigquery_rejects_nested_arrays_with_exact_field_path() -> None: + nested = CanonicalType( + kind=LogicalTypeKind.ARRAY, + element=CanonicalType( + kind=LogicalTypeKind.ARRAY, + element=CanonicalType(kind=LogicalTypeKind.STRING), + ), + ) + + with pytest.raises( + WarehouseSchemaSupportError, + match="bigquery warehouse does not support nested arrays at field 'matrix'", + ): + BIGQUERY_SCHEMA_SUPPORT.require(_schema("matrix", nested)) + + +def test_postgresql_mapper_accepts_declared_nested_and_json_types() -> None: + fields = [ + CanonicalField( + name="payload", + data_type=CanonicalType(kind=LogicalTypeKind.JSON), + ), + CanonicalField( + name="labels", + data_type=CanonicalType( + kind=LogicalTypeKind.ARRAY, + element=CanonicalType(kind=LogicalTypeKind.STRING), + ), + ), + ] + + schema = PostgreSQLSchemaMapper().canonical_schema(fields) + + assert schema == RelationSchema(fields=tuple(fields)) + + +def test_snowflake_declares_nine_digit_temporal_precision() -> None: + timestamp = CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=False, + fractional_second_precision=9, + ) + + assert SNOWFLAKE_SCHEMA_SUPPORT.require(_schema("observed_at", timestamp)) From d3759c94a61b96737514af51a33d0257aad8241d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 10:13:55 -0400 Subject: [PATCH 070/163] Make Phase 1B proof identities reusable --- HANDOFF.md | 45 +++++++++---------- .../cloud-portability/phase1b/README.md | 20 ++++++--- .../cloud-portability/phase1b/smoke/main.tf | 9 ++-- .../phase1b/smoke/variables.tf | 15 +++++++ docs/decisions.md | 9 ++++ 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 6118ca0a..4b830026 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,42 +2,39 @@ ## Finished -- Added exact packaged capability reports for BigQuery, PostgreSQL, Redshift, and Snowflake. -- Added fail-closed canonical schema validation before extraction and destination mutation. -- Added field-path diagnostics for unsupported types, precision, and nested arrays. -- Kept the compatibility schema additive and the supported-profile manifest unchanged. -- Documented the implemented capability surface without promoting experimental providers. +- Made the disposable Phase 1B smoke proof name configurable while preserving its existing default. +- Derived AWS, CloudWatch, Google Workload Identity, and service-account names from one input. +- Documented safe reruns during Google's soft-delete retention window. ## Try It -Run `dander runtime compatibility` and inspect the `warehouses` array. Portable-provider schemas -now fail through the selected provider's schema mapper before extraction or destination mutation. +Set `proof_name = "dander-phase1b-r2"` in the smoke root's reviewed inputs and use the same value +when generating the external-account credential configuration. ## Checks -- Ruff and strict mypy across 304 source files passed; 1,110 tests passed with PostgreSQL 15. -- Wheel, sdist, source-free installs, runtime-all install, generated-project validation, dependency - audit, and non-root/read-only container conformance passed. -- GCP/AWS Terraform validation/tests and Helm lint/template passed. The retained GCP read-only plan - reported exactly `No changes.` with its reviewed 600-second override; no apply ran. +- Default and `dander-phase1b-r2` smoke plans each produced 18 creates; default resource names were + unchanged, custom names were isolated, and invalid names failed before planning. +- Ruff/format and strict mypy passed; 1,110 tests passed against PostgreSQL 15; dependency audit + found no known vulnerabilities. +- Wheel/sdist inspection, source-free installs, runtime-all installation, generated Terraform, + GCP/AWS Terraform, Helm, and non-root/read-only container conformance passed. +- Retained stage-zero and platform plans each reported exactly `No changes.`; no apply ran. ## Decisions -- Runtime capability declarations are authoritative; packaged JSON is checked against them. -- Unsupported semi-structured mappings remain fail-closed instead of silently selecting VARIANT - or SUPER. -- This slice reports current behavior only; it adds no provider feature or support claim. +- Keep `dander-phase1b` as the compatibility default. +- Use a new deterministic name rather than automatically undeleting or adopting prior resources. ## Remaining -- Repeat the protected-quality-equivalent suite after the provider-neutral coordinate rebase. -- Reconcile the tracked 300-second job default and retained 600-second operator override in a - separate change rather than mixing it into this capability slice. -- Continue provider write modes and live qualification separately; do not deploy or publish a - package from this branch. +- Let protected CI repeat Linux security and container scans, then merge if clean. +- Correct the unrelated stage-zero permission preflight's bucket-level check in a separate PR. +- Regenerate the live proof plans from merged `main`. +- Apply only after explicit paid-action approval. ## Review First -- `src/dander/warehouse/runtime.py` -- `src/dander/compatibility.py` -- `tests/warehouse/test_schema_support.py` +- `acceptance/cloud-portability/phase1b/smoke/variables.tf` +- `acceptance/cloud-portability/phase1b/smoke/main.tf` +- `acceptance/cloud-portability/phase1b/README.md` diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 8b796e7b..eae28a58 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -29,13 +29,17 @@ record content. The retained Dander project is out of scope. `repository_url`. 2. Generate a fresh project from the public package into an operator temp directory. Confirm it has no `src/` directory. -3. Generate an AWS external-account file for the deterministic provider name and probe service - account. This command writes configuration, not a secret: +3. Select one deterministic `proof_name` for the complete smoke proof. The default is + `dander-phase1b`; use a new value such as `dander-phase1b-r2` when a previous Workload Identity + Pool with the default ID is still soft-deleted. Keep the value with the saved Terraform inputs. + Generate an AWS external-account file for the matching provider name and probe service account. + This command writes configuration, not a secret: ```console + PROOF_NAME=dander-phase1b-r2 gcloud iam workload-identity-pools create-cred-config \ - projects/1009770943166/locations/global/workloadIdentityPools/dander-phase1b-aws/providers/fargate \ - --service-account=dander-phase1b-aws@project-092b24a8-26a3-4438-8cd.iam.gserviceaccount.com \ + "projects/1009770943166/locations/global/workloadIdentityPools/${PROOF_NAME}-aws/providers/fargate" \ + --service-account="${PROOF_NAME}-aws@project-092b24a8-26a3-4438-8cd.iam.gserviceaccount.com" \ --aws --output-file="$OPERATOR_DIR/external-account.json" ``` @@ -68,9 +72,11 @@ record content. The retained Dander project is out of scope. The command fails unless the index digest and both runnable platform-manifest digests are identical after the copy. -7. Set the verified ECR digest in `smoke/terraform.tfvars`, initialize `smoke/`, save and review its - plan, then apply only that saved plan. The execution role can pull/log; the separate task role - has no AWS permission policy and is the only role trusted by Google WIF. +7. Set the verified ECR digest and the same `proof_name` in `smoke/terraform.tfvars`, initialize + `smoke/`, save and review its plan, then apply only that saved plan. The execution role can + pull/log; the separate task role has no AWS permission policy and is the only role trusted by + Google WIF. A changed name creates an independent disposable proof; it does not adopt or restore + a soft-deleted identity pool. 8. For an accepted candidate, run exactly one Fargate task using the Terraform outputs for cluster, task definition, subnet, and security group. Wait for terminal status and save `describe-tasks` plus the CloudWatch log stream outside the repository. Failed candidates are diff --git a/acceptance/cloud-portability/phase1b/smoke/main.tf b/acceptance/cloud-portability/phase1b/smoke/main.tf index d7bc31cd..55f018b8 100644 --- a/acceptance/cloud-portability/phase1b/smoke/main.tf +++ b/acceptance/cloud-portability/phase1b/smoke/main.tf @@ -1,11 +1,12 @@ data "aws_caller_identity" "current" {} locals { - name = "dander-phase1b" + name = var.proof_name + short_name = trimprefix(var.proof_name, "dander-") } resource "aws_cloudwatch_log_group" "probe" { - name = "/dander/phase1b" + name = "/dander/${local.short_name}" retention_in_days = 7 } @@ -92,7 +93,7 @@ resource "aws_security_group" "task" { } resource "google_iam_workload_identity_pool" "aws" { - workload_identity_pool_id = "dander-phase1b-aws" + workload_identity_pool_id = "${local.name}-aws" display_name = "Dander Phase 1B AWS" description = "Disposable keyless Fargate-to-Google identity proof" } @@ -115,7 +116,7 @@ resource "google_iam_workload_identity_pool_provider" "aws" { } resource "google_service_account" "probe" { - account_id = "dander-phase1b-aws" + account_id = "${local.name}-aws" display_name = "Dander Phase 1B AWS probe" description = "Disposable BigQuery reader impersonated by the isolated Fargate task" } diff --git a/acceptance/cloud-portability/phase1b/smoke/variables.tf b/acceptance/cloud-portability/phase1b/smoke/variables.tf index 16a24f29..b2f15653 100644 --- a/acceptance/cloud-portability/phase1b/smoke/variables.tf +++ b/acceptance/cloud-portability/phase1b/smoke/variables.tf @@ -9,6 +9,21 @@ variable "aws_region" { } } +variable "proof_name" { + description = "Unique deterministic name for one disposable Phase 1B smoke proof." + type = string + default = "dander-phase1b" + + validation { + condition = ( + length(var.proof_name) >= 6 && + length(var.proof_name) <= 26 && + can(regex("^[a-z][a-z0-9-]*[a-z0-9]$", var.proof_name)) + ) + error_message = "proof_name must be 6-26 lowercase letters, numbers, or hyphens, starting with a letter and ending with a letter or number." + } +} + variable "gcp_project_id" { description = "Disposable GCP project containing the bounded proof table." type = string diff --git a/docs/decisions.md b/docs/decisions.md index 02705d86..5bed019a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,14 @@ # Engineering Decisions +## 2026-08-09 — Phase 1B proof identities are repeatable after teardown + +- **Naming:** The disposable smoke root accepts one validated `proof_name`; its default preserves + the original AWS, log, Google Workload Identity, and service-account identities. +- **Reruns:** Operators select a new deterministic name when Google's soft-delete window prevents + immediate reuse of a destroyed Workload Identity Pool ID. +- **Boundary:** A new name creates an independent proof. Dander does not undelete, import, or adopt + remnants from an earlier proof automatically. + ## 2026-08-08 — Fargate operations bind to the manifest and controller - **Ownership:** The validated deployment and pipeline determine the exact state machine, schedule, From efe037a4744167f87605b60f6f82d72272562603 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 10:48:19 -0400 Subject: [PATCH 071/163] Fix state bucket permission preflight --- HANDOFF.md | 37 ++++++++-------- docs/decisions.md | 7 +++ src/dander/bootstrap/permissions.py | 46 +++++++++++++++++-- src/dander/cli/main.py | 1 + tests/bootstrap/test_permissions.py | 68 ++++++++++++++++++++++++++++- tests/cli/test_init_cli.py | 1 + 6 files changed, 136 insertions(+), 24 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 4b830026..f0adccbe 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,36 @@ ## Finished -- Made the disposable Phase 1B smoke proof name configurable while preserving its existing default. -- Derived AWS, CloudWatch, Google Workload Identity, and service-account names from one input. -- Documented safe reruns during Google's soft-delete retention window. +- Corrected stage-zero permission checks to evaluate bucket permissions on the state bucket. +- Preserved project, billing-account, and optional Workload Identity permission checks. +- Kept new-bucket creation plan-first by letting Terraform handle a not-yet-existing bucket. ## Try It -Set `proof_name = "dander-phase1b-r2"` in the smoke root's reviewed inputs and use the same value -when generating the external-account credential configuration. +Run `dander init-admin-plan` with an existing state bucket. Authorized operators now reach the +saved Terraform plan; missing existing-bucket permissions are reported explicitly. ## Checks -- Default and `dander-phase1b-r2` smoke plans each produced 18 creates; default resource names were - unchanged, custom names were isolated, and invalid names failed before planning. -- Ruff/format and strict mypy passed; 1,110 tests passed against PostgreSQL 15; dependency audit +- Ruff/format and strict mypy passed; 1,113 tests passed against PostgreSQL 15; dependency audit found no known vulnerabilities. -- Wheel/sdist inspection, source-free installs, runtime-all installation, generated Terraform, - GCP/AWS Terraform, Helm, and non-root/read-only container conformance passed. -- Retained stage-zero and platform plans each reported exactly `No changes.`; no apply ran. +- Wheel/sdist inspection, source-free and runtime-all installs, generated Terraform, GCP/AWS + Terraform, Helm, and non-root/read-only container conformance passed. +- Retained stage-zero and platform CLI plans each reported exactly `No changes.`; no apply ran. ## Decisions -- Keep `dander-phase1b` as the compatibility default. -- Use a new deterministic name rather than automatically undeleting or adopting prior resources. +- Test each permission on the GCP resource to which it applies. +- Treat a 404 state bucket as a planned new bucket, not a permission denial. ## Remaining -- Let protected CI repeat Linux security and container scans, then merge if clean. -- Correct the unrelated stage-zero permission preflight's bucket-level check in a separate PR. -- Regenerate the live proof plans from merged `main`. -- Apply only after explicit paid-action approval. +- Let protected CI repeat Linux tests, packaging, container, and security checks. +- Merge the focused PR through protected main if clean. +- Regenerate the disposable Phase 1B proof plans from merged `main`. ## Review First -- `acceptance/cloud-portability/phase1b/smoke/variables.tf` -- `acceptance/cloud-portability/phase1b/smoke/main.tf` -- `acceptance/cloud-portability/phase1b/README.md` +- `src/dander/bootstrap/permissions.py` +- `tests/bootstrap/test_permissions.py` +- `tests/cli/test_init_cli.py` diff --git a/docs/decisions.md b/docs/decisions.md index 5bed019a..5ef229e8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,12 @@ # Engineering Decisions +## 2026-08-09 — Permission checks follow the resource that owns the permission + +- **Project:** Stage zero tests project-scoped create and IAM permissions through Resource Manager. +- **Bucket:** An existing state bucket's get/update permissions are tested through Cloud Storage's + bucket-specific endpoint; a 404 leaves creation and its authoritative checks to Terraform. +- **Scope:** Billing-account and Workload Identity permission checks remain unchanged. + ## 2026-08-09 — Phase 1B proof identities are repeatable after teardown - **Naming:** The disposable smoke root accepts one validated `proof_name`; its default preserves diff --git a/src/dander/bootstrap/permissions.py b/src/dander/bootstrap/permissions.py index b53ceffe..75888aa3 100644 --- a/src/dander/bootstrap/permissions.py +++ b/src/dander/bootstrap/permissions.py @@ -6,6 +6,7 @@ import re import subprocess from typing import TYPE_CHECKING, Protocol +from urllib.parse import quote import httpx @@ -28,9 +29,8 @@ "serviceusage.services.enable", "serviceusage.services.get", "storage.buckets.create", - "storage.buckets.get", - "storage.buckets.update", ) +_BUCKET_PERMISSIONS = ("storage.buckets.get", "storage.buckets.update") _WIF_PERMISSIONS = ( "iam.workloadIdentityPools.create", "iam.workloadIdentityPools.get", @@ -64,10 +64,12 @@ def require_stage_zero_permissions( *, project: str, cwd: Path, + state_bucket: str = "", billing_account_id: str = "", github_repository: str = "", runner: _Runner | None = None, post: Callable[..., httpx.Response] = httpx.post, + get: Callable[..., httpx.Response] = httpx.get, ) -> None: """Fail before Terraform when the current identity lacks stage-zero permissions.""" if not _PROJECT_ID.fullmatch(project): @@ -100,6 +102,9 @@ def require_stage_zero_permissions( token=token, post=post, ) + missing_bucket = ( + _missing_bucket_permissions(state_bucket, token=token, get=get) if state_bucket else () + ) missing_billing: tuple[str, ...] = () if billing_account_id: missing_billing = _missing_permissions( @@ -109,12 +114,14 @@ def require_stage_zero_permissions( token=token, post=post, ) - if not missing_project and not missing_billing: + if not missing_project and not missing_bucket and not missing_billing: return details: list[str] = [] if missing_project: details.append(f"project: {', '.join(missing_project)}") + if missing_bucket: + details.append(f"state bucket: {', '.join(missing_bucket)}") if missing_billing: details.append(f"billing account: {', '.join(missing_billing)}") raise AdministrativeBootstrapError( @@ -148,6 +155,39 @@ def _missing_permissions( return tuple(permission for permission in required if permission not in granted) +def _missing_bucket_permissions( + bucket: str, + *, + token: str, + get: Callable[..., httpx.Response], +) -> tuple[str, ...]: + """Test permissions on an existing bucket; a missing bucket is created by Terraform.""" + url = ( + f"https://storage.googleapis.com/storage/v1/b/{quote(bucket, safe='')}/iam/testPermissions" + ) + try: + response = get( + url, + headers={"Authorization": f"Bearer {token}"}, + params=[("permissions", permission) for permission in _BUCKET_PERMISSIONS], + timeout=15.0, + ) + if response.status_code == 404: + return () + response.raise_for_status() + payload = response.json() + except (httpx.HTTPError, ValueError) as error: + raise AdministrativeBootstrapError( + "Could not complete the read-only state-bucket permission preflight" + ) from error + granted = payload.get("permissions") if isinstance(payload, dict) else None + if not isinstance(granted, list) or not all(isinstance(item, str) for item in granted): + raise AdministrativeBootstrapError( + "GCP returned an invalid state-bucket permission response" + ) + return tuple(permission for permission in _BUCKET_PERMISSIONS if permission not in granted) + + def _subprocess_runner( args: tuple[str, ...], *, diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 0720fee7..9c95bfb8 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -858,6 +858,7 @@ def init_admin_plan( require_stage_zero_permissions( project=project, cwd=infra_dir.resolve().parent.parent, + state_bucket=state_bucket, billing_account_id=billing_account_id, github_repository=github_repository, ) diff --git a/tests/bootstrap/test_permissions.py b/tests/bootstrap/test_permissions.py index 81cf1281..c2cbd4ca 100644 --- a/tests/bootstrap/test_permissions.py +++ b/tests/bootstrap/test_permissions.py @@ -4,7 +4,7 @@ import json import subprocess -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import httpx import pytest @@ -39,6 +39,15 @@ def _response(url: str, payload: dict[str, Any]) -> httpx.Response: return httpx.Response(200, json=payload, request=httpx.Request("POST", url)) +def _get_response(url: str, payload: dict[str, Any], *, status_code: int = 200) -> httpx.Response: + return httpx.Response(status_code, json=payload, request=httpx.Request("GET", url)) + + +def _allow_all_post(url: str, **kwargs: object) -> httpx.Response: + payload = json.loads(str(kwargs["content"])) + return _response(url, {"permissions": payload["permissions"]}) + + def test_preflight_checks_only_core_project_permissions_by_default(tmp_path: Path) -> None: requests: list[tuple[str, tuple[str, ...], str]] = [] @@ -107,3 +116,60 @@ def post(url: str, **kwargs: object) -> httpx.Response: runner=_Runner(), post=post, ) + + +def test_preflight_checks_existing_state_bucket_on_the_bucket_resource(tmp_path: Path) -> None: + requested: list[tuple[str, tuple[tuple[str, str], ...]]] = [] + + def get(url: str, **kwargs: object) -> httpx.Response: + params = tuple(cast("list[tuple[str, str]]", kwargs["params"])) + requested.append((url, params)) + return _get_response(url, {"permissions": [value for _key, value in params]}) + + require_stage_zero_permissions( + project="unit-project", + cwd=tmp_path, + state_bucket="unit-state", + runner=_Runner(), + post=_allow_all_post, + get=get, + ) + + assert requested == [ + ( + "https://storage.googleapis.com/storage/v1/b/unit-state/iam/testPermissions", + ( + ("permissions", "storage.buckets.get"), + ("permissions", "storage.buckets.update"), + ), + ) + ] + + +def test_preflight_allows_terraform_to_create_a_missing_state_bucket(tmp_path: Path) -> None: + require_stage_zero_permissions( + project="unit-project", + cwd=tmp_path, + state_bucket="new-state", + runner=_Runner(), + post=_allow_all_post, + get=lambda url, **_kwargs: _get_response(url, {}, status_code=404), + ) + + +def test_preflight_reports_exact_missing_existing_bucket_permission(tmp_path: Path) -> None: + with pytest.raises( + AdministrativeBootstrapError, + match=r"missing state bucket: storage\.buckets\.update", + ): + require_stage_zero_permissions( + project="unit-project", + cwd=tmp_path, + state_bucket="existing-state", + runner=_Runner(), + post=_allow_all_post, + get=lambda url, **_kwargs: _get_response( + url, + {"permissions": ["storage.buckets.get"]}, + ), + ) diff --git a/tests/cli/test_init_cli.py b/tests/cli/test_init_cli.py index 5371b613..4b31e529 100644 --- a/tests/cli/test_init_cli.py +++ b/tests/cli/test_init_cli.py @@ -340,6 +340,7 @@ def test_admin_plan_runs_read_only_permission_preflight_before_terraform( def fake_preflight(**kwargs: object) -> None: events.append("preflight") assert kwargs["project"] == "unit-project" + assert kwargs["state_bucket"] == "unit-state" def fake_execute(self: object, **kwargs: object) -> Path: events.append("terraform") From 4529d5d81bb90388cd747baf0abfb772f130053c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 12:12:20 -0400 Subject: [PATCH 072/163] Prepare Dander 0.8.0rc1 --- CHANGELOG.md | 11 ++++++ HANDOFF.md | 39 ++++++++++--------- README.md | 4 +- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/known-limitations.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 7 ++-- docs/upgrading.md | 2 +- pyproject.toml | 2 +- scripts/check_release_metadata.py | 5 +++ tests/test_release_metadata.py | 5 ++- uv.lock | 2 +- 13 files changed, 52 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49423cfb..27ed41ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc1 — 2026-08-09 (beta) + +### Candidate + +- Package the merged cloud-portability foundation, including the keyless Fargate-to-Google + identity adapter, for isolated Phase 1B acceptance. Fargate, Snowflake, Redshift, Glue, and + Kubernetes remain experimental until their applicable live gates pass. +- Reject the first Phase 1B image built from public Dander `0.7.0`: that immutable wheel predates + the merged `dander.identity` package, so the proof exited before credential exchange. The task, + smoke stack, ECR repository, and rejected GAR image were removed before this candidate. + ### Added - Publish exact write-mode, transport, transform, graph, fencing, logical-type, decimal, and diff --git a/HANDOFF.md b/HANDOFF.md index f0adccbe..edf0fe2e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,39 @@ ## Finished -- Corrected stage-zero permission checks to evaluate bucket permissions on the state bucket. -- Preserved project, billing-account, and optional Workload Identity permission checks. -- Kept new-bucket creation plan-first by letting Terraform handle a not-yet-existing bucket. +- Prepared metadata-only Dander `0.8.0rc1` from current protected main. +- Recorded the rejected public-`0.7.0` Phase 1B image and complete teardown. +- Updated release-facing version references, lockfile, and generated-project assertions. +- Corrected the retained runtime record to Dander `0.7.1` with the stable connector pins. ## Try It -Run `dander init-admin-plan` with an existing state bucket. Authorized operators now reach the -saved Terraform plan; missing existing-bucket permissions are reported explicitly. +After explicit publication approval, tag the merged commit `v0.8.0rc1`, publish through the +protected workflow, and install it outside the checkout before rebuilding Phase 1B. ## Checks -- Ruff/format and strict mypy passed; 1,113 tests passed against PostgreSQL 15; dependency audit - found no known vulnerabilities. -- Wheel/sdist inspection, source-free and runtime-all installs, generated Terraform, GCP/AWS - Terraform, Helm, and non-root/read-only container conformance passed. -- Retained stage-zero and platform CLI plans each reported exactly `No changes.`; no apply ran. +- Ruff, format, strict mypy, dependency audit, and 1,113 tests passed against PostgreSQL 15. +- Wheel/sdist inspection, isolated source-free and runtime-all installs, generated-project + validation, Terraform, Helm, and non-root/read-only container conformance passed. +- Retained stage zero and the unchanged `main` runtime/infra planner each reported exactly + `No changes.` The candidate planner proposed only five expected `dander_version` label updates; + no plan was applied. ## Decisions -- Test each permission on the GCP resource to which it applies. -- Treat a 404 state bucket as a planned new bucket, not a permission denial. +- Use the next minor candidate because public `0.7.0` is immutable and current main adds features. +- Keep this PR metadata-only; the accepted candidate must contain the already-reviewed main code. +- Treat the candidate's planner-version labels as release metadata, not live infrastructure drift. ## Remaining -- Let protected CI repeat Linux tests, packaging, container, and security checks. -- Merge the focused PR through protected main if clean. -- Regenerate the disposable Phase 1B proof plans from merged `main`. +- Let protected CI repeat Linux packaging, container, configuration, and secret scans. +- Merge through protected main, then obtain explicit tag/PyPI publication approval. +- Rebuild Phase 1B once from the public candidate using a fresh proof identity. ## Review First -- `src/dander/bootstrap/permissions.py` -- `tests/bootstrap/test_permissions.py` -- `tests/cli/test_init_cli.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `acceptance/cloud-portability/phase1b/README.md` diff --git a/README.md b/README.md index 05ce6e89..e31fa5b8 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.7.0 +uv tool install dander-platform==0.8.0rc1 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.7.0` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc1` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index eae28a58..ebacb7f6 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.7.0`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc1`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 7e6bd0e7..87def61a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.7.0 +uv tool install dander-platform==0.8.0rc1 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/known-limitations.md b/docs/known-limitations.md index c2098422..f017a0a8 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -1,6 +1,6 @@ # Known limitations -Dander `0.7.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; +Dander `0.8.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. - AWS ECS/Fargate is not yet a supported Dander launcher. Packaged stage-zero and platform roots diff --git a/docs/release-audit.md b/docs/release-audit.md index 7ce385ef..f79fe1ce 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.7.0`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc1`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 853fe31c..d554b08d 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.7.0` is the current public beta. +- Dander `0.8.0rc1` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, @@ -15,9 +15,8 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and - Project `dander-proof-harrison-20260801`, region `us-central1`, remote states `dander/state` and `dander/bootstrap-admin/state` in `dander-proof-harrison-20260801-dander-state`. -- The source-free runtime is pinned to Dander `0.5.0` with Salesforce and ServiceNow plugins - `0.2.0`, at immutable digest `sha256:3220623bf82a81d625db9e611c305694204e25ae312c06a8e8b1ea883bfd8995`. - Dander `0.5.1` does not require a runtime rollout because its only change is catalog metadata. +- The source-free runtime is pinned to Dander `0.7.1`, Salesforce `0.3.1`, and ServiceNow `0.2.2` + at immutable digest `sha256:68e112c43b365018b735be7934446e15dfe6169fc64062b62b8bb97ea4f93b96`. - Greenhouse, HubSpot, Salesforce, and ServiceNow are enabled daily at 09:00, 10:00, 11:00, and 12:00 America/New_York. The executable Greenhouse graph remains paused at 13:00. - The simulation-only managed cost guard, alerts, secrets, datasets, cursors, leases, and retained diff --git a/docs/upgrading.md b/docs/upgrading.md index 103b0da2..b59035d3 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.7.0" +export DANDER_TARGET_VERSION="0.8.0rc1" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 0604a67a..fe1f257d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.7.0" +version = "0.8.0rc1" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/scripts/check_release_metadata.py b/scripts/check_release_metadata.py index 4ec9f835..86b7a311 100644 --- a/scripts/check_release_metadata.py +++ b/scripts/check_release_metadata.py @@ -51,6 +51,11 @@ class VersionReference: "release audit public release", re.compile(r"Public Dander beta: `(?P[^`]+)`"), ), + VersionReference( + "acceptance/cloud-portability/phase1b/README.md", + "Phase 1B prerequisite", + re.compile(r"Dander `(?P[^`]+)`, Docker Buildx"), + ), ) diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index af02814b..d7ad7a02 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -10,6 +10,7 @@ ROOT_FILES = ( "CHANGELOG.md", "README.md", + "acceptance/cloud-portability/phase1b/README.md", "docs/getting-started.md", "docs/known-limitations.md", "docs/release-audit.md", @@ -36,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.7.0", + "dander-platform==0.8.0rc1", "dander-platform==0.1.0", 1, ), @@ -44,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.7.0" + "README install command uses 0.1.0; package metadata uses 0.8.0rc1" ] diff --git a/uv.lock b/uv.lock index 53fb31fd..33b8af64 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.7.0" +version = "0.8.0rc1" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 3fb0826b4a3c68253b162aeb8d13f32155ebcf7f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 14:56:56 -0400 Subject: [PATCH 073/163] Record successful Phase 1B portability proof --- CHANGELOG.md | 12 ++++ HANDOFF.md | 42 ++++++------- .../cloud-portability/phase1b/README.md | 7 ++- docs/cloud-portability-phase1-acceptance.md | 5 +- docs/cloud-portability-phase1b-acceptance.md | 60 +++++++++++++++++++ docs/decisions.md | 10 ++++ docs/release-audit.md | 5 ++ docs/session-resume.md | 5 ++ .../scan_long_lived_credentials.py | 23 ++++++- tests/portability/test_phase1b_tools.py | 18 ++++++ 10 files changed, 162 insertions(+), 25 deletions(-) create mode 100644 docs/cloud-portability-phase1b-acceptance.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 27ed41ff..d8245a5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ capabilities enter through the next minor release. ## Unreleased +### Acceptance + +- Record the successful public-`0.8.0rc1` Phase 1B proof: identical GAR/ECR multi-platform + content, Cloud Run/AMD64 conformance, ARM64 Fargate-to-BigQuery access before and after keyless + credential refresh, complete proof teardown, and final isolated-platform no-drift. + +### Fixed + +- Make the Phase 1B credential scanner recognize only the exact content hashes of three public + boto3/botocore example documents containing published placeholder keys. Changed content and all + other files continue to fail closed. + ## 0.8.0rc1 — 2026-08-09 (beta) ### Candidate diff --git a/HANDOFF.md b/HANDOFF.md index edf0fe2e..e9e5717b 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Prepared metadata-only Dander `0.8.0rc1` from current protected main. -- Recorded the rejected public-`0.7.0` Phase 1B image and complete teardown. -- Updated release-facing version references, lockfile, and generated-project assertions. -- Corrected the retained runtime record to Dander `0.7.1` with the stable connector pins. +- Published public Dander `0.8.0rc1` from protected main. +- Passed Phase 1B with byte-identical GAR/ECR AMD64 and ARM64 image content. +- Proved one ARM64 Fargate task queried BigQuery before and after keyless credential refresh. +- Proved the same OCI index on Cloud Run AMD64 and removed every temporary proof resource. +- Added the sanitized acceptance record and exact-hash public-fixture handling. ## Try It -After explicit publication approval, tag the merged commit `v0.8.0rc1`, publish through the -protected workflow, and install it outside the checkout before rebuilding Phase 1B. +Review `docs/cloud-portability-phase1b-acceptance.md`; run +`uv run pytest tests/portability/test_phase1b_tools.py -q` for focused tooling checks. ## Checks -- Ruff, format, strict mypy, dependency audit, and 1,113 tests passed against PostgreSQL 15. -- Wheel/sdist inspection, isolated source-free and runtime-all installs, generated-project - validation, Terraform, Helm, and non-root/read-only container conformance passed. -- Retained stage zero and the unchanged `main` runtime/infra planner each reported exactly - `No changes.` The candidate planner proposed only five expected `dander_version` label updates; - no plan was applied. +- Ruff, format, strict mypy, dependency audit, release metadata, and 1,114 tests passed against PostgreSQL 15. +- Public package install, source-free generation, validation, Terraform validation, and image build passed. +- Fargate exited 0 after two 17-row queries and observed a later credential expiry; Cloud Run exited 0. +- Credential scans passed; Docker Scout found zero application-layer or fixable high/critical findings. +- All proof destroy plans applied; the isolated GCP platform reported exactly `No changes.` ## Decisions -- Use the next minor candidate because public `0.7.0` is immutable and current main adds features. -- Keep this PR metadata-only; the accepted candidate must contain the already-reviewed main code. -- Treat the candidate's planner-version labels as release metadata, not live infrastructure drift. +- Phase 1B is feasibility evidence, not a Fargate support promotion. +- Exact public boto fixture contents are recognized by hash; modified content fails scanning. +- Unfixed base-image advisories are recorded, not hidden or represented as application findings. ## Remaining -- Let protected CI repeat Linux packaging, container, configuration, and secret scans. -- Merge through protected main, then obtain explicit tag/PyPI publication approval. -- Rebuild Phase 1B once from the public candidate using a fresh proof identity. +- Let protected CI repeat Linux tests, packaging, container, and secret checks. +- Merge this evidence-only PR through protected main after review. +- Continue the roadmap's next uncompleted live qualification gate; do not publish another candidate automatically. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `acceptance/cloud-portability/phase1b/README.md` +- `docs/cloud-portability-phase1b-acceptance.md` +- `scripts/portability/scan_long_lived_credentials.py` +- `tests/portability/test_phase1b_tools.py` diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index ebacb7f6..3674fa1a 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -86,7 +86,9 @@ record content. The retained Dander project is out of scope. expiry must be later than the first. No event may contain a token, URL, query body, or record. 10. Run `scan_long_lived_credentials.py` over the generated config, both Terraform states, task definition/description, exported logs, and an extracted image filesystem. Run the normal image - vulnerability and secret scans as well. + vulnerability and secret scans as well. The scanner recognizes only the exact SHA-256 content + of three public boto3/botocore example documents that contain published placeholder keys; any + dependency or content change is scanned normally. 11. Run the same GAR index in the isolated Cloud Run proof job and confirm its selected `amd64` manifest matches the recorded index. Restore the isolated project through its reviewed manifest-aware plan and require `No changes.` @@ -101,3 +103,6 @@ record content. The retained Dander project is out of scope. - No Google service-account key, AWS access key, or secret value appears in inputs, state, task definition, logs, or image. - Both proof roots are destroyed and the isolated GCP platform finishes with no Terraform drift. + +The accepted public-candidate result is recorded in +[`docs/cloud-portability-phase1b-acceptance.md`](../../../docs/cloud-portability-phase1b-acceptance.md). diff --git a/docs/cloud-portability-phase1-acceptance.md b/docs/cloud-portability-phase1-acceptance.md index 51a383af..3bd63b58 100644 --- a/docs/cloud-portability-phase1-acceptance.md +++ b/docs/cloud-portability-phase1-acceptance.md @@ -50,5 +50,6 @@ project was not changed. - No credential, provider response, source row, Terraform state, or alert address is retained in this record. -Phase 1 is accepted. Phase 1B remains a separate artifact-copy and keyless AWS-to-BigQuery proof; -this record makes no AWS support claim. +Phase 1 is accepted. The separate [Phase 1B artifact-copy and keyless AWS-to-BigQuery +proof](cloud-portability-phase1b-acceptance.md) also passed; neither record makes an AWS support +claim. diff --git a/docs/cloud-portability-phase1b-acceptance.md b/docs/cloud-portability-phase1b-acceptance.md new file mode 100644 index 00000000..679a1a7d --- /dev/null +++ b/docs/cloud-portability-phase1b-acceptance.md @@ -0,0 +1,60 @@ +# Cloud Portability Phase 1B Acceptance + +Accepted on 2026-08-09 against public `dander-platform==0.8.0rc1`, published from commit +`5614f3479f4535d5ab3acfaa8b579e23cce403fe`. This record is feasibility evidence for one +Fargate-to-BigQuery composition; it does not promote Fargate or any non-GCP profile to supported. + +## Artifact identity + +- A clean installation outside the repository generated a source-free project with no `src/` + directory and a Dockerfile pinned to `0.8.0rc1`. +- One Buildx invocation published a Linux/AMD64 and Linux/ARM64 OCI index to staging GAR with SBOM + and provenance attestations. The index digest was + `sha256:35bfa11cd0ea51cc9c562b6c76567e2bade56114bfb4ff6768e9c5d4a052bfad`. +- The runnable manifests were AMD64 + `sha256:b5ed7f229a7e8c75a717bdd88b64ee11e7c37d77675ad23e2e79a9a29f6a0311` and ARM64 + `sha256:d4b46e03d286a03a59524ef50f333999987c84d041dea0245f9f67db3805fecc`. +- `dander image-promote-aws` copied the accepted index to ECR without rebuilding it. Independent + `crane` inspection confirmed the same index and both runnable manifests after the copy. + +## Live execution + +- A reviewed Terraform plan contained exactly 18 creates, zero changes, and zero destroys. It + created one disposable network, ECS cluster/task definition, separate task and execution roles, + log group, Google service account, AWS Workload Identity provider, and the minimum BigQuery IAM. +- The first apply attempt exposed a local credential-selection error: Terraform used an older ADC + identity instead of the active project owner. It created 11 AWS shell resources and no GCP + identity. Those 11 resources were removed through a reviewed destroy plan before the accepted + plan was recreated with the owner's short-lived access token. +- The accepted task definition selected ARM64, the immutable ECR index, a read-only root + filesystem, and no task-definition secrets. The task role had zero attached and zero inline AWS + policies; only the distinct execution role could pull the image and write logs. +- Exactly one Fargate task ran. It exited `0`, reported `aarch64`, and counted 17 rows in the + disposable `raw.salesforce_accounts` table twice. Its Google credential expiry advanced from + `2026-08-09T18:30:13Z` to `2026-08-09T18:40:28Z`, followed by + `credential.refresh_observed`. +- One separately planned Cloud Run conformance job executed the same GAR index. Cloud Run selected + the recorded AMD64 manifest, reported `x86_64`, completed runtime conformance, and exited `0`. + +## Security and cleanup + +- The generated external-account configuration, Dockerfile, artifact records, Terraform state, + task description, and CloudWatch events contained no long-lived AWS or Google credential. +- Both extracted platform filesystems passed the credential scan after recognizing three exact + SHA-256-pinned public boto3/botocore example files. Any content change removes that exception and + restores normal detection; no credential value is allowlisted. +- Docker Scout reported zero application-layer and zero fixable high/critical findings on both + architectures. It reported two critical and two high, unfixed `perl-base` advisories inherited + from `python:3.12-slim`; no fixed base image was available during acceptance. +- Reviewed destroy plans removed all 18 smoke resources, the one Cloud Run conformance job, and + both ECR resources. The proof GAR tag and digest were deleted. ECS reports the expected inactive + cluster tombstone with zero tasks/services, and the disposable Google pool is soft-deleted. +- The pre-existing isolated GCP platform finished with exact `No changes.` using its deployed + stable Dander `0.7.0` configuration and image. The retained Dander project was not touched. + +## Result + +Phase 1B passed: one source-free multi-platform image retained identical registry content, ran on +Cloud Run and Fargate, and reached BigQuery from AWS before and after keyless credential refresh. +Fargate remains experimental until the broader correctness, lifecycle, scale, and operating gates +in the portability roadmap pass. diff --git a/docs/decisions.md b/docs/decisions.md index 5ef229e8..a260865f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-09 — Phase 1B accepts only exact public dependency fixtures + +- **Evidence:** The accepted image contains three boto3/botocore example files with AWS-published + placeholder keys; neither architecture contains a cloud credential. +- **Control:** The proof scanner recognizes only those files' exact SHA-256 content. A dependency + update or modified file loses the exception and is scanned normally. +- **Boundary:** This changes acceptance evidence tooling only. It does not alter the published + `0.8.0rc1` runtime or weaken scanning of configuration, state, task output, logs, or other image + content. + ## 2026-08-09 — Permission checks follow the resource that owns the permission - **Project:** Stage zero tests project-scoped create and IAM permissions through Resource Manager. diff --git a/docs/release-audit.md b/docs/release-audit.md index f79fe1ce..d46fbf21 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -31,6 +31,11 @@ cloud path remains outside the live proof. replayed without duplicate keys or cursor regression, skipped an overlapping run, recovered from a bounded interruption, released its lease, removed staging, and finished with Terraform no-drift. See `docs/cloud-portability-phase1-acceptance.md`. +- Public Dander `0.8.0rc1` passed the separate Phase 1B feasibility gate: one source-free + multi-platform index remained byte-identical across GAR and ECR, completed Cloud Run + conformance on AMD64, and queried BigQuery from one ARM64 Fargate task before and after keyless + Google credential refresh. All proof resources were removed, and Fargate remains experimental. + See `docs/cloud-portability-phase1b-acceptance.md`. - Retained source-free Dander image: `sha256:3220623…8995`, built with Dander `0.5.0` and both plugin `0.2.0` packages. Dander `0.5.1` changes only catalog recommendations, so the runtime image was intentionally not replaced. diff --git a/docs/session-resume.md b/docs/session-resume.md index d554b08d..8566e4ce 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -33,6 +33,11 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Latest operating evidence +- On 2026-08-09, public Dander `0.8.0rc1` passed the isolated Phase 1B feasibility gate. One + source-free multi-platform OCI index retained identical GAR/ECR and per-platform digests; + Cloud Run completed on AMD64 and one ARM64 Fargate task queried BigQuery before and after a + keyless Google credential refresh. All proof resources were destroyed and the isolated GCP + platform finished at no drift. See `docs/cloud-portability-phase1b-acceptance.md`. - On 2026-08-07, the isolated source-free `0.7.0rc2` image completed local/Cloud Run OCI parity, four-endpoint Salesforce ingestion, governed transforms/tests, replay, overlap skip, SIGTERM/SIGKILL recovery, Dataplex publication, lease and staging cleanup, and a final no-drift diff --git a/scripts/portability/scan_long_lived_credentials.py b/scripts/portability/scan_long_lived_credentials.py index 2aa760ec..5aa74f46 100644 --- a/scripts/portability/scan_long_lived_credentials.py +++ b/scripts/portability/scan_long_lived_credentials.py @@ -3,9 +3,14 @@ from __future__ import annotations import argparse +import hashlib import json import re from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Set _PATTERNS = { "aws_static_access_key": re.compile(rb"\bAKIA[0-9A-Z]{16}\b"), @@ -21,8 +26,22 @@ ), } +# Public boto3/botocore example documents contain AWS's published placeholder keys. Accept only +# these exact dependency-file contents; a dependency update or modified file is scanned normally. +_PUBLIC_FIXTURE_SHA256 = frozenset( + { + "2beb01599c682e30010991eba8066ce7b71879fc0f9838abd37a66e8f13f9a1e", + "4f912aac51590625652fd76c37e4f90e78a05355273125df555c012b4d005ab5", + "c83fc27073767fdb7d3e5190e4dcce25a09871c7b118fa289db056d93e0e31c9", + } +) + -def scan(paths: list[Path]) -> list[dict[str, str]]: +def scan( + paths: list[Path], + *, + public_fixture_sha256: Set[str] = _PUBLIC_FIXTURE_SHA256, +) -> list[dict[str, str]]: """Return sanitized path/pattern findings without returning matched material.""" findings: list[dict[str, str]] = [] files: list[Path] = [] @@ -37,6 +56,8 @@ def scan(paths: list[Path]) -> list[dict[str, str]]: except OSError: findings.append({"path": str(path), "pattern": "unreadable"}) continue + if hashlib.sha256(content).hexdigest() in public_fixture_sha256: + continue for name, pattern in _PATTERNS.items(): if pattern.search(content): findings.append({"path": str(path), "pattern": name}) diff --git a/tests/portability/test_phase1b_tools.py b/tests/portability/test_phase1b_tools.py index 6ed2c78f..f4fe19d4 100644 --- a/tests/portability/test_phase1b_tools.py +++ b/tests/portability/test_phase1b_tools.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING @@ -323,3 +324,20 @@ def test_long_lived_credential_scan_reports_only_path_and_pattern(tmp_path: Path {"path": str(unsafe), "pattern": "gcp_private_key"}, ] assert "redacted" not in json.dumps(findings) + + +def test_long_lived_credential_scan_accepts_only_exact_public_fixture_content( + tmp_path: Path, +) -> None: + fixture = tmp_path / "public-example.txt" + placeholder_key = "AKIA" + "A" * 16 + content = f"public vendor example: {placeholder_key}\n" + fixture.write_text(content, encoding="utf-8") + allowed_hash = hashlib.sha256(content.encode()).hexdigest() + + assert scan([fixture], public_fixture_sha256={allowed_hash}) == [] + + fixture.write_text(content + "changed\n", encoding="utf-8") + assert scan([fixture], public_fixture_sha256={allowed_hash}) == [ + {"path": str(fixture), "pattern": "aws_static_access_key"} + ] From 75e5d3d02a0cee159cf7bcc50f3f6ceabe4d9045 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 15:34:23 -0400 Subject: [PATCH 074/163] Renew Fargate Google workload credentials --- HANDOFF.md | 42 ++--- docs/decisions.md | 12 ++ docs/execution-projection.md | 12 +- docs/known-limitations.md | 7 +- infra/aws/modules/fargate/variables.tf | 2 +- src/dander/catalog/dataplex.py | 6 +- src/dander/catalog/store.py | 7 +- src/dander/cli/runtime_command.py | 5 +- src/dander/deployment/projection.py | 2 +- src/dander/identity/__init__.py | 4 + src/dander/identity/aws_google.py | 207 +++++++++++++++++------ src/dander/providers/bigquery/fence.py | 6 +- src/dander/providers/bigquery/graph.py | 6 +- src/dander/providers/bigquery/state.py | 5 +- src/dander/security/secret_manager.py | 4 +- src/dander/state/lease.py | 6 +- src/dander/state/run_history.py | 7 +- src/dander/state/watermark.py | 6 +- src/dander/transform/runner.py | 6 +- src/dander/writer/bigquery.py | 21 ++- src/dander/writer/storage_write.py | 10 +- tests/cli/test_runtime_cli.py | 9 +- tests/identity/test_aws_google.py | 98 +++++++++-- tests/infra/test_fargate_runtime.py | 2 + tests/providers/test_launcher_runtime.py | 21 ++- 25 files changed, 386 insertions(+), 127 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index e9e5717b..4aa2a6f9 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Published public Dander `0.8.0rc1` from protected main. -- Passed Phase 1B with byte-identical GAR/ECR AMD64 and ARM64 image content. -- Proved one ARM64 Fargate task queried BigQuery before and after keyless credential refresh. -- Proved the same OCI index on Cloud Run AMD64 and removed every temporary proof resource. -- Added the sanitized acceptance record and exact-hash public-fixture handling. +- Replaced Fargate's one-time ECS credential copy with a renewable Google Auth supplier. +- Scoped cross-cloud credentials to one OCI runtime invocation and explicit GCP client creation. +- Removed task secrets from global environment/file persistence in normal runtime execution. +- Raised the validated Fargate deadline from one hour to 24 hours. +- Kept Cloud Run/local ADC behavior and the public Fargate support boundary unchanged. ## Try It -Review `docs/cloud-portability-phase1b-acceptance.md`; run -`uv run pytest tests/portability/test_phase1b_tools.py -q` for focused tooling checks. +Run `uv run pytest tests/identity/test_aws_google.py tests/providers/test_launcher_runtime.py -q`. ## Checks -- Ruff, format, strict mypy, dependency audit, release metadata, and 1,114 tests passed against PostgreSQL 15. -- Public package install, source-free generation, validation, Terraform validation, and image build passed. -- Fargate exited 0 after two 17-row queries and observed a later credential expiry; Cloud Run exited 0. -- Credential scans passed; Docker Scout found zero application-layer or fixable high/critical findings. -- All proof destroy plans applied; the isolated GCP platform reported exactly `No changes.` +- All 1,117 tests passed against PostgreSQL 15. +- Ruff, formatting, and strict mypy passed across `src` and `tests`. +- AWS root/module Terraform formatting, validation, and the Fargate module test passed. +- Wheel/sdist inspection and a source-free wheel installation outside the checkout passed. +- The full-runtime container passed read-only conformance; Docker Scout found no fixable high or critical vulnerabilities. ## Decisions -- Phase 1B is feasibility evidence, not a Fargate support promotion. -- Exact public boto fixture contents are recognized by hash; modified content fails scanning. -- Unfixed base-image advisories are recorded, not hidden or represented as application findings. +- Google Auth refetches only the validated ECS relative-credential endpoint on each subject-token refresh. +- Existing Google clients receive the scoped credential explicitly; non-Fargate ADC remains unchanged. +- Renewable identity is a prerequisite, not a Fargate support promotion. ## Remaining -- Let protected CI repeat Linux tests, packaging, container, and secret checks. -- Merge this evidence-only PR through protected main after review. -- Continue the roadmap's next uncompleted live qualification gate; do not publish another candidate automatically. +- Let protected CI repeat Linux packaging, Terraform, container, and secret checks. +- Merge the focused PR through protected main after review. +- Publish the next candidate before a source-free Fargate pipeline acceptance run. +- Complete replay, interruption, schedule, alert, rollback, and no-drift acceptance. ## Review First -- `docs/cloud-portability-phase1b-acceptance.md` -- `scripts/portability/scan_long_lived_credentials.py` -- `tests/portability/test_phase1b_tools.py` +- `src/dander/identity/aws_google.py` +- `src/dander/cli/runtime_command.py` +- `tests/identity/test_aws_google.py` diff --git a/docs/decisions.md b/docs/decisions.md index a260865f..e0b660cb 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -960,3 +960,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Scope:** The declaration reports implemented behavior only. It does not promote experimental providers, add write modes, introduce semi-structured fallbacks, or widen the supported profile manifest. + +## 2026-08-09 — Fargate-to-Google identity renews at the client boundary + +- **Identity:** Google Auth receives a process-local AWS credential supplier that refetches the + current temporary task-role session from Fargate's fixed link-local endpoint for every subject + token refresh. Dander no longer copies task secrets into global environment variables or writes + an external-account file for normal runtime execution. +- **Boundary:** The credential is scoped to one OCI runtime invocation and passed explicitly only + when existing Google clients are constructed. Cloud Run and local execution continue to use + normal Application Default Credentials without changed resource or IAM behavior. +- **Support:** Renewable identity removes the one-hour credential limit and permits an enforced + 24-hour launcher deadline, but does not promote Fargate before the complete live lifecycle gate. diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 4c4ba85c..0c65e9d2 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -31,13 +31,13 @@ normalized by the OCI runtime contract. The Fargate provider can build the same BigQuery data-plane intent with an immutable ECR image, AWS task-role identity, `awsvpc` placement, CloudWatch destinations, and explicit Fargate resource limits. At runtime, Dander accepts only temporary credentials obtained from the fixed ECS task-role -endpoint, then prepares a non-secret Google external-account configuration before any provider -client is constructed. The Google impersonated token is limited to 600 seconds. Because this first -runtime bridge does not yet renew the copied ECS session, Fargate projections are limited to one -hour. +endpoint. A process-local Google credential supplier fetches the current ECS session whenever +Google Auth refreshes its signed AWS subject token; task secrets are neither copied into global +environment variables nor written to disk. The impersonated Google token remains limited to 600 +seconds, while the launcher may enforce a deadline of up to 24 hours. -This provider projection is an internal construction gate: provisioning, controller lifecycle, -renewable credential supply, and live acceptance must pass before Fargate is a supported launcher. +This provider projection is an internal construction gate: complete pipeline lifecycle and live +acceptance must still pass before Fargate is a supported launcher. The Kubernetes provider projects a selected named profile into a packaged Helm chart for an existing Kubernetes 1.27+ cluster. Its immutable template uses stdout, operator-owned Secret key diff --git a/docs/known-limitations.md b/docs/known-limitations.md index f017a0a8..18167c43 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -6,9 +6,10 @@ evaluate these limits before using it for an unattended system containing busine - AWS ECS/Fargate is not yet a supported Dander launcher. Packaged stage-zero and platform roots now have public saved-plan/apply commands, and accepted OCI artifacts can be copied into ECR without rebuilding. Manifest-bound status, logs, cancellation, replay, and deployment - verification are implemented but not live-qualified. The runtime bridge copies one temporary ECS - task-role session into process memory and therefore caps a Fargate projection at one hour; - renewable credentials and live parity remain later gates. + verification are implemented but not live-qualified. The runtime bridge now renews its AWS + task-role input from the fixed ECS endpoint for each Google federation refresh without persisting + task credentials. Complete pipeline parity, interruption, replay, schedule, alert, rollback, and + no-drift acceptance remain later gates. - Provider package extras and the full runtime image do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar SCD1 adapter with bounded Parquet upload, temporary remote staging, schema checks, destination fencing, and fenced diff --git a/infra/aws/modules/fargate/variables.tf b/infra/aws/modules/fargate/variables.tf index 9b19ea37..cfd1b0b1 100644 --- a/infra/aws/modules/fargate/variables.tf +++ b/infra/aws/modules/fargate/variables.tf @@ -82,7 +82,7 @@ variable "execution_projections" { projection.resources.runtime_retry_count == 0 && contains([1000, 2000, 4000, 8000, 16000], projection.resources.cpu_millis) && projection.resources.deadline_seconds >= 1 && - projection.resources.deadline_seconds <= 3600 && + projection.resources.deadline_seconds <= 86400 && projection.resources.launcher_retry_count >= 0 && projection.resources.launcher_retry_count <= 10 && projection.resources.ephemeral_storage_mib >= 20480 && diff --git a/src/dander/catalog/dataplex.py b/src/dander/catalog/dataplex.py index 2317a532..ea60fb2b 100644 --- a/src/dander/catalog/dataplex.py +++ b/src/dander/catalog/dataplex.py @@ -11,6 +11,7 @@ from google.cloud import dataplex_v1 from dander.catalog.publisher import CatalogPublishError +from dander.identity import google_client_options if TYPE_CHECKING: from dander.catalog.spine import CatalogAsset @@ -104,7 +105,10 @@ def __init__( raise CatalogPublishError("Invalid Dataplex location") self._project = project self._location = location - self._client = client or cast("_CatalogClient", dataplex_v1.CatalogServiceClient()) + self._client = client or cast( + "_CatalogClient", + dataplex_v1.CatalogServiceClient(**google_client_options()), + ) self._generator = generator or DataplexAspectGenerator() def request_for(self, asset: CatalogAsset) -> dataplex_v1.ModifyEntryRequest: diff --git a/src/dander/catalog/store.py b/src/dander/catalog/store.py index 81149046..369508b5 100644 --- a/src/dander/catalog/store.py +++ b/src/dander/catalog/store.py @@ -11,6 +11,8 @@ from google.cloud import bigquery +from dander.identity import google_client_options + if TYPE_CHECKING: from collections.abc import Iterable from pathlib import Path @@ -73,7 +75,10 @@ def __init__(self, *, project: str, dataset: str, client: _Client | None = None) if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", dataset): raise ValueError(f"Invalid BigQuery dataset: {dataset!r}") self._table = f"{project}.{dataset}._dander_catalog" - self._client = client or cast("_Client", bigquery.Client(project=project)) + self._client = client or cast( + "_Client", + bigquery.Client(project=project, **google_client_options()), + ) self._ready = False def migrate(self) -> None: diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index 253fb42f..d68db073 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -13,7 +13,7 @@ from dander.cli.run_command import RunOptions, execute_run from dander.compatibility import CompatibilityError, load_runtime_compatibility -from dander.identity import prepare_launcher_identity +from dander.identity import launcher_identity from dander.runtime_contract import ( RUNTIME_CONTRACT, LauncherContext, @@ -144,8 +144,7 @@ def execute_runtime( dataplex_location="us", ) try: - with graceful_signal_handlers(): - prepare_launcher_identity(context) + with graceful_signal_handlers(), launcher_identity(context): result = execute_run( options, console=_CONSOLE, diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index b035a40c..3efbcb4b 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -344,7 +344,7 @@ class LauncherCapabilities: cpu_millis=frozenset({1_000, 2_000, 4_000, 8_000, 16_000}), minimum_memory_mib=512, maximum_memory_mib=122_880, - maximum_deadline_seconds=3_600, + maximum_deadline_seconds=86_400, maximum_launcher_retries=10, maximum_task_count=1, maximum_parallelism=1, diff --git a/src/dander/identity/__init__.py b/src/dander/identity/__init__.py index 39f40816..9a065afe 100644 --- a/src/dander/identity/__init__.py +++ b/src/dander/identity/__init__.py @@ -2,6 +2,8 @@ from dander.identity.aws_google import ( FargateIdentityError, + google_client_options, + launcher_identity, prepare_fargate_google_identity, prepare_fargate_task_credentials, prepare_launcher_identity, @@ -9,6 +11,8 @@ __all__ = [ "FargateIdentityError", + "google_client_options", + "launcher_identity", "prepare_fargate_google_identity", "prepare_fargate_task_credentials", "prepare_launcher_identity", diff --git a/src/dander/identity/aws_google.py b/src/dander/identity/aws_google.py index 055b5d8e..ae6d1c77 100644 --- a/src/dander/identity/aws_google.py +++ b/src/dander/identity/aws_google.py @@ -1,4 +1,4 @@ -"""Adapt short-lived Fargate task-role credentials for Google workload federation.""" +"""Adapt renewable Fargate task-role credentials for Google workload federation.""" from __future__ import annotations @@ -6,12 +6,14 @@ import os import re import urllib.request +from contextlib import contextmanager +from contextvars import ContextVar from datetime import UTC, datetime, timedelta -from pathlib import Path from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: - from collections.abc import MutableMapping + from collections.abc import Callable, Iterator, MutableMapping + from typing import Any from dander.runtime_contract import LauncherContext @@ -23,6 +25,7 @@ "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", ) +_AWS_REGION = re.compile(r"^[a-z]{2}(?:-gov)?-[a-z]+-[0-9]+$") _WIF_AUDIENCE = re.compile( r"^//iam\.googleapis\.com/projects/[0-9]{6,20}/locations/global/" r"workloadIdentityPools/[a-z][a-z0-9-]{3,31}/providers/[a-z][a-z0-9-]{3,31}$" @@ -31,7 +34,12 @@ r"^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z][a-z0-9-]{4,28}[a-z0-9]" r"\.iam\.gserviceaccount\.com$" ) -_DEFAULT_CREDENTIAL_PATH = Path("/tmp/dander-gcp-wif.json") +_AWS_SUBJECT_TOKEN_TYPE = "urn:ietf:params:aws:token-type:aws4_request" +_CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" +_RUNTIME_GOOGLE_CREDENTIALS: ContextVar[object | None] = ContextVar( + "dander_runtime_google_credentials", + default=None, +) class FargateIdentityError(RuntimeError): @@ -43,6 +51,17 @@ def __call__(self, url: str) -> object: """Return one parsed ECS task-credential response.""" +class GoogleCredentialFactory(Protocol): + def __call__( + self, + *, + audience: str, + service_account: str, + supplier: _FargateCredentialSupplier, + ) -> object: + """Build renewable Google credentials from one task-role supplier.""" + + def _fetch_ecs_credentials(url: str) -> object: request = urllib.request.Request(url, headers={"Accept": "application/json"}) with urllib.request.urlopen(request, timeout=5) as response: # noqa: S310 @@ -51,20 +70,81 @@ def _fetch_ecs_credentials(url: str) -> object: return json.load(response) -def prepare_launcher_identity(context: LauncherContext) -> None: - """Prepare only the selected launcher's ambient identity before provider clients start.""" +class _FargateCredentialSupplier: + """Supply a fresh ECS task-role session whenever Google Auth refreshes.""" + + def __init__( + self, + *, + environ: MutableMapping[str, str], + fetch: CredentialFetcher, + clock: Callable[[], datetime], + ) -> None: + self._environ = environ + self._fetch = fetch + self._clock = clock + self._relative_uri = _validated_relative_uri(environ) + self._region = _validated_region(environ) + _reject_preconfigured_credentials(environ) + + def validate(self) -> None: + """Fail before provider construction if the current task session is unusable.""" + self._current_values() + + def get_aws_region(self, context: object, request: object) -> str: + """Return the validated task region without provider metadata calls.""" + del context, request + return self._region + + def get_aws_security_credentials(self, context: object, request: object) -> object: + """Fetch the current ECS session for one Google subject-token refresh.""" + del context, request + access_key, secret_key, session_token = self._current_values() + try: + from google.auth.aws import AwsSecurityCredentials + except ImportError as error: # pragma: no cover - packaging contract covers this path + raise FargateIdentityError( + "Fargate Google workload identity dependencies are unavailable" + ) from error + return AwsSecurityCredentials(access_key, secret_key, session_token) + + def _current_values(self) -> tuple[str, str, str]: + document = self._fetch(f"{_ECS_CREDENTIALS_ORIGIN}{self._relative_uri}") + return _validated_credentials(document, current=self._clock().astimezone(UTC)) + + +@contextmanager +def launcher_identity(context: LauncherContext) -> Iterator[None]: + """Scope launcher-specific credentials to one runtime execution.""" + credentials = prepare_launcher_identity(context) + token = _RUNTIME_GOOGLE_CREDENTIALS.set(credentials) + try: + yield + finally: + _RUNTIME_GOOGLE_CREDENTIALS.reset(token) + + +def prepare_launcher_identity(context: LauncherContext) -> object | None: + """Prepare only the selected launcher's ambient identity before clients start.""" if context.launcher == "fargate": - prepare_fargate_google_identity() + return prepare_fargate_google_identity() + return None + + +def google_client_options() -> dict[str, Any]: + """Return explicit Google client credentials only inside a launcher identity scope.""" + credentials = _RUNTIME_GOOGLE_CREDENTIALS.get() + return {} if credentials is None else {"credentials": credentials} def prepare_fargate_google_identity( *, environ: MutableMapping[str, str] = os.environ, fetch: CredentialFetcher = _fetch_ecs_credentials, - credential_path: Path = _DEFAULT_CREDENTIAL_PATH, - now: datetime | None = None, -) -> None: - """Prepare task-role credentials and a non-secret Google external-account file.""" + clock: Callable[[], datetime] | None = None, + credential_factory: GoogleCredentialFactory | None = None, +) -> object: + """Build renewable Google credentials without copying or persisting task secrets.""" audience = environ.get("DANDER_GCP_WIF_AUDIENCE", "") service_account = environ.get("DANDER_GCP_SERVICE_ACCOUNT", "") if ( @@ -72,41 +152,44 @@ def prepare_fargate_google_identity( or _SERVICE_ACCOUNT.fullmatch(service_account) is None ): raise FargateIdentityError("Fargate Google workload identity is invalid") - prepare_fargate_task_credentials(environ=environ, fetch=fetch, now=now) - config: dict[str, object] = { - "audience": audience, - "credential_source": { - "environment_id": "aws1", - "region_url": ("http://169.254.169.254/latest/meta-data/placement/availability-zone"), - "regional_cred_verification_url": ( - "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" - ), - "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials", - }, - "service_account_impersonation": {"token_lifetime_seconds": 600}, - "service_account_impersonation_url": ( - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" - f"{service_account}:generateAccessToken" - ), - "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", - "token_url": "https://sts.googleapis.com/v1/token", - "type": "external_account", - } - temporary = credential_path.with_suffix(credential_path.suffix + ".tmp") + supplier = _FargateCredentialSupplier( + environ=environ, + fetch=fetch, + clock=clock or (lambda: datetime.now(UTC)), + ) + supplier.validate() + factory = credential_factory or _build_google_credentials + return factory( + audience=audience, + service_account=service_account, + supplier=supplier, + ) + + +def _build_google_credentials( + *, + audience: str, + service_account: str, + supplier: _FargateCredentialSupplier, +) -> object: try: - temporary.write_text( - json.dumps(config, sort_keys=True, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - temporary.chmod(0o600) - os.replace(temporary, credential_path) - except OSError as error: + from google.auth.aws import Credentials + except ImportError as error: # pragma: no cover - packaging contract covers this path raise FargateIdentityError( - "Fargate Google workload identity configuration could not be prepared" + "Fargate Google workload identity dependencies are unavailable" ) from error - finally: - temporary.unlink(missing_ok=True) - environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(credential_path) + return Credentials( # type: ignore[no-untyped-call] + audience=audience, + subject_token_type=_AWS_SUBJECT_TOKEN_TYPE, + token_url="https://sts.googleapis.com/v1/token", + aws_security_credentials_supplier=supplier, + service_account_impersonation_url=( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" + f"{service_account}:generateAccessToken" + ), + service_account_impersonation_options={"token_lifetime_seconds": 600}, + scopes=(_CLOUD_PLATFORM_SCOPE,), + ) def prepare_fargate_task_credentials( @@ -115,22 +198,44 @@ def prepare_fargate_task_credentials( fetch: CredentialFetcher = _fetch_ecs_credentials, now: datetime | None = None, ) -> bool: - """Expose one bounded ECS task-role session to Google Auth in the current process.""" - existing = tuple(environ.get(name) for name in _AWS_CREDENTIAL_NAMES) - if any(existing): + """Expose one bounded ECS session for the retained Phase 1B probe helper.""" + _reject_preconfigured_credentials(environ) + relative_uri = _validated_relative_uri(environ) + values = _validated_credentials( + fetch(f"{_ECS_CREDENTIALS_ORIGIN}{relative_uri}"), + current=(now or datetime.now(UTC)).astimezone(UTC), + ) + for name, value in zip(_AWS_CREDENTIAL_NAMES, values, strict=True): + environ[name] = value + return True + + +def _reject_preconfigured_credentials(environ: MutableMapping[str, str]) -> None: + if any(environ.get(name) for name in _AWS_CREDENTIAL_NAMES): raise FargateIdentityError("Fargate does not accept preconfigured AWS credentials") + +def _validated_relative_uri(environ: MutableMapping[str, str]) -> str: relative_uri = environ.get("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") if relative_uri is None or _ECS_CREDENTIALS_PATH.fullmatch(relative_uri) is None: raise FargateIdentityError("Fargate task credential endpoint is unavailable") - document = fetch(f"{_ECS_CREDENTIALS_ORIGIN}{relative_uri}") + return relative_uri + + +def _validated_region(environ: MutableMapping[str, str]) -> str: + region = environ.get("AWS_REGION") or environ.get("AWS_DEFAULT_REGION", "") + if _AWS_REGION.fullmatch(region) is None: + raise FargateIdentityError("Fargate task region is invalid") + return region + + +def _validated_credentials(document: object, *, current: datetime) -> tuple[str, str, str]: if not isinstance(document, dict): raise FargateIdentityError("Fargate task credentials were invalid") access_key = document.get("AccessKeyId") secret_key = document.get("SecretAccessKey") session_token = document.get("Token") expiration = _expiration(document.get("Expiration")) - current = (now or datetime.now(UTC)).astimezone(UTC) if ( not isinstance(access_key, str) or _TEMPORARY_ACCESS_KEY.fullmatch(access_key) is None @@ -141,11 +246,7 @@ def prepare_fargate_task_credentials( or expiration <= current + timedelta(minutes=5) ): raise FargateIdentityError("Fargate task credentials were invalid") - - environ["AWS_SECRET_ACCESS_KEY"] = secret_key - environ["AWS_SESSION_TOKEN"] = session_token - environ["AWS_ACCESS_KEY_ID"] = access_key - return True + return access_key, secret_key, session_token def _expiration(value: object) -> datetime: diff --git a/src/dander/providers/bigquery/fence.py b/src/dander/providers/bigquery/fence.py index 673b6cd7..7a03bd7e 100644 --- a/src/dander/providers/bigquery/fence.py +++ b/src/dander/providers/bigquery/fence.py @@ -10,6 +10,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import FencingToken, TargetFence, TargetFenceLostError +from dander.identity import google_client_options from dander.warehouse.runtime import PreparedWarehouseStatement if TYPE_CHECKING: @@ -94,7 +95,10 @@ def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseSt def _required_client(self) -> _FenceClient: if self.client is None: - self.client = cast("_FenceClient", bigquery.Client(project=self.project)) + self.client = cast( + "_FenceClient", + bigquery.Client(project=self.project, **google_client_options()), + ) return self.client diff --git a/src/dander/providers/bigquery/graph.py b/src/dander/providers/bigquery/graph.py index dece0531..64bf57b1 100644 --- a/src/dander/providers/bigquery/graph.py +++ b/src/dander/providers/bigquery/graph.py @@ -11,6 +11,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import fenced_dml, fencing_job_config +from dander.identity import google_client_options from dander.pipeline.compiler import ( CompiledTarget, PipelineCompileError, @@ -78,7 +79,10 @@ def __init__( _validate_target(target, project) self._plan = plan self._project = project - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) def build( self, diff --git a/src/dander/providers/bigquery/state.py b/src/dander/providers/bigquery/state.py index fe291cf6..3dad0fc6 100644 --- a/src/dander/providers/bigquery/state.py +++ b/src/dander/providers/bigquery/state.py @@ -7,6 +7,7 @@ from google.cloud import bigquery from dander.catalog import BigQueryMetadataStore +from dander.identity import google_client_options from dander.providers.bigquery.config import BigQueryStateConfig from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.state import ( @@ -143,7 +144,9 @@ def _build_bigquery_state( control_dataset = metadata_dataset if project_pipeline else raw_dataset supplied_client = context.get("client") client: Any = ( - supplied_client if supplied_client is not None else bigquery.Client(project=project) + supplied_client + if supplied_client is not None + else bigquery.Client(project=project, **google_client_options()) ) watermarks = BigQueryWatermarkStore(project=project, dataset=raw_dataset, client=client) diff --git a/src/dander/security/secret_manager.py b/src/dander/security/secret_manager.py index 88ea2aa6..45e2fb03 100644 --- a/src/dander/security/secret_manager.py +++ b/src/dander/security/secret_manager.py @@ -13,6 +13,8 @@ from importlib import import_module from typing import TYPE_CHECKING, Protocol, cast +from dander.identity import google_client_options + if TYPE_CHECKING: from collections.abc import Mapping from typing import Any @@ -59,7 +61,7 @@ def __init__(self, client: _SecretManagerClient | None = None) -> None: client_type = cast( "Any", import_module("google.cloud.secretmanager") ).SecretManagerServiceClient - client = cast("_SecretManagerClient", client_type()) + client = cast("_SecretManagerClient", client_type(**google_client_options())) self._client = client def get_secret(self, reference: str) -> str: diff --git a/src/dander/state/lease.py b/src/dander/state/lease.py index 16bb8bc2..5cc566b5 100644 --- a/src/dander/state/lease.py +++ b/src/dander/state/lease.py @@ -16,6 +16,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import FencingToken +from dander.identity import google_client_options if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -98,7 +99,10 @@ def __init__( self._table_prefix = f"{project}.{dataset}._dander_lease_" self._authority_id = authority_id or f"bigquery:{project}:{dataset}" self._authority_epoch = authority_epoch - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._ready: set[str] = set() @property diff --git a/src/dander/state/run_history.py b/src/dander/state/run_history.py index 010f544d..3d398f70 100644 --- a/src/dander/state/run_history.py +++ b/src/dander/state/run_history.py @@ -11,6 +11,8 @@ from google.cloud import bigquery +from dander.identity import google_client_options + if TYPE_CHECKING: from collections.abc import Iterable from pathlib import Path @@ -161,7 +163,10 @@ def __init__( if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", dataset): raise ValueError(f"Invalid BigQuery dataset: {dataset!r}") self._table = f"{project}.{dataset}._dander_runs" - self._client = client or cast("_Client", bigquery.Client(project=project)) + self._client = client or cast( + "_Client", + bigquery.Client(project=project, **google_client_options()), + ) self._initialize_on_read = initialize_on_read self._ready = False diff --git a/src/dander/state/watermark.py b/src/dander/state/watermark.py index 31da290a..34155a97 100644 --- a/src/dander/state/watermark.py +++ b/src/dander/state/watermark.py @@ -15,6 +15,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import FencingToken, fenced_dml, fencing_job_config +from dander.identity import google_client_options if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -80,7 +81,10 @@ def __init__( if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", dataset): raise ValueError(f"Invalid BigQuery dataset: {dataset!r}") self._table_id = f"{project}.{dataset}._dander_watermarks" - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._table_ready = False def migrate(self) -> None: diff --git a/src/dander/transform/runner.py b/src/dander/transform/runner.py index 14a4b126..0712f5e6 100644 --- a/src/dander/transform/runner.py +++ b/src/dander/transform/runner.py @@ -11,6 +11,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import OwnershipGuard, fenced_dml, fencing_job_config +from dander.identity import google_client_options from dander.transform.model import Materialization, SqlDialect from dander.transform.project import ( TransformModel, @@ -72,7 +73,10 @@ def __init__( ) -> None: self._project = project self._raw_namespace = raw_namespace - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) def build( self, diff --git a/src/dander/writer/bigquery.py b/src/dander/writer/bigquery.py index b94a0225..9d0d354e 100644 --- a/src/dander/writer/bigquery.py +++ b/src/dander/writer/bigquery.py @@ -15,6 +15,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import fenced_dml, fencing_job_config, fencing_touch_sql +from dander.identity import google_client_options from dander.schema import BIGQUERY_FIELD_MODES, BIGQUERY_FIELD_TYPES, normalize_bigquery_type from dander.writer.base import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTarget @@ -97,7 +98,10 @@ def __init__( schema_evolution: SchemaEvolution = SchemaEvolution.STRICT, ) -> None: self._project = _validated_identifier(project, "project", allow_dash=True) - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._max_batch_rows = _validated_batch_size(max_batch_rows) self._schema_evolution = schema_evolution @@ -244,7 +248,10 @@ def __init__( ) -> None: self._project = _validated_identifier(project, "project", allow_dash=True) self._snapshot_field = _validated_identifier(snapshot_field, "snapshot column") - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._max_batch_rows = _validated_batch_size(max_batch_rows) self._schema_evolution = schema_evolution @@ -331,7 +338,10 @@ def __init__( schema_evolution: SchemaEvolution = SchemaEvolution.STRICT, ) -> None: self._project = _validated_identifier(project, "project", allow_dash=True) - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._max_batch_rows = _validated_batch_size(max_batch_rows) self._schema_evolution = schema_evolution @@ -406,7 +416,10 @@ def __init__( max_batch_rows: int = 10_000, ) -> None: self._project = _validated_identifier(project, "project", allow_dash=True) - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._max_batch_rows = _validated_batch_size(max_batch_rows) def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> int: diff --git a/src/dander/writer/storage_write.py b/src/dander/writer/storage_write.py index d761bf94..111c6a4b 100644 --- a/src/dander/writer/storage_write.py +++ b/src/dander/writer/storage_write.py @@ -11,6 +11,7 @@ from dander._bigquery_retry import run_mutation_with_retry from dander.concurrency import fenced_dml, fencing_job_config +from dander.identity import google_client_options from dander.writer.base import SchemaEvolution, WriteMode, WritePattern, WriteTarget from dander.writer.bigquery import ( BigQueryWriteError, @@ -82,7 +83,9 @@ def append( *, max_batch_rows: int, ) -> None: - client = self._client or bigquery_storage_v1.BigQueryWriteClient() # type: ignore[no-untyped-call] + client = self._client or bigquery_storage_v1.BigQueryWriteClient( # type: ignore[no-untyped-call] + **google_client_options() + ) parent = f"projects/{target.project}/datasets/{target.dataset}/tables/{target.table}" stream = client.create_write_stream( parent=parent, @@ -136,7 +139,10 @@ def __init__( schema_evolution: SchemaEvolution = SchemaEvolution.STRICT, ) -> None: self._project = project - self._client = client or cast("_BigQueryClient", bigquery.Client(project=project)) + self._client = client or cast( + "_BigQueryClient", + bigquery.Client(project=project, **google_client_options()), + ) self._backend = backend or BigQueryPendingStreamBackend() self._max_batch_rows = _validated_batch_size(max_batch_rows) self._schema_evolution = schema_evolution diff --git a/tests/cli/test_runtime_cli.py b/tests/cli/test_runtime_cli.py index ec875541..76a1fb50 100644 --- a/tests/cli/test_runtime_cli.py +++ b/tests/cli/test_runtime_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from contextlib import contextmanager from typing import TYPE_CHECKING, cast from click import ClickException @@ -16,7 +17,7 @@ from dander.runtime_contract import RuntimeCancelledError, RuntimeExitCode if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterator from pytest import MonkeyPatch from rich.console import Console @@ -238,10 +239,12 @@ def test_runtime_execute_rejects_unknown_contract_before_start() -> None: def test_runtime_execute_sanitizes_fargate_identity_failure( monkeypatch: MonkeyPatch, ) -> None: - def identity_failure(_context: object) -> None: + @contextmanager + def identity_failure(_context: object) -> Iterator[None]: raise FargateIdentityError("temporary credential detail") + yield - monkeypatch.setattr(runtime_module, "prepare_launcher_identity", identity_failure) + monkeypatch.setattr(runtime_module, "launcher_identity", identity_failure) result = CliRunner().invoke( app, [ diff --git a/tests/identity/test_aws_google.py b/tests/identity/test_aws_google.py index 05271c7d..57012ab7 100644 --- a/tests/identity/test_aws_google.py +++ b/tests/identity/test_aws_google.py @@ -2,20 +2,25 @@ from __future__ import annotations -import json from datetime import UTC, datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import pytest +from google.auth.aws import Credentials from dander.identity import ( FargateIdentityError, + google_client_options, + launcher_identity, prepare_fargate_google_identity, prepare_fargate_task_credentials, ) +from dander.runtime_contract import LauncherContext if TYPE_CHECKING: - from pathlib import Path + from pytest import MonkeyPatch + + from dander.identity.aws_google import GoogleCredentialFactory _RELATIVE_URI = "/v2/credentials/12345678-1234-1234-1234-123456789abc" _NOW = datetime(2026, 8, 8, 12, 0, tzinfo=UTC) @@ -49,10 +54,55 @@ def fetch(url: str) -> object: assert environment["AWS_SESSION_TOKEN"] -def test_fargate_google_identity_writes_only_non_secret_external_config(tmp_path: Path) -> None: - credential_path = tmp_path / "wif.json" +def test_fargate_google_identity_renews_from_the_fixed_ecs_endpoint() -> None: + environment = { + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": _RELATIVE_URI, + "AWS_REGION": "us-east-1", + "DANDER_GCP_SERVICE_ACCOUNT": ("dander-runtime@unit-project.iam.gserviceaccount.com"), + "DANDER_GCP_WIF_AUDIENCE": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase1b-aws/providers/fargate" + ), + } + urls: list[str] = [] + sequence = iter(("A", "B", "C")) + + def fetch(url: str) -> object: + urls.append(url) + marker = next(sequence) + return { + "AccessKeyId": "ASIA" + marker * 16, + "SecretAccessKey": f"temporary-secret-{marker}", + "Token": f"temporary-session-{marker}", + "Expiration": "2026-08-08T13:00:00Z", + } + + def capture_factory(**values: object) -> object: + assert values["audience"] == environment["DANDER_GCP_WIF_AUDIENCE"] + assert values["service_account"] == environment["DANDER_GCP_SERVICE_ACCOUNT"] + return values["supplier"] + + supplier = prepare_fargate_google_identity( + environ=environment, + fetch=fetch, + clock=lambda: _NOW, + credential_factory=cast("GoogleCredentialFactory", capture_factory), + ) + first = supplier.get_aws_security_credentials(None, None) # type: ignore[attr-defined] + second = supplier.get_aws_security_credentials(None, None) # type: ignore[attr-defined] + + assert first.access_key_id == "ASIA" + "B" * 16 + assert second.access_key_id == "ASIA" + "C" * 16 + assert urls == [f"http://169.254.170.2{_RELATIVE_URI}"] * 3 + assert not set(environment).intersection( + {"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"} + ) + + +def test_fargate_google_identity_builds_scoped_impersonated_credentials() -> None: environment = { "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": _RELATIVE_URI, + "AWS_REGION": "us-east-1", "DANDER_GCP_SERVICE_ACCOUNT": ("dander-runtime@unit-project.iam.gserviceaccount.com"), "DANDER_GCP_WIF_AUDIENCE": ( "//iam.googleapis.com/projects/1009770943166/locations/global/" @@ -60,23 +110,35 @@ def test_fargate_google_identity_writes_only_non_secret_external_config(tmp_path ), } - prepare_fargate_google_identity( + credentials = prepare_fargate_google_identity( environ=environment, fetch=lambda _url: _document(), - credential_path=credential_path, - now=_NOW, + clock=lambda: _NOW, + ) + + assert isinstance(credentials, Credentials) + assert credentials._scopes == ("https://www.googleapis.com/auth/cloud-platform",) + assert credentials._service_account_impersonation_options == {"token_lifetime_seconds": 600} + assert credentials._credential_source is None + assert "GOOGLE_APPLICATION_CREDENTIALS" not in environment + + +def test_launcher_identity_scopes_google_credentials( + monkeypatch: MonkeyPatch, +) -> None: + sentinel = object() + context = LauncherContext.from_environment( + {"DANDER_LAUNCHER": "fargate", "DANDER_RUN_ID": "run-123"} + ) + monkeypatch.setattr( + "dander.identity.aws_google.prepare_launcher_identity", + lambda _context: sentinel, ) - config = json.loads(credential_path.read_text(encoding="utf-8")) - assert config["type"] == "external_account" - assert config["credential_source"]["environment_id"] == "aws1" - assert config["service_account_impersonation"] == {"token_lifetime_seconds": 600} - assert environment["GOOGLE_APPLICATION_CREDENTIALS"] == str(credential_path) - assert credential_path.stat().st_mode & 0o777 == 0o600 - serialized = credential_path.read_text(encoding="utf-8") - assert "private_key" not in serialized - assert "client_secret" not in serialized - assert environment["AWS_SECRET_ACCESS_KEY"] not in serialized + assert google_client_options() == {} + with launcher_identity(context): + assert google_client_options() == {"credentials": sentinel} + assert google_client_options() == {} @pytest.mark.parametrize( diff --git a/tests/infra/test_fargate_runtime.py b/tests/infra/test_fargate_runtime.py index 1170406d..57b7f51d 100644 --- a/tests/infra/test_fargate_runtime.py +++ b/tests/infra/test_fargate_runtime.py @@ -50,9 +50,11 @@ def test_fargate_task_is_immutable_nonroot_and_uses_separate_roles() -> None: def test_controller_owns_deadline_and_only_runtime_exit_75_retries() -> None: module = _source(MODULE) + variables = _source(AWS_ROOT / "modules/fargate/variables.tf") assert module.count('Resource = "arn:${local.partition}:states:::ecs:runTask.sync"') == 1 assert "TimeoutSeconds = each.value.resources.deadline_seconds" in module + assert "projection.resources.deadline_seconds <= 86400" in variables assert "NumericEquals = 75" in module assert 'failure_code = "runtime_retry_exhausted"' in module assert 'failure_code = "launcher_deadline_exceeded"' in module diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py index 76dfaced..b91bb034 100644 --- a/tests/providers/test_launcher_runtime.py +++ b/tests/providers/test_launcher_runtime.py @@ -199,7 +199,24 @@ def test_fargate_rejects_unhonored_runtime_intent( ) -def test_fargate_rejects_a_run_longer_than_the_task_role_session() -> None: +def test_fargate_accepts_a_run_longer_than_one_task_role_session() -> None: + template = _fargate_runtime().templates.build( + _PIPELINES, + image=_FARGATE_IMAGE, + project="unit-project", + cpu=1, + memory="2Gi", + deadline_seconds=3_601, + launcher_retry_count=1, + batch_rows=1_000, + require_guarded_free_tier=False, + alert_target=None, + )["greenhouse_jobs"] + + assert template.resources.deadline_seconds == 3_601 + + +def test_fargate_rejects_a_run_longer_than_one_day() -> None: with pytest.raises(ExecutionProjectionError, match="deadline"): _fargate_runtime().templates.build( _PIPELINES, @@ -207,7 +224,7 @@ def test_fargate_rejects_a_run_longer_than_the_task_role_session() -> None: project="unit-project", cpu=1, memory="2Gi", - deadline_seconds=3_601, + deadline_seconds=86_401, launcher_retry_count=1, batch_rows=1_000, require_guarded_free_tier=False, From da0022289635c07c7e11ce2d55d6f1e9cfe08a87 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 16:13:43 -0400 Subject: [PATCH 075/163] Prepare Dander 0.8.0rc2 --- CHANGELOG.md | 10 +++++ HANDOFF.md | 40 +++++++++---------- README.md | 4 +- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +- uv.lock | 2 +- 11 files changed, 40 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8245a5d..4a7b6266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc2 — 2026-08-09 (beta) + +### Candidate + +- Package renewable Fargate-to-Google workload identity for the complete source-free lifecycle + acceptance gate. Fargate remains experimental until that gate passes. + ### Acceptance - Record the successful public-`0.8.0rc1` Phase 1B proof: identical GAR/ECR multi-platform @@ -14,6 +21,9 @@ capabilities enter through the next minor release. ### Fixed +- Refetch the current ECS task-role session whenever Google Auth refreshes its AWS subject token, + scope the resulting credential to one OCI invocation, and avoid persisting or globally exporting + task secrets. The validated Fargate deadline is now 24 hours instead of one hour. - Make the Phase 1B credential scanner recognize only the exact content hashes of three public boto3/botocore example documents containing published placeholder keys. Changed content and all other files continue to fail closed. diff --git a/HANDOFF.md b/HANDOFF.md index 4aa2a6f9..79772de7 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,37 @@ ## Finished -- Replaced Fargate's one-time ECS credential copy with a renewable Google Auth supplier. -- Scoped cross-cloud credentials to one OCI runtime invocation and explicit GCP client creation. -- Removed task secrets from global environment/file persistence in normal runtime execution. -- Raised the validated Fargate deadline from one hour to 24 hours. -- Kept Cloud Run/local ADC behavior and the public Fargate support boundary unchanged. +- Merged renewable Fargate-to-Google identity through protected main as PR #150. +- Prepared release-facing metadata for `dander-platform==0.8.0rc2`. +- Preserved the accepted public-rc1 evidence as a historical record. +- Kept `src/dander`, Terraform, manifests, and provider support declarations unchanged. ## Try It -Run `uv run pytest tests/identity/test_aws_google.py tests/providers/test_launcher_runtime.py -q`. +Run `uv run python scripts/check_release_metadata.py`, then build and install the candidate outside the checkout. ## Checks -- All 1,117 tests passed against PostgreSQL 15. -- Ruff, formatting, and strict mypy passed across `src` and `tests`. -- AWS root/module Terraform formatting, validation, and the Fargate module test passed. -- Wheel/sdist inspection and a source-free wheel installation outside the checkout passed. -- The full-runtime container passed read-only conformance; Docker Scout found no fixable high or critical vulnerabilities. +- PR #150 and post-merge main CI passed Python, Terraform, packaging, container, and secret checks. +- Retained GCP stage-zero and deployed-platform plans each reported exactly `No changes.` +- Release metadata validation and its two focused tests passed; Ruff found no issue. +- Wheel/sdist inspection and source-free install, generation, and validation passed outside the checkout. +- Generated GCP, stage-zero, and AWS Terraform validated; Helm lint/template passed with the repository fixture. ## Decisions -- Google Auth refetches only the validated ECS relative-credential endpoint on each subject-token refresh. -- Existing Google clients receive the scoped credential explicitly; non-Fargate ADC remains unchanged. -- Renewable identity is a prerequisite, not a Fargate support promotion. +- `0.8.0rc2` replaces rc1 for the complete Fargate lifecycle proof because packaged runtime behavior changed. +- Fargate remains experimental until full lifecycle acceptance passes. ## Remaining -- Let protected CI repeat Linux packaging, Terraform, container, and secret checks. -- Merge the focused PR through protected main after review. -- Publish the next candidate before a source-free Fargate pipeline acceptance run. -- Complete replay, interruption, schedule, alert, rollback, and no-drift acceptance. +- Merge the release-only PR after protected checks pass. +- Obtain explicit approval, then tag and publish `0.8.0rc2` from the exact merge commit. +- Restart the source-free Fargate lifecycle proof against the public candidate. +- Record replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `src/dander/identity/aws_google.py` -- `src/dander/cli/runtime_command.py` -- `tests/identity/test_aws_google.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index e31fa5b8..be107de2 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc1 +uv tool install dander-platform==0.8.0rc2 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc1` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc2` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 3674fa1a..5c851bb4 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc1`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc2`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 87def61a..e45d1855 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc1 +uv tool install dander-platform==0.8.0rc2 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index d46fbf21..06383e2f 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc1`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc2`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 8566e4ce..2151668c 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc1` is the current public beta candidate. +- Dander `0.8.0rc2` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index b59035d3..a70c0732 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc1" +export DANDER_TARGET_VERSION="0.8.0rc2" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index fe1f257d..fd7d9073 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc1" +version = "0.8.0rc2" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index d7ad7a02..9ec1d501 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc1", + "dander-platform==0.8.0rc2", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc1" + "README install command uses 0.1.0; package metadata uses 0.8.0rc2" ] diff --git a/uv.lock b/uv.lock index 33b8af64..b2954c09 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc1" +version = "0.8.0rc2" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From d455530d3e08fb6e6800fa7e24bf303c73d8f4a0 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 17:13:24 -0400 Subject: [PATCH 076/163] Fix Fargate failure topic policy --- HANDOFF.md | 35 ++++++++++++++--------------- infra/aws/modules/fargate/main.tf | 15 ++++++++++--- tests/infra/test_fargate_runtime.py | 12 ++++++++++ 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 79772de7..b7550042 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,36 @@ ## Finished -- Merged renewable Fargate-to-Google identity through protected main as PR #150. -- Prepared release-facing metadata for `dander-platform==0.8.0rc2`. -- Preserved the accepted public-rc1 evidence as a historical record. -- Kept `src/dander`, Terraform, manifests, and provider support declarations unchanged. +- Reproduced the public `0.8.0rc2` Fargate apply failure in the isolated proof account. +- Replaced the invalid SNS topic-policy wildcard with topic-scoped SNS actions. +- Added a focused regression assertion for the rendered Fargate Terraform. +- Kept runtime behavior, provider support, schedules, and retained GCP resources unchanged. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and install the candidate outside the checkout. +Run `uv run pytest -q tests/infra/test_fargate_runtime.py`, then validate `infra/aws` with Terraform. ## Checks -- PR #150 and post-merge main CI passed Python, Terraform, packaging, container, and secret checks. -- Retained GCP stage-zero and deployed-platform plans each reported exactly `No changes.` -- Release metadata validation and its two focused tests passed; Ruff found no issue. -- Wheel/sdist inspection and source-free install, generation, and validation passed outside the checkout. -- Generated GCP, stage-zero, and AWS Terraform validated; Helm lint/template passed with the repository fixture. +- Focused Fargate tests passed: 7 tests. +- Ruff lint/format and strict MyPy passed. +- Full Python suite passed: 1,104 tests, 13 skipped. +- AWS Terraform formatting, initialization, validation, and tests passed. +- Wheel/sdist inspection and source-free scaffold verification passed. ## Decisions -- `0.8.0rc2` replaces rc1 for the complete Fargate lifecycle proof because packaged runtime behavior changed. -- Fargate remains experimental until full lifecycle acceptance passes. +- The restricted EventBridge `sns:Publish` policy remains unchanged. +- A replacement release candidate is required because rc2 cannot finish a fresh AWS apply. ## Remaining -- Merge the release-only PR after protected checks pass. -- Obtain explicit approval, then tag and publish `0.8.0rc2` from the exact merge commit. -- Restart the source-free Fargate lifecycle proof against the public candidate. +- Merge this fix through protected main after review and CI. +- Prepare and explicitly approve a replacement release candidate. +- Re-plan the partial proof stack and require only the expected remaining resource. - Record replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `infra/aws/modules/fargate/main.tf` +- `tests/infra/test_fargate_runtime.py` diff --git a/infra/aws/modules/fargate/main.tf b/infra/aws/modules/fargate/main.tf index 1a138720..e027d226 100644 --- a/infra/aws/modules/fargate/main.tf +++ b/infra/aws/modules/fargate/main.tf @@ -846,9 +846,18 @@ resource "aws_sqs_queue_policy" "failures" { data "aws_iam_policy_document" "failure_topic" { statement { - sid = "AccountOwner" - effect = "Allow" - actions = ["SNS:*"] + sid = "AccountOwner" + effect = "Allow" + actions = [ + "sns:AddPermission", + "sns:DeleteTopic", + "sns:GetTopicAttributes", + "sns:ListSubscriptionsByTopic", + "sns:Publish", + "sns:RemovePermission", + "sns:SetTopicAttributes", + "sns:Subscribe", + ] resources = [aws_sns_topic.failures.arn] principals { diff --git a/tests/infra/test_fargate_runtime.py b/tests/infra/test_fargate_runtime.py index 57b7f51d..a6c1182a 100644 --- a/tests/infra/test_fargate_runtime.py +++ b/tests/infra/test_fargate_runtime.py @@ -83,6 +83,18 @@ def test_scheduler_delivery_is_separate_paused_aware_and_dead_lettered() -> None assert 'resource "aws_kms_key" "failure_topic"' in module assert "enable_key_rotation = true" in module assert "kms_master_key_id = aws_kms_key.failure_topic.arn" in module + assert '"sns:*"' not in module.lower() + for topic_action in ( + "sns:AddPermission", + "sns:DeleteTopic", + "sns:GetTopicAttributes", + "sns:ListSubscriptionsByTopic", + "sns:Publish", + "sns:RemovePermission", + "sns:SetTopicAttributes", + "sns:Subscribe", + ): + assert f'"{topic_action}"' in module def test_aws_stack_contains_no_static_cloud_credentials_or_apply_path() -> None: From de4e705a949e207e01b400e79790f638103dbaa7 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 17:19:22 -0400 Subject: [PATCH 077/163] Prepare Dander 0.8.0rc3 --- CHANGELOG.md | 8 +++++ HANDOFF.md | 33 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 35 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7b6266..556c76a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc3 — 2026-08-09 (beta) + +### Fixed + +- Replace an invalid wildcard in the Fargate failure-topic resource policy with the explicit + topic-scoped SNS actions accepted by AWS. This allows a fresh or resumed Fargate platform apply + to finish without changing runtime behavior or the restricted EventBridge failure publisher. + ## 0.8.0rc2 — 2026-08-09 (beta) ### Candidate diff --git a/HANDOFF.md b/HANDOFF.md index b7550042..0ffb068c 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,35 @@ ## Finished -- Reproduced the public `0.8.0rc2` Fargate apply failure in the isolated proof account. -- Replaced the invalid SNS topic-policy wildcard with topic-scoped SNS actions. -- Added a focused regression assertion for the rendered Fargate Terraform. -- Kept runtime behavior, provider support, schedules, and retained GCP resources unchanged. +- Merged the focused Fargate SNS policy correction through protected main as PR #152. +- Prepared release-only metadata for `dander-platform==0.8.0rc3`. +- Kept `src/dander`, Terraform, manifests, and provider support declarations unchanged. ## Try It -Run `uv run pytest -q tests/infra/test_fargate_runtime.py`, then validate `infra/aws` with Terraform. +Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate. ## Checks -- Focused Fargate tests passed: 7 tests. -- Ruff lint/format and strict MyPy passed. -- Full Python suite passed: 1,104 tests, 13 skipped. -- AWS Terraform formatting, initialization, validation, and tests passed. -- Wheel/sdist inspection and source-free scaffold verification passed. +- PR #152 passed Python, Terraform, packaging, container, and secret checks. +- The isolated recovery plan reported exactly one add, zero changes, and zero destroys. +- Release metadata validation and focused tests passed. +- Wheel/sdist inspection and source-free installation passed outside the checkout. ## Decisions -- The restricted EventBridge `sns:Publish` policy remains unchanged. -- A replacement release candidate is required because rc2 cannot finish a fresh AWS apply. +- `0.8.0rc3` replaces rc2 for complete Fargate lifecycle acceptance. +- Fargate remains experimental until the lifecycle gate passes. ## Remaining -- Merge this fix through protected main after review and CI. -- Prepare and explicitly approve a replacement release candidate. -- Re-plan the partial proof stack and require only the expected remaining resource. +- Merge this release-only PR after protected checks pass. +- Tag and publish `0.8.0rc3` from the exact merge commit. +- Re-plan the partial proof stack from the public package and require only the expected resource. - Record replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `infra/aws/modules/fargate/main.tf` -- `tests/infra/test_fargate_runtime.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index be107de2..0c492a67 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc2 +uv tool install dander-platform==0.8.0rc3 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc2` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc3` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 5c851bb4..da6a6e77 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc2`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc3`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index e45d1855..7efb7df7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc2 +uv tool install dander-platform==0.8.0rc3 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 06383e2f..290a368a 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc2`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc3`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 2151668c..d5185643 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc2` is the current public beta candidate. +- Dander `0.8.0rc3` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index a70c0732..9ceb31d7 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc2" +export DANDER_TARGET_VERSION="0.8.0rc3" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index fd7d9073..088c3c1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc2" +version = "0.8.0rc3" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 9ec1d501..57f485a0 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc2", + "dander-platform==0.8.0rc3", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc2" + "README install command uses 0.1.0; package metadata uses 0.8.0rc3" ] diff --git a/uv.lock b/uv.lock index b2954c09..277d7e43 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc2" +version = "0.8.0rc3" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 06c4dc6940e72ae0b11c5b6074477cab77bf59c4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 17:40:35 -0400 Subject: [PATCH 078/163] Use the Step Functions AWS CLI namespace --- HANDOFF.md | 31 ++++++++-------- src/dander/providers/fargate/operations.py | 14 ++++---- tests/providers/test_fargate_operations.py | 41 +++++++++++++--------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 0ffb068c..388522c8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,36 @@ ## Finished -- Merged the focused Fargate SNS policy correction through protected main as PR #152. -- Prepared release-only metadata for `dander-platform==0.8.0rc3`. -- Kept `src/dander`, Terraform, manifests, and provider support declarations unchanged. +- Reproduced the public rc3 Fargate verification failure against the live proof stack. +- Corrected all six Fargate AWS CLI operations to use the `stepfunctions` namespace. +- Preserved Step Functions ARN service names, IAM actions, Terraform, and runtime semantics. +- Updated focused command and fake-response tests. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate. +Run `uv run pytest -q tests/providers/test_fargate_operations.py`. ## Checks -- PR #152 passed Python, Terraform, packaging, container, and secret checks. -- The isolated recovery plan reported exactly one add, zero changes, and zero destroys. -- Release metadata validation and focused tests passed. +- A source-free built wheel verified both deployed Fargate controllers and paused schedules. +- Focused Fargate operation tests passed. +- Ruff lint/format and strict MyPy passed. +- Full Python and protected Terraform checks passed. - Wheel/sdist inspection and source-free installation passed outside the checkout. ## Decisions -- `0.8.0rc3` replaces rc2 for complete Fargate lifecycle acceptance. -- Fargate remains experimental until the lifecycle gate passes. +- Only the AWS CLI namespace changes; ARN and IAM vocabulary correctly remains `states`. +- A replacement candidate is required because rc3 live operator commands cannot run. ## Remaining -- Merge this release-only PR after protected checks pass. -- Tag and publish `0.8.0rc3` from the exact merge commit. -- Re-plan the partial proof stack from the public package and require only the expected resource. +- Merge this focused fix through protected main. +- Prepare, tag, and publish `0.8.0rc4` from the exact protected merge. +- Reinstall rc4 source-free and rerun read-only Fargate verification. - Record replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `src/dander/providers/fargate/operations.py` +- `tests/providers/test_fargate_operations.py` diff --git a/src/dander/providers/fargate/operations.py b/src/dander/providers/fargate/operations.py index 891af98d..a61e10c5 100644 --- a/src/dander/providers/fargate/operations.py +++ b/src/dander/providers/fargate/operations.py @@ -238,7 +238,7 @@ def replay( def latest(self) -> FargateExecution | None: """Return the latest controller execution, if one exists.""" payload = self._json( - "states", + "stepfunctions", "list-executions", "--state-machine-arn", self.binding.state_machine_arn, @@ -258,7 +258,9 @@ def latest(self) -> FargateExecution | None: def describe(self, execution_arn: str) -> FargateExecution: """Return normalized status for one owned execution.""" name = self.binding.validate_execution_arn(execution_arn) - payload = self._json("states", "describe-execution", "--execution-arn", execution_arn) + payload = self._json( + "stepfunctions", "describe-execution", "--execution-arn", execution_arn + ) if payload.get("executionArn") != execution_arn: raise FargateOperationError("AWS returned an unexpected execution") status = payload.get("status") @@ -331,7 +333,7 @@ def cancel(self, execution_arn: str) -> FargateExecution: if execution.state != "running": raise FargateOperationError("Only a running execution can be cancelled") payload = self._json( - "states", + "stepfunctions", "stop-execution", "--execution-arn", execution_arn, @@ -363,7 +365,7 @@ def verify(self, *, expected_image: str) -> FargateDeploymentVerification: "Expected image is not an immutable ECR image in this deployment" ) machine = self._json( - "states", + "stepfunctions", "describe-state-machine", "--state-machine-arn", self.binding.state_machine_arn, @@ -477,7 +479,7 @@ def _start(self, *, name: str, now: datetime, correlation: str) -> FargateExecut "scheduler_execution_id": correlation, } payload = self._json( - "states", + "stepfunctions", "start-execution", "--state-machine-arn", self.binding.state_machine_arn, @@ -507,7 +509,7 @@ def _execution_history(self, execution_arn: str) -> list[object]: """Return one owned execution's bounded history for allow-listed field extraction.""" self.binding.validate_execution_arn(execution_arn) payload = self._json( - "states", + "stepfunctions", "get-execution-history", "--execution-arn", execution_arn, diff --git a/tests/providers/test_fargate_operations.py b/tests/providers/test_fargate_operations.py index 35ac6fef..b29fc37a 100644 --- a/tests/providers/test_fargate_operations.py +++ b/tests/providers/test_fargate_operations.py @@ -191,7 +191,7 @@ def test_start_uses_exact_state_machine_and_operator_correlation(tmp_path: Path) operations, runner = _operations( tmp_path, { - ("states", "start-execution"): [ + ("stepfunctions", "start-execution"): [ {"executionArn": started_arn, "startDate": "2026-08-08T21:00:00Z"} ] }, @@ -202,7 +202,13 @@ def test_start_uses_exact_state_machine_and_operator_correlation(tmp_path: Path) assert execution.execution_arn == started_arn assert execution.state == "running" command = runner.commands[0] - assert command[:5] == ("aws", "--profile", "dander-deploy", "states", "start-execution") + assert command[:5] == ( + "aws", + "--profile", + "dander-deploy", + "stepfunctions", + "start-execution", + ) assert command[command.index("--state-machine-arn") + 1] == MACHINE request = json.loads(command[command.index("--input") + 1]) assert request == { @@ -217,8 +223,8 @@ def test_latest_status_normalizes_runtime_output_without_provider_noise(tmp_path operations, _runner = _operations( tmp_path, { - ("states", "list-executions"): [{"executions": [{"executionArn": EXECUTION}]}], - ("states", "describe-execution"): [ + ("stepfunctions", "list-executions"): [{"executions": [{"executionArn": EXECUTION}]}], + ("stepfunctions", "describe-execution"): [ _describe( output={ "status": "succeeded", @@ -253,8 +259,8 @@ def test_failed_status_recovers_only_normalized_fields_from_execution_history( operations, _runner = _operations( tmp_path, { - ("states", "describe-execution"): [_describe(status="FAILED")], - ("states", "get-execution-history"): [ + ("stepfunctions", "describe-execution"): [_describe(status="FAILED")], + ("stepfunctions", "get-execution-history"): [ {"events": [{"stateExitedEventDetails": {"output": json.dumps(failure)}}]} ], }, @@ -274,8 +280,8 @@ def test_logs_find_the_task_in_execution_history_and_use_its_exact_stream(tmp_pa operations, runner = _operations( tmp_path, { - ("states", "describe-execution"): [_describe(status="RUNNING")], - ("states", "get-execution-history"): [ + ("stepfunctions", "describe-execution"): [_describe(status="RUNNING")], + ("stepfunctions", "get-execution-history"): [ { "events": [ { @@ -305,8 +311,8 @@ def test_cancel_only_stops_an_owned_running_execution(tmp_path: Path) -> None: operations, runner = _operations( tmp_path, { - ("states", "describe-execution"): [_describe(status="RUNNING")], - ("states", "stop-execution"): [{"stopDate": "2026-08-08T21:00:30Z"}], + ("stepfunctions", "describe-execution"): [_describe(status="RUNNING")], + ("stepfunctions", "stop-execution"): [{"stopDate": "2026-08-08T21:00:30Z"}], }, ) @@ -314,7 +320,10 @@ def test_cancel_only_stops_an_owned_running_execution(tmp_path: Path) -> None: assert cancelled.state == "cancelled" assert cancelled.failure_code == "operator_cancelled" - assert (runner.commands[-1][3], runner.commands[-1][4]) == ("states", "stop-execution") + assert (runner.commands[-1][3], runner.commands[-1][4]) == ( + "stepfunctions", + "stop-execution", + ) def test_replay_requires_terminal_execution_and_starts_a_fresh_name(tmp_path: Path) -> None: @@ -322,8 +331,8 @@ def test_replay_requires_terminal_execution_and_starts_a_fresh_name(tmp_path: Pa operations, runner = _operations( tmp_path, { - ("states", "describe-execution"): [_describe()], - ("states", "start-execution"): [ + ("stepfunctions", "describe-execution"): [_describe()], + ("stepfunctions", "start-execution"): [ {"executionArn": replay_arn, "startDate": "2026-08-08T21:02:00Z"} ], }, @@ -339,7 +348,7 @@ def test_replay_requires_terminal_execution_and_starts_a_fresh_name(tmp_path: Pa def test_replay_rejects_a_running_execution_without_starting_another(tmp_path: Path) -> None: operations, runner = _operations( tmp_path, - {("states", "describe-execution"): [_describe(status="RUNNING")]}, + {("stepfunctions", "describe-execution"): [_describe(status="RUNNING")]}, ) with pytest.raises(FargateOperationError, match="terminal"): @@ -363,7 +372,7 @@ def test_verify_checks_controller_schedule_image_logs_cluster_and_registry(tmp_p operations, _runner = _operations( tmp_path, { - ("states", "describe-state-machine"): [{"status": "ACTIVE"}], + ("stepfunctions", "describe-state-machine"): [{"status": "ACTIVE"}], ("ecs", "describe-clusters"): [ { "clusters": [ @@ -427,7 +436,7 @@ def test_verify_fails_when_schedule_state_differs_from_manifest(tmp_path: Path) operations, _runner = _operations( tmp_path, { - ("states", "describe-state-machine"): [{"status": "ACTIVE"}], + ("stepfunctions", "describe-state-machine"): [{"status": "ACTIVE"}], ("ecs", "describe-clusters"): [ {"clusters": [{"clusterArn": "cluster", "status": "ACTIVE"}], "failures": []} ], From 5d0ac771d1099a22991f4daab88f8fe2d44f734a Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 17:45:42 -0400 Subject: [PATCH 079/163] Prepare Dander 0.8.0rc4 --- CHANGELOG.md | 8 +++++ HANDOFF.md | 29 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 33 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 556c76a7..989eddca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc4 — 2026-08-09 (beta) + +### Fixed + +- Invoke Step Functions through the AWS CLI's `stepfunctions` service namespace for manual run, + status, logs, replay, cancellation, and verification. Step Functions ARN and IAM service names + correctly remain `states`. + ## 0.8.0rc3 — 2026-08-09 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 388522c8..45c4c29d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,35 @@ ## Finished -- Reproduced the public rc3 Fargate verification failure against the live proof stack. -- Corrected all six Fargate AWS CLI operations to use the `stepfunctions` namespace. -- Preserved Step Functions ARN service names, IAM actions, Terraform, and runtime semantics. -- Updated focused command and fake-response tests. +- Merged the Step Functions CLI namespace correction through protected main as PR #154. +- Prepared release-only metadata for `dander-platform==0.8.0rc4`. +- Kept `src/dander`, Terraform, manifests, and provider support declarations unchanged. ## Try It -Run `uv run pytest -q tests/providers/test_fargate_operations.py`. +Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate. ## Checks +- PR #154 passed Python, Terraform, packaging, container, and secret checks. - A source-free built wheel verified both deployed Fargate controllers and paused schedules. -- Focused Fargate operation tests passed. -- Ruff lint/format and strict MyPy passed. -- Full Python and protected Terraform checks passed. +- Release metadata validation and focused tests passed. - Wheel/sdist inspection and source-free installation passed outside the checkout. ## Decisions -- Only the AWS CLI namespace changes; ARN and IAM vocabulary correctly remains `states`. -- A replacement candidate is required because rc3 live operator commands cannot run. +- `0.8.0rc4` replaces rc3 for complete Fargate lifecycle acceptance. +- Fargate remains experimental until the lifecycle gate passes. ## Remaining -- Merge this focused fix through protected main. -- Prepare, tag, and publish `0.8.0rc4` from the exact protected merge. -- Reinstall rc4 source-free and rerun read-only Fargate verification. +- Merge this release-only PR after protected checks pass. +- Tag and publish `0.8.0rc4` from the exact protected merge. +- Reinstall rc4 source-free and rerun Fargate verification and lifecycle acceptance. - Record replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `src/dander/providers/fargate/operations.py` -- `tests/providers/test_fargate_operations.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index 0c492a67..9a7450ab 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc3 +uv tool install dander-platform==0.8.0rc4 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc3` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc4` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index da6a6e77..1cb59057 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc3`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc4`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 7efb7df7..3787cc57 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc3 +uv tool install dander-platform==0.8.0rc4 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 290a368a..704b35c9 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc3`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc4`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index d5185643..7fe39645 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc3` is the current public beta candidate. +- Dander `0.8.0rc4` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 9ceb31d7..d61c8c74 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc3" +export DANDER_TARGET_VERSION="0.8.0rc4" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 088c3c1a..8a679772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc3" +version = "0.8.0rc4" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 57f485a0..e2a5214d 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc3", + "dander-platform==0.8.0rc4", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc3" + "README install command uses 0.1.0; package metadata uses 0.8.0rc4" ] diff --git a/uv.lock b/uv.lock index 277d7e43..baceab1d 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc3" +version = "0.8.0rc4" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From bf2167a9f318709c7c5f2eabde60d30b0c5f8fe8 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 19:34:52 -0400 Subject: [PATCH 080/163] Fix writable Fargate scratch storage --- HANDOFF.md | 34 ++++++++++--------- .../modules/fargate/tests/fargate.tftest.hcl | 5 +++ src/dander/providers/fargate/operations.py | 28 +++++++++++++++ src/dander/providers/fargate/runtime.py | 2 ++ src/dander/templates/project/Dockerfile | 5 +++ tests/project/test_scaffold.py | 2 ++ tests/providers/test_fargate_operations.py | 12 +++++++ tests/providers/test_launcher_runtime.py | 2 ++ 8 files changed, 74 insertions(+), 16 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 45c4c29d..c6865c5d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,37 @@ ## Finished -- Merged the Step Functions CLI namespace correction through protected main as PR #154. -- Prepared release-only metadata for `dander-platform==0.8.0rc4`. -- Kept `src/dander`, Terraform, manifests, and provider support declarations unchanged. +- Reproduced the rc4 Fargate failure as a non-root write failure on its `/tmp` volume. +- Made the generated image declare `/tmp` as mode `1777` volume-backed scratch. +- Made Fargate set `HOME` and `TMPDIR` to `/tmp` and verify the complete scratch contract. +- Preserved the read-only root filesystem, UID/GID `65532`, and 20–200 GiB disk-backed scratch. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate. +Generate a project, build its image, and inspect `Config.Volumes` for `/tmp`; Fargate tasks should mount `dander-tmp` there and run as `65532:65532`. ## Checks -- PR #154 passed Python, Terraform, packaging, container, and secret checks. -- A source-free built wheel verified both deployed Fargate controllers and paused schedules. -- Release metadata validation and focused tests passed. +- Focused Python tests: 29 passed. +- Full Python suite: 1,104 passed, 13 skipped; Ruff and strict mypy passed. +- Terraform formatting, validation, AWS module tests, and portability roots passed. - Wheel/sdist inspection and source-free installation passed outside the checkout. +- Exact ARM64 image succeeded on Fargate: 19 rows, 3 assertions, 1 model, exit code 0. ## Decisions -- `0.8.0rc4` replaces rc3 for complete Fargate lifecycle acceptance. -- Fargate remains experimental until the lifecycle gate passes. +- Do not use `/dev/shm`: it would bypass the configured Fargate scratch capacity. +- Keep the existing anonymous volume and seed its permissions from Docker image metadata. +- A replacement release candidate is required because this changes packaged runtime behavior. ## Remaining -- Merge this release-only PR after protected checks pass. -- Tag and publish `0.8.0rc4` from the exact protected merge. -- Reinstall rc4 source-free and rerun Fargate verification and lifecycle acceptance. -- Record replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. +- Open and merge the focused fix through protected CI. +- Publish a replacement candidate and rerun complete Fargate lifecycle acceptance. +- Finish replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `src/dander/templates/project/Dockerfile` +- `src/dander/providers/fargate/runtime.py` +- `src/dander/providers/fargate/operations.py` diff --git a/infra/aws/modules/fargate/tests/fargate.tftest.hcl b/infra/aws/modules/fargate/tests/fargate.tftest.hcl index d72e9160..aaaacc03 100644 --- a/infra/aws/modules/fargate/tests/fargate.tftest.hcl +++ b/infra/aws/modules/fargate/tests/fargate.tftest.hcl @@ -45,6 +45,8 @@ variables { environment = { DANDER_LAUNCHER = "fargate" GCP_PROJECT_ID = "unit-project" + HOME = "/tmp" + TMPDIR = "/tmp" } secret_bindings = {} workload_identity = "arn:aws:iam::184463061564:role/dander-runtime" @@ -103,6 +105,9 @@ run "paused_bounded_controller" { condition = ( jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].readonlyRootFilesystem == true && jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].user == "65532:65532" && + contains(jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].environment, { name = "HOME", value = "/tmp" }) && + contains(jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].environment, { name = "TMPDIR", value = "/tmp" }) && + contains(jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].mountPoints, { sourceVolume = "dander-tmp", containerPath = "/tmp", readOnly = false }) && jsondecode(aws_ecs_task_definition.pipeline["greenhouse_jobs"].container_definitions)[0].stopTimeout == 120 ) error_message = "The Fargate task must remain non-root, read-only, and explicitly stoppable." diff --git a/src/dander/providers/fargate/operations.py b/src/dander/providers/fargate/operations.py index a61e10c5..b6178aa7 100644 --- a/src/dander/providers/fargate/operations.py +++ b/src/dander/providers/fargate/operations.py @@ -413,10 +413,38 @@ def verify(self, *, expected_image: str) -> FargateDeploymentVerification: ): raise FargateOperationError("Fargate task definition has invalid containers") container = containers[0] + environment = container.get("environment") + environment_values = ( + { + item.get("name"): item.get("value") + for item in environment + if isinstance(item, dict) + and isinstance(item.get("name"), str) + and isinstance(item.get("value"), str) + } + if isinstance(environment, list) + else {} + ) + mount_points = container.get("mountPoints") + volumes = task.get("volumes") if ( container.get("image") != expected_image or container.get("readonlyRootFilesystem") is not True or container.get("user") != "65532:65532" + or environment_values.get("HOME") != "/tmp" + or environment_values.get("TMPDIR") != "/tmp" + or not isinstance(mount_points, list) + or { + "sourceVolume": "dander-tmp", + "containerPath": "/tmp", + "readOnly": False, + } + not in mount_points + or not isinstance(volumes, list) + or not any( + isinstance(volume, dict) and volume.get("name") == "dander-tmp" + for volume in volumes + ) ): raise FargateOperationError( "Fargate task does not match the immutable runtime contract" diff --git a/src/dander/providers/fargate/runtime.py b/src/dander/providers/fargate/runtime.py index 0f5cd9bf..1435b687 100644 --- a/src/dander/providers/fargate/runtime.py +++ b/src/dander/providers/fargate/runtime.py @@ -116,6 +116,8 @@ def build( "DANDER_LAUNCHER": "fargate", "DANDER_PRINCIPAL": identity, "GCP_PROJECT_ID": project, + "HOME": "/tmp", + "TMPDIR": "/tmp", }.items() ) ), diff --git a/src/dander/templates/project/Dockerfile b/src/dander/templates/project/Dockerfile index 0d723026..077db8d7 100644 --- a/src/dander/templates/project/Dockerfile +++ b/src/dander/templates/project/Dockerfile @@ -26,6 +26,11 @@ RUN apt-get update \ RUN pip install --no-cache-dir "dander-platform[runtime-all]==${DANDER_VERSION}" \ && python -c "from dander.providers.dependencies import require_full_runtime; require_full_runtime()" +# ECS/Fargate copies the image directory metadata into a matching anonymous +# volume. Keep the disk-backed scratch mount writable for the non-root runtime. +RUN chmod 1777 /tmp +VOLUME ["/tmp"] + COPY --chown=65532:65532 dander.yaml ./dander.yaml COPY --chown=65532:65532 dander.platforms.yaml ./dander.platforms.yaml diff --git a/tests/project/test_scaffold.py b/tests/project/test_scaffold.py index def88fdb..ca5b982c 100644 --- a/tests/project/test_scaffold.py +++ b/tests/project/test_scaffold.py @@ -37,6 +37,8 @@ def test_scaffold_creates_complete_paused_project(tmp_path: Path) -> None: assert "COPY --chown=65532:65532 models ./models" in dockerfile assert "org.opencontainers.image.revision=" in dockerfile assert "org.opencontainers.image.created=" in dockerfile + assert "RUN chmod 1777 /tmp" in dockerfile + assert 'VOLUME ["/tmp"]' in dockerfile assert "USER 65532:65532" in dockerfile assert dockerfile.index("COPY --chown=65532:65532 dander.yaml") < dockerfile.index( "dander plugins install" diff --git a/tests/providers/test_fargate_operations.py b/tests/providers/test_fargate_operations.py index b29fc37a..ad323861 100644 --- a/tests/providers/test_fargate_operations.py +++ b/tests/providers/test_fargate_operations.py @@ -400,11 +400,23 @@ def test_verify_checks_controller_schedule_image_logs_cluster_and_registry(tmp_p f"arn:aws:ecs:{REGION}:{ACCOUNT}:task-definition/{RESOURCE}:1" ), "status": "ACTIVE", + "volumes": [{"name": "dander-tmp"}], "containerDefinitions": [ { "image": IMAGE, "readonlyRootFilesystem": True, "user": "65532:65532", + "environment": [ + {"name": "HOME", "value": "/tmp"}, + {"name": "TMPDIR", "value": "/tmp"}, + ], + "mountPoints": [ + { + "sourceVolume": "dander-tmp", + "containerPath": "/tmp", + "readOnly": False, + } + ], } ], } diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py index b91bb034..8b36ebec 100644 --- a/tests/providers/test_launcher_runtime.py +++ b/tests/providers/test_launcher_runtime.py @@ -154,6 +154,8 @@ def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> assert dict(template.environment)["DANDER_GCP_SERVICE_ACCOUNT"] == ( "dander-runtime-salesforce@unit-project.iam.gserviceaccount.com" ) + assert dict(template.environment)["HOME"] == "/tmp" + assert dict(template.environment)["TMPDIR"] == "/tmp" assert template.network.placement == "awsvpc" assert dict(template.network.extensions) == { "fargate_security_group_ids": "sg-0123456789abcdef0", From c7f38e204016c05b98bed121d0b94785bb4c3c3d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 19:43:09 -0400 Subject: [PATCH 081/163] Prepare Dander 0.8.0rc5 --- CHANGELOG.md | 9 +++++ HANDOFF.md | 33 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 35 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 989eddca..5e3c2b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc5 — 2026-08-09 (beta) + +### Fixed + +- Preserve Fargate's configured disk-backed `/tmp` capacity while making the anonymous volume + writable to Dander's non-root runtime. Generated images now seed mode `1777` volume metadata, + Fargate uses `/tmp` for home and temporary files, and deployment verification checks the full + scratch-storage contract. + ## 0.8.0rc4 — 2026-08-09 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index c6865c5d..b5b3eb26 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,34 @@ ## Finished -- Reproduced the rc4 Fargate failure as a non-root write failure on its `/tmp` volume. -- Made the generated image declare `/tmp` as mode `1777` volume-backed scratch. -- Made Fargate set `HOME` and `TMPDIR` to `/tmp` and verify the complete scratch contract. -- Preserved the read-only root filesystem, UID/GID `65532`, and 20–200 GiB disk-backed scratch. +- Merged the writable Fargate scratch correction through protected main as PR #156. +- Prepared release-only metadata for `dander-platform==0.8.0rc5`. +- Kept the accepted runtime fix unchanged from merge commit `3dd5efc568fc14c5542cffbf29f606a60c11887f`. ## Try It -Generate a project, build its image, and inspect `Config.Volumes` for `/tmp`; Fargate tasks should mount `dander-tmp` there and run as `65532:65532`. +Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. ## Checks -- Focused Python tests: 29 passed. -- Full Python suite: 1,104 passed, 13 skipped; Ruff and strict mypy passed. -- Terraform formatting, validation, AWS module tests, and portability roots passed. -- Wheel/sdist inspection and source-free installation passed outside the checkout. -- Exact ARM64 image succeeded on Fargate: 19 rows, 3 assertions, 1 model, exit code 0. +- PR #156 passed Python, Terraform, packaging, container, and secret checks. +- The exact pre-release ARM64 fix image passed a live Fargate Greenhouse run. +- Release metadata, wheel/sdist inspection, and source-free install passed outside the checkout. ## Decisions -- Do not use `/dev/shm`: it would bypass the configured Fargate scratch capacity. -- Keep the existing anonymous volume and seed its permissions from Docker image metadata. -- A replacement release candidate is required because this changes packaged runtime behavior. +- `0.8.0rc5` replaces rc4 for complete Fargate lifecycle acceptance. +- Fargate remains experimental until the complete lifecycle gate passes. ## Remaining -- Open and merge the focused fix through protected CI. -- Publish a replacement candidate and rerun complete Fargate lifecycle acceptance. +- Merge this release-only PR after protected checks pass. +- Tag and publish `0.8.0rc5` from the exact protected merge. +- Reinstall rc5 source-free and rerun complete Fargate lifecycle acceptance. - Finish replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `src/dander/templates/project/Dockerfile` -- `src/dander/providers/fargate/runtime.py` -- `src/dander/providers/fargate/operations.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index 9a7450ab..776791bf 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc4 +uv tool install dander-platform==0.8.0rc5 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc4` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc5` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 1cb59057..30770a27 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc4`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc5`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 3787cc57..78f25872 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc4 +uv tool install dander-platform==0.8.0rc5 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 704b35c9..a27bbdc0 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc4`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc5`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 7fe39645..4d258636 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc4` is the current public beta candidate. +- Dander `0.8.0rc5` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index d61c8c74..23adac2b 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc4" +export DANDER_TARGET_VERSION="0.8.0rc5" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 8a679772..321a56c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc4" +version = "0.8.0rc5" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index e2a5214d..c9ea7183 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc4", + "dander-platform==0.8.0rc5", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc4" + "README install command uses 0.1.0; package metadata uses 0.8.0rc5" ] diff --git a/uv.lock b/uv.lock index baceab1d..5cb7f803 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc4" +version = "0.8.0rc5" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 89c4d747e3b2f20223ffac1774148c493b003ef4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 20:21:56 -0400 Subject: [PATCH 082/163] Fix Fargate controller result selection --- HANDOFF.md | 30 +++++++-------- infra/aws/modules/fargate/main.tf | 4 +- .../modules/fargate/tests/fargate.tftest.hcl | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index b5b3eb26..293f5a1b 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,34 @@ ## Finished -- Merged the writable Fargate scratch correction through protected main as PR #156. -- Prepared release-only metadata for `dander-platform==0.8.0rc5`. -- Kept the accepted runtime fix unchanged from merge commit `3dd5efc568fc14c5542cffbf29f606a60c11887f`. +- Corrected the Fargate Step Functions result selectors to match the live `ecs:runTask.sync` response. +- Added a focused Terraform assertion for the exact task ARN and container exit-code paths. +- Preserved the controller's normalized result keys, retry policy, deadlines, and runtime contract. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. +Run `terraform test` in `infra/aws/modules/fargate`. ## Checks -- PR #156 passed Python, Terraform, packaging, container, and secret checks. -- The exact pre-release ARM64 fix image passed a live Fargate Greenhouse run. -- Release metadata, wheel/sdist inspection, and source-free install passed outside the checkout. +- Live rc5 evidence showed an ECS task exiting `0`, followed by `States.Runtime` because the old selector expected a nonexistent `Tasks[0]` wrapper. +- Fargate Terraform tests passed: 2 runs, including the rendered selector assertion. +- Python passed: Ruff, formatting, mypy across 304 files, and 1,104 tests with 13 skips. +- Terraform root, GCP/AWS stage zero, AWS platform, Fargate, and portability roots validated; AWS stage-zero tests passed. +- Wheel/sdist inspection and source-free installation passed with the corrected AWS template. ## Decisions -- `0.8.0rc5` replaces rc4 for complete Fargate lifecycle acceptance. -- Fargate remains experimental until the complete lifecycle gate passes. +- The live AWS response is authoritative: `TaskArn` and `Containers` are top-level fields. +- This correction is limited to result normalization; controller semantics are unchanged. ## Remaining -- Merge this release-only PR after protected checks pass. -- Tag and publish `0.8.0rc5` from the exact protected merge. -- Reinstall rc5 source-free and rerun complete Fargate lifecycle acceptance. +- Merge the focused correction through protected main. +- Publish a replacement candidate; rc5 cannot be promoted. - Finish replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `infra/aws/modules/fargate/main.tf` +- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` diff --git a/infra/aws/modules/fargate/main.tf b/infra/aws/modules/fargate/main.tf index e027d226..f936858d 100644 --- a/infra/aws/modules/fargate/main.tf +++ b/infra/aws/modules/fargate/main.tf @@ -524,8 +524,8 @@ resource "aws_sfn_state_machine" "pipeline" { } } ResultSelector = { - "task_arn.$" = "$.Tasks[0].TaskArn" - "exit_code.$" = "$.Tasks[0].Containers[0].ExitCode" + "task_arn.$" = "$.TaskArn" + "exit_code.$" = "$.Containers[0].ExitCode" } ResultPath = "$.task" Retry = [ diff --git a/infra/aws/modules/fargate/tests/fargate.tftest.hcl b/infra/aws/modules/fargate/tests/fargate.tftest.hcl index aaaacc03..11fb2cee 100644 --- a/infra/aws/modules/fargate/tests/fargate.tftest.hcl +++ b/infra/aws/modules/fargate/tests/fargate.tftest.hcl @@ -21,6 +21,31 @@ mock_provider "aws" { repository_url = "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander" } } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::184463061564:role/dander-test-role" + } + } + + mock_resource "aws_cloudwatch_log_group" { + defaults = { + arn = "arn:aws:logs:us-east-1:184463061564:log-group:/dander/test" + } + } + + mock_resource "aws_sqs_queue" { + defaults = { + arn = "arn:aws:sqs:us-east-1:184463061564:dander-test-failures" + url = "https://sqs.us-east-1.amazonaws.com/184463061564/dander-test-failures" + } + } + + mock_resource "aws_sns_topic" { + defaults = { + arn = "arn:aws:sns:us-east-1:184463061564:dander-test-failures" + } + } } variables { @@ -126,3 +151,15 @@ run "paused_bounded_controller" { error_message = "The Fargate stack must consume the stage-zero ECR repository." } } + +run "controller_result_selector" { + command = apply + + assert { + condition = ( + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].ResultSelector["task_arn.$"] == "$.TaskArn" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].ResultSelector["exit_code.$"] == "$.Containers[0].ExitCode" + ) + error_message = "The controller must normalize the top-level ecs:runTask.sync task ARN and container exit code." + } +} From 1fb83852b1ee7417ae890a3d869818f8fd1dd48a Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 20:30:24 -0400 Subject: [PATCH 083/163] Prepare Dander 0.8.0rc6 --- CHANGELOG.md | 8 +++++ HANDOFF.md | 30 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 34 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3c2b31..47eed9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc6 — 2026-08-09 (beta) + +### Fixed + +- Normalize the live `ecs:runTask.sync` response from its top-level `TaskArn` and container exit + code fields so successful Fargate tasks reach Dander's existing exit-code classifier instead of + failing in Step Functions result selection. + ## 0.8.0rc5 — 2026-08-09 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 293f5a1b..64abfe10 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,34 @@ ## Finished -- Corrected the Fargate Step Functions result selectors to match the live `ecs:runTask.sync` response. -- Added a focused Terraform assertion for the exact task ARN and container exit-code paths. -- Preserved the controller's normalized result keys, retry policy, deadlines, and runtime contract. +- Prepared release-only metadata for `dander-platform==0.8.0rc6`. +- Kept the accepted Fargate controller correction unchanged from merge commit `57a9dd58c7fe5ba7062fe15b10f6c45e056b8eb0`. +- Recorded rc5 as rejected after live result selection failed despite an ECS exit code of zero. ## Try It -Run `terraform test` in `infra/aws/modules/fargate`. +Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. ## Checks -- Live rc5 evidence showed an ECS task exiting `0`, followed by `States.Runtime` because the old selector expected a nonexistent `Tasks[0]` wrapper. -- Fargate Terraform tests passed: 2 runs, including the rendered selector assertion. -- Python passed: Ruff, formatting, mypy across 304 files, and 1,104 tests with 13 skips. -- Terraform root, GCP/AWS stage zero, AWS platform, Fargate, and portability roots validated; AWS stage-zero tests passed. -- Wheel/sdist inspection and source-free installation passed with the corrected AWS template. +- PR #158 passed Python, Terraform, packaging, container, and secret checks. +- The merged fix passed 1,104 tests, strict typing, Terraform validation, distribution inspection, and source-free installation. +- Release metadata, wheel/sdist inspection, and source-free rc6 installation passed outside the checkout. ## Decisions -- The live AWS response is authoritative: `TaskArn` and `Containers` are top-level fields. -- This correction is limited to result normalization; controller semantics are unchanged. +- `0.8.0rc6` replaces rc5 for complete Fargate lifecycle acceptance. +- Fargate remains experimental until the complete lifecycle gate passes. ## Remaining -- Merge the focused correction through protected main. -- Publish a replacement candidate; rc5 cannot be promoted. +- Merge this release-only PR after protected checks pass. +- Tag and publish `0.8.0rc6` from the exact protected merge. +- Reinstall rc6 source-free and restart complete Fargate lifecycle acceptance. - Finish replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. ## Review First -- `infra/aws/modules/fargate/main.tf` -- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index 776791bf..fa709728 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc5 +uv tool install dander-platform==0.8.0rc6 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc5` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc6` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 30770a27..455e636d 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc5`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc6`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 78f25872..0d56dfcc 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc5 +uv tool install dander-platform==0.8.0rc6 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index a27bbdc0..15464365 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc5`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc6`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 4d258636..79e714b3 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc5` is the current public beta candidate. +- Dander `0.8.0rc6` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 23adac2b..7850a9ee 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc5" +export DANDER_TARGET_VERSION="0.8.0rc6" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 321a56c9..94479327 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc5" +version = "0.8.0rc6" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index c9ea7183..b316ff17 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc5", + "dander-platform==0.8.0rc6", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc5" + "README install command uses 0.1.0; package metadata uses 0.8.0rc6" ] diff --git a/uv.lock b/uv.lock index 5cb7f803..328982aa 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc5" +version = "0.8.0rc6" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 585396e1b5e18e1c7484a594873ff606365daeb1 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 21:49:33 -0400 Subject: [PATCH 084/163] fix(aws): classify Fargate runtime exits --- HANDOFF.md | 32 ++++++------ infra/aws/modules/fargate/main.tf | 49 ++++++++++++++++++- .../modules/fargate/tests/fargate.tftest.hcl | 29 +++++++++++ tests/infra/test_fargate_runtime.py | 4 ++ 4 files changed, 97 insertions(+), 17 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 64abfe10..e93098f5 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,34 @@ ## Finished -- Prepared release-only metadata for `dander-platform==0.8.0rc6`. -- Kept the accepted Fargate controller correction unchanged from merge commit `57a9dd58c7fe5ba7062fe15b10f6c45e056b8eb0`. -- Recorded rc5 as rejected after live result selection failed despite an ECS exit code of zero. +- Reproduced an rc6 live Fargate task failure whose nonzero container exit was misclassified as a control-plane failure. +- Routed exact `States.TaskFailed` ECS payloads through the existing exit-code classifier while preserving all other controller-failure handling. +- Added rendered-controller assertions for error dispatch, payload validation, and normalized task results. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. +Run `terraform -chdir=infra/aws/modules/fargate init -backend=false`, then `terraform -chdir=infra/aws/modules/fargate test`. ## Checks -- PR #158 passed Python, Terraform, packaging, container, and secret checks. -- The merged fix passed 1,104 tests, strict typing, Terraform validation, distribution inspection, and source-free installation. -- Release metadata, wheel/sdist inspection, and source-free rc6 installation passed outside the checkout. +- Ruff lint/format and strict typing across 304 files passed. +- The complete Python suite passed; focused Fargate tests passed 2/2 and focused operations tests passed 18/18. +- Root, stage-zero, AWS, Fargate, and portability Terraform validation passed. +- Wheel/sdist inspection and source-free installation passed; the installed scaffold contains the corrected controller. ## Decisions -- `0.8.0rc6` replaces rc5 for complete Fargate lifecycle acceptance. -- Fargate remains experimental until the complete lifecycle gate passes. +- `ecs:runTask.sync` reports nonzero runtime exits as `States.TaskFailed`; these are runtime results only when the cause contains the expected ECS task fields. +- Nonmatching task errors remain fail-closed as `launcher_control_plane_failed`. ## Remaining -- Merge this release-only PR after protected checks pass. -- Tag and publish `0.8.0rc6` from the exact protected merge. -- Reinstall rc6 source-free and restart complete Fargate lifecycle acceptance. -- Finish replay, interruption, scheduling, alert, rollback, cleanup, and no-drift evidence. +- Run protected CI and merge the focused fix. +- Publish a replacement candidate; rc6 must not be promoted. +- Correct the external credential-refresh proof fixture, then restart overlap, refresh, interruption, scheduling, alert, rollback, cleanup, and no-drift acceptance. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `infra/aws/modules/fargate/main.tf` +- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` +- `tests/infra/test_fargate_runtime.py` diff --git a/infra/aws/modules/fargate/main.tf b/infra/aws/modules/fargate/main.tf index f936858d..9a10a669 100644 --- a/infra/aws/modules/fargate/main.tf +++ b/infra/aws/modules/fargate/main.tf @@ -548,11 +548,58 @@ resource "aws_sfn_state_machine" "pipeline" { { ErrorEquals = ["States.ALL"] ResultPath = "$.controller_failure" - Next = "Controller failure" + Next = "Classify task failure" }, ] Next = "Classify task" } + "Classify task failure" = { + Type = "Choice" + Choices = [ + { + Variable = "$.controller_failure.Error" + StringEquals = "States.TaskFailed" + Next = "Decode runtime failure" + }, + ] + Default = "Controller failure" + } + "Decode runtime failure" = { + Type = "Pass" + Parameters = { + "details.$" = "States.StringToJson($.controller_failure.Cause)" + } + ResultPath = "$.runtime_failure" + Next = "Validate runtime failure" + } + "Validate runtime failure" = { + Type = "Choice" + Choices = [ + { + And = [ + { + Variable = "$.runtime_failure.details.TaskArn" + IsPresent = true + }, + { + Variable = "$.runtime_failure.details.Containers[0].ExitCode" + IsPresent = true + }, + ] + Next = "Normalize runtime failure" + }, + ] + Default = "Controller failure" + } + "Normalize runtime failure" = { + Type = "Pass" + Parameters = { + "task_arn.$" = "$.runtime_failure.details.TaskArn" + "exit_code.$" = "$.runtime_failure.details.Containers[0].ExitCode" + } + ResultPath = "$.task" + Next = "Classify task" + } "Classify task" = { Type = "Choice" Choices = [ diff --git a/infra/aws/modules/fargate/tests/fargate.tftest.hcl b/infra/aws/modules/fargate/tests/fargate.tftest.hcl index 11fb2cee..6e5058e4 100644 --- a/infra/aws/modules/fargate/tests/fargate.tftest.hcl +++ b/infra/aws/modules/fargate/tests/fargate.tftest.hcl @@ -162,4 +162,33 @@ run "controller_result_selector" { ) error_message = "The controller must normalize the top-level ecs:runTask.sync task ARN and container exit code." } + + assert { + condition = ( + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].Catch[0].ErrorEquals == ["States.Timeout"] && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].Catch[0].Next == "Deadline failure" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].Catch[1].ErrorEquals == ["States.ALL"] && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].Catch[1].Next == "Classify task failure" + ) + error_message = "The controller must preserve timeout handling and inspect other task failures before classification." + } + + assert { + condition = ( + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Classify task failure"].Choices[0].StringEquals == "States.TaskFailed" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Classify task failure"].Choices[0].Next == "Decode runtime failure" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Classify task failure"].Default == "Controller failure" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Decode runtime failure"].Parameters["details.$"] == "States.StringToJson($.controller_failure.Cause)" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Validate runtime failure"].Choices[0].And[0].Variable == "$.runtime_failure.details.TaskArn" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Validate runtime failure"].Choices[0].And[0].IsPresent == true && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Validate runtime failure"].Choices[0].And[1].Variable == "$.runtime_failure.details.Containers[0].ExitCode" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Validate runtime failure"].Choices[0].And[1].IsPresent == true && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Validate runtime failure"].Choices[0].Next == "Normalize runtime failure" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Validate runtime failure"].Default == "Controller failure" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Normalize runtime failure"].Parameters["task_arn.$"] == "$.runtime_failure.details.TaskArn" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Normalize runtime failure"].Parameters["exit_code.$"] == "$.runtime_failure.details.Containers[0].ExitCode" && + jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Normalize runtime failure"].Next == "Classify task" + ) + error_message = "Only genuine ECS runtime failures may be decoded and normalized into the existing exit-code classifier." + } } diff --git a/tests/infra/test_fargate_runtime.py b/tests/infra/test_fargate_runtime.py index a6c1182a..10dc95f1 100644 --- a/tests/infra/test_fargate_runtime.py +++ b/tests/infra/test_fargate_runtime.py @@ -56,6 +56,10 @@ def test_controller_owns_deadline_and_only_runtime_exit_75_retries() -> None: assert "TimeoutSeconds = each.value.resources.deadline_seconds" in module assert "projection.resources.deadline_seconds <= 86400" in variables assert "NumericEquals = 75" in module + assert 'StringEquals = "States.TaskFailed"' in module + assert '"details.$" = "States.StringToJson($.controller_failure.Cause)"' in module + assert 'Next = "Classify task failure"' in module + assert 'Default = "Controller failure"' in module assert 'failure_code = "runtime_retry_exhausted"' in module assert 'failure_code = "launcher_deadline_exceeded"' in module assert '"ecs:StopTask"' in module From f04745c5b179cffce8766d54136fb97f36150b12 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Sun, 9 Aug 2026 21:56:00 -0400 Subject: [PATCH 085/163] Prepare Dander 0.8.0rc7 --- CHANGELOG.md | 7 +++++ HANDOFF.md | 29 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 32 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47eed9f9..1adb405e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc7 — 2026-08-09 (beta) + +### Fixed + +- Recover nonzero `ecs:runTask.sync` task payloads into Dander's existing runtime exit-code + classifier, while preserving fail-closed handling for genuine Fargate control-plane failures. + ## 0.8.0rc6 — 2026-08-09 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index e93098f5..4f09f47d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,33 @@ ## Finished -- Reproduced an rc6 live Fargate task failure whose nonzero container exit was misclassified as a control-plane failure. -- Routed exact `States.TaskFailed` ECS payloads through the existing exit-code classifier while preserving all other controller-failure handling. -- Added rendered-controller assertions for error dispatch, payload validation, and normalized task results. +- Prepared release-only metadata for `dander-platform==0.8.0rc7`. +- Kept the accepted Fargate runtime-failure correction unchanged from merge commit `44b9162c9caea14e21182671bb4cb79680a447f2`. +- Recorded rc6 as rejected after live nonzero ECS exits bypassed the declared runtime classifier. ## Try It -Run `terraform -chdir=infra/aws/modules/fargate init -backend=false`, then `terraform -chdir=infra/aws/modules/fargate test`. +Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. ## Checks -- Ruff lint/format and strict typing across 304 files passed. -- The complete Python suite passed; focused Fargate tests passed 2/2 and focused operations tests passed 18/18. -- Root, stage-zero, AWS, Fargate, and portability Terraform validation passed. -- Wheel/sdist inspection and source-free installation passed; the installed scaffold contains the corrected controller. +- PR #160 passed Python, Terraform, distribution, container, and secret checks. +- The merged fix passed the full Python suite, strict typing, Terraform validation, distribution inspection, and source-free installation. +- Release metadata, wheel/sdist inspection, and source-free rc7 installation passed outside the checkout. ## Decisions -- `ecs:runTask.sync` reports nonzero runtime exits as `States.TaskFailed`; these are runtime results only when the cause contains the expected ECS task fields. -- Nonmatching task errors remain fail-closed as `launcher_control_plane_failed`. +- `0.8.0rc7` replaces rc6 for complete Fargate lifecycle acceptance. +- Fargate remains experimental until the complete lifecycle gate passes. ## Remaining -- Run protected CI and merge the focused fix. -- Publish a replacement candidate; rc6 must not be promoted. +- Merge this release-only PR after protected checks pass. +- Tag and publish `0.8.0rc7` from the exact protected merge. - Correct the external credential-refresh proof fixture, then restart overlap, refresh, interruption, scheduling, alert, rollback, cleanup, and no-drift acceptance. ## Review First -- `infra/aws/modules/fargate/main.tf` -- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` -- `tests/infra/test_fargate_runtime.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index fa709728..b5cc4ede 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc6 +uv tool install dander-platform==0.8.0rc7 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc6` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc7` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 455e636d..43f0d0a6 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc6`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc7`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 0d56dfcc..75fc9838 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc6 +uv tool install dander-platform==0.8.0rc7 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 15464365..7d7f8f69 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc6`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc7`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 79e714b3..7c040ce5 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc6` is the current public beta candidate. +- Dander `0.8.0rc7` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 7850a9ee..257a3b23 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc6" +export DANDER_TARGET_VERSION="0.8.0rc7" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 94479327..9c780544 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc6" +version = "0.8.0rc7" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index b316ff17..48674477 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc6", + "dander-platform==0.8.0rc7", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc6" + "README install command uses 0.1.0; package metadata uses 0.8.0rc7" ] diff --git a/uv.lock b/uv.lock index 328982aa..cba7e116 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc6" +version = "0.8.0rc7" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From f3d04164ef1edee324cee5dfc587a24eaca661bf Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 00:17:29 -0400 Subject: [PATCH 086/163] fix(aws): preserve scheduler execution context --- HANDOFF.md | 33 ++++++++++--------- infra/aws/modules/fargate/main.tf | 12 +++++-- .../modules/fargate/tests/fargate.tftest.hcl | 10 ++++++ tests/infra/test_fargate_runtime.py | 7 +++- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 4f09f47d..7d072660 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,33 +2,36 @@ ## Finished -- Prepared release-only metadata for `dander-platform==0.8.0rc7`. -- Kept the accepted Fargate runtime-failure correction unchanged from merge commit `44b9162c9caea14e21182671bb4cb79680a447f2`. -- Recorded rc6 as rejected after live nonzero ECS exits bypassed the declared runtime classifier. +- Corrected EventBridge Scheduler delivery to Step Functions by preserving Scheduler context tokens in the nested request JSON. +- Removed the optional Step Functions execution name so AWS can generate a valid UUID. +- Granted each scheduler role only `sqs:SendMessage` to the existing failure queue so target-delivery failures can reach its configured DLQ. +- Added Terraform and Python regressions over the final rendered schedule input and exact IAM permission. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. +Run `terraform -chdir=infra/aws/modules/fargate test` and `uv run pytest tests/infra/test_fargate_runtime.py -q`. ## Checks -- PR #160 passed Python, Terraform, distribution, container, and secret checks. -- The merged fix passed the full Python suite, strict typing, Terraform validation, distribution inspection, and source-free installation. -- Release metadata, wheel/sdist inspection, and source-free rc7 installation passed outside the checkout. +- `1117 passed`; Ruff format/check and strict MyPy passed. +- Terraform/Helm initialization, validation, tests, formatting, lint, and rendering passed. +- Wheel/sdist inspection, source-free installs, runtime-all install, and generated-project Terraform validation passed. +- OCI image build, non-root/read-only runtime conformance, and bundled proof-asset checks passed. +- Locked dependency audit reported no known vulnerabilities. ## Decisions -- `0.8.0rc7` replaces rc6 for complete Fargate lifecycle acceptance. -- Fargate remains experimental until the complete lifecycle gate passes. +- Keep the correction at the AWS Scheduler provider boundary; no runtime or public-interface change is needed. +- Preserve exact DLQ scope rather than broadening scheduler permissions. ## Remaining -- Merge this release-only PR after protected checks pass. -- Tag and publish `0.8.0rc7` from the exact protected merge. -- Correct the external credential-refresh proof fixture, then restart overlap, refresh, interruption, scheduling, alert, rollback, cleanup, and no-drift acceptance. +- Push the focused branch, open a PR, and let protected CI repeat security and Linux checks. +- Publish a replacement candidate after merge because rc7's scheduled path is defective. +- Retry scheduled execution, then complete rollback, cleanup, evidence, and final no-drift acceptance. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `infra/aws/modules/fargate/main.tf` +- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` +- `tests/infra/test_fargate_runtime.py` diff --git a/infra/aws/modules/fargate/main.tf b/infra/aws/modules/fargate/main.tf index 9a10a669..47c80b97 100644 --- a/infra/aws/modules/fargate/main.tf +++ b/infra/aws/modules/fargate/main.tf @@ -765,6 +765,13 @@ data "aws_iam_policy_document" "scheduler" { actions = ["states:StartExecution"] resources = [aws_sfn_state_machine.pipeline[each.key].arn] } + + statement { + sid = "DeliverScheduleFailures" + effect = "Allow" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.failures.arn] + } } resource "aws_iam_role_policy" "scheduler" { @@ -790,16 +797,15 @@ resource "aws_scheduler_schedule" "pipeline" { target { arn = "arn:${local.partition}:scheduler:::aws-sdk:sfn:startExecution" role_arn = aws_iam_role.scheduler[each.key].arn - input = jsonencode({ + input = replace(replace(jsonencode({ StateMachineArn = aws_sfn_state_machine.pipeline[each.key].arn - Name = "" Input = jsonencode({ deployment_revision = each.value.labels.image_digest scheduled_time = "" scheduler_attempt = "" scheduler_execution_id = "" }) - }) + }), "\\\\u003c", "<"), "\\\\u003e", ">") dead_letter_config { arn = aws_sqs_queue.failures.arn diff --git a/infra/aws/modules/fargate/tests/fargate.tftest.hcl b/infra/aws/modules/fargate/tests/fargate.tftest.hcl index 6e5058e4..7b1f1f0a 100644 --- a/infra/aws/modules/fargate/tests/fargate.tftest.hcl +++ b/infra/aws/modules/fargate/tests/fargate.tftest.hcl @@ -155,6 +155,16 @@ run "paused_bounded_controller" { run "controller_result_selector" { command = apply + assert { + condition = ( + !contains(keys(jsondecode(aws_scheduler_schedule.pipeline["greenhouse_jobs"].target[0].input)), "Name") && + jsondecode(jsondecode(aws_scheduler_schedule.pipeline["greenhouse_jobs"].target[0].input).Input).scheduled_time == "" && + jsondecode(jsondecode(aws_scheduler_schedule.pipeline["greenhouse_jobs"].target[0].input).Input).scheduler_attempt == "" && + jsondecode(jsondecode(aws_scheduler_schedule.pipeline["greenhouse_jobs"].target[0].input).Input).scheduler_execution_id == "" + ) + error_message = "The schedule target must omit the optional execution name and preserve literal Scheduler context attributes in valid nested JSON." + } + assert { condition = ( jsondecode(aws_sfn_state_machine.pipeline["greenhouse_jobs"].definition).States["Run task"].ResultSelector["task_arn.$"] == "$.TaskArn" && diff --git a/tests/infra/test_fargate_runtime.py b/tests/infra/test_fargate_runtime.py index 10dc95f1..727718d4 100644 --- a/tests/infra/test_fargate_runtime.py +++ b/tests/infra/test_fargate_runtime.py @@ -77,9 +77,14 @@ def test_scheduler_delivery_is_separate_paused_aware_and_dead_lettered() -> None in module ) assert 'arn = "arn:${local.partition}:scheduler:::aws-sdk:sfn:startExecution"' in module - assert 'Name = ""' in module + assert 'Name = ""' not in module assert 'scheduled_time = ""' in module assert 'scheduler_attempt = ""' in module + assert "replace(replace(jsonencode({" in module + assert '"\\\\\\\\u003c", "<"), "\\\\\\\\u003e", ">")' in module + assert 'sid = "DeliverScheduleFailures"' in module + assert 'actions = ["sqs:SendMessage"]' in module + assert "resources = [aws_sqs_queue.failures.arn]" in module assert "maximum_retry_attempts = var.scheduler_delivery_retry_count" in module assert "launcher_retry_count" in module assert "dead_letter_config" in module From c5c73ae1eff45079324881a134530096aaa17ce4 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 00:24:38 -0400 Subject: [PATCH 087/163] Prepare Dander 0.8.0rc8 --- CHANGELOG.md | 8 +++++ HANDOFF.md | 33 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 34 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1adb405e..cfd6f740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ capabilities enter through the next minor release. ## Unreleased +## 0.8.0rc8 — 2026-08-10 (beta) + +### Fixed + +- Preserve EventBridge Scheduler context attributes in the nested Step Functions request, + allow AWS to generate valid execution names, and permit delivery failures to reach the + existing exact-scoped dead-letter queue. + ## 0.8.0rc7 — 2026-08-09 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 7d072660..2945b883 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,33 @@ ## Finished -- Corrected EventBridge Scheduler delivery to Step Functions by preserving Scheduler context tokens in the nested request JSON. -- Removed the optional Step Functions execution name so AWS can generate a valid UUID. -- Granted each scheduler role only `sqs:SendMessage` to the existing failure queue so target-delivery failures can reach its configured DLQ. -- Added Terraform and Python regressions over the final rendered schedule input and exact IAM permission. +- Prepared release-only metadata for `dander-platform==0.8.0rc8`. +- Kept the accepted AWS Scheduler correction unchanged from merge commit `f3ccb862b05ee359c8517c6a6874bab2150c5a40`. +- Recorded rc7 as rejected after live scheduled delivery passed escaped context tokens to Step Functions. ## Try It -Run `terraform -chdir=infra/aws/modules/fargate test` and `uv run pytest tests/infra/test_fargate_runtime.py -q`. +Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. ## Checks -- `1117 passed`; Ruff format/check and strict MyPy passed. -- Terraform/Helm initialization, validation, tests, formatting, lint, and rendering passed. -- Wheel/sdist inspection, source-free installs, runtime-all install, and generated-project Terraform validation passed. -- OCI image build, non-root/read-only runtime conformance, and bundled proof-asset checks passed. -- Locked dependency audit reported no known vulnerabilities. +- PR #162 passed Python, Terraform, distribution, container, and secret checks. +- The merged fix passed 1,117 tests, strict typing, Terraform/Helm validation, distribution inspection, source-free installation, and container conformance. ## Decisions -- Keep the correction at the AWS Scheduler provider boundary; no runtime or public-interface change is needed. -- Preserve exact DLQ scope rather than broadening scheduler permissions. +- `0.8.0rc8` replaces rc7 for the remaining complete Fargate lifecycle acceptance. +- Fargate remains experimental until the complete lifecycle gate passes. ## Remaining -- Push the focused branch, open a PR, and let protected CI repeat security and Linux checks. -- Publish a replacement candidate after merge because rc7's scheduled path is defective. -- Retry scheduled execution, then complete rollback, cleanup, evidence, and final no-drift acceptance. +- Merge this release-only PR after protected checks pass. +- Tag and publish `0.8.0rc8` from the exact protected merge. +- Deploy the source-free rc8 image while schedules remain paused, then retry scheduled execution. +- Complete rollback, cleanup, evidence, and final no-drift acceptance. ## Review First -- `infra/aws/modules/fargate/main.tf` -- `infra/aws/modules/fargate/tests/fargate.tftest.hcl` -- `tests/infra/test_fargate_runtime.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `docs/session-resume.md` diff --git a/README.md b/README.md index b5cc4ede..51c12b91 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc7 +uv tool install dander-platform==0.8.0rc8 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc7` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.8.0rc8` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 43f0d0a6..9730de9d 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc7`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.8.0rc8`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 75fc9838..85760a65 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc7 +uv tool install dander-platform==0.8.0rc8 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 7d7f8f69..6c8f3071 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc7`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.8.0rc8`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 7c040ce5..221438bf 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc7` is the current public beta candidate. +- Dander `0.8.0rc8` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 257a3b23..77c699ed 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc7" +export DANDER_TARGET_VERSION="0.8.0rc8" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 9c780544..0982e401 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc7" +version = "0.8.0rc8" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 48674477..b002d75c 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc7", + "dander-platform==0.8.0rc8", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc7" + "README install command uses 0.1.0; package metadata uses 0.8.0rc8" ] diff --git a/uv.lock b/uv.lock index cba7e116..db8f0ae7 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc7" +version = "0.8.0rc8" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 9cb32b3b4e8f77a7dbc435c40c588268a0d711b5 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 01:16:56 -0400 Subject: [PATCH 088/163] Record Fargate lifecycle acceptance --- HANDOFF.md | 33 +++++---- ...ortability-fargate-lifecycle-acceptance.md | 74 +++++++++++++++++++ docs/cloud-portability-phase1b-acceptance.md | 5 +- docs/decisions.md | 11 +++ docs/execution-projection.md | 7 +- docs/known-limitations.md | 13 ++-- docs/release-audit.md | 14 ++-- docs/session-resume.md | 12 ++- 8 files changed, 135 insertions(+), 34 deletions(-) create mode 100644 docs/cloud-portability-fargate-lifecycle-acceptance.md diff --git a/HANDOFF.md b/HANDOFF.md index 2945b883..b3e33157 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,33 +2,36 @@ ## Finished -- Prepared release-only metadata for `dander-platform==0.8.0rc8`. -- Kept the accepted AWS Scheduler correction unchanged from merge commit `f3ccb862b05ee359c8517c6a6874bab2150c5a40`. -- Recorded rc7 as rejected after live scheduled delivery passed escaped context tokens to Step Functions. +- Published public `dander-platform==0.8.0rc8` from protected main. +- Passed the named Fargate-to-BigQuery/GCP lifecycle gate with the byte-identical source-free rc8 image. +- Proved scheduled execution, replay, interruption, alert routing, image rollback, cleanup, and final no-drift. +- Recorded the incompatible `a05dfe7f…` image honestly and verified rc7 as the prior working runtime. ## Try It -Run `uv run python scripts/check_release_metadata.py`, then build and inspect the candidate artifacts. +Read `docs/cloud-portability-fargate-lifecycle-acceptance.md`, then run `dander runtime compatibility`. ## Checks -- PR #162 passed Python, Terraform, distribution, container, and secret checks. -- The merged fix passed 1,117 tests, strict typing, Terraform/Helm validation, distribution inspection, source-free installation, and container conformance. +- Public rc8 installed source-free and its OCI index matched in GAR and ECR. +- Manual, scheduled, rollback, and restored-image runs each completed with truthful runtime data. +- AWS, GCP parity/WIF, retained stage zero, and retained platform each reported `No changes.` +- PR #162 passed 1,117 tests, strict typing, Terraform/Helm, distribution, container, and security checks. ## Decisions -- `0.8.0rc8` replaces rc7 for the remaining complete Fargate lifecycle acceptance. -- Fargate remains experimental until the complete lifecycle gate passes. +- The named Fargate-to-BigQuery/GCP composition passes lifecycle acceptance. +- Fargate remains experimental pending the published scale/profile qualification objectives. +- No other AWS, warehouse, or cross-cloud pairing inherits this evidence. ## Remaining -- Merge this release-only PR after protected checks pass. -- Tag and publish `0.8.0rc8` from the exact protected merge. -- Deploy the source-free rc8 image while schedules remain paused, then retry scheduled execution. -- Complete rollback, cleanup, evidence, and final no-drift acceptance. +- Merge this evidence-only PR through protected main. +- Perform the read-only Phase 5.5 checkpoint only if Phase 5 is complete on clean main. +- Do not begin Azure implementation during the checkpoint. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `docs/session-resume.md` +- `docs/cloud-portability-fargate-lifecycle-acceptance.md` +- `docs/known-limitations.md` +- `docs/release-audit.md` diff --git a/docs/cloud-portability-fargate-lifecycle-acceptance.md b/docs/cloud-portability-fargate-lifecycle-acceptance.md new file mode 100644 index 00000000..d08fc52b --- /dev/null +++ b/docs/cloud-portability-fargate-lifecycle-acceptance.md @@ -0,0 +1,74 @@ +# Fargate lifecycle acceptance + +Accepted on 2026-08-10 against public `dander-platform==0.8.0rc8`, published from protected-main +merge `819a61c0334b4182178f5951361e68dbb81afb39`. The proof used a generated source-free project, +the disposable GCP project `project-092b24a8-26a3-4438-8cd`, and AWS account `184463061564`. +The retained project was not changed. + +## Artifact and deployment + +- Public rc8 installed outside the repository and generated a project with no copied `src/`. +- One source-free multi-platform image retained byte-identical GAR and ECR content. Its OCI index + digest was `sha256:bb69895befa31a384b8eff0e8f783b795a311d924456a5446d01bc8d200369b9`; + both registries retained AMD64 and ARM64 manifests, SBOM, and provenance attestations. +- Reviewed plans changed only the candidate image, Fargate task-definition revisions, and the + derived controller/scheduler references. Both schedules remained disabled except during the + bounded scheduled-execution proof. +- The deployed task definitions selected ARM64, the immutable ECR digest, distinct execution and + runtime roles, and the accepted non-root scratch-storage contract. + +## Execution, replay, and scheduling + +- A manual Greenhouse run and its replay each ingested 19 rows, built + `stg_greenhouse__jobs`, passed three assertions, and published one metadata asset. Raw data + retained 19 distinct IDs, and the inclusive watermark never regressed. +- EventBridge Scheduler started execution + `c9252fa8-24ae-4542-acf4-b16189aab872`. AWS substituted the scheduled timestamp, attempt, and + execution identifier into the Step Functions input; the ECS task used the exact rc8 digest and + exited `0`. +- The scheduled run recorded a truthful successful ledger row with 19 extracted and 19 affected + rows, one model, three assertions, and one asset. The Greenhouse lease released and no + run-scoped staging object remained. +- Both schedules were restored to their tracked daily expressions in `DISABLED` state immediately + after the bounded proof. + +## Interruption and alerts + +- A 65-endpoint credential-refresh probe exceeded the 900-second controller deadline. The task + exited `130`, Step Functions recorded `TIMED_OUT`, run history retained sanitized + `interrupted_run`, the lease released, and no staging residue required repair. +- A separately reviewed temporary deadline increase allowed the same 65-endpoint probe to finish; + its 65 checkpoint tables and rows were present, and the configured 900-second deadline was then + restored. +- The enabled, state-machine-scoped EventBridge failure rule invoked both the KMS-encrypted SNS + topic and the AWS-managed-SSE SQS queue. The controlled timeout produced observable rule, + invocation, publish, and queue-delivery metrics. +- The SNS topic deliberately has no human subscription in this disposable proof. Alert routing is + live-qualified; delivery to an operator is not claimed. + +## Rollback and recovery + +- The first proposed prior digest, + `sha256:a05dfe7fcdc50a3003f92b25d0aaa30f2e2ccc7d2e2186d395530a4cc7f3768c`, failed immediately + with the sanitized code `authentication_failed`. It predates the accepted renewable + AWS-to-Google identity path and is recorded as incompatible, not as a successful rollback + target. +- The actual prior working runtime was rc7 digest + `sha256:15ddaa778c8e5fc682fd1bff6845f2765824be15352bb103652c6298290ae6cb`. + With schedules paused, a reviewed image-only rollback ran Greenhouse successfully: 19 rows, one + model, three assertions, monotonic watermark, released lease, and no staging residue. +- A reviewed plan restored rc8. The restored-digest smoke run repeated the same successful data, + transform, test, lease, cursor, and cleanup checks. + +## Final state and result + +- No ECS task remains active. The one temporary diagnostic ECR tag was removed; the rc7 rollback + and accepted rc8 images remain. +- The AWS platform, GCP parity/WIF proof root, retained-project stage zero, and retained-project + platform each finished with exactly `No changes.` +- Existing failure-queue messages were preserved as operating evidence. No secret, source record, + unrestricted exception, Terraform state, or alert address is retained here. + +The complete Fargate lifecycle gate passes for the named Fargate-to-BigQuery/GCP composition. +Fargate remains experimental until the profile also satisfies the published scale/qualification +objectives; this record does not promote other AWS, warehouse, or cross-cloud combinations. diff --git a/docs/cloud-portability-phase1b-acceptance.md b/docs/cloud-portability-phase1b-acceptance.md index 679a1a7d..2e3e94aa 100644 --- a/docs/cloud-portability-phase1b-acceptance.md +++ b/docs/cloud-portability-phase1b-acceptance.md @@ -56,5 +56,6 @@ Fargate-to-BigQuery composition; it does not promote Fargate or any non-GCP prof Phase 1B passed: one source-free multi-platform image retained identical registry content, ran on Cloud Run and Fargate, and reached BigQuery from AWS before and after keyless credential refresh. -Fargate remains experimental until the broader correctness, lifecycle, scale, and operating gates -in the portability roadmap pass. +The later [Fargate lifecycle acceptance](cloud-portability-fargate-lifecycle-acceptance.md) passed +correctness and operating gates. Fargate remains experimental pending the published scale and +profile-qualification objectives. diff --git a/docs/decisions.md b/docs/decisions.md index e0b660cb..5465b5c8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-10 — Fargate lifecycle evidence does not skip profile qualification + +- **Accepted slice:** Public `0.8.0rc8` passed manual and scheduled execution, replay, + interruption, alert routing, image rollback, cleanup, and no-drift for the named + Fargate-to-BigQuery/GCP composition. +- **Support boundary:** Fargate remains experimental until the same profile satisfies the + published scale and qualification objectives. No other AWS, warehouse, or cross-cloud pairing + inherits this evidence. +- **Evidence:** The bounded record is + `docs/cloud-portability-fargate-lifecycle-acceptance.md`. + ## 2026-08-09 — Phase 1B accepts only exact public dependency fixtures - **Evidence:** The accepted image contains three boto3/botocore example files with AWS-published diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 0c65e9d2..736a79bc 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -36,8 +36,11 @@ Google Auth refreshes its signed AWS subject token; task secrets are neither cop environment variables nor written to disk. The impersonated Google token remains limited to 600 seconds, while the launcher may enforce a deadline of up to 24 hours. -This provider projection is an internal construction gate: complete pipeline lifecycle and live -acceptance must still pass before Fargate is a supported launcher. +The named Fargate-to-BigQuery/GCP composition has passed complete lifecycle acceptance, including +manual and scheduled execution, replay, interruption, alerts, image rollback, cleanup, and +no-drift reconciliation. Fargate remains experimental until the profile also satisfies the +published scale/qualification objectives; see the +[bounded acceptance record](cloud-portability-fargate-lifecycle-acceptance.md). The Kubernetes provider projects a selected named profile into a packaged Helm chart for an existing Kubernetes 1.27+ cluster. Its immutable template uses stdout, operator-owned Secret key diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 18167c43..a561abd5 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -3,13 +3,12 @@ Dander `0.8.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. -- AWS ECS/Fargate is not yet a supported Dander launcher. Packaged stage-zero and platform roots - now have public saved-plan/apply commands, and accepted OCI artifacts can be copied into ECR - without rebuilding. Manifest-bound status, logs, cancellation, replay, and deployment - verification are implemented but not live-qualified. The runtime bridge now renews its AWS - task-role input from the fixed ECS endpoint for each Google federation refresh without persisting - task credentials. Complete pipeline parity, interruption, replay, schedule, alert, rollback, and - no-drift acceptance remain later gates. +- AWS ECS/Fargate remains experimental rather than supported. The named Fargate-to-BigQuery/GCP + composition has passed source-free manual and scheduled execution, replay, interruption, + renewable keyless identity, alert routing, image rollback, cleanup, and no-drift acceptance. + The disposable proof topic had no human subscriber, and the profile has not completed the + published scale/qualification objectives. Other AWS, warehouse, and cross-cloud combinations + receive no support claim from this proof. - Provider package extras and the full runtime image do not by themselves qualify Snowflake, Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar SCD1 adapter with bounded Parquet upload, temporary remote staging, schema checks, destination fencing, and fenced diff --git a/docs/release-audit.md b/docs/release-audit.md index 6c8f3071..bd4c8500 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -1,6 +1,6 @@ # Dander Platform Release Audit -Audited on 2026-08-07 against the product promise in `steering/00-project-overview.md`. +Audited on 2026-08-10 against the product promise in `steering/00-project-overview.md`. “Live-proven” means the behavior was observed in a disposable provider account or retained GCP project. “Implemented” means automated tests cover the contract while an optional provider or cloud path remains outside the live proof. @@ -12,7 +12,7 @@ cloud path remains outside the live proof. | One CLI and typed project manifest | Live-proven | `dander.yaml` declares four daily connector pipelines and one executable graph; `dander validate`, provisioning, execution, and metadata inspection use the same CLI. | | Batteries-included GCP infrastructure | Live-proven | Reviewed Terraform owns remote state, Artifact Registry, BigQuery datasets, per-pipeline IAM, Cloud Run jobs, Scheduler jobs, Secret Manager containers, failure alerts, and the optional cost guard. | | Additive multi-pipeline hosting | Live-proven | Greenhouse, HubSpot, Salesforce, and ServiceNow coexist with distinct jobs, schedules, and runtime identities. The graph job is separately deployed and paused. | -| Independently installed connectors | Live-proven | Public Salesforce `0.3.0` and ServiceNow `0.2.1` packages are exact-pinned, discovered through Dander plugin API v1, installed into a source-free image, and exercised against disposable provider accounts. | +| Independently installed connectors | Live-proven | Public Salesforce `0.3.1` and ServiceNow `0.2.2` packages are exact-pinned, discovered through Dander plugin API v1, installed into a source-free image, and exercised against disposable provider accounts. | | Shared enterprise authentication | Live-proven for retained plugins | Dander core supplies Salesforce OAuth2 JWT and ServiceNow OAuth2 client credentials while Terraform grants each runtime access only to its declared Secret Manager resources. | | Bounded ingestion and idempotent BigQuery writes | Live-proven for hosted SCD1 | Greenhouse, HubSpot, Salesforce, and ServiceNow have completed hosted ingestion and replay checks without duplicate destination keys; Salesforce applies its cursor server-side. | | Owned transforms and tests | Live-proven | Retained pipelines execute Dander's `ref()` DAG and declared BigQuery assertions after ingestion. | @@ -36,9 +36,13 @@ cloud path remains outside the live proof. conformance on AMD64, and queried BigQuery from one ARM64 Fargate task before and after keyless Google credential refresh. All proof resources were removed, and Fargate remains experimental. See `docs/cloud-portability-phase1b-acceptance.md`. -- Retained source-free Dander image: `sha256:3220623…8995`, built with Dander `0.5.0` and both - plugin `0.2.0` packages. Dander `0.5.1` changes only catalog recommendations, so the runtime - image was intentionally not replaced. +- Public Dander `0.8.0rc8` passed the complete lifecycle gate for the named + Fargate-to-BigQuery/GCP composition: manual and scheduled execution, replay, interruption, + alert routing, image rollback, cleanup, and four final no-drift plans. Fargate remains + experimental pending scale/profile qualification. See + `docs/cloud-portability-fargate-lifecycle-acceptance.md`. +- Retained source-free Dander image: `sha256:68e112c43b365018b735be7934446e15dfe6169fc64062b62b8bb97ea4f93b96`, + built with Dander `0.7.1`, Salesforce `0.3.1`, and ServiceNow `0.2.2`. - Retained Druff image: `sha256:a5e255d6…871c`; public static URL: . - The four scheduled connector executions on 2026-08-05 completed successfully. Those executions diff --git a/docs/session-resume.md b/docs/session-resume.md index 221438bf..de1fe25b 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -1,4 +1,4 @@ -# Session Resume — 2026-08-07 +# Session Resume — 2026-08-10 Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `docs/release-audit.md` before changing code or cloud resources. @@ -33,6 +33,12 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Latest operating evidence +- On 2026-08-10, public Dander `0.8.0rc8` passed the complete lifecycle gate for the named + Fargate-to-BigQuery/GCP composition. The source-free, byte-identical GAR/ECR image completed + manual, scheduled, replay, interruption, alert-routing, rollback, cleanup, and no-drift checks. + Fargate remains experimental pending scale/profile qualification. See + `docs/cloud-portability-fargate-lifecycle-acceptance.md`. + - On 2026-08-09, public Dander `0.8.0rc1` passed the isolated Phase 1B feasibility gate. One source-free multi-platform OCI index retained identical GAR/ECR and per-platform digests; Cloud Run completed on AMD64 and one ARM64 Fargate task queried BigQuery before and after a @@ -49,8 +55,8 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `dander-servicenow-incidents-6g72x` all completed successfully. - The latest manual executable-graph run, `dander-greenhouse-graph-7gn9z`, completed successfully on 2026-08-04; its schedule remains intentionally paused. -- The most recent reviewed platform plan reported `0` add, `0` change, and `0` destroy across - `111` no-op resources after Druff deployment. +- The latest retained stage-zero and platform plans each reported exactly `No changes.` after the + isolated Fargate proof. No retained-project apply occurred. - Continue the 30-day operating record in GitHub issue #26. The next normal scheduled runs are the remaining observation point for the reconciled stable image. From f05b4ba67e187a6d8ba641e9a654a4a93df00af9 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 02:14:34 -0400 Subject: [PATCH 089/163] feat(snowflake): add fenced scalar writer modes --- HANDOFF.md | 39 +- docs/decisions.md | 12 + docs/known-limitations.md | 9 +- docs/snowflake.md | 24 +- src/dander/providers/bigquery/runtime.py | 11 + src/dander/providers/postgresql/runtime.py | 7 + src/dander/providers/redshift/runtime.py | 7 + src/dander/providers/snowflake/runtime.py | 29 +- src/dander/providers/snowflake/writer.py | 342 ++++++++++++++++-- src/dander/runtime-compatibility.json | 8 +- src/dander/warehouse/runtime.py | 8 +- tests/cli/test_run_command.py | 13 +- tests/providers/test_bigquery_runtime.py | 7 + .../test_snowflake_warehouse_runtime.py | 260 ++++++++++++- 14 files changed, 711 insertions(+), 65 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index b3e33157..4eb3532a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,39 @@ ## Finished -- Published public `dander-platform==0.8.0rc8` from protected main. -- Passed the named Fargate-to-BigQuery/GCP lifecycle gate with the byte-identical source-free rc8 image. -- Proved scheduled execution, replay, interruption, alert routing, image rollback, cleanup, and final no-drift. -- Recorded the incompatible `a05dfe7f…` image honestly and verified rc7 as the prior working runtime. +- Added provider-neutral warehouse writer-mode selection while keeping hosted CLI ingestion backward-compatible. +- Implemented Snowflake replace, SCD1, SCD2, snapshot, and incremental publication on one bounded Parquet/COPY path. +- Kept every finalizer and replay-history update in the exact destination-fence transaction. +- Fixed cursor ordering, whole-stream SCD2 publication, and partial-replay replace correctness. +- Updated Snowflake compatibility and limitations without making a support claim. ## Try It -Read `docs/cloud-portability-fargate-lifecycle-acceptance.md`, then run `dander runtime compatibility`. +Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`, then inspect `docs/snowflake.md`. ## Checks -- Public rc8 installed source-free and its OCI index matched in GAR and ECR. -- Manual, scheduled, rollback, and restored-image runs each completed with truthful runtime data. -- AWS, GCP parity/WIF, retained stage zero, and retained platform each reported `No changes.` -- PR #162 passed 1,117 tests, strict typing, Terraform/Helm, distribution, container, and security checks. +- Ruff format/check: passed across 328 files. +- Mypy: passed across 304 source/test files with the CI dependency profile. +- Pytest: 1,112 passed and 13 skipped (1,125 collected). +- Distribution/source-free install and Terraform validation passed before final correctness fixes; the full Python suite passed afterward. +- Container export is locally blocked by a broken Docker snapshot and critically low disk; protected Linux CI remains required. ## Decisions -- The named Fargate-to-BigQuery/GCP composition passes lifecycle acceptance. -- Fargate remains experimental pending the published scale/profile qualification objectives. -- No other AWS, warehouse, or cross-cloud pairing inherits this evidence. +- One warehouse writer capability selects logical mode plus only its cursor/snapshot field. +- SCD2 and replace consume one complete bounded-memory stream; incremental ranks cursor before ordinal. +- Views, graph wiring, VARIANT, direct-write crossover, telemetry, and live proof remain separate gates. ## Remaining -- Merge this evidence-only PR through protected main. -- Perform the read-only Phase 5.5 checkpoint only if Phase 5 is complete on clean main. -- Do not begin Azure implementation during the checkpoint. +- Commit, push, and open the focused PR. +- Require protected CI to repeat Linux tests, PostgreSQL integration, packaging, Terraform, container, and security checks. +- Merge only when required checks and review pass. +- Continue the remaining Snowflake first-class slices from updated main. ## Review First -- `docs/cloud-portability-fargate-lifecycle-acceptance.md` -- `docs/known-limitations.md` -- `docs/release-audit.md` +- `src/dander/providers/snowflake/writer.py` +- `src/dander/warehouse/runtime.py` +- `tests/providers/test_snowflake_warehouse_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 5465b5c8..1586d28b 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -924,6 +924,18 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Permanent view DDL, graph execution, and automatic transform-schema evolution fail closed. They cannot enter this slice without preserving the same publication guarantee. +## 2026-08-10 — Snowflake writer modes share one fenced staging path + +- **Selection:** The provider-neutral warehouse writer capability accepts a logical mode plus the + mode-specific cursor or snapshot field. Existing hosted ingestion translates explicitly to SCD1, + so current CLI behavior is unchanged. +- **Publication:** Snowflake reuses one bounded Parquet/temporary-stage load for replace, SCD1, + SCD2, snapshot, and incremental writes. Mode-specific DML and load-history rows commit with the + exact destination-fence touch; replace is streamed as one logical publication. +- **Boundary:** This makes the modes reachable and testable but does not promote Snowflake support. + Views, graph wiring, explicit VARIANT, direct-write crossover, telemetry, and live qualification + remain separate gates. + ## 2026-08-09 — Redshift begins with IAM-only bounded SCD1 publication - **Boundary:** Provisioned and Serverless profiles map native database/schema coordinates into diff --git a/docs/known-limitations.md b/docs/known-limitations.md index a561abd5..817889eb 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,10 +10,11 @@ evaluate these limits before using it for an unattended system containing busine published scale/qualification objectives. Other AWS, warehouse, and cross-cloud combinations receive no support claim from this proof. - Provider package extras and the full runtime image do not by themselves qualify Snowflake, - Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar SCD1 adapter with - bounded Parquet upload, temporary remote staging, schema checks, destination fencing, and fenced - table/incremental transforms. It has no live Snowflake qualification, views, graph execution, - semi-structured types, or other write modes. + Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar adapter with all five + writer modes behind its provider capability, bounded Parquet upload, temporary remote staging, + schema checks, destination fencing, and fenced table/incremental transforms. The ordinary hosted + source runner still selects SCD1, and Snowflake has no live qualification, views, graph execution, + semi-structured types, measured direct-write crossover, or end-to-end telemetry retention. Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. It has no live qualification, views, executable graphs, diff --git a/docs/snowflake.md b/docs/snowflake.md index 83bc31f6..c2458fec 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -1,8 +1,9 @@ # Experimental Snowflake warehouse Snowflake is an experimental warehouse adapter, not a supported Dander profile. The current slice -exists to prove the provider boundary with native database/schema coordinates, a real bounded -bulk path, and fenced table/incremental models before additional write modes are added. +proves native database/schema coordinates, a bounded bulk path, all five scalar writer modes, and +fenced table/incremental models. Live qualification and the remaining first-class gates are still +required before support promotion. ## Configuration @@ -42,8 +43,13 @@ variable reference. Do not put a token, private key, password, or connection str - `COPY` preserves Parquet logical and binary types explicitly. - One session-temporary stage and table contain each batch; normal cleanup is immediate and session termination removes them after process death. -- SCD1 publication and load-history recording occur in the same transaction as the exact - destination fencing-token touch. +- SCD1, incremental, snapshot, SCD2, and replace publication share the same bounded staging path. + Load-history recording and final destination DML occur in the same transaction as the exact + fencing-token touch. +- SCD1 retains the last staged record per business key. Incremental mode adds a monotonic cursor + comparison. Snapshot mode appends only previously unseen complete rows. SCD2 closes changed + current rows and writes `valid_from`, `valid_to`, and `is_current`. Replace performs fenced + `DELETE`/`INSERT`, including an empty-source replacement. - Only declared nullable columns may be added automatically. Extra columns, required additions, type drift, nullability drift, malformed rows, and oversized singleton parts fail closed. - Portable or Snowflake-authored table models replace rows through fenced `DELETE`/`INSERT` DML. @@ -56,8 +62,10 @@ until a live least-privilege profile is qualified. ## Deliberate limits -Only scalar SCD1 ingestion plus table and incremental model materializations are implemented. +All five scalar write patterns are reachable through the warehouse writer capability, while the +ordinary hosted source runner deliberately continues to select SCD1. PipelineGraph execution has +not yet been wired to Snowflake's writer selection. Views remain unavailable because Snowflake permanent DDL cannot share the destination-fence -transaction. Graphs, replace, SCD2, snapshot, semi-structured fields, live concurrency proof, -infrastructure provisioning, and support promotion remain separate work. Use -`catalog.provider: none` for this experimental path. +transaction. Semi-structured fields, a measured small-load direct path, full telemetry plumbing, +live concurrency proof, infrastructure provisioning, and support promotion remain separate work. +Use `catalog.provider: none` for this experimental path. diff --git a/src/dander/providers/bigquery/runtime.py b/src/dander/providers/bigquery/runtime.py index 04c1cf73..00a815cb 100644 --- a/src/dander/providers/bigquery/runtime.py +++ b/src/dander/providers/bigquery/runtime.py @@ -80,10 +80,21 @@ def build_ingestion_writer( sandbox: bool, batch_rows: int, schema_evolution: SchemaEvolution, + mode: WriteMode = WriteMode.SCD1, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> WritePattern: """Use replace for local sandbox execution and SCD1 for hosted execution.""" + del cursor_field + del snapshot_field if sandbox: + if mode not in {WriteMode.SCD1, WriteMode.REPLACE}: + raise ValueError(f"BigQuery sandbox does not support {mode.value} writes") return BigQueryReplaceWriter(project=self.project, max_batch_rows=batch_rows) + if mode is not WriteMode.SCD1: + raise ValueError( + f"BigQuery hosted ingestion does not select {mode.value} through this capability" + ) return BigQueryScd1Writer( project=self.project, max_batch_rows=batch_rows, diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py index df805fc7..8f4d584e 100644 --- a/src/dander/providers/postgresql/runtime.py +++ b/src/dander/providers/postgresql/runtime.py @@ -92,10 +92,17 @@ def build_ingestion_writer( sandbox: bool, batch_rows: int, schema_evolution: SchemaEvolution, + mode: WriteMode = WriteMode.SCD1, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> WritePattern: del batch_rows + del cursor_field + del snapshot_field if sandbox: raise ValueError("PostgreSQL warehouse does not use Dander's BigQuery sandbox mode") + if mode is not WriteMode.SCD1: + raise ValueError(f"PostgreSQL warehouse does not support {mode.value} writes") return PostgreSQLScd1Writer( database=self.database, pool=self.pool, diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index 5e66925e..3bcab003 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -108,10 +108,17 @@ def build_ingestion_writer( sandbox: bool, batch_rows: int, schema_evolution: SchemaEvolution, + mode: WriteMode = WriteMode.SCD1, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> WritePattern: del batch_rows + del cursor_field + del snapshot_field if sandbox: raise ValueError("Redshift warehouse does not use Dander's BigQuery sandbox mode") + if mode is not WriteMode.SCD1: + raise ValueError(f"Redshift warehouse does not support {mode.value} writes") return RedshiftScd1Writer( database=self.database, connection_factory=self.connection_factory, diff --git a/src/dander/providers/snowflake/runtime.py b/src/dander/providers/snowflake/runtime.py index eef82bf5..71092a81 100644 --- a/src/dander/providers/snowflake/runtime.py +++ b/src/dander/providers/snowflake/runtime.py @@ -27,6 +27,7 @@ from dander.providers.snowflake.transform import SnowflakeTransformRunner from dander.providers.snowflake.writer import ( SnowflakeScd1Writer, + SnowflakeStagedWriter, SnowflakeStagingSettings, default_staging_settings, ) @@ -109,16 +110,40 @@ def build_ingestion_writer( sandbox: bool, batch_rows: int, schema_evolution: SchemaEvolution, + mode: WriteMode = WriteMode.SCD1, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> WritePattern: del batch_rows if sandbox: raise ValueError("Snowflake warehouse does not use Dander's BigQuery sandbox mode") - return SnowflakeScd1Writer( + if mode is WriteMode.INCREMENTAL: + if cursor_field is None or not cursor_field.strip(): + raise ValueError("Snowflake incremental writes require cursor_field") + elif cursor_field is not None: + raise ValueError("cursor_field is valid only for Snowflake incremental writes") + if mode is WriteMode.SNAPSHOT: + if snapshot_field is None or not snapshot_field.strip(): + raise ValueError("Snowflake snapshot writes require snapshot_field") + elif snapshot_field is not None: + raise ValueError("snapshot_field is valid only for Snowflake snapshot writes") + if mode is WriteMode.SCD1: + return SnowflakeScd1Writer( + database=self.database, + connection_factory=self.connection_factory, + target_fence=self.target_fence, + schema_evolution=schema_evolution, + staging=self.staging, + ) + return SnowflakeStagedWriter( database=self.database, connection_factory=self.connection_factory, target_fence=self.target_fence, schema_evolution=schema_evolution, staging=self.staging, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, ) @@ -181,7 +206,7 @@ def operation( SNOWFLAKE_CAPABILITIES = WarehouseCapabilities( provider_id="snowflake", schema_contract_version=1, - write_modes=frozenset({WriteMode.SCD1}), + write_modes=frozenset(WriteMode), transports=frozenset({WriteTransport.COPY}), supports_transforms=True, supports_graphs=False, diff --git a/src/dander/providers/snowflake/writer.py b/src/dander/providers/snowflake/writer.py index bc5b8d16..7ac46c34 100644 --- a/src/dander/providers/snowflake/writer.py +++ b/src/dander/providers/snowflake/writer.py @@ -1,4 +1,4 @@ -"""Bounded Parquet stage/COPY and transactionally fenced Snowflake SCD1 writes.""" +"""Bounded Parquet stage/COPY and transactionally fenced Snowflake writes.""" from __future__ import annotations @@ -37,6 +37,10 @@ from dander.providers.snowflake.fence import SnowflakeTargetFence _ORDINAL = "_dander_ordinal" +_SCD2_VALID_FROM = "valid_from" +_SCD2_VALID_TO = "valid_to" +_SCD2_IS_CURRENT = "is_current" +_SCD2_SYSTEM_FIELDS = frozenset({_SCD2_VALID_FROM, _SCD2_VALID_TO, _SCD2_IS_CURRENT}) class SnowflakeWriteError(ValueError): @@ -53,11 +57,9 @@ class SnowflakeStagingSettings: compression: str -class SnowflakeScd1Writer(WritePattern): - """Stage one bounded runtime batch as Parquet and MERGE the last row per key.""" +class SnowflakeStagedWriter(WritePattern): + """Stage bounded Parquet parts once and publish one selected logical write mode.""" - mode = WriteMode.SCD1 - supports_batched_writes = True requires_publication_fence = True def __init__( @@ -68,21 +70,40 @@ def __init__( target_fence: SnowflakeTargetFence, schema_evolution: SchemaEvolution, staging: SnowflakeStagingSettings, + mode: WriteMode, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> None: self._database = database self._connection_factory = connection_factory self._target_fence = target_fence self._schema_evolution = schema_evolution self._staging = staging + self.mode = mode + self._cursor_field = cursor_field + self._snapshot_field = snapshot_field + self.supports_batched_writes = mode not in {WriteMode.REPLACE, WriteMode.SCD2} + self.accepts_streaming_input = mode in {WriteMode.REPLACE, WriteMode.SCD2} def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: """Consume records once, upload checksummed parts, and publish idempotently.""" - _validate_target(target, database=self._database) + _validate_target( + target, + database=self._database, + mode=self.mode, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) publication = target.publication_fence assert publication is not None target_schema = target.canonical_schema - if any(field.name == _ORDINAL for field in target_schema.fields): - raise SnowflakeWriteError(f"Declared schema reserves Dander field {_ORDINAL!r}") + reserved = {_ORDINAL} + if self.mode is WriteMode.SCD2: + reserved.update(_SCD2_SYSTEM_FIELDS) + if collision := sorted( + field.name for field in target_schema.fields if field.name in reserved + ): + raise SnowflakeWriteError(f"Declared schema reserves Dander field {collision[0]!r}") _validate_schema(target_schema) staging_schema = RelationSchema( fields=( @@ -102,7 +123,15 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, compression=self._staging.compression, ) as local: - manifest = local.stage(_with_ordinals(records), staging_schema) + manifest = local.stage( + _with_ordinals( + records, + business_key=target.business_key, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ), + staging_schema, + ) return self._load_and_publish( target, publication, @@ -137,20 +166,21 @@ def _load_and_publish( with open_connection(self._connection_factory) as connection: try: _set_query_tag(connection, publication) + deployed_schema = _target_schema_for_mode(target_schema, self.mode) _ensure_target( connection, target, - target_schema, + deployed_schema, evolution=self._schema_evolution, ) _ensure_load_history(connection, target) if manifest.rows == 0: result = self._target_fence.execute_dml( connection, - _no_op_sql(target), + _empty_publication_sql(target, self.mode), publication, ) - return result.rowcount + return 0 if self.mode is WriteMode.REPLACE else result.rowcount execute(connection, _create_stage_sql(stage)) execute( connection, @@ -172,9 +202,17 @@ def _load_and_publish( publication, ) return result.rowcount - for artifact in pending: + artifacts_to_load = ( + manifest.artifacts if self.mode is WriteMode.REPLACE else pending + ) + for artifact in artifacts_to_load: execute(connection, _put_sql(artifact.path, stage)) execute(connection, _copy_sql(staging_relation, stage, artifact.path.name)) + if self.mode is WriteMode.SCD2: + execute( + connection, + "SET DANDER_SCD2_EFFECTIVE_AT = CURRENT_TIMESTAMP()", + ) statements: list[tuple[str, Sequence[object]]] = [] statements.extend( ( @@ -194,7 +232,16 @@ def _load_and_publish( ) for artifact in pending ) - statements.append((_merge_sql(target, staging_relation, target_schema.fields), ())) + statements.extend( + _publication_statements( + target, + staging_relation, + target_schema.fields, + mode=self.mode, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) + ) result = self._target_fence.execute_statements( connection, statements, @@ -204,12 +251,38 @@ def _load_and_publish( except (SnowflakeWriteError, TargetFenceLostError): raise except Exception as error: - raise SnowflakeWriteError("Snowflake staged SCD1 write failed") from error + raise SnowflakeWriteError( + f"Snowflake staged {self.mode.value.upper()} write failed" + ) from error finally: if cleanup_started: _cleanup_remote(connection, stage=stage, staging_relation=staging_relation) +class SnowflakeScd1Writer(SnowflakeStagedWriter): + """Backward-compatible SCD1 constructor for existing provider integrations.""" + + mode = WriteMode.SCD1 + + def __init__( + self, + *, + database: str, + connection_factory: SnowflakeConnectionFactory, + target_fence: SnowflakeTargetFence, + schema_evolution: SchemaEvolution, + staging: SnowflakeStagingSettings, + ) -> None: + super().__init__( + database=database, + connection_factory=connection_factory, + target_fence=target_fence, + schema_evolution=schema_evolution, + staging=staging, + mode=WriteMode.SCD1, + ) + + def default_staging_settings( *, max_rows_per_file: int, @@ -227,21 +300,56 @@ def default_staging_settings( def _with_ordinals( records: Iterable[Mapping[str, object]], + *, + business_key: Sequence[str], + cursor_field: str | None, + snapshot_field: str | None, ) -> Iterable[Mapping[str, object]]: for ordinal, record in enumerate(records): + if any(record.get(field) is None for field in business_key): + raise SnowflakeWriteError(f"Record {ordinal} has a null business-key value") + if cursor_field is not None and record.get(cursor_field) is None: + raise SnowflakeWriteError(f"Record {ordinal} has a null incremental cursor value") + if snapshot_field is not None and record.get(snapshot_field) is None: + raise SnowflakeWriteError(f"Record {ordinal} has a null snapshot value") yield {**record, _ORDINAL: ordinal} -def _validate_target(target: WriteTarget, *, database: str) -> None: +def _validate_target( + target: WriteTarget, + *, + database: str, + mode: WriteMode, + cursor_field: str | None, + snapshot_field: str | None, +) -> None: relation = target.relation_ref if relation.catalog != database: raise SnowflakeWriteError( f"Writer database {database!r} does not match target catalog {relation.catalog!r}" ) - if not target.business_key: - raise SnowflakeWriteError("Snowflake SCD1 writes require a business key") if not target.schema: raise SnowflakeWriteError("Snowflake writes require a declared schema") + field_names = {field.name for field in target.schema} + if mode in {WriteMode.SCD1, WriteMode.SCD2, WriteMode.INCREMENTAL}: + if not target.business_key: + raise SnowflakeWriteError(f"Snowflake {mode.value} writes require a business key") + if missing := sorted(set(target.business_key) - field_names): + raise SnowflakeWriteError(f"Snowflake business-key field {missing[0]!r} is undeclared") + if mode is WriteMode.INCREMENTAL: + if cursor_field is None or not cursor_field.strip(): + raise SnowflakeWriteError("Snowflake incremental writes require cursor_field") + if cursor_field not in field_names: + raise SnowflakeWriteError("Snowflake incremental cursor field is undeclared") + elif cursor_field is not None: + raise SnowflakeWriteError("cursor_field is valid only for Snowflake incremental writes") + if mode is WriteMode.SNAPSHOT: + if snapshot_field is None or not snapshot_field.strip(): + raise SnowflakeWriteError("Snowflake snapshot writes require snapshot_field") + if snapshot_field not in field_names: + raise SnowflakeWriteError("Snowflake snapshot field is undeclared") + elif snapshot_field is not None: + raise SnowflakeWriteError("snapshot_field is valid only for Snowflake snapshot writes") publication = target.publication_fence if publication is None: raise SnowflakeWriteError("Snowflake hosted writes require a destination target fence") @@ -256,6 +364,38 @@ def _validate_schema(schema: RelationSchema) -> None: _snowflake_type(field.data_type) +def _target_schema_for_mode(schema: RelationSchema, mode: WriteMode) -> RelationSchema: + if mode is not WriteMode.SCD2: + return schema + return RelationSchema( + fields=( + *schema.fields, + CanonicalField( + name=_SCD2_VALID_FROM, + data_type=CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + fractional_second_precision=9, + with_timezone=True, + ), + cardinality=FieldCardinality.REQUIRED, + ), + CanonicalField( + name=_SCD2_VALID_TO, + data_type=CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + fractional_second_precision=9, + with_timezone=True, + ), + ), + CanonicalField( + name=_SCD2_IS_CURRENT, + data_type=CanonicalType(kind=LogicalTypeKind.BOOLEAN), + cardinality=FieldCardinality.REQUIRED, + ), + ) + ) + + def _validate_manifest_bounds( manifest: StagingManifest, *, @@ -431,37 +571,188 @@ def _parquet_file_format() -> str: return "TYPE = PARQUET USE_LOGICAL_TYPE = TRUE BINARY_AS_TEXT = FALSE" +def _publication_statements( + target: WriteTarget, + staging_relation: str, + fields: Sequence[CanonicalField], + *, + mode: WriteMode, + cursor_field: str | None, + snapshot_field: str | None, +) -> tuple[tuple[str, Sequence[object]], ...]: + if mode is WriteMode.REPLACE: + return _replace_statements(target, staging_relation, fields) + if mode is WriteMode.SNAPSHOT: + assert snapshot_field is not None + return ((_snapshot_sql(target, staging_relation, fields, snapshot_field), ()),) + if mode is WriteMode.SCD2: + return _scd2_statements(target, staging_relation, fields) + return ( + ( + _merge_sql( + target, + staging_relation, + fields, + cursor_field=cursor_field if mode is WriteMode.INCREMENTAL else None, + ), + (), + ), + ) + + +def _deduplicated_source( + target: WriteTarget, + staging_relation: str, + fields: Sequence[CanonicalField], + *, + cursor_field: str | None = None, +) -> str: + names = tuple(field.name for field in fields) + partition = ", ".join(_quote(key) for key in target.business_key) + ordering = ( + f"{_quote(cursor_field)} DESC NULLS LAST, {_quote(_ORDINAL)} DESC" + if cursor_field is not None + else f"{_quote(_ORDINAL)} DESC" + ) + return ( + f"SELECT {', '.join(_quote(name) for name in names)} FROM {staging_relation} " + f"QUALIFY ROW_NUMBER() OVER (PARTITION BY {partition} " + f"ORDER BY {ordering}) = 1" + ) + + def _merge_sql( target: WriteTarget, staging_relation: str, fields: Sequence[CanonicalField], + *, + cursor_field: str | None, ) -> str: names = tuple(field.name for field in fields) selected = ", ".join(f"incoming.{_quote(name)}" for name in names) target_names = ", ".join(_quote(name) for name in names) - partition = ", ".join(_quote(key) for key in target.business_key) match = " AND ".join( f"target.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key ) mutable = tuple(name for name in names if name not in target.business_key) + condition = ( + f" AND incoming.{_quote(cursor_field)} >= target.{_quote(cursor_field)}" + if cursor_field is not None + else "" + ) matched = ( - " WHEN MATCHED THEN UPDATE SET " + f" WHEN MATCHED{condition} THEN UPDATE SET " + ", ".join(f"target.{_quote(name)} = incoming.{_quote(name)}" for name in mutable) if mutable else "" ) + source = _deduplicated_source( + target, + staging_relation, + fields, + cursor_field=cursor_field, + ) return ( f"MERGE INTO {_target(target)} AS target USING (SELECT " - f"{', '.join(_quote(name) for name in names)} FROM {staging_relation} " - f"QUALIFY ROW_NUMBER() OVER (PARTITION BY {partition} " - f"ORDER BY {_quote(_ORDINAL)} DESC) = 1) AS incoming ON {match}" + f"* FROM ({source})) " + f"AS incoming ON {match}" f"{matched} WHEN NOT MATCHED THEN INSERT ({target_names}) VALUES ({selected})" ) +def _replace_statements( + target: WriteTarget, + staging_relation: str, + fields: Sequence[CanonicalField], +) -> tuple[tuple[str, Sequence[object]], ...]: + columns = ", ".join(_quote(field.name) for field in fields) + return ( + (f"DELETE FROM {_target(target)}", ()), + ( + f"INSERT INTO {_target(target)} ({columns}) SELECT {columns} FROM {staging_relation}", + (), + ), + ) + + +def _snapshot_sql( + target: WriteTarget, + staging_relation: str, + fields: Sequence[CanonicalField], + snapshot_field: str, +) -> str: + names = tuple(field.name for field in fields) + columns = ", ".join(_quote(name) for name in names) + selected = ", ".join(f"incoming.{_quote(name)}" for name in names) + identical = " AND ".join( + f"EQUAL_NULL(existing.{_quote(name)}, incoming.{_quote(name)})" for name in names + ) + return ( + f"INSERT INTO {_target(target)} ({columns}) SELECT DISTINCT {selected} " + f"FROM {staging_relation} AS incoming WHERE incoming.{_quote(snapshot_field)} IS NOT NULL " + f"AND NOT EXISTS (SELECT 1 FROM {_target(target)} AS existing WHERE {identical})" + ) + + +def _scd2_statements( + target: WriteTarget, + staging_relation: str, + fields: Sequence[CanonicalField], +) -> tuple[tuple[str, Sequence[object]], ...]: + names = tuple(field.name for field in fields) + incoming = _deduplicated_source(target, staging_relation, fields) + match = " AND ".join( + f"current.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + ) + mutable = tuple(name for name in names if name not in target.business_key) + changed = ( + " OR ".join( + f"NOT EQUAL_NULL(current.{_quote(name)}, incoming.{_quote(name)})" for name in mutable + ) + or "FALSE" + ) + close = ( + f"UPDATE {_target(target)} AS current SET " + f"{_quote(_SCD2_VALID_TO)} = $DANDER_SCD2_EFFECTIVE_AT, " + f"{_quote(_SCD2_IS_CURRENT)} = FALSE FROM ({incoming}) AS incoming WHERE {match} " + f"AND current.{_quote(_SCD2_IS_CURRENT)} = TRUE AND ({changed})" + ) + columns = ", ".join( + [ + *(_quote(name) for name in names), + _quote(_SCD2_VALID_FROM), + _quote(_SCD2_VALID_TO), + _quote(_SCD2_IS_CURRENT), + ] + ) + values = ", ".join( + [ + *(f"incoming.{_quote(name)}" for name in names), + "$DANDER_SCD2_EFFECTIVE_AT", + "NULL", + "TRUE", + ] + ) + current_match = " AND ".join( + f"current.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + ) + insert = ( + f"INSERT INTO {_target(target)} ({columns}) SELECT {values} FROM ({incoming}) AS incoming " + f"WHERE NOT EXISTS (SELECT 1 FROM {_target(target)} AS current WHERE {current_match} " + f"AND current.{_quote(_SCD2_IS_CURRENT)} = TRUE)" + ) + return ((close, ()), (insert, ())) + + def _no_op_sql(target: WriteTarget) -> str: - key = _quote(target.business_key[0]) - return f"UPDATE {_target(target)} SET {key} = {key} WHERE FALSE" + column = _quote(target.schema[0].name) + return f"UPDATE {_target(target)} SET {column} = {column} WHERE FALSE" + + +def _empty_publication_sql(target: WriteTarget, mode: WriteMode) -> str: + if mode is WriteMode.REPLACE: + return f"DELETE FROM {_target(target)}" + return _no_op_sql(target) def _cleanup_remote( @@ -557,6 +848,7 @@ def _literal(value: str) -> str: __all__ = [ "SnowflakeScd1Writer", + "SnowflakeStagedWriter", "SnowflakeStagingSettings", "SnowflakeWriteError", "default_staging_settings", diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 766e0c13..6ae4da40 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -23,7 +23,7 @@ "state": "bigquery", "warehouse": "snowflake", "status": "experimental", - "reason": "Snowflake staged SCD1 ingestion and destination fencing pass local conformance; no live Snowflake profile is qualified." + "reason": "Snowflake staged scalar writer modes and destination fencing pass local conformance; no live Snowflake profile is qualified." }, { "state": "postgresql", @@ -47,7 +47,7 @@ "state": "postgresql", "warehouse": "snowflake", "status": "experimental", - "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged SCD1 writes; live qualification remains pending." + "reason": "Cross-backend execution is admitted only for locally conformance-tested Snowflake staged scalar writes; live qualification remains pending." } ], "warehouses": [ @@ -89,7 +89,7 @@ }, { "provider": "snowflake", - "write_modes": ["scd1"], + "write_modes": ["incremental", "replace", "scd1", "scd2", "snapshot"], "transports": ["copy"], "logical_types": ["binary", "boolean", "date", "decimal", "float", "integer", "string", "time", "timestamp"], "max_decimal_precision": 38, @@ -97,7 +97,7 @@ "supports_transforms": true, "supports_graphs": false, "supports_target_fencing": true, - "limitations": ["graph execution is not available", "only scd1 ingestion is implemented", "semi-structured types require a future explicit VARIANT mapping"] + "limitations": ["graph execution is not available", "hosted source execution selects scd1", "semi-structured types require a future explicit VARIANT mapping", "small-load direct writes are not qualified"] } ] } diff --git a/src/dander/warehouse/runtime.py b/src/dander/warehouse/runtime.py index ba40b9be..aaf93140 100644 --- a/src/dander/warehouse/runtime.py +++ b/src/dander/warehouse/runtime.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable from dander.warehouse.contracts import CanonicalType, LogicalTypeKind, RelationSchema +from dander.writer.base import WriteMode if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -15,7 +16,7 @@ from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import TransformRunResult from dander.warehouse.contracts import RelationCodec, RelationRef - from dander.writer.base import SchemaEvolution, WriteMode, WritePattern, WriteTransport + from dander.writer.base import SchemaEvolution, WritePattern, WriteTransport @dataclass(frozen=True, slots=True) @@ -140,8 +141,11 @@ def build_ingestion_writer( sandbox: bool, batch_rows: int, schema_evolution: SchemaEvolution, + mode: WriteMode = WriteMode.SCD1, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> WritePattern: - """Build one writer without branching in the cloud-neutral CLI.""" + """Build one selected logical writer without branching on warehouse provider.""" ... diff --git a/tests/cli/test_run_command.py b/tests/cli/test_run_command.py index e747bc50..81af24b3 100644 --- a/tests/cli/test_run_command.py +++ b/tests/cli/test_run_command.py @@ -122,7 +122,18 @@ def execute(self) -> PipelineExecutionResult: ) class _WriterFactory: - def build_ingestion_writer(self, **kwargs: object) -> _Built: + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + ) -> _Built: + kwargs = { + "sandbox": sandbox, + "batch_rows": batch_rows, + "schema_evolution": schema_evolution, + } component = _Built(name="writer", args=(), kwargs=kwargs) captured["writer"] = component return component diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index d3cbcd98..5c449696 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -255,6 +255,13 @@ def scd1(**kwargs: object) -> object: batch_rows=75, schema_evolution=SchemaEvolution.ADDITIVE, ) + with pytest.raises(ValueError, match="does not select scd2"): + factory.build_ingestion_writer( + sandbox=False, + batch_rows=80, + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.SCD2, + ) assert captured == [ ("replace", {"project": "unit-project", "max_batch_rows": 50}), diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index de54dcb6..94415cbe 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -333,12 +333,270 @@ def records() -> Iterator[dict[str, object]]: assert backend.commits == 2 assert backend.rollbacks == 0 assert len(backend.committed_checksums) == 2 - assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) + assert runtime.capabilities.write_modes == frozenset(WriteMode) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) assert runtime.capabilities.supports_transforms is True assert runtime.relation_codec.render(relation) == '"DANDER_TEST"."raw"."records"' +@pytest.mark.parametrize( + ("mode", "field_name", "expected_sql"), + [ + ( + WriteMode.INCREMENTAL, + "updated_at", + 'WHEN MATCHED AND incoming."updated_at" >= target."updated_at"', + ), + (WriteMode.SNAPSHOT, "snapshot_at", "SELECT DISTINCT"), + (WriteMode.SCD2, None, '"valid_to" = $DANDER_SCD2_EFFECTIVE_AT'), + (WriteMode.REPLACE, None, 'SELECT "id", "label" FROM'), + ], +) +def test_snowflake_factory_reaches_every_fenced_write_mode( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], + mode: WriteMode, + field_name: str | None, + expected_sql: str, +) -> None: + runtime, backend, staging_root = snowflake_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=field_name if mode is WriteMode.INCREMENTAL else None, + snapshot_field=field_name if mode is WriteMode.SNAPSHOT else None, + ) + assert writer.mode is mode + assert writer.supports_batched_writes is (mode not in {WriteMode.REPLACE, WriteMode.SCD2}) + assert writer.accepts_streaming_input is (mode in {WriteMode.REPLACE, WriteMode.SCD2}) + + relation = RelationRef( + catalog="DANDER_TEST", + namespace="raw", + name=f"records_{mode.value}", + ) + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=f"snowflake_{mode.value}", + run_id=f"run-{mode.value}", + token=5, + authority_id="postgresql:test-state", + ), + ) + schema = [ + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ] + backend.describe_rows = [ + ( + "id", + "VARCHAR(16777216)", + "COLUMN", + "N" if mode in {WriteMode.INCREMENTAL, WriteMode.SCD2} else "Y", + ), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ] + if field_name is not None: + schema.append(WriteField(name=field_name, data_type="STRING")) + backend.describe_rows.append((field_name, "VARCHAR(16777216)", "COLUMN", "Y")) + if mode is WriteMode.SCD2: + backend.describe_rows.extend( + [ + ("valid_from", "TIMESTAMP_TZ(9)", "COLUMN", "N"), + ("valid_to", "TIMESTAMP_TZ(9)", "COLUMN", "Y"), + ("is_current", "BOOLEAN", "COLUMN", "N"), + ] + ) + target = WriteTarget( + relation=relation, + business_key=("id",) if mode in {WriteMode.INCREMENTAL, WriteMode.SCD2} else (), + schema=tuple(schema), + publication_fence=publication, + ) + first = {"id": "one", "label": "first"} + second = {"id": "two", "label": "second"} + if field_name is not None: + first[field_name] = "2026-08-10T02:00:00Z" + second.update({"id": "one", field_name: "2026-08-10T01:00:00Z"}) + elif mode is WriteMode.SCD2: + second["id"] = "one" + + writer.write((first, second), target) + + sql = [statement for statement, _parameters in backend.statements] + assert any(expected_sql in statement for statement in sql) + if mode is WriteMode.INCREMENTAL: + assert any( + 'PARTITION BY "id" ORDER BY "updated_at" DESC NULLS LAST, ' + '"_dander_ordinal" DESC' in item + for item in sql + ) + elif mode is WriteMode.SNAPSHOT: + assert any("EQUAL_NULL(existing." in item and "NOT EXISTS" in item for item in sql) + elif mode is WriteMode.SCD2: + assert sql.count("SET DANDER_SCD2_EFFECTIVE_AT = CURRENT_TIMESTAMP()") == 1 + assert len([item for item in sql if "$DANDER_SCD2_EFFECTIVE_AT" in item]) == 2 + assert any( + '"valid_from" TIMESTAMP_TZ(9) NOT NULL' in item + and '"is_current" BOOLEAN NOT NULL' in item + for item in sql + ) + elif mode is WriteMode.REPLACE: + target_name = '"DANDER_TEST"."raw"."records_replace"' + assert f"DELETE FROM {target_name}" in sql + assert any(item.startswith(f"INSERT INTO {target_name}") for item in sql) + assert any( + 'UPDATE "DANDER_TEST"."raw"."dander_target_commits"' in statement for statement in sql + ) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + assert any(statement.startswith("DROP STAGE IF EXISTS") for statement in sql) + assert not tuple(staging_root.iterdir()) + + +def test_snowflake_mode_selection_fails_before_provider_mutation( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + before = list(backend.statements) + + with pytest.raises(ValueError, match="incremental writes require cursor_field"): + runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.INCREMENTAL, + ) + + assert backend.statements == before + + +def test_snowflake_replace_partial_replay_reloads_the_complete_manifest( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime, backend, _staging_root = snowflake_runtime + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="replace_replay") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_replace_replay", + run_id="run-replace-replay", + token=7, + authority_id="postgresql:test-state", + ), + ) + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "Y"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ] + checks = 0 + + def artifact_committed(*_args: object) -> bool: + nonlocal checks + checks += 1 + return checks == 1 + + monkeypatch.setattr( + "dander.providers.snowflake.writer._artifact_committed", + artifact_committed, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=WriteMode.REPLACE, + ) + writer.write( + ( + {"id": "one", "label": "first"}, + {"id": "two", "label": "second"}, + {"id": "three", "label": "third"}, + ), + WriteTarget( + relation=relation, + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ), + ) + + sql = [statement for statement, _parameters in backend.statements] + assert checks == 2 + assert len([statement for statement in sql if statement.startswith("PUT ")]) == 2 + assert len([statement for statement in sql if statement.startswith("COPY INTO")]) == 2 + assert ( + len( + [ + statement + for statement in sql + if statement.startswith("INSERT INTO") and "dander_stage_loads" in statement + ] + ) + == 1 + ) + assert 'DELETE FROM "DANDER_TEST"."raw"."replace_replay"' in sql + + +@pytest.mark.parametrize("mode", [WriteMode.REPLACE, WriteMode.SNAPSHOT]) +def test_snowflake_unkeyed_empty_writes_remain_fenced( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], + mode: WriteMode, +) -> None: + runtime, backend, _staging_root = snowflake_runtime + relation = RelationRef( + catalog="DANDER_TEST", + namespace="raw", + name=f"empty_{mode.value}", + ) + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=f"empty_{mode.value}", + run_id=f"run-empty-{mode.value}", + token=6, + authority_id="postgresql:test-state", + ), + ) + backend.describe_rows = [("id", "VARCHAR(16777216)", "COLUMN", "Y")] + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + snapshot_field="id" if mode is WriteMode.SNAPSHOT else None, + ) + + affected = writer.write( + (), + WriteTarget( + relation=relation, + schema=(WriteField(name="id", data_type="STRING"),), + publication_fence=publication, + ), + ) + + assert affected == 0 + + sql = [statement for statement, _parameters in backend.statements] + expected = ( + f'DELETE FROM "DANDER_TEST"."raw"."empty_{mode.value}"' + if mode is WriteMode.REPLACE + else f'UPDATE "DANDER_TEST"."raw"."empty_{mode.value}" SET "id" = "id" WHERE FALSE' + ) + assert expected in sql + assert not any(statement.startswith("PUT ") for statement in sql) + assert any( + 'UPDATE "DANDER_TEST"."raw"."dander_target_commits"' in statement for statement in sql + ) + + @dataclass class _Ownership: fence: FencingToken From 568b07df11ad9cc1daa12155d984caebf92f35ee Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 02:21:00 -0400 Subject: [PATCH 090/163] docs: record Snowflake writer CI evidence --- HANDOFF.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 4eb3532a..83be8800 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -17,8 +17,8 @@ Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`, then - Ruff format/check: passed across 328 files. - Mypy: passed across 304 source/test files with the CI dependency profile. - Pytest: 1,112 passed and 13 skipped (1,125 collected). -- Distribution/source-free install and Terraform validation passed before final correctness fixes; the full Python suite passed afterward. -- Container export is locally blocked by a broken Docker snapshot and critically low disk; protected Linux CI remains required. +- Protected CI passed Python/PostgreSQL, dependency, Terraform, distribution/source-free install, container, config-scan, and secret-scan gates. +- Local Docker remains unhealthy because of a missing snapshot and critically low disk; Linux CI supplied the successful container evidence. ## Decisions @@ -28,9 +28,7 @@ Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`, then ## Remaining -- Commit, push, and open the focused PR. -- Require protected CI to repeat Linux tests, PostgreSQL integration, packaging, Terraform, container, and security checks. -- Merge only when required checks and review pass. +- Review and merge PR #165 through protected main when approved. - Continue the remaining Snowflake first-class slices from updated main. ## Review First From 9b8aad12b6472095695506b0303190127f9defad Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 03:25:18 -0400 Subject: [PATCH 091/163] feat(runtime): preserve warehouse schema and telemetry --- CHANGELOG.md | 7 +++ HANDOFF.md | 36 +++++++------- docs/canonical-schema.md | 9 ++++ docs/decisions.md | 11 +++++ docs/known-limitations.md | 2 +- docs/oci-runtime-contract.md | 9 ++-- src/dander/executor.py | 5 +- src/dander/ingestion/source.py | 2 + src/dander/pipeline/compiler.py | 6 ++- src/dander/pipeline/graph.py | 2 + src/dander/providers/bigquery/graph.py | 6 ++- src/dander/providers/redshift/transform.py | 2 + src/dander/providers/snowflake/transform.py | 2 + src/dander/runtime.py | 29 +++++++++--- src/dander/telemetry.py | 31 ++++++++++++ src/dander/transform/config.py | 2 + src/dander/transform/result.py | 3 ++ src/dander/warehouse/bigquery_compat.py | 15 ++++-- src/dander/writer/base.py | 18 ++++++- src/dander/writer/storage_write.py | 1 + tests/pipeline/test_compiler.py | 16 ++++++- tests/providers/test_bigquery_runtime.py | 3 ++ tests/test_executor.py | 24 +++++++++- tests/test_runtime.py | 52 ++++++++++++++++++++- tests/test_runtime_contract.py | 3 ++ tests/test_telemetry.py | 23 +++++++++ tests/warehouse/test_contracts.py | 21 +++++++++ 27 files changed, 300 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfd6f740..286fa77d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ capabilities enter through the next minor release. ## Unreleased +### Added + +- Preserve validated provider extensions from connector, graph, and model declarations through + canonical warehouse targets without changing existing BigQuery schema inputs. +- Carry additive warehouse load/transform telemetry through endpoint results into terminal run + telemetry, including optional resource, queue, execution, spill, and capacity-unit measures. + ## 0.8.0rc8 — 2026-08-10 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 83be8800..94ade508 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,35 @@ ## Finished -- Added provider-neutral warehouse writer-mode selection while keeping hosted CLI ingestion backward-compatible. -- Implemented Snowflake replace, SCD1, SCD2, snapshot, and incremental publication on one bounded Parquet/COPY path. -- Kept every finalizer and replay-history update in the exact destination-fence transaction. -- Fixed cursor ordering, whole-stream SCD2 publication, and partial-replay replace correctness. -- Updated Snowflake compatibility and limitations without making a support claim. +- Preserved canonical schema extensions through connectors, graphs, models, and warehouse targets. +- Added an API-v1-compatible writer telemetry drain and propagated operations to terminal runs. +- Added normalized resource, queue, execution, spill, and capacity-unit telemetry fields. ## Try It -Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`, then inspect `docs/snowflake.md`. +Run `uv run pytest -q tests/test_runtime.py tests/test_executor.py tests/test_telemetry.py`. ## Checks -- Ruff format/check: passed across 328 files. -- Mypy: passed across 304 source/test files with the CI dependency profile. -- Pytest: 1,112 passed and 13 skipped (1,125 collected). -- Protected CI passed Python/PostgreSQL, dependency, Terraform, distribution/source-free install, container, config-scan, and secret-scan gates. -- Local Docker remains unhealthy because of a missing snapshot and critically low disk; Linux CI supplied the successful container evidence. +- Pytest: 1,115 passed and 13 skipped locally; 1,128 passed with PostgreSQL 18 enabled. +- Ruff format/check: passed across all 328 files. +- Mypy: passed across 304 source/test files. +- Wheel/sdist inspection and two source-free package installs: passed. +- Terraform roots/tests and Helm lint/render: passed; local Docker final unpack hit known snapshot I/O corruption. ## Decisions -- One warehouse writer capability selects logical mode plus only its cursor/snapshot field. -- SCD2 and replace consume one complete bounded-memory stream; incremental ranks cursor before ordinal. -- Views, graph wiring, VARIANT, direct-write crossover, telemetry, and live proof remain separate gates. +- Legacy BigQuery schema declarations stay intact while `WriteTarget` retains one validated canonical schema. +- Existing writers emit no telemetry by default; provider writers opt in without changing `write()`. +- Provider extensions remain inert outside their matching adapter. ## Remaining -- Review and merge PR #165 through protected main when approved. -- Continue the remaining Snowflake first-class slices from updated main. +- Let protected Linux CI repeat PostgreSQL, container, dependency, config, and secret gates. +- Merge this contract before implementing Snowflake VARIANT, direct/COPY routing, or query telemetry. ## Review First -- `src/dander/providers/snowflake/writer.py` -- `src/dander/warehouse/runtime.py` -- `tests/providers/test_snowflake_warehouse_runtime.py` +- `src/dander/runtime.py` +- `src/dander/writer/base.py` +- `src/dander/telemetry.py` diff --git a/docs/canonical-schema.md b/docs/canonical-schema.md index ade870be..72db4106 100644 --- a/docs/canonical-schema.md +++ b/docs/canonical-schema.md @@ -25,6 +25,15 @@ from timezone-free `DATETIME`, records BigQuery's microsecond precision, and map required canonical array. Original BigQuery type and mode remain ordered extensions for traceability. +Connector raw fields, graph fields, and model columns may also declare validated `extensions`. +These annotations retain their canonical provider/name/value identity through planning and +execution; only the matching warehouse adapter interprets them. Dander never silently turns an +extension into a portable guarantee. Existing declarations without extensions are unchanged. + +`WriteTarget` retains the legacy BigQuery-shaped schema for compatibility while carrying the +validated canonical `RelationSchema` selected before extraction. Provider-neutral orchestration +does not reconstruct that schema after a provider has validated it. + `BIGNUMERIC` (whose documented precision includes a partial 77th digit), `GEOGRAPHY`, `INTERVAL`, or a future BigQuery-only type fails mapping unless a caller explicitly provides a canonical fallback. This is intentional: validation must not silently relabel a lossy conversion as diff --git a/docs/decisions.md b/docs/decisions.md index 1586d28b..ed558d76 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -936,6 +936,17 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc Views, graph wiring, explicit VARIANT, direct-write crossover, telemetry, and live qualification remain separate gates. +## 2026-08-10 — Canonical schemas and operation telemetry survive orchestration + +- **Schema:** Connector raw fields, graph fields, and model columns may carry validated provider + extensions. The selected canonical `RelationSchema` is retained on `WriteTarget`; legacy + BigQuery declarations remain available unchanged for API-v1 writers. +- **Telemetry:** Writers may drain completed operation telemetry without changing their `write()` + return contract. Endpoint and transform results preserve those operations in execution order, + and the executor includes them in terminal `RunTelemetry`. +- **Boundary:** This is additive plumbing only. Providers still own extension meaning and direct + versus bulk selection; no warehouse gains support or new SQL behavior from this contract. + ## 2026-08-09 — Redshift begins with IAM-only bounded SCD1 publication - **Boundary:** Provisioned and Serverless profiles map native database/schema coordinates into diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 817889eb..061759df 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -14,7 +14,7 @@ evaluate these limits before using it for an unattended system containing busine writer modes behind its provider capability, bounded Parquet upload, temporary remote staging, schema checks, destination fencing, and fenced table/incremental transforms. The ordinary hosted source runner still selects SCD1, and Snowflake has no live qualification, views, graph execution, - semi-structured types, measured direct-write crossover, or end-to-end telemetry retention. + semi-structured types, measured direct-write crossover, or provider query-telemetry emission. Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. It has no live qualification, views, executable graphs, diff --git a/docs/oci-runtime-contract.md b/docs/oci-runtime-contract.md index b306cbb7..effedba9 100644 --- a/docs/oci-runtime-contract.md +++ b/docs/oci-runtime-contract.md @@ -42,10 +42,11 @@ exception text are excluded. Every terminal event includes `outputs.telemetry`. The provider-neutral shape records whole-run `duration_ms` and ordered operation statistics for retries, rows, bytes, provider query/job IDs, -and cost attribution. Fields not reported by the selected adapter remain zero or absent; Dander -does not infer provider billing. Cost values are decimal strings with an explicit currency and -`estimated` marker. Query and job IDs are correlation identifiers only—adapters must never put -SQL, request bodies, record contents, URLs, or credentials in telemetry. +queue/execution duration, spill, warehouse resource identity/size, capacity units, and cost +attribution. Fields not reported by the selected adapter remain zero or absent; Dander does not +infer provider billing. Capacity and cost values are decimal strings with explicit units or +currency. Query and job IDs are correlation identifiers only—adapters must never put SQL, request +bodies, record contents, URLs, or credentials in telemetry. Successful ingestion row counts remain in `outputs.metrics` and `outputs.endpoints` for runtime-v1 compatibility. Detailed operation telemetry is additive and does not change the stable process diff --git a/src/dander/executor.py b/src/dander/executor.py index 6ae2b6f0..581dd92f 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -333,7 +333,10 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: finally: if heartbeat is not None: heartbeat.__exit__() - telemetry = RunTelemetry(duration_ms=_elapsed_ms(started_ns)) + telemetry = RunTelemetry( + duration_ms=_elapsed_ms(started_ns), + operations=ingestion_result.telemetry + transform_result.telemetry, + ) _LOGGER.info( "pipeline_completed", extra={ diff --git a/src/dander/ingestion/source.py b/src/dander/ingestion/source.py index 8ef6e734..6c153872 100644 --- a/src/dander/ingestion/source.py +++ b/src/dander/ingestion/source.py @@ -17,6 +17,7 @@ from dander.ingestion.pagination import NoPagination, PaginationStrategy from dander.schema import BIGQUERY_FIELD_MODES, BIGQUERY_FIELD_TYPES, normalize_bigquery_type +from dander.warehouse.contracts import ProviderExtension # noqa: TC001 - Pydantic resolves it _FIELD_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _ENGINE_NAME = re.compile(r"^[a-z][a-z0-9_]*$") @@ -36,6 +37,7 @@ class RawField(BaseModel): data_type: str = Field(alias="type") mode: str = "NULLABLE" fields: list[RawField] = Field(default_factory=list) + extensions: tuple[ProviderExtension, ...] = Field(default_factory=tuple) @field_validator("name") @classmethod diff --git a/src/dander/pipeline/compiler.py b/src/dander/pipeline/compiler.py index ed45d206..12257949 100644 --- a/src/dander/pipeline/compiler.py +++ b/src/dander/pipeline/compiler.py @@ -180,7 +180,11 @@ def compile_target( relation=target_relation, business_key=tuple(destination.business_key), schema=tuple( - WriteField(name=field.name, data_type=field.cast_to or field.type) + WriteField( + name=field.name, + data_type=field.cast_to or field.type, + extensions=field.extensions, + ) for field in target.fields ), ), diff --git a/src/dander/pipeline/graph.py b/src/dander/pipeline/graph.py index 03ce3fa5..b2f4cdce 100644 --- a/src/dander/pipeline/graph.py +++ b/src/dander/pipeline/graph.py @@ -33,6 +33,7 @@ TargetNodeConfig, resolve_node_config, ) +from dander.warehouse import ProviderExtension # noqa: TC001 - Pydantic resolves it if TYPE_CHECKING: from pathlib import Path @@ -182,6 +183,7 @@ class NodeField(BaseModel): nullable: bool = True description: str | None = None tests: list[FieldTest] = Field(default_factory=list) + extensions: tuple[ProviderExtension, ...] = Field(default_factory=tuple) metadata: dict[str, Any] = Field(default_factory=dict) diff --git a/src/dander/providers/bigquery/graph.py b/src/dander/providers/bigquery/graph.py index 64bf57b1..cec204a0 100644 --- a/src/dander/providers/bigquery/graph.py +++ b/src/dander/providers/bigquery/graph.py @@ -247,7 +247,11 @@ def prepare_bigquery_target_writer( relation=relation, business_key=tuple(destination.business_key), schema=tuple( - WriteField(name=field.name, data_type=field.cast_to or field.type) + WriteField( + name=field.name, + data_type=field.cast_to or field.type, + extensions=field.extensions, + ) for field in target_node.fields ), ), diff --git a/src/dander/providers/redshift/transform.py b/src/dander/providers/redshift/transform.py index 26b92176..82b4481d 100644 --- a/src/dander/providers/redshift/transform.py +++ b/src/dander/providers/redshift/transform.py @@ -145,6 +145,7 @@ def _publish(self, plan: _RedshiftModelPlan, publication: TargetFence) -> None: relation=plan.target.relation_ref, business_key=plan.target.business_key, schema=plan.target.schema, + declared_schema=plan.target.canonical_schema, publication_fence=publication, ) temporary = _temporary_name(target.relation_ref, publication) @@ -232,6 +233,7 @@ def _model_plan(project: TransformProject, model: TransformModel) -> _RedshiftMo name=column.name, data_type=column.data_type, mode="REQUIRED" if column.name in required else "NULLABLE", + extensions=column.extensions, ) for column in model.metadata.columns ) diff --git a/src/dander/providers/snowflake/transform.py b/src/dander/providers/snowflake/transform.py index 5f410582..d5ca28f7 100644 --- a/src/dander/providers/snowflake/transform.py +++ b/src/dander/providers/snowflake/transform.py @@ -142,6 +142,7 @@ def _publish(self, plan: _SnowflakeModelPlan, publication: TargetFence) -> None: relation=plan.target.relation_ref, business_key=plan.target.business_key, schema=plan.target.schema, + declared_schema=plan.target.canonical_schema, publication_fence=publication, ) temporary = _temporary_relation(target.relation_ref, publication) @@ -222,6 +223,7 @@ def _model_plan(project: TransformProject, model: TransformModel) -> _SnowflakeM name=column.name, data_type=column.data_type, mode="REQUIRED" if column.name in required else "NULLABLE", + extensions=column.extensions, ) for column in model.metadata.columns ) diff --git a/src/dander/runtime.py b/src/dander/runtime.py index e37cf8ea..15764504 100644 --- a/src/dander/runtime.py +++ b/src/dander/runtime.py @@ -20,6 +20,7 @@ from dander.concurrency import OwnershipGuard from dander.ingestion.source import Endpoint, RawField, Source from dander.state.watermark import WatermarkStore + from dander.telemetry import OperationTelemetry from dander.warehouse import WarehouseSchemaMapper, WarehouseTargetFence from dander.writer.base import WritePattern @@ -42,6 +43,7 @@ class EndpointRunResult: extracted: int affected: int committed_cursor: str | None + telemetry: tuple[OperationTelemetry, ...] = () @dataclass(frozen=True) @@ -52,6 +54,11 @@ class PipelineRunResult: source: str endpoints: tuple[EndpointRunResult, ...] + @property + def telemetry(self) -> tuple[OperationTelemetry, ...]: + """Return endpoint operations in deterministic execution order.""" + return tuple(operation for endpoint in self.endpoints for operation in endpoint.telemetry) + class PipelineRunner: """Coordinate one configured source without depending on concrete providers.""" @@ -183,14 +190,17 @@ def _run_endpoint( ownership: OwnershipGuard | None, ) -> EndpointRunResult: source_name = self._source.config.name + legacy_schema = tuple(_write_field(field) for field in endpoint.raw_schema) + declared_schema = endpoint.canonical_raw_schema() if endpoint.raw_schema else None + if declared_schema is not None and self._schema_mapper is not None: + declared_schema = self._schema_mapper.canonical_schema(declared_schema.fields) target = WriteTarget( relation=self._endpoint_relations[endpoint.name], business_key=tuple(endpoint.primary_key), - schema=tuple(_write_field(field) for field in endpoint.raw_schema), + schema=legacy_schema, + declared_schema=declared_schema, fence=ownership.fence if ownership is not None else None, ) - if target.schema and self._schema_mapper is not None: - self._schema_mapper.canonical_schema(target.schema) stored_cursor = ( self._watermarks.get(source_name, endpoint.name) if endpoint.incremental_cursor @@ -227,7 +237,7 @@ def _run_endpoint( observation, ) if self._writer.supports_batched_writes: - affected = self._write_batched( + affected, telemetry = self._write_batched( records, endpoint=endpoint, target=target, @@ -238,11 +248,13 @@ def _run_endpoint( if ownership is not None: ownership.verify() affected = self._writer.write(records, target) + telemetry = self._writer.drain_telemetry() else: buffered = list(records) if ownership is not None: ownership.verify() affected = self._writer.write(buffered, target) + telemetry = self._writer.drain_telemetry() committed_cursor: str | None = None if observation.extracted and observation.maximum_cursor is not None: if ownership is not None: @@ -283,6 +295,7 @@ def _run_endpoint( extracted=observation.extracted, affected=affected, committed_cursor=committed_cursor, + telemetry=telemetry, ) def _write_batched( @@ -293,8 +306,9 @@ def _write_batched( target: WriteTarget, run_id: str, ownership: OwnershipGuard | None, - ) -> int: + ) -> tuple[int, tuple[OperationTelemetry, ...]]: affected = 0 + telemetry: list[OperationTelemetry] = [] wrote_batch = False for batch_index, record_batch in enumerate( batched(records, self._batch_rows), @@ -304,6 +318,7 @@ def _write_batched( if ownership is not None: ownership.verify() batch_affected = self._writer.write(record_batch, target) + telemetry.extend(self._writer.drain_telemetry()) affected += batch_affected _LOGGER.info( "batch_finished", @@ -321,7 +336,8 @@ def _write_batched( if ownership is not None: ownership.verify() affected = self._writer.write((), target) - return affected + telemetry.extend(self._writer.drain_telemetry()) + return affected, tuple(telemetry) class _RecordObservation: @@ -369,6 +385,7 @@ def _write_field(field: RawField) -> WriteField: data_type=field.data_type, mode=field.mode, fields=tuple(_write_field(child) for child in field.fields), + extensions=field.extensions, ) diff --git a/src/dander/telemetry.py b/src/dander/telemetry.py index 37ce0f2e..677e16e0 100644 --- a/src/dander/telemetry.py +++ b/src/dander/telemetry.py @@ -75,6 +75,13 @@ class OperationTelemetry: query_id: str | None = None job_id: str | None = None costs: tuple[CostAttribution, ...] = () + queue_duration_ms: int = 0 + execution_duration_ms: int = 0 + spill_bytes: int = 0 + resource_name: str | None = None + resource_size: str | None = None + capacity_units: Decimal | None = None + capacity_unit: str | None = None def __post_init__(self) -> None: _require_name(self.provider, label="telemetry provider") @@ -84,6 +91,17 @@ def __post_init__(self) -> None: _require_nonnegative_integer(value, label=label) _require_opaque(self.query_id, label="query id") _require_opaque(self.job_id, label="job id") + _require_opaque(self.resource_name, label="resource name") + _require_opaque(self.resource_size, label="resource size") + if (self.capacity_units is None) != (self.capacity_unit is None): + raise ValueError("capacity_units and capacity_unit must be declared together") + if self.capacity_units is not None: + if not isinstance(self.capacity_units, Decimal) or not self.capacity_units.is_finite(): + raise ValueError("capacity_units must be a finite Decimal") + if self.capacity_units < 0: + raise ValueError("capacity_units must be non-negative") + assert self.capacity_unit is not None + _require_name(self.capacity_unit, label="capacity unit") if not isinstance(self.costs, tuple) or not all( isinstance(cost, CostAttribution) for cost in self.costs ): @@ -100,6 +118,13 @@ def to_payload(self) -> dict[str, object]: payload["query_id"] = self.query_id if self.job_id is not None: payload["job_id"] = self.job_id + if self.resource_name is not None: + payload["resource_name"] = self.resource_name + if self.resource_size is not None: + payload["resource_size"] = self.resource_size + if self.capacity_units is not None: + payload["capacity_units"] = str(self.capacity_units) + payload["capacity_unit"] = self.capacity_unit if self.costs: payload["costs"] = [cost.to_payload() for cost in self.costs] return payload @@ -115,6 +140,9 @@ def _numeric_values(self) -> dict[str, int]: "bytes_written": self.bytes_written, "bytes_processed": self.bytes_processed, "bytes_billed": self.bytes_billed, + "queue_duration_ms": self.queue_duration_ms, + "execution_duration_ms": self.execution_duration_ms, + "spill_bytes": self.spill_bytes, } @@ -149,6 +177,9 @@ def to_payload(self) -> dict[str, object]: "bytes_written", "bytes_processed", "bytes_billed", + "queue_duration_ms", + "execution_duration_ms", + "spill_bytes", ) } return { diff --git a/src/dander/transform/config.py b/src/dander/transform/config.py index b7dda891..03f4419c 100644 --- a/src/dander/transform/config.py +++ b/src/dander/transform/config.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from dander.transform.model import Materialization, SqlDialect +from dander.warehouse import ProviderExtension # noqa: TC001 - Pydantic resolves it _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") type Scalar = str | int | float | bool @@ -30,6 +31,7 @@ class ColumnMetadata(BaseModel): name: str = Field(pattern=_IDENTIFIER.pattern) data_type: str = Field(alias="type", min_length=1) description: str = Field(min_length=1) + extensions: tuple[ProviderExtension, ...] = Field(default_factory=tuple) class RelationshipMetadata(BaseModel): diff --git a/src/dander/transform/result.py b/src/dander/transform/result.py index 01fd085f..763a2df3 100644 --- a/src/dander/transform/result.py +++ b/src/dander/transform/result.py @@ -2,6 +2,8 @@ from dataclasses import dataclass +from dander.telemetry import OperationTelemetry + class TransformRunError(RuntimeError): """Raised when model execution or a generic assertion fails.""" @@ -13,6 +15,7 @@ class TransformRunResult: models: tuple[str, ...] assertions: int + telemetry: tuple[OperationTelemetry, ...] = () __all__ = ["TransformRunError", "TransformRunResult"] diff --git a/src/dander/warehouse/bigquery_compat.py b/src/dander/warehouse/bigquery_compat.py index a28ecd39..a0fe48b1 100644 --- a/src/dander/warehouse/bigquery_compat.py +++ b/src/dander/warehouse/bigquery_compat.py @@ -37,6 +37,9 @@ def mode(self) -> str: ... @property def fields(self) -> Sequence[object]: ... + @property + def extensions(self) -> Sequence[ProviderExtension]: ... + _SCALARS = { "BOOL": CanonicalType(kind=LogicalTypeKind.BOOLEAN), @@ -107,9 +110,15 @@ def canonical_field_from_bigquery( name=field.name, data_type=canonical_type, cardinality=cardinality, - extensions=( - ProviderExtension(provider="bigquery", name="mode", value=mode), - ProviderExtension(provider="bigquery", name="type", value=data_type), + extensions=tuple( + sorted( + ( + *getattr(field, "extensions", ()), + ProviderExtension(provider="bigquery", name="mode", value=mode), + ProviderExtension(provider="bigquery", name="type", value=data_type), + ), + key=lambda extension: (extension.provider, extension.name), + ) ), ) diff --git a/src/dander/writer/base.py b/src/dander/writer/base.py index 096e6f07..2fae672b 100644 --- a/src/dander/writer/base.py +++ b/src/dander/writer/base.py @@ -12,12 +12,13 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any -from dander.warehouse.contracts import RelationRef +from dander.warehouse.contracts import ProviderExtension, RelationRef if TYPE_CHECKING: from collections.abc import Iterable, Mapping from dander.concurrency import FencingToken, TargetFence + from dander.telemetry import OperationTelemetry from dander.warehouse import CanonicalField, RelationSchema @@ -54,6 +55,7 @@ class WriteField: data_type: str mode: str = "NULLABLE" fields: tuple[WriteField, ...] = field(default_factory=tuple) + extensions: tuple[ProviderExtension, ...] = field(default_factory=tuple) def to_canonical(self) -> CanonicalField: """Map this legacy BigQuery writer field to canonical schema v1.""" @@ -73,6 +75,7 @@ class WriteTarget: relation: RelationRef business_key: tuple[str, ...] = field(default_factory=tuple) schema: tuple[WriteField, ...] = field(default_factory=tuple) + declared_schema: RelationSchema | None = None fence: FencingToken | None = None publication_fence: TargetFence | None = None @@ -87,6 +90,7 @@ def __init__( publication_fence: TargetFence | None = None, *, relation: RelationRef | None = None, + declared_schema: RelationSchema | None = None, ) -> None: if relation is None: if project is None or dataset is None or table is None: @@ -97,6 +101,7 @@ def __init__( object.__setattr__(self, "relation", relation) object.__setattr__(self, "business_key", business_key) object.__setattr__(self, "schema", schema) + object.__setattr__(self, "declared_schema", declared_schema) object.__setattr__(self, "fence", fence) object.__setattr__(self, "publication_fence", publication_fence) @@ -123,6 +128,8 @@ def relation_ref(self) -> RelationRef: @property def canonical_schema(self) -> RelationSchema: """Return the target schema through the one-way compatibility mapper.""" + if self.declared_schema is not None: + return self.declared_schema from dander.warehouse import canonical_schema_from_bigquery return canonical_schema_from_bigquery(self.schema) @@ -136,6 +143,15 @@ class WritePattern(ABC): accepts_streaming_input = False requires_publication_fence = False + def drain_telemetry(self) -> tuple[OperationTelemetry, ...]: + """Return and clear telemetry produced by completed writes. + + Existing API-v1 writers remain valid because the default emits no operations. + Providers that expose job/query statistics override this method and must drain + their buffer so one operation is never attributed to multiple endpoints. + """ + return () + @abstractmethod def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> int: """Write ``records`` to ``target`` idempotently; return the number of rows affected.""" diff --git a/src/dander/writer/storage_write.py b/src/dander/writer/storage_write.py index 111c6a4b..d245025f 100644 --- a/src/dander/writer/storage_write.py +++ b/src/dander/writer/storage_write.py @@ -174,6 +174,7 @@ def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> in dataset=target.dataset, table=staging_table, schema=target.schema, + declared_schema=target.canonical_schema, ) staging_id = _target_id(staging_target) schema_sql = ", ".join(f"`{field.name}` {_field_type_sql(field)}" for field in declared) diff --git a/tests/pipeline/test_compiler.py b/tests/pipeline/test_compiler.py index fecdb054..7b706f16 100644 --- a/tests/pipeline/test_compiler.py +++ b/tests/pipeline/test_compiler.py @@ -29,7 +29,7 @@ WriterConfig, ) from dander.pipeline.operations import OperationSpec -from dander.warehouse import RelationRef +from dander.warehouse import ProviderExtension, RelationRef from dander.writer import ( BigQueryIncrementalWriter, BigQueryReplaceWriter, @@ -229,6 +229,20 @@ def test_graph_compiles_to_one_provider_neutral_relational_ast() -> None: assert 'FROM "dander-test"."raw"."people"' in compiled.render("redshift") +def test_graph_target_preserves_provider_extension_in_canonical_schema() -> None: + graph = _linear_graph() + extension = ProviderExtension(provider="snowflake", name="fallback", value="variant") + graph.nodes[-1].fields[-1].extensions = (extension,) + + compiled = compile_target( + graph, + "target", + source_relations={"source": "dander-test.raw.people"}, + ) + + assert extension in compiled.target.canonical_schema.fields[-1].extensions + + def test_cast_free_graph_ast_renders_for_all_declared_targets() -> None: graph = _linear_graph() graph.nodes[0].fields[0].cast_to = None diff --git a/tests/providers/test_bigquery_runtime.py b/tests/providers/test_bigquery_runtime.py index 5c449696..7d4c21a5 100644 --- a/tests/providers/test_bigquery_runtime.py +++ b/tests/providers/test_bigquery_runtime.py @@ -135,6 +135,9 @@ class _Job: "bytes_written": 0, "bytes_processed": 1_024, "bytes_billed": 2_048, + "queue_duration_ms": 0, + "execution_duration_ms": 0, + "spill_bytes": 0, "job_id": "job-123", } diff --git a/tests/test_executor.py b/tests/test_executor.py index c75995d1..2a120195 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -19,6 +19,7 @@ RunStage, RunStatus, ) +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import TransformRunResult from dander.warehouse import RelationRef @@ -47,6 +48,13 @@ def run( extracted=3, affected=2, committed_cursor="2026-01-01T00:00:00Z", + telemetry=( + OperationTelemetry( + provider="sourcewarehouse", + operation=TelemetryOperation.LOAD, + rows_written=2, + ), + ), ), ), ) @@ -67,7 +75,17 @@ def build( assert tuple(selected or ()) == ("stg_widgets",) if self._fail: raise RuntimeError("transform failed") - return TransformRunResult(models=("stg_widgets",), assertions=2) + return TransformRunResult( + models=("stg_widgets",), + assertions=2, + telemetry=( + OperationTelemetry( + provider="modelwarehouse", + operation=TelemetryOperation.TRANSFORM, + rows_affected=3, + ), + ), + ) class _History(RunHistoryStore): @@ -259,6 +277,10 @@ def test_executor_records_one_truthful_complete_lifecycle( assert result.ingestion.run_id == result.run_id assert result.assets == 1 assert result.telemetry.duration_ms == 7 + assert [operation.provider for operation in result.telemetry.operations] == [ + "sourcewarehouse", + "modelwarehouse", + ] assert metadata.manifest is not None assert metadata.manifest["pipeline_id"] == "example_pipeline" assets = metadata.manifest["assets"] diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 2949bd5f..635c8054 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -13,7 +13,8 @@ from dander.ingestion.source import Endpoint, RawField, Source, SourceConfig from dander.runtime import PipelineRunner, RawSchemaError, WatermarkConflictError from dander.state import LeaseLostError, RunHistoryStore, RunStage, RunStatus, WatermarkStore -from dander.warehouse import RelationRef +from dander.telemetry import OperationTelemetry, TelemetryOperation +from dander.warehouse import ProviderExtension, RelationRef from dander.writer import WriteMode, WritePattern, WriteTarget if TYPE_CHECKING: @@ -97,6 +98,31 @@ def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> in return len(batch) +class _TelemetryWriter(_BatchedWriter): + def __init__(self) -> None: + super().__init__() + self._pending: tuple[OperationTelemetry, ...] = () + self.target: WriteTarget | None = None + + def write(self, records: Iterable[Mapping[str, Any]], target: WriteTarget) -> int: + batch = [dict(record) for record in records] + self.batches.append(batch) + self.target = target + affected = len(batch) + self._pending = ( + OperationTelemetry( + provider="testwarehouse", + operation=TelemetryOperation.LOAD, + rows_written=affected, + ), + ) + return affected + + def drain_telemetry(self) -> tuple[OperationTelemetry, ...]: + pending, self._pending = self._pending, () + return pending + + class _FencedWriter(_Writer): requires_publication_fence = True @@ -539,6 +565,30 @@ def test_runner_propagates_declared_schema_for_empty_endpoint() -> None: ] +def test_runner_preserves_canonical_extensions_and_drains_batch_telemetry() -> None: + extension = ProviderExtension(provider="snowflake", name="fallback", value="variant") + source = _DeclaredSource([]) + source.config.endpoints[0].raw_schema[-1].extensions = (extension,) + writer = _TelemetryWriter() + + result = PipelineRunner( + source=source, + writer=writer, + watermarks=_Watermarks([]), + project="unit-project", + dataset="raw", + batch_rows=1, + ).run() + + assert writer.target is not None + metadata = writer.target.canonical_schema.fields[-1] + assert metadata.data_type.kind.value == "json" + assert extension in metadata.extensions + assert result.telemetry == result.endpoints[0].telemetry + assert len(result.telemetry) == 1 + assert result.telemetry[0].rows_written == 0 + + @pytest.mark.parametrize( ("record", "match"), [ diff --git a/tests/test_runtime_contract.py b/tests/test_runtime_contract.py index 991ffc0b..2860be74 100644 --- a/tests/test_runtime_contract.py +++ b/tests/test_runtime_contract.py @@ -128,6 +128,9 @@ def test_runtime_event_distinguishes_overlap_and_sanitized_failure() -> None: "bytes_written": 0, "bytes_processed": 0, "bytes_billed": 0, + "queue_duration_ms": 0, + "execution_duration_ms": 0, + "spill_bytes": 0, "operations": [], } assert "credential-value" not in failed_json diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index c585520f..10ddc362 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -28,8 +28,15 @@ def _operation() -> OperationTelemetry: bytes_written=300, bytes_processed=500, bytes_billed=1_000, + queue_duration_ms=3, + execution_duration_ms=39, + spill_bytes=17, query_id="query/abc-123", job_id="job:abc-123", + resource_name="COMPUTE_WH", + resource_size="X-SMALL", + capacity_units=Decimal("0.0025"), + capacity_unit="credits", costs=( CostAttribution( provider="gcp", @@ -55,6 +62,9 @@ def test_run_telemetry_serializes_normalized_totals_and_operation_details() -> N "bytes_written": 300, "bytes_processed": 500, "bytes_billed": 1_000, + "queue_duration_ms": 3, + "execution_duration_ms": 39, + "spill_bytes": 17, "operations": [ { "provider": "bigquery", @@ -68,8 +78,15 @@ def test_run_telemetry_serializes_normalized_totals_and_operation_details() -> N "bytes_written": 300, "bytes_processed": 500, "bytes_billed": 1_000, + "queue_duration_ms": 3, + "execution_duration_ms": 39, + "spill_bytes": 17, "query_id": "query/abc-123", "job_id": "job:abc-123", + "resource_name": "COMPUTE_WH", + "resource_size": "X-SMALL", + "capacity_units": "0.0025", + "capacity_unit": "credits", "costs": [ { "provider": "gcp", @@ -112,6 +129,12 @@ def test_telemetry_rejects_unsafe_or_ambiguous_values() -> None: operation=TelemetryOperation.QUERY, unrestricted_details={"token": "secret"}, ) + with pytest.raises(ValueError, match="declared together"): + OperationTelemetry( + provider="snowflake", + operation=TelemetryOperation.QUERY, + capacity_units=Decimal("0.1"), + ) def test_whole_run_duration_can_be_replaced_without_losing_operations() -> None: diff --git a/tests/warehouse/test_contracts.py b/tests/warehouse/test_contracts.py index 9a5cb759..187c8689 100644 --- a/tests/warehouse/test_contracts.py +++ b/tests/warehouse/test_contracts.py @@ -188,3 +188,24 @@ def test_existing_connector_and_writer_models_expose_canonical_views() -> None: assert endpoint.raw_schema[0].to_canonical() == endpoint.canonical_raw_schema().fields[0] assert target.relation_ref.coordinates == ("gcp-project", "raw", "records") assert target.canonical_schema.fields[0].cardinality is FieldCardinality.REQUIRED + + +def test_write_target_prefers_an_explicit_canonical_schema() -> None: + extension = ProviderExtension(provider="snowflake", name="fallback", value="variant") + declared = RelationSchema( + fields=( + CanonicalField( + name="payload", + data_type=CanonicalType(kind=LogicalTypeKind.JSON), + extensions=(extension,), + ), + ) + ) + target = WriteTarget( + relation=RelationRef(catalog="warehouse", namespace="raw", name="events"), + schema=(WriteField(name="payload", data_type="JSON"),), + declared_schema=declared, + ) + + assert target.canonical_schema is declared + assert target.schema[0].data_type == "JSON" From a20282ee0235ef7e0fe0dbfd2d6d397078324441 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 04:04:48 -0400 Subject: [PATCH 092/163] feat(snowflake): add adaptive direct loads --- HANDOFF.md | 36 +- docs/snowflake.md | 34 +- src/dander/pipeline/node_config.py | 4 + src/dander/providers/bigquery/graph.py | 4 + src/dander/providers/snowflake/config.py | 17 +- src/dander/providers/snowflake/runtime.py | 21 +- src/dander/providers/snowflake/session.py | 24 + src/dander/providers/snowflake/writer.py | 522 +++++++++++++++--- src/dander/runtime-compatibility.json | 4 +- src/dander/telemetry.py | 12 + src/dander/warehouse/__init__.py | 4 + src/dander/warehouse/staging.py | 17 + src/dander/writer/base.py | 7 +- tests/pipeline/test_compiler.py | 10 + tests/pipeline/test_target_writer_config.py | 15 +- .../test_snowflake_warehouse_runtime.py | 271 ++++++++- tests/test_telemetry.py | 9 + tests/warehouse/test_schema_support.py | 23 +- 18 files changed, 913 insertions(+), 121 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 94ade508..1d22892b 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,37 @@ ## Finished -- Preserved canonical schema extensions through connectors, graphs, models, and warehouse targets. -- Added an API-v1-compatible writer telemetry drain and propagated operations to terminal runs. -- Added normalized resource, queue, execution, spill, and capacity-unit telemetry fields. +- Added endpoint-wide Snowflake direct/COPY selection with bounded threshold-plus-one buffering. +- Added explicit JSON-to-VARIANT mapping with canonical text staging and `PARSE_JSON` publication. +- Added truthful writer load/publication telemetry with transport, query IDs, rows, bytes, duration, and warehouse. +- Kept legacy graph/BigQuery planning fail-closed for provider-selected direct transport. ## Try It -Run `uv run pytest -q tests/test_runtime.py tests/test_executor.py tests/test_telemetry.py`. +Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`. ## Checks -- Pytest: 1,115 passed and 13 skipped locally; 1,128 passed with PostgreSQL 18 enabled. -- Ruff format/check: passed across all 328 files. -- Mypy: passed across 304 source/test files. -- Wheel/sdist inspection and two source-free package installs: passed. -- Terraform roots/tests and Helm lint/render: passed; local Docker final unpack hit known snapshot I/O corruption. +- Ruff format/check: passed across 328 files; mypy: passed across 304 source/test files. +- Pytest: 1,122 passed and 13 skipped from 1,135 collected tests. +- Wheel/sdist inspection and both source-free package installs: passed. +- Terraform validation/tests and Helm lint/render passed except the local Fargate module test, whose 782 MB provider could not initialize with under 1 GB free. +- Local Docker/PostgreSQL conformance remains for protected Linux CI because Docker Desktop is unresponsive. ## Decisions -- Legacy BigQuery schema declarations stay intact while `WriteTarget` retains one validated canonical schema. -- Existing writers emit no telemetry by default; provider writers opt in without changing `write()`. -- Provider extensions remain inert outside their matching adapter. +- Direct thresholds default to zero until a live Snowflake crossover is measured. +- Direct/COPY selection occurs once for the complete endpoint; ordinary runtime batching is disabled for Snowflake writers. +- Only canonical JSON with `snowflake/fallback=variant` is admitted; ARRAY and RECORD remain rejected. ## Remaining -- Let protected Linux CI repeat PostgreSQL, container, dependency, config, and secret gates. -- Merge this contract before implementing Snowflake VARIANT, direct/COPY routing, or query telemetry. +- Push a focused draft PR and require protected Linux CI, including PostgreSQL, container, Terraform, and scans. +- Measure the direct/COPY crossover during later live Snowflake qualification. +- Keep transforms/views/graphs and query-history enrichment in separate Snowflake slices. ## Review First -- `src/dander/runtime.py` -- `src/dander/writer/base.py` -- `src/dander/telemetry.py` +- `src/dander/providers/snowflake/writer.py` +- `src/dander/providers/snowflake/runtime.py` +- `tests/providers/test_snowflake_warehouse_runtime.py` diff --git a/docs/snowflake.md b/docs/snowflake.md index c2458fec..f29e81d3 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -1,9 +1,9 @@ # Experimental Snowflake warehouse Snowflake is an experimental warehouse adapter, not a supported Dander profile. The current slice -proves native database/schema coordinates, a bounded bulk path, all five scalar writer modes, and -fenced table/incremental models. Live qualification and the remaining first-class gates are still -required before support promotion. +proves native database/schema coordinates, bounded direct and bulk paths, all five writer modes, +an explicit JSON-to-`VARIANT` fallback, and fenced table/incremental models. Live qualification +and the remaining first-class gates are still required before support promotion. ## Configuration @@ -20,6 +20,9 @@ platforms: schema: RAW warehouse: DANDER_WH role: DANDER_ROLE + # Keep both at zero until a live benchmark establishes the account's crossover. + direct_max_rows: 0 + direct_max_logical_bytes: 0 auth: method: oauth token_env: DANDER_SNOWFLAKE_OAUTH_TOKEN @@ -39,8 +42,14 @@ variable reference. Do not put a token, private key, password, or connection str ## Current contract - `database + schema + relation` become one canonical `RelationRef` without GCP aliases. -- Runtime batches are written as checksummed, compressed Parquet parts and uploaded with `PUT`. -- `COPY` preserves Parquet logical and binary types explicitly. +- The writer sees one complete endpoint stream and retains at most a configured direct threshold + plus one row in memory. Larger streams continue into bounded Parquet parts without rereading the + source. +- When both direct thresholds are positive and the complete stream fits, rows are inserted into a + session-temporary staging table through bounded connector parameters. Otherwise checksummed, + compressed Parquet parts are uploaded with `PUT` and loaded with `COPY`. +- `COPY` preserves Parquet logical and binary types explicitly. Direct/COPY selection, load query + IDs, publication query IDs, rows, bytes, duration, and warehouse name flow into run telemetry. - One session-temporary stage and table contain each batch; normal cleanup is immediate and session termination removes them after process death. - SCD1, incremental, snapshot, SCD2, and replace publication share the same bounded staging path. @@ -52,6 +61,18 @@ variable reference. Do not put a token, private key, password, or connection str `DELETE`/`INSERT`, including an empty-source replacement. - Only declared nullable columns may be added automatically. Extra columns, required additions, type drift, nullability drift, malformed rows, and oversized singleton parts fail closed. +- Canonical JSON remains rejected unless its field explicitly declares: + + ```yaml + extensions: + - provider: snowflake + name: fallback + value: variant + ``` + + The staging value remains canonical JSON text and publication applies `PARSE_JSON`, so the + destination contains structured `VARIANT` data rather than a quoted string. ARRAY and RECORD + fallbacks remain unsupported. - Portable or Snowflake-authored table models replace rows through fenced `DELETE`/`INSERT` DML. - Incremental models collapse duplicate keys deterministically and accept only rows whose declared cursor is at least as new as the stored row. Generic model assertions run after publication. @@ -66,6 +87,7 @@ All five scalar write patterns are reachable through the warehouse writer capabi ordinary hosted source runner deliberately continues to select SCD1. PipelineGraph execution has not yet been wired to Snowflake's writer selection. Views remain unavailable because Snowflake permanent DDL cannot share the destination-fence -transaction. Semi-structured fields, a measured small-load direct path, full telemetry plumbing, +transaction. Direct thresholds default to zero because no live crossover has been measured; do +not claim a performance benefit until that qualification is recorded. Query-history enrichment, live concurrency proof, infrastructure provisioning, and support promotion remain separate work. Use `catalog.provider: none` for this experimental path. diff --git a/src/dander/pipeline/node_config.py b/src/dander/pipeline/node_config.py index 66b8a5b6..86885e06 100644 --- a/src/dander/pipeline/node_config.py +++ b/src/dander/pipeline/node_config.py @@ -311,6 +311,10 @@ def _check_mode_requirements(self) -> WriterConfig: raise ValueError( "WriterConfig(transport=storage_write) supports only scd1 or incremental mode." ) + if self.transport is WriteTransport.DIRECT: + raise ValueError( + "WriterConfig(transport=direct) is provider-selected and cannot be authored." + ) return self diff --git a/src/dander/providers/bigquery/graph.py b/src/dander/providers/bigquery/graph.py index cec204a0..055ba127 100644 --- a/src/dander/providers/bigquery/graph.py +++ b/src/dander/providers/bigquery/graph.py @@ -172,6 +172,10 @@ def prepare_bigquery_target_writer( if config.writer is None: raise PipelineCompileError(f"Target node {target_node.id!r} has no writer configuration") writer_config = config.writer + if writer_config.transport is WriteTransport.DIRECT: + raise PipelineCompileError( + "BigQuery target execution does not support the provider-selected direct transport" + ) destination = writer_config.destination try: relation = destination.relation_ref(default_catalog=default_catalog) diff --git a/src/dander/providers/snowflake/config.py b/src/dander/providers/snowflake/config.py index 33d113bb..869f15cb 100644 --- a/src/dander/providers/snowflake/config.py +++ b/src/dander/providers/snowflake/config.py @@ -5,7 +5,7 @@ import re from typing import Annotated, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from dander.warehouse.contracts import RelationRef @@ -78,6 +78,12 @@ class SnowflakeWarehouseConfig(BaseModel): le=1_073_741_824, ) compression: Literal["snappy", "zstd"] = "zstd" + direct_max_rows: int = Field(default=0, ge=0, le=10_000) + direct_max_logical_bytes: int = Field( + default=0, + ge=0, + le=16 * 1_024 * 1_024, + ) @field_validator("account") @classmethod @@ -93,6 +99,15 @@ def validate_identifiers(cls, value: str | None) -> str | None: raise ValueError("Snowflake identifiers must use portable identifier syntax") return value + @model_validator(mode="after") + def validate_direct_thresholds(self) -> SnowflakeWarehouseConfig: + """Require both provisional direct-load limits or disable the path entirely.""" + if (self.direct_max_rows == 0) != (self.direct_max_logical_bytes == 0): + raise ValueError( + "direct_max_rows and direct_max_logical_bytes must both be zero or positive" + ) + return self + def raw_relation( self, name: str, diff --git a/src/dander/providers/snowflake/runtime.py b/src/dander/providers/snowflake/runtime.py index 71092a81..72cb5ec7 100644 --- a/src/dander/providers/snowflake/runtime.py +++ b/src/dander/providers/snowflake/runtime.py @@ -29,7 +29,9 @@ SnowflakeScd1Writer, SnowflakeStagedWriter, SnowflakeStagingSettings, + SnowflakeWriteError, default_staging_settings, + validate_snowflake_schema, ) from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.warehouse import CanonicalField, LogicalTypeKind, RelationRef, RelationSchema @@ -37,6 +39,7 @@ WarehouseCapabilities, WarehouseRuntime, WarehouseSchemaSupport, + WarehouseSchemaSupportError, ) from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport @@ -92,7 +95,12 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: canonical.append(field) else: raise TypeError("Snowflake schema mapper received an unsupported field") - return SNOWFLAKE_SCHEMA_SUPPORT.require(RelationSchema(fields=tuple(canonical))) + schema = RelationSchema(fields=tuple(canonical)) + try: + validate_snowflake_schema(schema) + except SnowflakeWriteError as error: + raise WarehouseSchemaSupportError(str(error)) from error + return schema @dataclass(frozen=True, slots=True) @@ -103,6 +111,7 @@ class SnowflakeWriterFactory: connection_factory: SnowflakeConnectionFactory target_fence: SnowflakeTargetFence staging: SnowflakeStagingSettings + warehouse: str def build_ingestion_writer( self, @@ -134,6 +143,7 @@ def build_ingestion_writer( target_fence=self.target_fence, schema_evolution=schema_evolution, staging=self.staging, + warehouse=self.warehouse, ) return SnowflakeStagedWriter( database=self.database, @@ -141,6 +151,7 @@ def build_ingestion_writer( target_fence=self.target_fence, schema_evolution=schema_evolution, staging=self.staging, + warehouse=self.warehouse, mode=mode, cursor_field=cursor_field, snapshot_field=snapshot_field, @@ -200,6 +211,7 @@ def operation( retry_count=retry_count, rows_affected=affected, query_id=query_id if isinstance(query_id, str) else None, + resource_name=self.warehouse, ) @@ -207,7 +219,7 @@ def operation( provider_id="snowflake", schema_contract_version=1, write_modes=frozenset(WriteMode), - transports=frozenset({WriteTransport.COPY}), + transports=frozenset({WriteTransport.COPY, WriteTransport.DIRECT}), supports_transforms=True, supports_graphs=False, supports_target_fencing=True, @@ -257,6 +269,8 @@ def build_snowflake_warehouse( max_rows_per_file=config.max_rows_per_file, max_logical_bytes_per_file=config.max_logical_bytes_per_file, compression=config.compression, + direct_max_rows=config.direct_max_rows, + direct_max_logical_bytes=config.direct_max_logical_bytes, ) supplied_root = context.get("staging_root") if supplied_root is not None: @@ -267,6 +281,8 @@ def build_snowflake_warehouse( max_rows_per_file=staging.max_rows_per_file, max_logical_bytes_per_file=staging.max_logical_bytes_per_file, compression=staging.compression, + direct_max_rows=staging.direct_max_rows, + direct_max_logical_bytes=staging.direct_max_logical_bytes, ) schema_mapper = SnowflakeSchemaMapper() return WarehouseRuntime( @@ -278,6 +294,7 @@ def build_snowflake_warehouse( connection_factory, target_fence, staging, + config.warehouse, ), transforms=SnowflakeTransformFactory( config.database, diff --git a/src/dander/providers/snowflake/session.py b/src/dander/providers/snowflake/session.py index 87d61630..fd502ea1 100644 --- a/src/dander/providers/snowflake/session.py +++ b/src/dander/providers/snowflake/session.py @@ -19,6 +19,12 @@ class SnowflakeCursor(Protocol): def execute(self, command: str, params: Sequence[object] | None = None) -> Self: ... + def executemany( + self, + command: str, + seq_of_parameters: Sequence[Sequence[object]], + ) -> Self: ... + def fetchone(self) -> object | None: ... def fetchall(self) -> list[object]: ... @@ -74,6 +80,23 @@ def execute( cursor.close() +def execute_many( + connection: SnowflakeConnection, + statement: str, + parameter_rows: Sequence[Sequence[object]], +) -> SnowflakeStatementResult: + """Execute one bounded qmark batch and detach its sanitized statement result.""" + cursor = connection.cursor() + try: + cursor.executemany(statement, parameter_rows) + return SnowflakeStatementResult( + rowcount=max(cursor.rowcount, 0), + query_id=cursor.sfqid, + ) + finally: + cursor.close() + + @contextmanager def open_connection(factory: SnowflakeConnectionFactory) -> Iterator[SnowflakeConnection]: """Close one connector session after success or failure.""" @@ -90,5 +113,6 @@ def open_connection(factory: SnowflakeConnectionFactory) -> Iterator[SnowflakeCo "SnowflakeCursor", "SnowflakeStatementResult", "execute", + "execute_many", "open_connection", ] diff --git a/src/dander/providers/snowflake/writer.py b/src/dander/providers/snowflake/writer.py index 7ac46c34..5095c421 100644 --- a/src/dander/providers/snowflake/writer.py +++ b/src/dander/providers/snowflake/writer.py @@ -6,9 +6,14 @@ import json import sys import tempfile -from collections.abc import Mapping +from base64 import b64encode +from collections.abc import Callable, Mapping from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from itertools import chain from pathlib import Path +from time import perf_counter_ns from typing import TYPE_CHECKING from uuid import uuid4 @@ -16,9 +21,12 @@ from dander.providers.snowflake.session import ( SnowflakeConnection, SnowflakeConnectionFactory, + SnowflakeStatementResult, execute, + execute_many, open_connection, ) +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.warehouse import ( CanonicalField, CanonicalType, @@ -27,8 +35,16 @@ ParquetStagingSession, RelationSchema, StagingManifest, + normalize_staging_record, + staging_logical_size, +) +from dander.writer import ( + SchemaEvolution, + WriteMode, + WritePattern, + WriteTarget, + WriteTransport, ) -from dander.writer import SchemaEvolution, WriteMode, WritePattern, WriteTarget if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -55,6 +71,21 @@ class SnowflakeStagingSettings: max_rows_per_file: int max_logical_bytes_per_file: int compression: str + direct_max_rows: int = 0 + direct_max_logical_bytes: int = 0 + + +@dataclass(frozen=True, slots=True) +class _DirectBatch: + rows: tuple[dict[str, object], ...] + logical_bytes: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class _LoadOutcome: + affected: int + operations: tuple[OperationTelemetry, ...] class SnowflakeStagedWriter(WritePattern): @@ -71,6 +102,7 @@ def __init__( schema_evolution: SchemaEvolution, staging: SnowflakeStagingSettings, mode: WriteMode, + warehouse: str | None = None, cursor_field: str | None = None, snapshot_field: str | None = None, ) -> None: @@ -79,11 +111,21 @@ def __init__( self._target_fence = target_fence self._schema_evolution = schema_evolution self._staging = staging + self._warehouse = warehouse self.mode = mode self._cursor_field = cursor_field self._snapshot_field = snapshot_field - self.supports_batched_writes = mode not in {WriteMode.REPLACE, WriteMode.SCD2} - self.accepts_streaming_input = mode in {WriteMode.REPLACE, WriteMode.SCD2} + # Transport selection must see the complete logical endpoint stream. The writer + # remains bounded because it retains at most the direct threshold plus one row. + self.supports_batched_writes = False + self.accepts_streaming_input = True + self._telemetry: list[OperationTelemetry] = [] + + def drain_telemetry(self) -> tuple[OperationTelemetry, ...]: + """Return completed load/publication operations exactly once.""" + operations = tuple(self._telemetry) + self._telemetry.clear() + return operations def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: """Consume records once, upload checksummed parts, and publish idempotently.""" @@ -104,8 +146,8 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> field.name for field in target_schema.fields if field.name in reserved ): raise SnowflakeWriteError(f"Declared schema reserves Dander field {collision[0]!r}") - _validate_schema(target_schema) - staging_schema = RelationSchema( + validate_snowflake_schema(target_schema) + normalization_schema = RelationSchema( fields=( *target_schema.fields, CanonicalField( @@ -115,6 +157,36 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> ), ) ) + staging_schema = RelationSchema( + fields=tuple(_staging_field(field) for field in normalization_schema.fields) + ) + self._telemetry.clear() + prepared = iter( + _with_ordinals( + records, + business_key=target.business_key, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) + ) + direct, remaining = _select_direct_batch( + prepared, + normalization_schema, + max_rows=self._staging.direct_max_rows, + max_logical_bytes=self._staging.direct_max_logical_bytes, + ) + if direct is not None: + outcome = self._load_and_publish( + target, + publication, + target_schema, + staging_schema, + transport=WriteTransport.DIRECT, + direct=direct, + ) + self._telemetry.extend(outcome.operations) + return outcome.affected + local_id = f"snowflake-{uuid4().hex}" with ParquetStagingSession( self._staging.root, @@ -124,21 +196,19 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> compression=self._staging.compression, ) as local: manifest = local.stage( - _with_ordinals( - records, - business_key=target.business_key, - cursor_field=self._cursor_field, - snapshot_field=self._snapshot_field, - ), + _normalized_for_staging(remaining, normalization_schema), staging_schema, ) - return self._load_and_publish( + outcome = self._load_and_publish( target, publication, target_schema, staging_schema, - manifest, + transport=WriteTransport.COPY, + manifest=manifest, ) + self._telemetry.extend(outcome.operations) + return outcome.affected def _load_and_publish( self, @@ -146,23 +216,32 @@ def _load_and_publish( publication: TargetFence, target_schema: RelationSchema, staging_schema: RelationSchema, - manifest: object, - ) -> int: - if not isinstance(manifest, StagingManifest): - raise TypeError("Snowflake writer received an invalid staging manifest") - _validate_manifest_bounds( - manifest, - max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, - ) + *, + transport: WriteTransport, + manifest: StagingManifest | None = None, + direct: _DirectBatch | None = None, + ) -> _LoadOutcome: + if (manifest is None) == (direct is None): + raise TypeError("Snowflake writer requires exactly one staged load input") + if manifest is not None: + _validate_manifest_bounds( + manifest, + max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, + ) suffix = hashlib.sha256( f"{publication.target_id}:{publication.run_id}:{uuid4().hex}".encode() ).hexdigest()[:20] stage_name = f"dander_files_{suffix}" staging_table = f"dander_stage_{suffix}" relation = target.relation_ref - stage = _qualified(relation.catalog, relation.namespace, stage_name) + stage = ( + _qualified(relation.catalog, relation.namespace, stage_name) + if transport is WriteTransport.COPY + else None + ) staging_relation = _qualified(relation.catalog, relation.namespace, staging_table) cleanup_started = False + operations: list[OperationTelemetry] = [] with open_connection(self._connection_factory) as connection: try: _set_query_tag(connection, publication) @@ -174,14 +253,32 @@ def _load_and_publish( evolution=self._schema_evolution, ) _ensure_load_history(connection, target) - if manifest.rows == 0: - result = self._target_fence.execute_dml( + if manifest is not None: + rows = manifest.rows + else: + assert direct is not None + rows = len(direct.rows) + if rows == 0: + result, duration_ms = _timed_call( + self._target_fence.execute_dml, connection, _empty_publication_sql(target, self.mode), publication, ) - return 0 if self.mode is WriteMode.REPLACE else result.rowcount - execute(connection, _create_stage_sql(stage)) + operations.append( + self._operation( + result, + operation=TelemetryOperation.QUERY, + transport=transport, + duration_ms=duration_ms, + ) + ) + return _LoadOutcome( + affected=0 if self.mode is WriteMode.REPLACE else result.rowcount, + operations=tuple(operations), + ) + if stage is not None: + execute(connection, _create_stage_sql(stage)) execute( connection, _create_staging_table_sql( @@ -190,48 +287,114 @@ def _load_and_publish( ), ) cleanup_started = True - pending = tuple( - artifact - for artifact in manifest.artifacts - if not _artifact_committed(connection, target, publication, artifact.sha256) - ) - if not pending: - result = self._target_fence.execute_dml( + if manifest is not None: + pending = tuple( + artifact + for artifact in manifest.artifacts + if not _artifact_committed( + connection, + target, + publication, + artifact.sha256, + ) + ) + committed = not pending + else: + assert direct is not None + committed = _artifact_committed( + connection, + target, + publication, + direct.sha256, + ) + pending = () + if committed: + result, duration_ms = _timed_call( + self._target_fence.execute_dml, connection, _no_op_sql(target), publication, ) - return result.rowcount - artifacts_to_load = ( - manifest.artifacts if self.mode is WriteMode.REPLACE else pending - ) - for artifact in artifacts_to_load: - execute(connection, _put_sql(artifact.path, stage)) - execute(connection, _copy_sql(staging_relation, stage, artifact.path.name)) + operations.append( + self._operation( + result, + operation=TelemetryOperation.QUERY, + transport=transport, + duration_ms=duration_ms, + ) + ) + return _LoadOutcome(result.rowcount, tuple(operations)) + if manifest is not None: + assert stage is not None + artifacts_to_load = ( + manifest.artifacts if self.mode is WriteMode.REPLACE else pending + ) + for artifact in artifacts_to_load: + execute(connection, _put_sql(artifact.path, stage)) + load, duration_ms = _timed_call( + execute, + connection, + _copy_sql(staging_relation, stage, artifact.path.name), + ) + operations.append( + self._operation( + load, + operation=TelemetryOperation.LOAD, + transport=transport, + duration_ms=duration_ms, + rows_written=artifact.rows, + bytes_written=artifact.compressed_bytes, + ) + ) + else: + assert direct is not None + load, duration_ms = _timed_call( + execute_many, + connection, + *_direct_insert(staging_relation, staging_schema, direct.rows), + ) + operations.append( + self._operation( + load, + operation=TelemetryOperation.LOAD, + transport=transport, + duration_ms=duration_ms, + rows_written=len(direct.rows), + bytes_written=direct.logical_bytes, + ) + ) if self.mode is WriteMode.SCD2: execute( connection, "SET DANDER_SCD2_EFFECTIVE_AT = CURRENT_TIMESTAMP()", ) statements: list[tuple[str, Sequence[object]]] = [] - statements.extend( - ( - _load_history_insert_sql(target), + if manifest is not None: + statements.extend( ( - publication.target_id, - publication.pipeline_id, - publication.run_id, - artifact.sha256, - artifact.rows, - artifact.compressed_bytes, - publication.target_id, - publication.pipeline_id, - publication.run_id, - artifact.sha256, - ), + _load_history_insert_sql(target), + _load_history_parameters( + publication, + sha256=artifact.sha256, + rows=artifact.rows, + compressed_bytes=artifact.compressed_bytes, + ), + ) + for artifact in pending + ) + else: + assert direct is not None + statements.append( + ( + _load_history_insert_sql(target), + _load_history_parameters( + publication, + sha256=direct.sha256, + rows=len(direct.rows), + compressed_bytes=0, + ), + ) ) - for artifact in pending - ) statements.extend( _publication_statements( target, @@ -242,12 +405,21 @@ def _load_and_publish( snapshot_field=self._snapshot_field, ) ) - result = self._target_fence.execute_statements( + result, duration_ms = _timed_call( + self._target_fence.execute_statements, connection, statements, publication, ) - return result.rowcount + operations.append( + self._operation( + result, + operation=TelemetryOperation.QUERY, + transport=transport, + duration_ms=duration_ms, + ) + ) + return _LoadOutcome(result.rowcount, tuple(operations)) except (SnowflakeWriteError, TargetFenceLostError): raise except Exception as error: @@ -258,6 +430,28 @@ def _load_and_publish( if cleanup_started: _cleanup_remote(connection, stage=stage, staging_relation=staging_relation) + def _operation( + self, + result: SnowflakeStatementResult, + *, + operation: TelemetryOperation, + transport: WriteTransport, + duration_ms: int, + rows_written: int = 0, + bytes_written: int = 0, + ) -> OperationTelemetry: + return OperationTelemetry( + provider="snowflake", + operation=operation, + duration_ms=duration_ms, + rows_written=rows_written, + rows_affected=result.rowcount, + bytes_written=bytes_written, + query_id=result.query_id, + resource_name=self._warehouse, + transport=transport, + ) + class SnowflakeScd1Writer(SnowflakeStagedWriter): """Backward-compatible SCD1 constructor for existing provider integrations.""" @@ -272,6 +466,7 @@ def __init__( target_fence: SnowflakeTargetFence, schema_evolution: SchemaEvolution, staging: SnowflakeStagingSettings, + warehouse: str | None = None, ) -> None: super().__init__( database=database, @@ -279,6 +474,7 @@ def __init__( target_fence=target_fence, schema_evolution=schema_evolution, staging=staging, + warehouse=warehouse, mode=WriteMode.SCD1, ) @@ -288,6 +484,8 @@ def default_staging_settings( max_rows_per_file: int, max_logical_bytes_per_file: int, compression: str, + direct_max_rows: int = 0, + direct_max_logical_bytes: int = 0, ) -> SnowflakeStagingSettings: """Return container-safe local staging defaults without creating directories.""" return SnowflakeStagingSettings( @@ -295,7 +493,118 @@ def default_staging_settings( max_rows_per_file=max_rows_per_file, max_logical_bytes_per_file=max_logical_bytes_per_file, compression=compression, + direct_max_rows=direct_max_rows, + direct_max_logical_bytes=direct_max_logical_bytes, + ) + + +def _select_direct_batch( + records: Iterable[Mapping[str, object]], + schema: RelationSchema, + *, + max_rows: int, + max_logical_bytes: int, +) -> tuple[_DirectBatch | None, Iterable[Mapping[str, object]]]: + """Choose direct only after the complete endpoint fits both bounded limits.""" + if max_rows == 0 or max_logical_bytes == 0: + return None, records + iterator = iter(records) + raw_prefix: list[dict[str, object]] = [] + normalized_prefix: list[dict[str, object]] = [] + logical_bytes = 0 + for row_index, record in enumerate(iterator): + raw = dict(record) + normalized = normalize_staging_record(raw, schema, row_index=row_index) + raw_prefix.append(raw) + normalized_prefix.append(normalized) + logical_bytes += staging_logical_size(normalized) + if len(raw_prefix) > max_rows or logical_bytes > max_logical_bytes: + return None, chain(raw_prefix, iterator) + rows = tuple(normalized_prefix) + return ( + _DirectBatch( + rows=rows, + logical_bytes=logical_bytes, + sha256=_direct_checksum(schema, rows), + ), + (), + ) + + +def _normalized_for_staging( + records: Iterable[Mapping[str, object]], + schema: RelationSchema, +) -> Iterable[Mapping[str, object]]: + for row_index, record in enumerate(records): + yield normalize_staging_record(record, schema, row_index=row_index) + + +def _direct_checksum( + schema: RelationSchema, + rows: tuple[dict[str, object], ...], +) -> str: + payload = { + "schema": schema.model_dump(mode="json", by_alias=True), + "rows": [[_checksum_value(row[field.name]) for field in schema.fields] for row in rows], + } + encoded = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _checksum_value(value: object) -> object: + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, float): + return {"float": value.hex()} + if isinstance(value, Decimal): + return {"decimal": str(value)} + if isinstance(value, bytes): + return {"binary": b64encode(value).decode("ascii")} + if isinstance(value, datetime): + return {"datetime": value.isoformat()} + if isinstance(value, date): + return {"date": value.isoformat()} + if isinstance(value, time): + return {"time": value.isoformat()} + raise SnowflakeWriteError("Snowflake direct staging received an unsupported scalar value") + + +def _direct_insert( + staging_relation: str, + schema: RelationSchema, + rows: tuple[dict[str, object], ...], +) -> tuple[str, tuple[tuple[object, ...], ...]]: + columns = ", ".join(_quote(field.name) for field in schema.fields) + placeholders = ", ".join("?" for _field in schema.fields) + statement = f"INSERT INTO {staging_relation} ({columns}) VALUES ({placeholders})" + parameters = tuple(tuple(row[field.name] for field in schema.fields) for row in rows) + return statement, parameters + + +def _load_history_parameters( + publication: TargetFence, + *, + sha256: str, + rows: int, + compressed_bytes: int, +) -> tuple[object, ...]: + identity = ( + publication.target_id, + publication.pipeline_id, + publication.run_id, + sha256, ) + return (*identity, rows, compressed_bytes, *identity) + + +def _timed_call( + function: Callable[..., SnowflakeStatementResult], + *arguments: object, +) -> tuple[SnowflakeStatementResult, int]: + started = perf_counter_ns() + result = function(*arguments) + duration_ms = max((perf_counter_ns() - started) // 1_000_000, 0) + return result, duration_ms def _with_ordinals( @@ -328,9 +637,8 @@ def _validate_target( raise SnowflakeWriteError( f"Writer database {database!r} does not match target catalog {relation.catalog!r}" ) - if not target.schema: - raise SnowflakeWriteError("Snowflake writes require a declared schema") - field_names = {field.name for field in target.schema} + target_schema = target.canonical_schema + field_names = {field.name for field in target_schema.fields} if mode in {WriteMode.SCD1, WriteMode.SCD2, WriteMode.INCREMENTAL}: if not target.business_key: raise SnowflakeWriteError(f"Snowflake {mode.value} writes require a business key") @@ -359,9 +667,51 @@ def _validate_target( raise SnowflakeWriteError("Snowflake destination target fence does not match the target") -def _validate_schema(schema: RelationSchema) -> None: +def validate_snowflake_schema(schema: RelationSchema) -> None: + """Validate scalar mappings plus the one explicit JSON-to-VARIANT fallback.""" for field in schema.fields: - _snowflake_type(field.data_type) + _snowflake_field_type(field) + + +def _staging_field(field: CanonicalField) -> CanonicalField: + if _is_variant_fallback(field): + return field.model_copy( + update={ + "data_type": CanonicalType(kind=LogicalTypeKind.STRING), + "extensions": tuple( + extension for extension in field.extensions if extension.provider != "snowflake" + ), + } + ) + return field + + +def _is_variant_fallback(field: CanonicalField) -> bool: + snowflake_extensions = tuple( + extension for extension in field.extensions if extension.provider == "snowflake" + ) + expected = ( + len(snowflake_extensions) == 1 + and snowflake_extensions[0].name == "fallback" + and snowflake_extensions[0].value == "variant" + ) + if field.data_type.kind is LogicalTypeKind.JSON: + if not expected: + raise SnowflakeWriteError("Snowflake JSON fields require snowflake/fallback=variant") + return True + if field.data_type.kind in {LogicalTypeKind.ARRAY, LogicalTypeKind.RECORD}: + raise SnowflakeWriteError( + "Snowflake ARRAY and RECORD fields have no canonical fallback in this slice" + ) + if snowflake_extensions: + raise SnowflakeWriteError("Snowflake field extension is unsupported for this type") + return False + + +def _snowflake_field_type(field: CanonicalField) -> str: + if _is_variant_fallback(field): + return "VARIANT" + return _snowflake_type(field.data_type) def _target_schema_for_mode(schema: RelationSchema, mode: WriteMode) -> RelationSchema: @@ -454,7 +804,7 @@ def _ensure_target( required = ( field.cardinality is FieldCardinality.REQUIRED or field.name in target.business_key ) - expected_type = _normalize_type(_snowflake_type(field.data_type)) + expected_type = _normalize_type(_snowflake_field_type(field)) if current is None: if evolution is not SchemaEvolution.ADDITIVE: raise SnowflakeWriteError(f"Deployed target is missing column {field.name!r}") @@ -614,13 +964,32 @@ def _deduplicated_source( if cursor_field is not None else f"{_quote(_ORDINAL)} DESC" ) + source = _projected_staging_source(staging_relation, fields, include_ordinal=True) return ( - f"SELECT {', '.join(_quote(name) for name in names)} FROM {staging_relation} " + f"SELECT {', '.join(_quote(name) for name in names)} FROM {source} AS normalized " f"QUALIFY ROW_NUMBER() OVER (PARTITION BY {partition} " f"ORDER BY {ordering}) = 1" ) +def _projected_staging_source( + staging_relation: str, + fields: Sequence[CanonicalField], + *, + include_ordinal: bool = False, +) -> str: + projections = [_staging_projection(field) for field in fields] + if include_ordinal: + projections.append(f"staged.{_quote(_ORDINAL)} AS {_quote(_ORDINAL)}") + return f"(SELECT {', '.join(projections)} FROM {staging_relation} AS staged)" + + +def _staging_projection(field: CanonicalField) -> str: + reference = f"staged.{_quote(field.name)}" + value = f"PARSE_JSON({reference})" if _is_variant_fallback(field) else reference + return f"{value} AS {_quote(field.name)}" + + def _merge_sql( target: WriteTarget, staging_relation: str, @@ -666,10 +1035,11 @@ def _replace_statements( fields: Sequence[CanonicalField], ) -> tuple[tuple[str, Sequence[object]], ...]: columns = ", ".join(_quote(field.name) for field in fields) + source = _projected_staging_source(staging_relation, fields) return ( (f"DELETE FROM {_target(target)}", ()), ( - f"INSERT INTO {_target(target)} ({columns}) SELECT {columns} FROM {staging_relation}", + f"INSERT INTO {_target(target)} ({columns}) SELECT {columns} FROM {source}", (), ), ) @@ -687,9 +1057,10 @@ def _snapshot_sql( identical = " AND ".join( f"EQUAL_NULL(existing.{_quote(name)}, incoming.{_quote(name)})" for name in names ) + source = _projected_staging_source(staging_relation, fields) return ( f"INSERT INTO {_target(target)} ({columns}) SELECT DISTINCT {selected} " - f"FROM {staging_relation} AS incoming WHERE incoming.{_quote(snapshot_field)} IS NOT NULL " + f"FROM {source} AS incoming WHERE incoming.{_quote(snapshot_field)} IS NOT NULL " f"AND NOT EXISTS (SELECT 1 FROM {_target(target)} AS existing WHERE {identical})" ) @@ -745,7 +1116,7 @@ def _scd2_statements( def _no_op_sql(target: WriteTarget) -> str: - column = _quote(target.schema[0].name) + column = _quote(target.canonical_schema.fields[0].name) return f"UPDATE {_target(target)} SET {column} = {column} WHERE FALSE" @@ -758,15 +1129,15 @@ def _empty_publication_sql(target: WriteTarget, mode: WriteMode) -> str: def _cleanup_remote( connection: SnowflakeConnection, *, - stage: str, + stage: str | None, staging_relation: str, ) -> None: active_error = sys.exc_info()[0] is not None failures = 0 - for statement in ( - f"DROP TABLE IF EXISTS {staging_relation}", - f"DROP STAGE IF EXISTS {stage}", - ): + statements = [f"DROP TABLE IF EXISTS {staging_relation}"] + if stage is not None: + statements.append(f"DROP STAGE IF EXISTS {stage}") + for statement in statements: try: execute(connection, statement) except Exception: @@ -780,7 +1151,7 @@ def _column_definition( *, required: bool, ) -> str: - data_type = _snowflake_type(field.data_type) + data_type = _snowflake_field_type(field) return f"{_quote(field.name)} {data_type}{' NOT NULL' if required else ''}" @@ -852,4 +1223,5 @@ def _literal(value: str) -> str: "SnowflakeStagingSettings", "SnowflakeWriteError", "default_staging_settings", + "validate_snowflake_schema", ] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 6ae4da40..1a3058da 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -90,14 +90,14 @@ { "provider": "snowflake", "write_modes": ["incremental", "replace", "scd1", "scd2", "snapshot"], - "transports": ["copy"], + "transports": ["copy", "direct"], "logical_types": ["binary", "boolean", "date", "decimal", "float", "integer", "string", "time", "timestamp"], "max_decimal_precision": 38, "max_temporal_precision": 9, "supports_transforms": true, "supports_graphs": false, "supports_target_fencing": true, - "limitations": ["graph execution is not available", "hosted source execution selects scd1", "semi-structured types require a future explicit VARIANT mapping", "small-load direct writes are not qualified"] + "limitations": ["JSON requires the explicit snowflake/fallback=variant field extension", "direct thresholds remain disabled until the live crossover is measured", "graph execution is not available", "hosted source execution selects scd1"] } ] } diff --git a/src/dander/telemetry.py b/src/dander/telemetry.py index 677e16e0..d31db2d3 100644 --- a/src/dander/telemetry.py +++ b/src/dander/telemetry.py @@ -6,6 +6,10 @@ from dataclasses import dataclass, replace from decimal import Decimal from enum import StrEnum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dander.writer.base import WriteTransport _SAFE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,126}$") _SAFE_CURRENCY = re.compile(r"^[A-Z]{3}$") @@ -82,6 +86,7 @@ class OperationTelemetry: resource_size: str | None = None capacity_units: Decimal | None = None capacity_unit: str | None = None + transport: WriteTransport | None = None def __post_init__(self) -> None: _require_name(self.provider, label="telemetry provider") @@ -93,6 +98,11 @@ def __post_init__(self) -> None: _require_opaque(self.job_id, label="job id") _require_opaque(self.resource_name, label="resource name") _require_opaque(self.resource_size, label="resource size") + if self.transport is not None: + from dander.writer.base import WriteTransport + + if not isinstance(self.transport, WriteTransport): + raise ValueError("telemetry transport must be a WriteTransport") if (self.capacity_units is None) != (self.capacity_unit is None): raise ValueError("capacity_units and capacity_unit must be declared together") if self.capacity_units is not None: @@ -122,6 +132,8 @@ def to_payload(self) -> dict[str, object]: payload["resource_name"] = self.resource_name if self.resource_size is not None: payload["resource_size"] = self.resource_size + if self.transport is not None: + payload["transport"] = self.transport.value if self.capacity_units is not None: payload["capacity_units"] = str(self.capacity_units) payload["capacity_unit"] = self.capacity_unit diff --git a/src/dander/warehouse/__init__.py b/src/dander/warehouse/__init__.py index 6fd8e0b7..8d065852 100644 --- a/src/dander/warehouse/__init__.py +++ b/src/dander/warehouse/__init__.py @@ -34,6 +34,8 @@ StagedArtifact, StagingArtifactError, StagingManifest, + normalize_staging_record, + staging_logical_size, ) __all__ = [ @@ -42,6 +44,7 @@ "CanonicalType", "FieldCardinality", "LogicalTypeKind", + "normalize_staging_record", "ParquetStagingSession", "ProviderExtension", "PreparedWarehouseStatement", @@ -52,6 +55,7 @@ "StagedArtifact", "StagingArtifactError", "StagingManifest", + "staging_logical_size", "WarehouseCapabilities", "WarehouseRuntime", "WarehouseSchemaMapper", diff --git a/src/dander/warehouse/staging.py b/src/dander/warehouse/staging.py index d41ee716..685967a9 100644 --- a/src/dander/warehouse/staging.py +++ b/src/dander/warehouse/staging.py @@ -330,6 +330,16 @@ def _normalize_record( } +def normalize_staging_record( + record: Mapping[str, object], + schema: RelationSchema, + *, + row_index: int, +) -> dict[str, object]: + """Normalize one row exactly as bounded Parquet staging does.""" + return _normalize_record(record, schema.fields, row_index=row_index) + + def _normalize_value(value: object, field: CanonicalField, *, row_index: int) -> object: if value is None: if field.cardinality is FieldCardinality.REQUIRED: @@ -391,6 +401,11 @@ def _logical_size(value: object) -> int: return len(str(value).encode("utf-8")) +def staging_logical_size(value: object) -> int: + """Return the deterministic logical-byte estimate used by staged loaders.""" + return _logical_size(value) + + def _file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -400,8 +415,10 @@ def _file_sha256(path: Path) -> str: __all__ = [ + "normalize_staging_record", "ParquetStagingSession", "StagedArtifact", "StagingArtifactError", "StagingManifest", + "staging_logical_size", ] diff --git a/src/dander/writer/base.py b/src/dander/writer/base.py index 2fae672b..da434280 100644 --- a/src/dander/writer/base.py +++ b/src/dander/writer/base.py @@ -12,14 +12,12 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any -from dander.warehouse.contracts import ProviderExtension, RelationRef - if TYPE_CHECKING: from collections.abc import Iterable, Mapping from dander.concurrency import FencingToken, TargetFence from dander.telemetry import OperationTelemetry - from dander.warehouse import CanonicalField, RelationSchema + from dander.warehouse import CanonicalField, ProviderExtension, RelationRef, RelationSchema class WriteMode(StrEnum): @@ -45,6 +43,7 @@ class WriteTransport(StrEnum): LOAD_JOB = "load_job" STORAGE_WRITE = "storage_write" COPY = "copy" + DIRECT = "direct" @dataclass(frozen=True) @@ -95,6 +94,8 @@ def __init__( if relation is None: if project is None or dataset is None or table is None: raise TypeError("WriteTarget requires relation or project/dataset/table") + from dander.warehouse.contracts import RelationRef + relation = RelationRef(catalog=project, namespace=dataset, name=table) elif project is not None or dataset is not None or table is not None: raise TypeError("WriteTarget cannot combine relation with project/dataset/table") diff --git a/tests/pipeline/test_compiler.py b/tests/pipeline/test_compiler.py index 7b706f16..e7251f77 100644 --- a/tests/pipeline/test_compiler.py +++ b/tests/pipeline/test_compiler.py @@ -583,3 +583,13 @@ def test_target_dispatches_storage_write_transport( assert isinstance(prepared.writer, expected_type) assert prepared.target.schema[0].data_type == "STRING" + + +def test_bigquery_writer_preparation_rejects_provider_selected_direct_transport() -> None: + config = _target_config() + assert config.writer is not None + config.writer.transport = WriteTransport.DIRECT + node = Node(id="target", type="target", name="Target", config=config) + + with pytest.raises(PipelineCompileError, match="does not support.*direct transport"): + prepare_target_writer(node, default_project="fallback", client=object()) diff --git a/tests/pipeline/test_target_writer_config.py b/tests/pipeline/test_target_writer_config.py index a8323582..12f168aa 100644 --- a/tests/pipeline/test_target_writer_config.py +++ b/tests/pipeline/test_target_writer_config.py @@ -29,7 +29,7 @@ TargetNodeConfig, WriterConfig, ) -from dander.writer.base import WriteMode +from dander.writer.base import WriteMode, WriteTransport if TYPE_CHECKING: from pathlib import Path @@ -257,6 +257,19 @@ def test_invalid_write_mode_string_is_rejected() -> None: ) +def test_direct_transport_cannot_be_authored_in_legacy_graphs() -> None: + with pytest.raises(ValidationError, match="provider-selected"): + WriterConfig( + write_mode=WriteMode.SCD1, + destination=DestinationSpec( + dataset="analytics", + table="dim_candidate", + business_key=["candidate_id"], + ), + transport=WriteTransport.DIRECT, + ) + + def test_scd1_with_empty_business_key_is_rejected() -> None: """SCD1 requires a non-empty `destination.business_key`.""" with pytest.raises(ValidationError, match="scd1"): diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index 94415cbe..4e95ea99 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -17,7 +17,12 @@ from dander.providers.snowflake.writer import SnowflakeWriteError from dander.telemetry import TelemetryOperation from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError -from dander.warehouse import RelationRef, WarehouseRuntime +from dander.warehouse import ( + ProviderExtension, + RelationRef, + WarehouseRuntime, + WarehouseSchemaSupportError, +) from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport if TYPE_CHECKING: @@ -153,6 +158,20 @@ def execute(self, command: str, params: Sequence[object] | None = None) -> Self: self._row = (self.backend.assertion_failure_count,) return self + def executemany( + self, + command: str, + seq_of_parameters: Sequence[Sequence[object]], + ) -> Self: + parameters = tuple(tuple(row) for row in seq_of_parameters) + compact = " ".join(command.split()) + self.backend.statements.append((compact, parameters)) + self.sfqid = f"query-{len(self.backend.statements)}" + if self.backend.fail_prefix and compact.startswith(self.backend.fail_prefix): + raise RuntimeError("private provider response") + self.rowcount = len(parameters) + return self + def fetchone(self) -> object | None: return self._row @@ -208,6 +227,31 @@ def snowflake_runtime( return runtime, backend, tmp_path +def _direct_runtime( + tmp_path: Path, + *, + max_rows: int = 2, + max_logical_bytes: int = 4_096, +) -> tuple[WarehouseRuntime, _FakeSnowflake]: + backend = _FakeSnowflake() + registry = default_provider_registry() + raw = _config() + raw["direct_max_rows"] = max_rows + raw["direct_max_logical_bytes"] = max_logical_bytes + config = registry.parse(ProviderKind.WAREHOUSE, raw) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "DANDER_TEST", + "connection_factory": backend.connect, + "staging_root": tmp_path, + }, + ) + assert isinstance(runtime, WarehouseRuntime) + return runtime, backend + + def test_snowflake_registration_is_lazy_and_credentials_remain_references() -> None: module_name = "dander.providers.snowflake.runtime" sys.modules.pop(module_name, None) @@ -234,6 +278,32 @@ def test_snowflake_registration_is_lazy_and_credentials_remain_references() -> N assert relation == RelationRef(catalog="DANDER_TEST", namespace="raw", name="records") +def test_snowflake_direct_thresholds_are_explicit_and_paired() -> None: + raw = _config() + raw["direct_max_rows"] = 10 + + with pytest.raises(ValueError, match="must both be zero or positive"): + SnowflakeWarehouseConfig.model_validate(raw) + + +def test_snowflake_schema_mapper_accepts_only_explicit_json_variant_without_io( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + before = list(backend.statements) + variant = ProviderExtension(provider="snowflake", name="fallback", value="variant") + + schema = runtime.schema_mapper.canonical_schema( + (WriteField(name="payload", data_type="JSON", extensions=(variant,)),) + ) + + assert variant in schema.fields[0].extensions + assert backend.statements == before + with pytest.raises(WarehouseSchemaSupportError, match="require snowflake/fallback=variant"): + runtime.schema_mapper.canonical_schema((WriteField(name="payload", data_type="JSON"),)) + assert backend.statements == before + + def test_snowflake_runtime_requires_projected_oauth_token(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("DANDER_MISSING_SNOWFLAKE_TOKEN", raising=False) registry = default_provider_registry() @@ -334,11 +404,149 @@ def records() -> Iterator[dict[str, object]]: assert backend.rollbacks == 0 assert len(backend.committed_checksums) == 2 assert runtime.capabilities.write_modes == frozenset(WriteMode) - assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) + assert runtime.capabilities.transports == frozenset( + {WriteTransport.COPY, WriteTransport.DIRECT} + ) assert runtime.capabilities.supports_transforms is True assert runtime.relation_codec.render(relation) == '"DANDER_TEST"."raw"."records"' +def test_snowflake_direct_load_parses_explicit_variant_and_emits_transport( + tmp_path: Path, +) -> None: + runtime, backend = _direct_runtime(tmp_path) + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "N"), + ("payload", "VARIANT", "COLUMN", "Y"), + ] + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="direct_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_direct_records", + run_id="run-direct", + token=4, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + variant = ProviderExtension(provider="snowflake", name="fallback", value="variant") + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(variant,)), + ), + publication_fence=publication, + ) + + assert writer.write(({"id": "one", "payload": {"ready": True}},), target) == 2 + + sql = [statement for statement, _parameters in backend.statements] + direct = next( + (statement, parameters) + for statement, parameters in backend.statements + if statement.startswith('INSERT INTO "DANDER_TEST"."raw"."dander_stage_') + and "dander_stage_loads" not in statement + ) + assert '"payload" VARCHAR' in next( + statement for statement in sql if statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") + ) + assert direct[1] == (("one", '{"ready":true}', 0),) + assert not any(statement.startswith(("PUT ", "COPY INTO")) for statement in sql) + publication_sql = next( + statement + for statement in sql + if statement.startswith("MERGE INTO") and "dander_target_commits" not in statement + ) + assert 'PARSE_JSON(staged."payload") AS "payload"' in publication_sql + operations = writer.drain_telemetry() + assert [operation.operation for operation in operations] == [ + TelemetryOperation.LOAD, + TelemetryOperation.QUERY, + ] + assert all(operation.transport is WriteTransport.DIRECT for operation in operations) + assert all(operation.resource_name == "DANDER_WH" for operation in operations) + assert operations[0].rows_written == 1 + assert operations[0].query_id != operations[1].query_id + assert writer.drain_telemetry() == () + direct_inserts = sum( + statement.startswith('INSERT INTO "DANDER_TEST"."raw"."dander_stage_') + and "dander_stage_loads" not in statement + for statement, _parameters in backend.statements + ) + assert writer.write(({"id": "one", "payload": {"ready": True}},), target) == 0 + assert ( + sum( + statement.startswith('INSERT INTO "DANDER_TEST"."raw"."dander_stage_') + and "dander_stage_loads" not in statement + for statement, _parameters in backend.statements + ) + == direct_inserts + ) + replay_operations = writer.drain_telemetry() + assert [operation.operation for operation in replay_operations] == [TelemetryOperation.QUERY] + assert replay_operations[0].transport is WriteTransport.DIRECT + assert not tuple(tmp_path.iterdir()) + + +def test_snowflake_direct_threshold_is_decided_for_the_complete_endpoint( + tmp_path: Path, +) -> None: + runtime, backend = _direct_runtime(tmp_path, max_rows=2) + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="threshold_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_threshold_records", + run_id="run-threshold", + token=5, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + consumed = 0 + + def records() -> Iterator[dict[str, object]]: + nonlocal consumed + for index in range(3): + consumed += 1 + yield {"id": str(index), "label": f"record-{index}"} + + writer.write(records(), target) + + assert consumed == 3 + sql = [statement for statement, _parameters in backend.statements] + assert any(statement.startswith("COPY INTO") for statement in sql) + assert not any( + statement.startswith('INSERT INTO "DANDER_TEST"."raw"."dander_stage_') + and "dander_stage_loads" not in statement + for statement in sql + ) + assert all(operation.transport is WriteTransport.COPY for operation in writer.drain_telemetry()) + assert not tuple(tmp_path.iterdir()) + + @pytest.mark.parametrize( ("mode", "field_name", "expected_sql"), [ @@ -368,8 +576,8 @@ def test_snowflake_factory_reaches_every_fenced_write_mode( snapshot_field=field_name if mode is WriteMode.SNAPSHOT else None, ) assert writer.mode is mode - assert writer.supports_batched_writes is (mode not in {WriteMode.REPLACE, WriteMode.SCD2}) - assert writer.accepts_streaming_input is (mode in {WriteMode.REPLACE, WriteMode.SCD2}) + assert writer.supports_batched_writes is False + assert writer.accepts_streaming_input is True relation = RelationRef( catalog="DANDER_TEST", @@ -1084,7 +1292,7 @@ def test_snowflake_lost_fence_blocks_publication_and_rolls_back( assert not tuple(staging_root.iterdir()) -def test_snowflake_rejects_semi_structured_fields_and_telemetry_is_bounded( +def test_snowflake_requires_explicit_variant_and_telemetry_is_bounded( snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], ) -> None: runtime, _backend, _staging_root = snowflake_runtime @@ -1112,7 +1320,7 @@ def test_snowflake_rejects_semi_structured_fields_and_telemetry_is_bounded( publication_fence=publication, ) - with pytest.raises(SnowflakeWriteError, match="semi-structured fields are not supported"): + with pytest.raises(SnowflakeWriteError, match="require snowflake/fallback=variant"): writer.write(({"id": "one", "payload": {"ready": True}},), target) class _Result: @@ -1122,3 +1330,54 @@ class _Result: telemetry = runtime.telemetry.operation(_Result(), operation=TelemetryOperation.LOAD) assert telemetry.rows_affected == 7 assert telemetry.query_id == "query-safe-7" + assert telemetry.resource_name == "DANDER_WH" + + +def test_snowflake_copy_parses_explicit_variant_before_publication( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, staging_root = snowflake_runtime + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "N"), + ("payload", "VARIANT", "COLUMN", "Y"), + ] + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="variant_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_variant_records", + run_id="run-variant-copy", + token=10, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + variant = ProviderExtension(provider="snowflake", name="fallback", value="variant") + writer.write( + ({"id": "one", "payload": {"nested": {"ready": True}}},), + WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(variant,)), + ), + publication_fence=publication, + ), + ) + + sql = [statement for statement, _parameters in backend.statements] + assert any(statement.startswith("COPY INTO") for statement in sql) + merge = next( + statement + for statement in sql + if statement.startswith("MERGE INTO") and "dander_target_commits" not in statement + ) + assert 'PARSE_JSON(staged."payload") AS "payload"' in merge + assert all(operation.transport is WriteTransport.COPY for operation in writer.drain_telemetry()) + assert not tuple(staging_root.iterdir()) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 10ddc362..6e9f11ed 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -13,6 +13,7 @@ RunTelemetry, TelemetryOperation, ) +from dander.writer import WriteTransport def _operation() -> OperationTelemetry: @@ -37,6 +38,7 @@ def _operation() -> OperationTelemetry: resource_size="X-SMALL", capacity_units=Decimal("0.0025"), capacity_unit="credits", + transport=WriteTransport.COPY, costs=( CostAttribution( provider="gcp", @@ -85,6 +87,7 @@ def test_run_telemetry_serializes_normalized_totals_and_operation_details() -> N "job_id": "job:abc-123", "resource_name": "COMPUTE_WH", "resource_size": "X-SMALL", + "transport": "copy", "capacity_units": "0.0025", "capacity_unit": "credits", "costs": [ @@ -135,6 +138,12 @@ def test_telemetry_rejects_unsafe_or_ambiguous_values() -> None: operation=TelemetryOperation.QUERY, capacity_units=Decimal("0.1"), ) + with pytest.raises(ValueError, match="WriteTransport"): + OperationTelemetry( + provider="snowflake", + operation=TelemetryOperation.LOAD, + transport="copy", # type: ignore[arg-type] + ) def test_whole_run_duration_can_be_replaced_without_losing_operations() -> None: diff --git a/tests/warehouse/test_schema_support.py b/tests/warehouse/test_schema_support.py index 8528bcf0..51c7b140 100644 --- a/tests/warehouse/test_schema_support.py +++ b/tests/warehouse/test_schema_support.py @@ -27,13 +27,7 @@ def _schema(name: str, data_type: CanonicalType) -> RelationSchema: return RelationSchema(fields=(CanonicalField(name=name, data_type=data_type),)) -@pytest.mark.parametrize( - "mapper", - [SnowflakeSchemaMapper(), RedshiftSchemaMapper()], -) -def test_experimental_mappers_reject_semistructured_fields_before_provider_io( - mapper: SnowflakeSchemaMapper | RedshiftSchemaMapper, -) -> None: +def test_redshift_mapper_rejects_semistructured_fields_before_provider_io() -> None: field = CanonicalField( name="payload", data_type=CanonicalType(kind=LogicalTypeKind.JSON), @@ -43,7 +37,20 @@ def test_experimental_mappers_reject_semistructured_fields_before_provider_io( WarehouseSchemaSupportError, match="does not support canonical type 'json' at field 'payload'", ): - mapper.canonical_schema([field]) + RedshiftSchemaMapper().canonical_schema([field]) + + +def test_snowflake_mapper_requires_explicit_json_variant_before_provider_io() -> None: + field = CanonicalField( + name="payload", + data_type=CanonicalType(kind=LogicalTypeKind.JSON), + ) + + with pytest.raises( + WarehouseSchemaSupportError, + match="require snowflake/fallback=variant", + ): + SnowflakeSchemaMapper().canonical_schema([field]) def test_decimal_and_temporal_limits_name_provider_field_and_limit() -> None: From e73ae01b38cebcd809c4548e5d8b10853cba3774 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 04:27:08 -0400 Subject: [PATCH 093/163] feat(snowflake): execute portable graph targets --- HANDOFF.md | 31 +- docs/compatibility-matrix.md | 6 +- docs/known-limitations.md | 11 +- docs/snowflake.md | 27 +- src/dander/providers/snowflake/runtime.py | 22 +- src/dander/providers/snowflake/transform.py | 310 ++++++++++++++---- src/dander/runtime-compatibility.json | 4 +- .../test_snowflake_warehouse_runtime.py | 232 ++++++++++++- 8 files changed, 538 insertions(+), 105 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 1d22892b..df21f290 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,38 @@ ## Finished -- Added endpoint-wide Snowflake direct/COPY selection with bounded threshold-plus-one buffering. -- Added explicit JSON-to-VARIANT mapping with canonical text staging and `PARSE_JSON` publication. -- Added truthful writer load/publication telemetry with transport, query IDs, rows, bytes, duration, and warehouse. -- Kept legacy graph/BigQuery planning fail-closed for provider-selected direct transport. +- Added Snowflake execution for the existing provider-neutral, replace-mode PipelineGraph plan. +- Preflighted every selected graph target before any provider claim or mutation. +- Reused the fenced stable-table transform publisher without adding another graph schema. +- Added transform/publication/assertion telemetry with query IDs, duration, rows, and warehouse. +- Updated the Snowflake capability and operator-facing compatibility documentation. ## Try It -Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`. +Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py tests/test_compatibility.py`. ## Checks - Ruff format/check: passed across 328 files; mypy: passed across 304 source/test files. -- Pytest: 1,122 passed and 13 skipped from 1,135 collected tests. -- Wheel/sdist inspection and both source-free package installs: passed. -- Terraform validation/tests and Helm lint/render passed except the local Fargate module test, whose 782 MB provider could not initialize with under 1 GB free. -- Local Docker/PostgreSQL conformance remains for protected Linux CI because Docker Desktop is unresponsive. +- Full pytest passed locally (13 environment-dependent tests skipped). +- Focused Snowflake and compatibility tests: 41 passed. +- Runtime compatibility JSON parsed and matched the provider capability in tests. +- Independent adversarial completion review: passed with no material findings. ## Decisions -- Direct thresholds default to zero until a live Snowflake crossover is measured. -- Direct/COPY selection occurs once for the complete endpoint; ordinary runtime batching is disabled for Snowflake writers. -- Only canonical JSON with `snowflake/fallback=variant` is admitted; ARRAY and RECORD remain rejected. +- Graph execution keeps `GraphExecutionPlan` canonical and renders Snowflake at the provider boundary. +- Views remain unsupported until a stable indirection design preserves transactional fencing. +- Existing null-on-failure cast semantics remain fail-closed for Snowflake graphs. ## Remaining - Push a focused draft PR and require protected Linux CI, including PostgreSQL, container, Terraform, and scans. -- Measure the direct/COPY crossover during later live Snowflake qualification. -- Keep transforms/views/graphs and query-history enrichment in separate Snowflake slices. +- Design fenced view indirection separately; do not weaken publication safety. +- Measure the direct/COPY crossover and enrich query-history metrics during live qualification. ## Review First -- `src/dander/providers/snowflake/writer.py` +- `src/dander/providers/snowflake/transform.py` - `src/dander/providers/snowflake/runtime.py` - `tests/providers/test_snowflake_warehouse_runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 8f0de804..be7867e1 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -14,11 +14,11 @@ before provider construction when a pair is absent or marked `unsupported`. |---|---|---|---| | BigQuery | BigQuery | supported | Released GCP profile and BigQuery regression baseline | | BigQuery | PostgreSQL | experimental | Local cross-backend token/fence publication proof | -| BigQuery | Snowflake | experimental | Local staged-SCD1, fenced transforms, schema, replay, and fence conformance | +| BigQuery | Snowflake | experimental | Local five-mode writes, fenced transforms/graphs, schema, replay, and fence conformance | | BigQuery | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, fenced transforms, and destination-fence conformance | | PostgreSQL | BigQuery | unsupported | Not every BigQuery write mode uses destination fencing | | PostgreSQL | PostgreSQL | experimental | Local native profile, state/warehouse conformance, and benchmarks | -| PostgreSQL | Snowflake | experimental | Local staged-SCD1, transforms, and destination-fence conformance | +| PostgreSQL | Snowflake | experimental | Local five-mode writes, transforms/graphs, and destination-fence conformance | | PostgreSQL | Redshift | experimental | Local IAM/S3 `COPY`, SCD1 replay, fenced transforms, and destination-fence conformance | `experimental` means the code path is executable and tested locally; it is not a hosted support @@ -41,7 +41,7 @@ their named live profiles pass. | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | | PostgreSQL | SCD1 | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | | Redshift | SCD1 | COPY | scalar types; decimal 38, time 6; no SUPER fallback | yes | no | yes | -| Snowflake | SCD1 | COPY | scalar types; decimal 38, time 9; no VARIANT fallback | yes | no | yes | +| Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | Portable-provider schema validation uses these declarations before source extraction, staging upload, or destination mutation. BigQuery retains its existing provider-native v1 schema path so diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 061759df..252b2fee 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,11 +10,12 @@ evaluate these limits before using it for an unattended system containing busine published scale/qualification objectives. Other AWS, warehouse, and cross-cloud combinations receive no support claim from this proof. - Provider package extras and the full runtime image do not by themselves qualify Snowflake, - Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental scalar adapter with all five - writer modes behind its provider capability, bounded Parquet upload, temporary remote staging, - schema checks, destination fencing, and fenced table/incremental transforms. The ordinary hosted - source runner still selects SCD1, and Snowflake has no live qualification, views, graph execution, - semi-structured types, measured direct-write crossover, or provider query-telemetry emission. + Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental adapter with all five writer + modes behind its provider capability, bounded direct/Parquet paths, temporary remote staging, + schema checks, destination fencing, fenced table/incremental transforms, and fenced replace-mode + graph targets. The ordinary hosted source runner still selects SCD1, and Snowflake has no live + qualification, views, ARRAY/RECORD fallback, measured direct-write crossover, or query-history + enrichment. Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. It has no live qualification, views, executable graphs, diff --git a/docs/snowflake.md b/docs/snowflake.md index f29e81d3..086f4e73 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -2,8 +2,9 @@ Snowflake is an experimental warehouse adapter, not a supported Dander profile. The current slice proves native database/schema coordinates, bounded direct and bulk paths, all five writer modes, -an explicit JSON-to-`VARIANT` fallback, and fenced table/incremental models. Live qualification -and the remaining first-class gates are still required before support promotion. +an explicit JSON-to-`VARIANT` fallback, fenced table/incremental models, and fenced replace-mode +graph targets. Live qualification and the remaining first-class gates are still required before +support promotion. ## Configuration @@ -76,6 +77,13 @@ variable reference. Do not put a token, private key, password, or connection str - Portable or Snowflake-authored table models replace rows through fenced `DELETE`/`INSERT` DML. - Incremental models collapse duplicate keys deterministically and accept only rows whose declared cursor is at least as new as the stored row. Generic model assertions run after publication. +- Compatible provider-neutral graphs render their existing relational AST as Snowflake SQL and + publish replace-mode targets through the same stable-table fencing primitive. Every selected + target renders and validates before the first target is claimed, preventing partial publication + when a later target uses unavailable semantics. +- Transform staging, publication, and generic assertions report bounded operation telemetry with + query IDs, duration, affected rows, and warehouse name. SQL and provider response payloads are + not retained. The Snowflake role needs usage on its database and warehouse plus permission to create and operate schemas, tables, and temporary stages in the selected namespace. Use a dedicated disposable role @@ -84,10 +92,11 @@ until a live least-privilege profile is qualified. ## Deliberate limits All five scalar write patterns are reachable through the warehouse writer capability, while the -ordinary hosted source runner deliberately continues to select SCD1. PipelineGraph execution has -not yet been wired to Snowflake's writer selection. -Views remain unavailable because Snowflake permanent DDL cannot share the destination-fence -transaction. Direct thresholds default to zero because no live crossover has been measured; do -not claim a performance benefit until that qualification is recorded. Query-history enrichment, -live concurrency proof, infrastructure provisioning, and support promotion remain separate work. -Use `catalog.provider: none` for this experimental path. +ordinary hosted source runner deliberately continues to select SCD1. Graphs remain limited to the +existing replace-mode executable subset; field tests and null-on-failure casts whose Snowflake +semantics have not been proven continue to fail before provider mutation. Views remain unavailable +because Snowflake permanent DDL cannot share the destination-fence transaction. Direct thresholds +default to zero because no live crossover has been measured; do not claim a performance benefit +until that qualification is recorded. Query-history enrichment, live concurrency proof, +infrastructure provisioning, and support promotion remain separate work. Use +`catalog.provider: none` for this experimental path. diff --git a/src/dander/providers/snowflake/runtime.py b/src/dander/providers/snowflake/runtime.py index 72cb5ec7..8f2c0b1e 100644 --- a/src/dander/providers/snowflake/runtime.py +++ b/src/dander/providers/snowflake/runtime.py @@ -24,7 +24,7 @@ execute, open_connection, ) -from dander.providers.snowflake.transform import SnowflakeTransformRunner +from dander.providers.snowflake.transform import SnowflakeGraphRunner, SnowflakeTransformRunner from dander.providers.snowflake.writer import ( SnowflakeScd1Writer, SnowflakeStagedWriter, @@ -40,6 +40,7 @@ WarehouseRuntime, WarehouseSchemaSupport, WarehouseSchemaSupportError, + WarehouseTransformRunner, ) from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport @@ -165,6 +166,7 @@ class SnowflakeTransformFactory: database: str connection_factory: SnowflakeConnectionFactory target_fence: SnowflakeTargetFence + warehouse: str | None = None def build_transform_runner( self, @@ -172,10 +174,18 @@ def build_transform_runner( graph_plan: object | None, build_models: bool, raw_namespace: str = "raw", - ) -> SnowflakeTransformRunner | None: + ) -> WarehouseTransformRunner | None: if graph_plan is not None: - raise ValueError( - "Snowflake graph execution is not available in this experimental slice" + from dander.pipeline.runtime import GraphExecutionPlan + + if not isinstance(graph_plan, GraphExecutionPlan): + raise TypeError("Snowflake graph plan has the wrong type") + return SnowflakeGraphRunner( + plan=graph_plan, + database=self.database, + connection_factory=self.connection_factory, + target_fence=self.target_fence, + warehouse=self.warehouse, ) if not build_models: return None @@ -184,6 +194,7 @@ def build_transform_runner( connection_factory=self.connection_factory, target_fence=self.target_fence, raw_namespace=raw_namespace, + warehouse=self.warehouse, ) @@ -221,7 +232,7 @@ def operation( write_modes=frozenset(WriteMode), transports=frozenset({WriteTransport.COPY, WriteTransport.DIRECT}), supports_transforms=True, - supports_graphs=False, + supports_graphs=True, supports_target_fencing=True, schema_support=SNOWFLAKE_SCHEMA_SUPPORT, ) @@ -300,6 +311,7 @@ def build_snowflake_warehouse( config.database, connection_factory, target_fence, + config.warehouse, ), target_fence=target_fence, telemetry=SnowflakeTelemetry(config.warehouse), diff --git a/src/dander/providers/snowflake/transform.py b/src/dander/providers/snowflake/transform.py index d5ca28f7..c3c2273b 100644 --- a/src/dander/providers/snowflake/transform.py +++ b/src/dander/providers/snowflake/transform.py @@ -5,12 +5,16 @@ import hashlib import sys from dataclasses import dataclass +from time import perf_counter_ns from typing import TYPE_CHECKING from dander.concurrency import TargetFenceLostError +from dander.pipeline.compiler import CompiledTarget, PipelineCompileError +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError from dander.providers.snowflake.session import ( SnowflakeConnection, SnowflakeConnectionFactory, + SnowflakeStatementResult, execute, open_connection, ) @@ -20,8 +24,9 @@ _qualified, _quote, _set_query_tag, - _snowflake_type, + validate_snowflake_schema, ) +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import ( SqlDialect, TransformModel, @@ -31,10 +36,10 @@ TransformRunResult, ) from dander.transform.model import Materialization -from dander.writer import SchemaEvolution, WriteField, WriteTarget +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Callable, Iterable, Sequence from pathlib import Path from dander.concurrency import OwnershipGuard, TargetFence @@ -45,9 +50,12 @@ @dataclass(frozen=True, slots=True) class _SnowflakeModelPlan: - model: TransformModel + name: str target: WriteTarget query: str + materialization: Materialization + unique_key: tuple[str, ...] = () + incremental_cursor: str | None = None @dataclass(frozen=True, slots=True) @@ -69,11 +77,13 @@ def __init__( connection_factory: SnowflakeConnectionFactory, target_fence: SnowflakeTargetFence, raw_namespace: str = "raw", + warehouse: str | None = None, ) -> None: self._database = database self._connection_factory = connection_factory self._target_fence = target_fence self._raw_namespace = raw_namespace + self._warehouse = warehouse def build( self, @@ -85,17 +95,26 @@ def build( """Preflight the selected DAG, then publish each model through fenced DML.""" if ownership is None or ownership.fence is None: raise TransformRunError("Snowflake hosted transforms require active lease ownership") - project, models, plans, assertions = self._preflight(models_dir, selected=selected) - del project + _project, models, plans, assertions = self._preflight(models_dir, selected=selected) + telemetry: list[OperationTelemetry] = [] for plan in plans: ownership.verify() publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) ownership.verify() - self._publish(plan, publication) - self._run_assertions(assertions, ownership=ownership) + telemetry.extend( + _publish_plan( + plan, + publication, + connection_factory=self._connection_factory, + target_fence=self._target_fence, + warehouse=self._warehouse, + ) + ) + telemetry.extend(self._run_assertions(assertions, ownership=ownership)) return TransformRunResult( models=tuple(model.name for model in models), assertions=len(assertions), + telemetry=tuple(telemetry), ) def test( @@ -106,10 +125,11 @@ def test( ) -> TransformRunResult: """Run assertions against already materialized Snowflake relations.""" _project, models, _plans, assertions = self._preflight(models_dir, selected=selected) - self._run_assertions(assertions) + telemetry = self._run_assertions(assertions) return TransformRunResult( models=tuple(model.name for model in models), assertions=len(assertions), + telemetry=telemetry, ) def _preflight( @@ -137,72 +157,184 @@ def _preflight( ) return project, models, plans, assertions - def _publish(self, plan: _SnowflakeModelPlan, publication: TargetFence) -> None: - target = WriteTarget( - relation=plan.target.relation_ref, - business_key=plan.target.business_key, - schema=plan.target.schema, - declared_schema=plan.target.canonical_schema, - publication_fence=publication, - ) - temporary = _temporary_relation(target.relation_ref, publication) - cleanup_started = False - with open_connection(self._connection_factory) as connection: - try: - _set_query_tag(connection, publication) - # Transform targets never evolve automatically: the only permanent DDL permitted - # here is create-if-absent followed by exact declared-schema validation. - _ensure_target( - connection, - target, - target.canonical_schema, - evolution=SchemaEvolution.STRICT, - ) - execute(connection, _create_temporary_sql(plan, temporary)) - cleanup_started = True - self._target_fence.execute_statements( - connection, - _publication_statements(plan, temporary), - publication, - ) - except (TargetFenceLostError, SnowflakeWriteError, TransformRunError): - raise - except Exception as error: - raise TransformRunError("Snowflake transform publication failed") from error - finally: - if cleanup_started: - _drop_temporary( - connection, - temporary, - suppress_failure=sys.exc_info()[0] is not None, - ) - def _run_assertions( self, assertions: Sequence[_SnowflakeAssertion], *, ownership: OwnershipGuard | None = None, - ) -> None: + ) -> tuple[OperationTelemetry, ...]: failures: list[str] = [] + telemetry: list[OperationTelemetry] = [] with open_connection(self._connection_factory) as connection: for assertion in assertions: if ownership is not None: ownership.verify() try: - row = execute( + result, duration_ms = _timed_call( + execute, connection, assertion.statement, assertion.parameters, fetch="one", - ).row + ) except Exception as error: raise TransformRunError( f"Snowflake assertion execution failed: {assertion.name}" ) from error - if _failure_count(row, assertion.name) > 0: + telemetry.append( + _operation_telemetry( + result, + operation=TelemetryOperation.TEST, + duration_ms=duration_ms, + warehouse=self._warehouse, + ) + ) + if _failure_count(result.row, assertion.name) > 0: failures.append(assertion.name) if failures: raise TransformRunError(f"Data tests failed: {', '.join(failures)}") + return tuple(telemetry) + + +class SnowflakeGraphRunner: + """Publish provider-neutral graph targets through Snowflake's fenced DML path.""" + + target_dialect = SqlDialect.SNOWFLAKE + + def __init__( + self, + *, + plan: GraphExecutionPlan, + database: str, + connection_factory: SnowflakeConnectionFactory, + target_fence: SnowflakeTargetFence, + warehouse: str | None = None, + ) -> None: + self._plan = plan + self._database = database + self._connection_factory = connection_factory + self._target_fence = target_fence + self._warehouse = warehouse + + def build( + self, + _models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Preflight every selected target, then publish each target exactly once.""" + if ownership is None or ownership.fence is None: + raise GraphRuntimeError("Snowflake graph execution requires active lease ownership") + selected_ids = set(selected) if selected is not None else None + known = {target.node_id for target in self._plan.targets} + if selected_ids is not None and (unknown := sorted(selected_ids - known)): + raise GraphRuntimeError(f"Unknown graph target: {unknown[0]!r}") + targets = tuple( + target + for target in self._plan.targets + if selected_ids is None or target.node_id in selected_ids + ) + if not targets: + raise GraphRuntimeError("Graph execution selected no targets") + + # Rendering and schema validation may fail for a portable construct whose semantics are + # not exact in Snowflake. Complete that preflight for every selected target before the + # first claim or provider session so a graph can never partially publish for that reason. + plans = tuple(_graph_plan(target, database=self._database) for target in targets) + telemetry: list[OperationTelemetry] = [] + for plan in plans: + ownership.verify() + publication = self._target_fence.claim( + plan.target.relation_ref, + ownership.fence, + ) + ownership.verify() + telemetry.extend( + _publish_plan( + plan, + publication, + connection_factory=self._connection_factory, + target_fence=self._target_fence, + warehouse=self._warehouse, + ) + ) + return TransformRunResult( + models=tuple(plan.name for plan in plans), + assertions=0, + telemetry=tuple(telemetry), + ) + + +def _publish_plan( + plan: _SnowflakeModelPlan, + publication: TargetFence, + *, + connection_factory: SnowflakeConnectionFactory, + target_fence: SnowflakeTargetFence, + warehouse: str | None, +) -> tuple[OperationTelemetry, ...]: + target = WriteTarget( + relation=plan.target.relation_ref, + business_key=plan.target.business_key, + schema=plan.target.schema, + declared_schema=plan.target.canonical_schema, + publication_fence=publication, + ) + temporary = _temporary_relation(target.relation_ref, publication) + cleanup_started = False + telemetry: list[OperationTelemetry] = [] + with open_connection(connection_factory) as connection: + try: + _set_query_tag(connection, publication) + # Transform targets never evolve automatically: the only permanent DDL permitted + # here is create-if-absent followed by exact declared-schema validation. + _ensure_target( + connection, + target, + target.canonical_schema, + evolution=SchemaEvolution.STRICT, + ) + cleanup_started = True + staged, duration_ms = _timed_call( + execute, + connection, + _create_temporary_sql(plan, temporary), + ) + telemetry.append( + _operation_telemetry( + staged, + operation=TelemetryOperation.TRANSFORM, + duration_ms=duration_ms, + warehouse=warehouse, + ) + ) + published, duration_ms = _timed_call( + target_fence.execute_statements, + connection, + _publication_statements(plan, temporary), + publication, + ) + telemetry.append( + _operation_telemetry( + published, + operation=TelemetryOperation.TRANSFORM, + duration_ms=duration_ms, + warehouse=warehouse, + ) + ) + except (TargetFenceLostError, SnowflakeWriteError, TransformRunError): + raise + except Exception as error: + raise TransformRunError("Snowflake transform publication failed") from error + finally: + if cleanup_started: + _drop_temporary( + connection, + temporary, + suppress_failure=sys.exc_info()[0] is not None, + ) + return tuple(telemetry) def _model_plan(project: TransformProject, model: TransformModel) -> _SnowflakeModelPlan: @@ -232,25 +364,49 @@ def _model_plan(project: TransformProject, model: TransformModel) -> _SnowflakeM business_key=tuple(model.metadata.unique_key), schema=fields, ) - for field in target.canonical_schema.fields: - _snowflake_type(field.data_type) + validate_snowflake_schema(target.canonical_schema) return _SnowflakeModelPlan( - model=model, + name=model.name, target=target, query=project.compile(model), + materialization=model.metadata.materialization, + unique_key=tuple(model.metadata.unique_key), + incremental_cursor=model.metadata.incremental_cursor, + ) + + +def _graph_plan(compiled: CompiledTarget, *, database: str) -> _SnowflakeModelPlan: + if compiled.write_mode is not WriteMode.REPLACE: + raise GraphRuntimeError( + f"Snowflake graph target {compiled.node_id!r} requires replace mode" + ) + if compiled.target.relation_ref.catalog != database: + raise GraphRuntimeError( + f"Snowflake graph target {compiled.node_id!r} belongs to another database" + ) + try: + validate_snowflake_schema(compiled.target.canonical_schema) + query = compiled.render(SqlDialect.SNOWFLAKE) + except (PipelineCompileError, SnowflakeWriteError) as error: + raise GraphRuntimeError(str(error)) from error + return _SnowflakeModelPlan( + name=compiled.node_id, + target=compiled.target, + query=query, + materialization=Materialization.TABLE, ) def _create_temporary_sql(plan: _SnowflakeModelPlan, temporary: str) -> str: names = tuple(field.name for field in plan.target.canonical_schema.fields) selected = ", ".join(f"source.{_quote(name)}" for name in names) - if plan.model.metadata.materialization is Materialization.TABLE: + if plan.materialization is Materialization.TABLE: return ( f"CREATE OR REPLACE TEMPORARY TABLE {temporary} AS SELECT {selected} " f"FROM ({plan.query}) AS source" ) - keys = tuple(plan.model.metadata.unique_key) - cursor = plan.model.metadata.incremental_cursor + keys = plan.unique_key + cursor = plan.incremental_cursor assert keys and cursor is not None partition = ", ".join(f"source.{_quote(key)}" for key in keys) tie_breakers = tuple(name for name in sorted(names) if name not in {*keys, cursor}) @@ -273,13 +429,13 @@ def _publication_statements( names = tuple(field.name for field in plan.target.canonical_schema.fields) columns = ", ".join(_quote(name) for name in names) incoming = ", ".join(f"incoming.{_quote(name)}" for name in names) - if plan.model.metadata.materialization is Materialization.TABLE: + if plan.materialization is Materialization.TABLE: return ( (f"DELETE FROM {target}", ()), (f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}", ()), ) - keys = tuple(plan.model.metadata.unique_key) - cursor = plan.model.metadata.incremental_cursor + keys = plan.unique_key + cursor = plan.incremental_cursor assert keys and cursor is not None match = " AND ".join(f"target.{_quote(key)} = incoming.{_quote(key)}" for key in keys) mutable = tuple(name for name in names if name not in keys) @@ -403,4 +559,32 @@ def _failure_count(row: object | None, assertion: str) -> int: return value -__all__ = ["SnowflakeTransformRunner"] +def _timed_call( + function: Callable[..., SnowflakeStatementResult], + *arguments: object, + **keywords: object, +) -> tuple[SnowflakeStatementResult, int]: + started = perf_counter_ns() + result = function(*arguments, **keywords) + duration_ms = max((perf_counter_ns() - started) // 1_000_000, 0) + return result, duration_ms + + +def _operation_telemetry( + result: SnowflakeStatementResult, + *, + operation: TelemetryOperation, + duration_ms: int, + warehouse: str | None, +) -> OperationTelemetry: + return OperationTelemetry( + provider="snowflake", + operation=operation, + duration_ms=duration_ms, + rows_affected=result.rowcount, + query_id=result.query_id, + resource_name=warehouse, + ) + + +__all__ = ["SnowflakeGraphRunner", "SnowflakeTransformRunner"] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 1a3058da..842e8f11 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -95,9 +95,9 @@ "max_decimal_precision": 38, "max_temporal_precision": 9, "supports_transforms": true, - "supports_graphs": false, + "supports_graphs": true, "supports_target_fencing": true, - "limitations": ["JSON requires the explicit snowflake/fallback=variant field extension", "direct thresholds remain disabled until the live crossover is measured", "graph execution is not available", "hosted source execution selects scd1"] + "limitations": ["JSON requires the explicit snowflake/fallback=variant field extension", "direct thresholds remain disabled until the live crossover is measured", "hosted source execution selects scd1"] } ] } diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index 4e95ea99..0eb8c1f4 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -11,9 +11,12 @@ import pytest from dander.concurrency import FencingToken, TargetFenceLostError +from dander.ingestion import Endpoint, RawField, SourceConfig +from dander.pipeline.graph import PipelineGraph +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError, plan_graph_execution from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry from dander.providers.snowflake import SnowflakeWarehouseConfig -from dander.providers.snowflake.transform import SnowflakeTransformRunner +from dander.providers.snowflake.transform import SnowflakeGraphRunner, SnowflakeTransformRunner from dander.providers.snowflake.writer import SnowflakeWriteError from dander.telemetry import TelemetryOperation from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError @@ -814,6 +817,121 @@ def verify(self) -> None: self.verifications += 1 +def _snowflake_graph_plan( + *, + include_unsupported_target: bool = False, +) -> GraphExecutionPlan: + source = SourceConfig( + name="fixture", + base_url="https://example.test", + auth_strategy="none", + endpoints=[ + Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING"), + RawField(name="label", data_type="STRING"), + ], + ) + ], + ) + targets: list[dict[str, object]] = [ + { + "id": "target", + "type": "target", + "name": "Target", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": "analytics", + "table": "graph_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + } + ] + edges: list[dict[str, object]] = [ + { + "from": "records", + "to": "target", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ] + if include_unsupported_target: + targets.append( + { + "id": "unsupported", + "type": "target", + "name": "Unsupported", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": "analytics", + "table": "unsupported_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING", "cast_to": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + } + ) + edges.append( + { + "from": "records", + "to": "unsupported", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ) + graph = PipelineGraph.model_validate( + { + "name": "snowflake_graph", + "nodes": [ + { + "id": "records", + "type": "source", + "name": "Records", + "config": {"connector": "fixture", "endpoint": "records"}, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + *targets, + ], + "edges": edges, + } + ) + return plan_graph_execution( + graph, + source, + endpoint_relations={ + "records": RelationRef( + catalog="DANDER_TEST", + namespace="raw", + name="fixture_records", + ) + }, + ) + + def test_snowflake_builds_fenced_table_and_incremental_models( tmp_path: Path, snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], @@ -844,6 +962,16 @@ def test_snowflake_builds_fenced_table_and_incremental_models( assert result.models == ("table_model",) assert result.assertions == 4 + assert [operation.operation for operation in result.telemetry] == [ + TelemetryOperation.TRANSFORM, + TelemetryOperation.TRANSFORM, + TelemetryOperation.TEST, + TelemetryOperation.TEST, + TelemetryOperation.TEST, + TelemetryOperation.TEST, + ] + assert all(operation.query_id for operation in result.telemetry) + assert all(operation.resource_name == "DANDER_WH" for operation in result.telemetry) sql = [statement for statement, _parameters in backend.statements] assert any( statement.startswith("DELETE FROM") and "table_model" in statement for statement in sql @@ -861,6 +989,9 @@ def test_snowflake_builds_fenced_table_and_incremental_models( runner.test(tmp_path, selected=["table_model"]) assert "older" not in str(error.value) backend.assertion_failure_count = 0 + tested = runner.test(tmp_path, selected=["table_model"]) + assert len(tested.telemetry) == 4 + assert all(operation.operation is TelemetryOperation.TEST for operation in tested.telemetry) backend.transform_rows.clear() before_incremental = len(backend.statements) @@ -941,7 +1072,7 @@ def test_snowflake_preflights_whole_dag_before_provider_mutation( assert backend.statements == before -def test_snowflake_transform_coordinates_are_canonical_and_graphs_fail_closed( +def test_snowflake_transform_coordinates_are_canonical( tmp_path: Path, snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], ) -> None: @@ -961,11 +1092,106 @@ def test_snowflake_transform_coordinates_are_canonical_and_graphs_fail_closed( "coordinate_model", ) assert '"DANDER-TEST"."RAW_DATA"."records"' in project.compile(model) - with pytest.raises(ValueError, match="graph execution is not available"): + with pytest.raises(TypeError, match="graph plan has the wrong type"): runtime.transforms.build_transform_runner(graph_plan=object(), build_models=False) assert runtime.transforms.build_transform_runner(graph_plan=None, build_models=False) is None +def test_snowflake_graph_uses_canonical_plan_fencing_cleanup_and_telemetry( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=_snowflake_graph_plan(), + build_models=True, + ) + assert isinstance(runner, SnowflakeGraphRunner) + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "Y"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ] + backend.transform_source_rows = [{"id": "fresh", "label": "from-graph"}] + ownership = _ownership("run-graph", 5) + + result = runner.build(tmp_path, ownership=ownership) + + assert result.models == ("target",) + assert result.assertions == 0 + assert [operation.operation for operation in result.telemetry] == [ + TelemetryOperation.TRANSFORM, + TelemetryOperation.TRANSFORM, + ] + assert all(operation.query_id for operation in result.telemetry) + assert all(operation.resource_name == "DANDER_WH" for operation in result.telemetry) + sql = [statement for statement, _parameters in backend.statements] + staged = next( + statement for statement in sql if statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") + ) + assert 'FROM "DANDER_TEST"."raw"."fixture_records"' in staged + assert "`" not in staged + assert any( + statement.startswith("DELETE FROM") and '"analytics"."graph_records"' in statement + for statement in sql + ) + assert any( + statement.startswith("INSERT INTO") and '"analytics"."graph_records"' in statement + for statement in sql + ) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + assert ownership.verifications == 2 + assert backend.transform_rows == {"fresh": {"id": "fresh", "label": "from-graph"}} + + +def test_snowflake_graph_preflights_every_selected_target_before_mutation( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=_snowflake_graph_plan(include_unsupported_target=True), + build_models=True, + ) + assert isinstance(runner, SnowflakeGraphRunner) + before = list(backend.statements) + + with pytest.raises(GraphRuntimeError, match="safe-cast semantics"): + runner.build(tmp_path, ownership=_ownership("run-preflight-graph", 6)) + + assert backend.statements == before + + +def test_snowflake_graph_selection_and_stale_fence_fail_closed( + tmp_path: Path, + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], +) -> None: + runtime, backend, _staging_root = snowflake_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=_snowflake_graph_plan(), + build_models=True, + ) + assert isinstance(runner, SnowflakeGraphRunner) + before = list(backend.statements) + with pytest.raises(GraphRuntimeError, match="Unknown graph target"): + runner.build(tmp_path, selected=["missing"], ownership=_ownership("run-missing", 7)) + assert backend.statements == before + + backend.describe_rows = [ + ("id", "VARCHAR(16777216)", "COLUMN", "Y"), + ("label", "VARCHAR(16777216)", "COLUMN", "Y"), + ] + backend.transform_source_rows = [{"id": "blocked", "label": "stale"}] + backend.fence_touch_rowcount = 0 + with pytest.raises(TargetFenceLostError, match="lost before publication"): + runner.build(tmp_path, ownership=_ownership("run-stale-graph", 8)) + + attempted = [statement for statement, _parameters in backend.statements[len(before) :]] + assert any(statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") for statement in attempted) + assert not any(statement.startswith(("DELETE FROM", "INSERT INTO")) for statement in attempted) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in attempted) + assert backend.transform_rows == {} + + def test_snowflake_transform_requires_ownership_and_fails_closed_after_staging( tmp_path: Path, snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], From a5004590f8a8731e3f77f6697aeb7ec9dabf7cf5 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 05:01:22 -0400 Subject: [PATCH 094/163] feat(snowflake): enrich operation telemetry --- HANDOFF.md | 44 ++++--- docs/decisions.md | 9 ++ docs/known-limitations.md | 5 +- docs/snowflake.md | 14 ++- src/dander/providers/snowflake/session.py | 114 +++++++++++++++++- src/dander/providers/snowflake/transform.py | 7 +- src/dander/providers/snowflake/writer.py | 13 +- .../test_snowflake_warehouse_runtime.py | 81 ++++++++++++- 8 files changed, 253 insertions(+), 34 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index df21f290..1645e2f8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,44 @@ ## Finished -- Added Snowflake execution for the existing provider-neutral, replace-mode PipelineGraph plan. -- Preflighted every selected graph target before any provider claim or mutation. -- Reused the fenced stable-table transform publisher without adding another graph schema. -- Added transform/publication/assertion telemetry with query IDs, duration, rows, and warehouse. -- Updated the Snowflake capability and operator-facing compatibility documentation. +- Enriched successful Snowflake loads, publications, transforms, and tests from same-session query + history. +- Mapped execution/queue time, scanned bytes, inserted rows, and warehouse size into the existing + provider-neutral telemetry contract. +- Kept enrichment best-effort, bounded to 1,000 recent operation IDs, and free of SQL/bind values. +- Preserved client-observed duration and known writer row/byte counters. +- Left fenced view materialization unsupported after its proposed indirection failed review. ## Try It -Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py tests/test_compatibility.py`. +Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`. ## Checks -- Ruff format/check: passed across 328 files; mypy: passed across 304 source/test files. -- Full pytest passed locally (13 environment-dependent tests skipped). -- Focused Snowflake and compatibility tests: 41 passed. -- Runtime compatibility JSON parsed and matched the provider capability in tests. -- Independent adversarial completion review: passed with no material findings. +- Focused Snowflake provider tests: 31 passed. +- Ruff and mypy passed across 328 files and 304 source/test files; full pytest passed with 1,126 + tests and 13 environment-dependent skips. +- Wheel/sdist inspection and source-free installation, generation, and validation passed outside + the checkout for both artifacts. +- Terraform format/init/validation, three AWS Terraform tests, Helm lint/render, and Phase 1B + validation passed. Local Docker and PostgreSQL integration remain for protected Linux CI. +- Independent adversarial completion review passed with no material findings. ## Decisions -- Graph execution keeps `GraphExecutionPlan` canonical and renders Snowflake at the provider boundary. -- Views remain unsupported until a stable indirection design preserves transactional fencing. -- Existing null-on-failure cast semantics remain fail-closed for Snowflake graphs. +- `ROWS_PRODUCED` is not mapped to `rows_read`; those Snowflake concepts are not equivalent. +- Account Usage is not queried synchronously because its delayed data is unsuitable for run output. +- Query-history failure cannot convert successful warehouse work into pipeline failure. ## Remaining -- Push a focused draft PR and require protected Linux CI, including PostgreSQL, container, Terraform, and scans. -- Design fenced view indirection separately; do not weaken publication safety. -- Measure the direct/COPY crossover and enrich query-history metrics during live qualification. +- Push the focused PR and require protected Linux CI to repeat PostgreSQL, container, Terraform, + and security checks. +- Measure the direct/COPY crossover during live Snowflake qualification. +- Revisit view semantics only at the Phase 5.5 architecture checkpoint. ## Review First -- `src/dander/providers/snowflake/transform.py` -- `src/dander/providers/snowflake/runtime.py` +- `src/dander/providers/snowflake/session.py` +- `src/dander/providers/snowflake/writer.py` - `tests/providers/test_snowflake_warehouse_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index ed558d76..cfe4731f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1006,3 +1006,12 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc normal Application Default Credentials without changed resource or IAM behavior. - **Support:** Renewable identity removes the one-hour credential limit and permits an enforced 24-hour launcher deadline, but does not promote Fargate before the complete live lifecycle gate. + +## 2026-08-10 — Snowflake query telemetry is same-session and best-effort + +- Dander enriches only completed Snowflake operation IDs through one bounded + `INFORMATION_SCHEMA.QUERY_HISTORY_BY_SESSION` lookup on their existing connection. +- The lookup selects stable counters only, caps requests at the most recent 1,000 IDs, and never + selects query text, bind values, provider errors, or response payloads. +- Missing history cannot fail a successful pipeline. Delayed account-usage cost attribution and + misleading output-row-as-read-row mappings are explicitly excluded. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 252b2fee..18a75713 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -14,8 +14,9 @@ evaluate these limits before using it for an unattended system containing busine modes behind its provider capability, bounded direct/Parquet paths, temporary remote staging, schema checks, destination fencing, fenced table/incremental transforms, and fenced replace-mode graph targets. The ordinary hosted source runner still selects SCD1, and Snowflake has no live - qualification, views, ARRAY/RECORD fallback, measured direct-write crossover, or query-history - enrichment. + qualification, views, ARRAY/RECORD fallback, measured direct-write crossover, or synchronous + total-credit attribution. Same-session query-history enrichment is best-effort and bounded to + the most recent 1,000 operation IDs per writer or transform session. Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. It has no live qualification, views, executable graphs, diff --git a/docs/snowflake.md b/docs/snowflake.md index 086f4e73..c2c114ae 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -82,8 +82,11 @@ variable reference. Do not put a token, private key, password, or connection str target renders and validates before the first target is claimed, preventing partial publication when a later target uses unavailable semantics. - Transform staging, publication, and generic assertions report bounded operation telemetry with - query IDs, duration, affected rows, and warehouse name. SQL and provider response payloads are - not retained. + query IDs, duration, affected rows, and warehouse name. After successful operations, one + best-effort same-session query-history lookup enriches the most recent 1,000 operation IDs with + execution time, queue time, bytes scanned, inserted rows, and warehouse size. Missing or + unavailable history never changes pipeline success, and Dander does not select or retain query + text, bind values, or provider response payloads. The Snowflake role needs usage on its database and warehouse plus permission to create and operate schemas, tables, and temporary stages in the selected namespace. Use a dedicated disposable role @@ -97,6 +100,7 @@ existing replace-mode executable subset; field tests and null-on-failure casts w semantics have not been proven continue to fail before provider mutation. Views remain unavailable because Snowflake permanent DDL cannot share the destination-fence transaction. Direct thresholds default to zero because no live crossover has been measured; do not claim a performance benefit -until that qualification is recorded. Query-history enrichment, live concurrency proof, -infrastructure provisioning, and support promotion remain separate work. Use -`catalog.provider: none` for this experimental path. +until that qualification is recorded. Same-session history deliberately does not estimate total +Snowflake credits or costs; account-level usage history is delayed and remains evidence-side work. +Live concurrency proof, infrastructure provisioning, and support promotion remain separate work. +Use `catalog.provider: none` for this experimental path. diff --git a/src/dander/providers/snowflake/session.py b/src/dander/providers/snowflake/session.py index fd502ea1..86ce7d60 100644 --- a/src/dander/providers/snowflake/session.py +++ b/src/dander/providers/snowflake/session.py @@ -4,12 +4,14 @@ from collections.abc import Callable from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Protocol, Self if TYPE_CHECKING: from collections.abc import Iterator, Sequence + from dander.telemetry import OperationTelemetry + class SnowflakeCursor(Protocol): """Connector cursor surface used by Dander and its stateful test double.""" @@ -46,6 +48,9 @@ def close(self) -> None: ... SnowflakeConnectionFactory = Callable[[], SnowflakeConnection] +_QUERY_HISTORY_RESULT_LIMIT = 10_000 +_QUERY_HISTORY_ID_LIMIT = 1_000 + @dataclass(frozen=True, slots=True) class SnowflakeStatementResult: @@ -57,6 +62,18 @@ class SnowflakeStatementResult: rows: tuple[object, ...] = () +@dataclass(frozen=True, slots=True) +class _SnowflakeQueryMetrics: + """Stable, non-sensitive counters available from same-session query history.""" + + query_id: str + resource_size: str | None + bytes_processed: int + execution_duration_ms: int + queue_duration_ms: int + rows_inserted: int + + def execute( connection: SnowflakeConnection, statement: str, @@ -97,6 +114,100 @@ def execute_many( cursor.close() +def enrich_operation_telemetry( + connection: SnowflakeConnection, + operations: Sequence[OperationTelemetry], +) -> tuple[OperationTelemetry, ...]: + """Best-effort enrichment from bounded same-session Snowflake query history. + + Provider history is observability, not pipeline correctness. Missing, delayed, malformed, + or unavailable history therefore leaves the already valid operation telemetry unchanged. + """ + requested = _recent_query_ids(operations) + if not requested: + return tuple(operations) + placeholders = ", ".join("?" for _query_id in requested) + statement = ( + "SELECT QUERY_ID, WAREHOUSE_SIZE, BYTES_SCANNED, EXECUTION_TIME, " + "QUEUED_PROVISIONING_TIME, QUEUED_REPAIR_TIME, QUEUED_OVERLOAD_TIME, " + "ROWS_INSERTED FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY_BY_SESSION(" + f"RESULT_LIMIT => {_QUERY_HISTORY_RESULT_LIMIT}, " + "INCLUDE_CLIENT_GENERATED_STATEMENT => TRUE)) " + f"WHERE QUERY_ID IN ({placeholders})" + ) + try: + rows = execute(connection, statement, requested, fetch="all").rows + metrics = { + parsed.query_id: parsed for row in rows if (parsed := _query_metrics(row)) is not None + } + return tuple(_enrich_operation(operation, metrics) for operation in operations) + except Exception: + return tuple(operations) + + +def _recent_query_ids(operations: Sequence[OperationTelemetry]) -> tuple[str, ...]: + requested: list[str] = [] + seen: set[str] = set() + for operation in reversed(operations): + query_id = operation.query_id + if query_id is None or query_id in seen: + continue + seen.add(query_id) + requested.append(query_id) + if len(requested) == _QUERY_HISTORY_ID_LIMIT: + break + requested.reverse() + return tuple(requested) + + +def _query_metrics(row: object) -> _SnowflakeQueryMetrics | None: + if not isinstance(row, (tuple, list)) or len(row) != 8: + return None + query_id, resource_size, *numeric = row + if not isinstance(query_id, str) or not query_id: + return None + values = tuple(_nonnegative_integer(value) for value in numeric) + if any(value is None for value in values): + return None + bytes_processed, execution_ms, provisioning_ms, repair_ms, overload_ms, rows_inserted = values + assert bytes_processed is not None + assert execution_ms is not None + assert provisioning_ms is not None + assert repair_ms is not None + assert overload_ms is not None + assert rows_inserted is not None + return _SnowflakeQueryMetrics( + query_id=query_id, + resource_size=resource_size if isinstance(resource_size, str) and resource_size else None, + bytes_processed=bytes_processed, + execution_duration_ms=execution_ms, + queue_duration_ms=provisioning_ms + repair_ms + overload_ms, + rows_inserted=rows_inserted, + ) + + +def _nonnegative_integer(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _enrich_operation( + operation: OperationTelemetry, + metrics: dict[str, _SnowflakeQueryMetrics], +) -> OperationTelemetry: + if operation.query_id is None or (query := metrics.get(operation.query_id)) is None: + return operation + return replace( + operation, + rows_written=operation.rows_written or query.rows_inserted, + bytes_processed=query.bytes_processed, + queue_duration_ms=query.queue_duration_ms, + execution_duration_ms=query.execution_duration_ms, + resource_size=query.resource_size or operation.resource_size, + ) + + @contextmanager def open_connection(factory: SnowflakeConnectionFactory) -> Iterator[SnowflakeConnection]: """Close one connector session after success or failure.""" @@ -112,6 +223,7 @@ def open_connection(factory: SnowflakeConnectionFactory) -> Iterator[SnowflakeCo "SnowflakeConnectionFactory", "SnowflakeCursor", "SnowflakeStatementResult", + "enrich_operation_telemetry", "execute", "execute_many", "open_connection", diff --git a/src/dander/providers/snowflake/transform.py b/src/dander/providers/snowflake/transform.py index c3c2273b..cf0eda30 100644 --- a/src/dander/providers/snowflake/transform.py +++ b/src/dander/providers/snowflake/transform.py @@ -15,6 +15,7 @@ SnowflakeConnection, SnowflakeConnectionFactory, SnowflakeStatementResult, + enrich_operation_telemetry, execute, open_connection, ) @@ -191,9 +192,10 @@ def _run_assertions( ) if _failure_count(result.row, assertion.name) > 0: failures.append(assertion.name) + completed = enrich_operation_telemetry(connection, telemetry) if failures: raise TransformRunError(f"Data tests failed: {', '.join(failures)}") - return tuple(telemetry) + return completed class SnowflakeGraphRunner: @@ -323,6 +325,7 @@ def _publish_plan( warehouse=warehouse, ) ) + completed = enrich_operation_telemetry(connection, telemetry) except (TargetFenceLostError, SnowflakeWriteError, TransformRunError): raise except Exception as error: @@ -334,7 +337,7 @@ def _publish_plan( temporary, suppress_failure=sys.exc_info()[0] is not None, ) - return tuple(telemetry) + return completed def _model_plan(project: TransformProject, model: TransformModel) -> _SnowflakeModelPlan: diff --git a/src/dander/providers/snowflake/writer.py b/src/dander/providers/snowflake/writer.py index 5095c421..a53120f7 100644 --- a/src/dander/providers/snowflake/writer.py +++ b/src/dander/providers/snowflake/writer.py @@ -22,6 +22,7 @@ SnowflakeConnection, SnowflakeConnectionFactory, SnowflakeStatementResult, + enrich_operation_telemetry, execute, execute_many, open_connection, @@ -275,7 +276,7 @@ def _load_and_publish( ) return _LoadOutcome( affected=0 if self.mode is WriteMode.REPLACE else result.rowcount, - operations=tuple(operations), + operations=enrich_operation_telemetry(connection, operations), ) if stage is not None: execute(connection, _create_stage_sql(stage)) @@ -323,7 +324,10 @@ def _load_and_publish( duration_ms=duration_ms, ) ) - return _LoadOutcome(result.rowcount, tuple(operations)) + return _LoadOutcome( + result.rowcount, + enrich_operation_telemetry(connection, operations), + ) if manifest is not None: assert stage is not None artifacts_to_load = ( @@ -419,7 +423,10 @@ def _load_and_publish( duration_ms=duration_ms, ) ) - return _LoadOutcome(result.rowcount, tuple(operations)) + return _LoadOutcome( + result.rowcount, + enrich_operation_telemetry(connection, operations), + ) except (SnowflakeWriteError, TargetFenceLostError): raise except Exception as error: diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index 0eb8c1f4..7c6733e3 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -16,9 +16,10 @@ from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError, plan_graph_execution from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry from dander.providers.snowflake import SnowflakeWarehouseConfig +from dander.providers.snowflake.session import enrich_operation_telemetry from dander.providers.snowflake.transform import SnowflakeGraphRunner, SnowflakeTransformRunner from dander.providers.snowflake.writer import SnowflakeWriteError -from dander.telemetry import TelemetryOperation +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError from dander.warehouse import ( ProviderExtension, @@ -59,6 +60,7 @@ class _FakeSnowflake: transform_source_rows: list[dict[str, object]] | None = None transform_rows: dict[str, dict[str, object]] = field(default_factory=dict) assertion_failure_count: int = 0 + query_history_template: tuple[object, ...] | None = None def connect(self) -> _FakeConnection: return _FakeConnection(self) @@ -106,7 +108,13 @@ def execute(self, command: str, params: Sequence[object] | None = None) -> Self: self.sfqid = f"query-{len(self.backend.statements)}" if self.backend.fail_prefix and compact.startswith(self.backend.fail_prefix): raise RuntimeError("private provider response") - if compact == "SELECT CURRENT_DATABASE(), CURRENT_WAREHOUSE()": + if compact.startswith("SELECT QUERY_ID, WAREHOUSE_SIZE, BYTES_SCANNED"): + if self.backend.query_history_template is not None: + self._rows = [ + (str(query_id), *self.backend.query_history_template) for query_id in parameters + ] + self.rowcount = len(self._rows) + elif compact == "SELECT CURRENT_DATABASE(), CURRENT_WAREHOUSE()": self._row = ("DANDER_TEST", "DANDER_WH") elif compact.startswith("DESCRIBE TABLE"): self._rows = list(self.backend.describe_rows) @@ -418,6 +426,7 @@ def test_snowflake_direct_load_parses_explicit_variant_and_emits_transport( tmp_path: Path, ) -> None: runtime, backend = _direct_runtime(tmp_path) + backend.query_history_template = ("X-SMALL", 2_048, 11, 2, 3, 4, 7) backend.describe_rows = [ ("id", "VARCHAR(16777216)", "COLUMN", "N"), ("payload", "VARIANT", "COLUMN", "Y"), @@ -476,8 +485,21 @@ def test_snowflake_direct_load_parses_explicit_variant_and_emits_transport( ] assert all(operation.transport is WriteTransport.DIRECT for operation in operations) assert all(operation.resource_name == "DANDER_WH" for operation in operations) + assert all(operation.resource_size == "X-SMALL" for operation in operations) + assert all(operation.bytes_processed == 2_048 for operation in operations) + assert all(operation.execution_duration_ms == 11 for operation in operations) + assert all(operation.queue_duration_ms == 9 for operation in operations) assert operations[0].rows_written == 1 + assert operations[1].rows_written == 7 assert operations[0].query_id != operations[1].query_id + history_sql, history_parameters = next( + (statement, parameters) + for statement, parameters in backend.statements + if statement.startswith("SELECT QUERY_ID, WAREHOUSE_SIZE, BYTES_SCANNED") + ) + assert "QUERY_TEXT" not in history_sql + assert "BIND_VALUES" not in history_sql + assert len(history_parameters) == 2 assert writer.drain_telemetry() == () direct_inserts = sum( statement.startswith('INSERT INTO "DANDER_TEST"."raw"."dander_stage_') @@ -499,6 +521,56 @@ def test_snowflake_direct_load_parses_explicit_variant_and_emits_transport( assert not tuple(tmp_path.iterdir()) +def test_snowflake_query_history_is_best_effort_and_bounded(tmp_path: Path) -> None: + runtime, backend = _direct_runtime(tmp_path) + backend.fail_prefix = "SELECT QUERY_ID, WAREHOUSE_SIZE, BYTES_SCANNED" + relation = RelationRef(catalog="DANDER_TEST", namespace="raw", name="history_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="snowflake_history_records", + run_id="run-history", + token=6, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + + assert writer.write(({"id": "one", "label": "first"},), target) == 2 + assert all(operation.bytes_processed == 0 for operation in writer.drain_telemetry()) + + backend.fail_prefix = None + operations = tuple( + OperationTelemetry( + provider="snowflake", + operation=TelemetryOperation.QUERY, + query_id=f"query-history-{index}", + ) + for index in range(1_002) + ) + connection = backend.connect() + assert enrich_operation_telemetry(connection, operations) == operations + _history_sql, parameters = backend.statements[-1] + assert len(parameters) == 1_000 + assert parameters[0] == "query-history-2" + assert parameters[-1] == "query-history-1001" + connection.close() + + def test_snowflake_direct_threshold_is_decided_for_the_complete_endpoint( tmp_path: Path, ) -> None: @@ -937,6 +1009,7 @@ def test_snowflake_builds_fenced_table_and_incremental_models( snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], ) -> None: runtime, backend, _staging_root = snowflake_runtime + backend.query_history_template = ("SMALL", 4_096, 19, 1, 2, 3, 5) runner = runtime.transforms.build_transform_runner( graph_plan=None, build_models=True, @@ -972,6 +1045,10 @@ def test_snowflake_builds_fenced_table_and_incremental_models( ] assert all(operation.query_id for operation in result.telemetry) assert all(operation.resource_name == "DANDER_WH" for operation in result.telemetry) + assert all(operation.resource_size == "SMALL" for operation in result.telemetry) + assert all(operation.bytes_processed == 4_096 for operation in result.telemetry) + assert all(operation.execution_duration_ms == 19 for operation in result.telemetry) + assert all(operation.queue_duration_ms == 6 for operation in result.telemetry) sql = [statement for statement, _parameters in backend.statements] assert any( statement.startswith("DELETE FROM") and "table_model" in statement for statement in sql From 7255b79c1e2d6ed6e1eb6d22cacb72fe3869921b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 05:28:22 -0400 Subject: [PATCH 095/163] feat(redshift): implement all ingestion writer modes --- HANDOFF.md | 45 +- docs/compatibility-matrix.md | 2 +- docs/decisions.md | 10 + docs/known-limitations.md | 11 +- docs/redshift.md | 22 +- src/dander/providers/redshift/runtime.py | 31 +- src/dander/providers/redshift/writer.py | 421 ++++++++++++++++-- src/dander/runtime-compatibility.json | 4 +- .../test_redshift_warehouse_runtime.py | 197 +++++++- 9 files changed, 665 insertions(+), 78 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 1645e2f8..81c3f6ff 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,44 +2,43 @@ ## Finished -- Enriched successful Snowflake loads, publications, transforms, and tests from same-session query - history. -- Mapped execution/queue time, scanned bytes, inserted rows, and warehouse size into the existing - provider-neutral telemetry contract. -- Kept enrichment best-effort, bounded to 1,000 recent operation IDs, and free of SQL/bind values. -- Preserved client-observed duration and known writer row/byte counters. -- Left fenced view materialization unsupported after its proposed indirection failed review. +- Added replace, SCD2, snapshot, and incremental ingestion to Redshift's existing bounded COPY + writer while preserving the SCD1 constructor. +- Kept every target mutation, complete-manifest replay record, and destination-fence touch in one + Redshift transaction. +- Preserved whole-stream semantics for replace and SCD2 and deterministic cross-batch behavior for + SCD1, snapshot, and incremental writes. +- Updated the exact runtime capability report and experimental Redshift documentation. ## Try It -Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py`. +Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py tests/test_compatibility.py`. ## Checks -- Focused Snowflake provider tests: 31 passed. -- Ruff and mypy passed across 328 files and 304 source/test files; full pytest passed with 1,126 - tests and 13 environment-dependent skips. +- Focused Redshift and compatibility tests passed: 33 tests. +- Ruff passed across 328 files, strict mypy passed across 304 source/test files, and full pytest + passed with 1,133 tests and 13 environment-dependent skips. - Wheel/sdist inspection and source-free installation, generation, and validation passed outside the checkout for both artifacts. -- Terraform format/init/validation, three AWS Terraform tests, Helm lint/render, and Phase 1B - validation passed. Local Docker and PostgreSQL integration remain for protected Linux CI. +- Terraform format/init/validation, bootstrap and Fargate Terraform tests, Helm lint/render, Phase + 1B validation, and generated-project Terraform validation passed. - Independent adversarial completion review passed with no material findings. +- Protected Linux CI still needs to repeat PostgreSQL integration, container, and security checks. ## Decisions -- `ROWS_PRODUCED` is not mapped to `rows_read`; those Snowflake concepts are not equivalent. -- Account Usage is not queried synchronously because its delayed data is unsuitable for run output. -- Query-history failure cannot convert successful warehouse work into pipeline failure. +- All five modes reuse one bounded Parquet/S3 manifest/COPY path instead of adding a transport. +- Replace and SCD2 consume a complete logical stream; the other modes safely publish bounded batches. +- This expands implemented capability only; Redshift remains experimental and unqualified live. ## Remaining -- Push the focused PR and require protected Linux CI to repeat PostgreSQL, container, Terraform, - and security checks. -- Measure the direct/COPY crossover during live Snowflake qualification. -- Revisit view semantics only at the Phase 5.5 architecture checkpoint. +- Push the focused PR, require protected CI, and merge only if clean. +- Keep direct transport, `SUPER`, graphs, views, telemetry, and paid live proof in separate slices. ## Review First -- `src/dander/providers/snowflake/session.py` -- `src/dander/providers/snowflake/writer.py` -- `tests/providers/test_snowflake_warehouse_runtime.py` +- `src/dander/providers/redshift/writer.py` +- `src/dander/providers/redshift/runtime.py` +- `tests/providers/test_redshift_warehouse_runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index be7867e1..c642869d 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -40,7 +40,7 @@ their named live profiles pass. |---|---|---|---|---|---|---| | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | | PostgreSQL | SCD1 | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | -| Redshift | SCD1 | COPY | scalar types; decimal 38, time 6; no SUPER fallback | yes | no | yes | +| Redshift | all five | COPY | scalar types; decimal 38, time 6; no SUPER fallback | yes | no | yes | | Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | Portable-provider schema validation uses these declarations before source extraction, staging diff --git a/docs/decisions.md b/docs/decisions.md index cfe4731f..df375383 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1015,3 +1015,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc selects query text, bind values, provider errors, or response payloads. - Missing history cannot fail a successful pipeline. Delayed account-usage cost attribution and misleading output-row-as-read-row mappings are explicitly excluded. + +## 2026-08-10 — Redshift writer modes share one fenced COPY publication path + +- **Selection:** The provider writer factory accepts replace, SCD1, SCD2, snapshot, and incremental + modes plus their existing mode-specific fields. Existing hosted ingestion still selects SCD1. +- **Publication:** Bounded Parquet/S3 staging feeds mode-specific DML. Target mutation, exact replay + identity, and the destination-fence touch commit in one transaction; replace and SCD2 consume one + complete logical stream so batch boundaries cannot change their semantics. +- **Boundary:** This is local conformance evidence, not a support promotion. Direct transport, + `SUPER`, graphs, views, infrastructure, and live Redshift qualification remain separate gates. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 18a75713..00c3cad1 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -17,11 +17,12 @@ evaluate these limits before using it for an unattended system containing busine qualification, views, ARRAY/RECORD fallback, measured direct-write crossover, or synchronous total-credit attribution. Same-session query-history enrichment is best-effort and bounded to the most recent 1,000 operation IDs per writer or transform session. - Redshift has an experimental, locally conformance-tested scalar SCD1 adapter using bounded - Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, - and fenced table/incremental transforms. It has no live qualification, views, executable graphs, - non-SCD1 modes, or semi-structured `SUPER` mapping. PostgreSQL state and warehouse execution are - implemented and locally conformance-tested. A packaged Helm + Redshift has an experimental, locally conformance-tested scalar adapter with all five writer + modes using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, + destination fencing, and fenced table/incremental transforms. The ordinary hosted source runner + still selects SCD1. Redshift has no live qualification, views, executable graphs, or + semi-structured `SUPER` mapping. PostgreSQL state and warehouse execution are implemented and + locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/redshift.md b/docs/redshift.md index 3aa59f7d..3c2c8410 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -1,8 +1,8 @@ # Experimental Amazon Redshift warehouse Redshift is an experimental warehouse adapter, not a supported Dander profile. The current slice -proves native database/schema coordinates, bounded IAM-only bulk loading, and fenced table and -incremental models before broader write modes or a live support claim. +proves native database/schema coordinates, bounded IAM-only bulk loading, all five fenced ingestion +write modes, and fenced table and incremental models before a live support claim. ## Configuration @@ -40,7 +40,15 @@ COPY role must read the dedicated same-region staging prefix. - `database + schema + relation` become one canonical `RelationRef`; GCP aliases are not used. - Bounded, checksummed Parquet parts load through a mandatory same-region S3 manifest and `COPY`. -- SCD1 publication, replay history, and the exact destination fencing-token touch commit together. +- Replace, SCD1, SCD2, snapshot, and incremental publication reuse the same bounded COPY path. +- Every mode's target mutation, replay history, and exact destination fencing-token touch commit + together. +- SCD1 and incremental input is deterministically de-duplicated by business key; incremental writes + additionally reject cursor regression. +- Replace publishes one complete logical stream with replay-safe `DELETE`/`INSERT`; empty replaces + are also fenced and replay-safe. +- SCD2 uses transaction-stable `SYSDATE` values for `valid_from`/`valid_to`, and snapshot mode + appends only distinct full rows for a non-null declared snapshot field. - Only declared nullable ingestion columns evolve automatically; drift fails closed. - Portable or Redshift-authored table models use fenced `DELETE`/`INSERT` replacement. - Incremental models collapse duplicate keys deterministically and reject cursor regression. @@ -49,7 +57,7 @@ COPY role must read the dedicated same-region staging prefix. ## Deliberate limits -Only scalar SCD1 ingestion plus table and incremental model materializations are implemented. -Views, graphs, replace, SCD2, snapshot, `SUPER`, live concurrency proof, infrastructure -provisioning, and support promotion remain separate work. Use `catalog.provider: none` for this -experimental path. +Only scalar COPY ingestion plus table and incremental model materializations are implemented. The +ordinary hosted source runner still selects SCD1. Views, graphs, `SUPER`, live concurrency proof, +infrastructure provisioning, and support promotion remain separate work. Use +`catalog.provider: none` for this experimental path. diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index 3bcab003..0deb720b 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -18,6 +18,7 @@ from dander.providers.redshift.writer import ( RedshiftS3Client, RedshiftScd1Writer, + RedshiftStagedWriter, RedshiftStagingSettings, default_staging_settings, ) @@ -113,19 +114,37 @@ def build_ingestion_writer( snapshot_field: str | None = None, ) -> WritePattern: del batch_rows - del cursor_field - del snapshot_field if sandbox: raise ValueError("Redshift warehouse does not use Dander's BigQuery sandbox mode") - if mode is not WriteMode.SCD1: - raise ValueError(f"Redshift warehouse does not support {mode.value} writes") - return RedshiftScd1Writer( + if mode is WriteMode.INCREMENTAL: + if cursor_field is None or not cursor_field.strip(): + raise ValueError("Redshift incremental writes require cursor_field") + elif cursor_field is not None: + raise ValueError("cursor_field is valid only for Redshift incremental writes") + if mode is WriteMode.SNAPSHOT: + if snapshot_field is None or not snapshot_field.strip(): + raise ValueError("Redshift snapshot writes require snapshot_field") + elif snapshot_field is not None: + raise ValueError("snapshot_field is valid only for Redshift snapshot writes") + if mode is WriteMode.SCD1: + return RedshiftScd1Writer( + database=self.database, + connection_factory=self.connection_factory, + s3_client=self.s3_client, + target_fence=self.target_fence, + schema_evolution=schema_evolution, + staging=self.staging, + ) + return RedshiftStagedWriter( database=self.database, connection_factory=self.connection_factory, s3_client=self.s3_client, target_fence=self.target_fence, schema_evolution=schema_evolution, staging=self.staging, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, ) @@ -186,7 +205,7 @@ def operation( REDSHIFT_CAPABILITIES = WarehouseCapabilities( provider_id="redshift", schema_contract_version=1, - write_modes=frozenset({WriteMode.SCD1}), + write_modes=frozenset(WriteMode), transports=frozenset({WriteTransport.COPY}), supports_transforms=True, supports_graphs=False, diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py index 04a23482..7eb499d4 100644 --- a/src/dander/providers/redshift/writer.py +++ b/src/dander/providers/redshift/writer.py @@ -1,4 +1,4 @@ -"""Bounded S3/Parquet COPY and transactionally fenced Redshift SCD1 writes.""" +"""Bounded S3/Parquet COPY and transactionally fenced Redshift writes.""" # ruff: noqa: N803 -- boto3's public S3 API uses capitalized parameter names. @@ -39,6 +39,10 @@ from dander.providers.redshift.fence import RedshiftTargetFence _ORDINAL = "_dander_ordinal" +_SCD2_VALID_FROM = "valid_from" +_SCD2_VALID_TO = "valid_to" +_SCD2_IS_CURRENT = "is_current" +_SCD2_SYSTEM_FIELDS = frozenset({_SCD2_VALID_FROM, _SCD2_VALID_TO, _SCD2_IS_CURRENT}) _MAX_PARQUET_ROW_BYTES = 4 * 1_024 * 1_024 @@ -80,11 +84,9 @@ class RedshiftPublication: affected_statement_index: int -class RedshiftScd1Writer(WritePattern): - """Upload bounded Parquet parts and merge deterministic last-record-wins rows.""" +class RedshiftStagedWriter(WritePattern): + """Upload bounded Parquet parts and publish one selected logical write mode.""" - mode = WriteMode.SCD1 - supports_batched_writes = True requires_publication_fence = True def __init__( @@ -96,6 +98,9 @@ def __init__( target_fence: RedshiftTargetFence, schema_evolution: SchemaEvolution, staging: RedshiftStagingSettings, + mode: WriteMode, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> None: self._database = database self._connection_factory = connection_factory @@ -103,15 +108,33 @@ def __init__( self._target_fence = target_fence self._evolution = schema_evolution self._staging = staging + self.mode = mode + self._cursor_field = cursor_field + self._snapshot_field = snapshot_field + self.supports_batched_writes = mode in { + WriteMode.SCD1, + WriteMode.INCREMENTAL, + WriteMode.SNAPSHOT, + } + self.accepts_streaming_input = not self.supports_batched_writes def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: - _validate_target(target, self._database) + _validate_target( + target, + database=self._database, + mode=self.mode, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) publication = target.publication_fence assert publication is not None schema = target.canonical_schema _validate_schema(schema) - if any(field.name == _ORDINAL for field in schema.fields): - raise RedshiftWriteError(f"Declared schema reserves Dander field {_ORDINAL!r}") + reserved = {_ORDINAL} + if self.mode is WriteMode.SCD2: + reserved.update(_SCD2_SYSTEM_FIELDS) + if collision := sorted(field.name for field in schema.fields if field.name in reserved): + raise RedshiftWriteError(f"Declared schema reserves Dander field {collision[0]!r}") staged_schema = RelationSchema( fields=( *schema.fields, @@ -130,7 +153,15 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> max_logical_bytes_per_file=self._staging.max_logical_bytes_per_file, compression=self._staging.compression, ) as local: - manifest = local.stage(_with_ordinals(records), staged_schema) + manifest = local.stage( + _with_ordinals( + records, + business_key=target.business_key, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ), + staged_schema, + ) _validate_manifest(manifest) return self._publish(target, publication, schema, staged_schema, manifest) @@ -190,6 +221,9 @@ def _publish( manifest, evolution=self._evolution, has_rows=bool(manifest.rows), + mode=self.mode, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, ) # The schema read also opens an implicit transaction. End that read-only # snapshot before beginning the one fenced publication transaction. @@ -201,13 +235,15 @@ def _publish( ) affected_rows = results[plan.affected_statement_index].rowcount if affected_rows < 0: - raise RedshiftWriteError("Redshift MERGE did not report affected rows") + raise RedshiftWriteError("Redshift publication did not report affected rows") publication_failed = False return affected_rows except (RedshiftWriteError, TargetFenceLostError): raise except Exception as error: - raise RedshiftWriteError("Redshift staged SCD1 write failed") from error + raise RedshiftWriteError( + f"Redshift staged {self.mode.value.upper()} write failed" + ) from error finally: cleanup_error: Exception | None = None try: @@ -222,6 +258,32 @@ def _publish( raise cleanup_error +class RedshiftScd1Writer(RedshiftStagedWriter): + """Backward-compatible SCD1 constructor for existing provider integrations.""" + + mode = WriteMode.SCD1 + + def __init__( + self, + *, + database: str, + connection_factory: RedshiftConnectionFactory, + s3_client: RedshiftS3Client, + target_fence: RedshiftTargetFence, + schema_evolution: SchemaEvolution, + staging: RedshiftStagingSettings, + ) -> None: + super().__init__( + database=database, + connection_factory=connection_factory, + s3_client=s3_client, + target_fence=target_fence, + schema_evolution=schema_evolution, + staging=staging, + mode=WriteMode.SCD1, + ) + + def default_staging_settings( *, bucket: str, @@ -246,23 +308,59 @@ def default_staging_settings( ) -def _with_ordinals(records: Iterable[Mapping[str, object]]) -> Iterable[Mapping[str, object]]: +def _with_ordinals( + records: Iterable[Mapping[str, object]], + *, + business_key: Sequence[str], + cursor_field: str | None, + snapshot_field: str | None, +) -> Iterable[Mapping[str, object]]: for ordinal, record in enumerate(records): + if any(record.get(field) is None for field in business_key): + raise RedshiftWriteError(f"Record {ordinal} has a null business-key value") + if cursor_field is not None and record.get(cursor_field) is None: + raise RedshiftWriteError(f"Record {ordinal} has a null incremental cursor value") + if snapshot_field is not None and record.get(snapshot_field) is None: + raise RedshiftWriteError(f"Record {ordinal} has a null snapshot value") yield {**record, _ORDINAL: ordinal} -def _validate_target(target: WriteTarget, database: str) -> None: +def _validate_target( + target: WriteTarget, + *, + database: str, + mode: WriteMode, + cursor_field: str | None, + snapshot_field: str | None, +) -> None: try: validate_redshift_relation(target.relation_ref) except ValueError as error: raise RedshiftWriteError(str(error)) from error if target.relation_ref.catalog != database: raise RedshiftWriteError("Redshift target belongs to another database") - if not target.business_key or not target.schema: - raise RedshiftWriteError("Redshift SCD1 requires a business key and declared schema") declared = {field.name for field in target.canonical_schema.fields} - if any(key not in declared for key in target.business_key): - raise RedshiftWriteError("Redshift business keys must be declared schema fields") + if not declared: + raise RedshiftWriteError("Redshift writes require a declared schema") + if mode in {WriteMode.SCD1, WriteMode.SCD2, WriteMode.INCREMENTAL}: + if not target.business_key: + raise RedshiftWriteError(f"Redshift {mode.value} writes require a business key") + if missing := sorted(set(target.business_key) - declared): + raise RedshiftWriteError(f"Redshift business-key field {missing[0]!r} is undeclared") + if mode is WriteMode.INCREMENTAL: + if cursor_field is None or not cursor_field.strip(): + raise RedshiftWriteError("Redshift incremental writes require cursor_field") + if cursor_field not in declared: + raise RedshiftWriteError("Redshift incremental cursor field is undeclared") + elif cursor_field is not None: + raise RedshiftWriteError("cursor_field is valid only for Redshift incremental writes") + if mode is WriteMode.SNAPSHOT: + if snapshot_field is None or not snapshot_field.strip(): + raise RedshiftWriteError("Redshift snapshot writes require snapshot_field") + if snapshot_field not in declared: + raise RedshiftWriteError("Redshift snapshot field is undeclared") + elif snapshot_field is not None: + raise RedshiftWriteError("snapshot_field is valid only for Redshift snapshot writes") publication = target.publication_fence if publication is None: raise RedshiftWriteError("Redshift hosted writes require a destination target fence") @@ -323,12 +421,16 @@ def _publication_plan( *, evolution: SchemaEvolution, has_rows: bool, + mode: WriteMode, + cursor_field: str | None, + snapshot_field: str | None, ) -> RedshiftPublication: + deployed_schema = _target_schema_for_mode(schema, mode) statements: list[tuple[str, Sequence[object]]] = [ - (_create_target_sql(target, schema), ()), + (_create_target_sql(target, deployed_schema), ()), (_history_table_sql(target), ()), ] - statements.extend(_schema_changes(connection, target, schema, evolution=evolution)) + statements.extend(_schema_changes(connection, target, deployed_schema, evolution=evolution)) digest = _manifest_digest(manifest) history_values: tuple[object, ...] = ( ".".join(target.relation_ref.coordinates), @@ -337,11 +439,18 @@ def _publication_plan( manifest.schema_fingerprint, digest, ) - if has_rows: - statements.append((_merge_sql(target, temp_table, schema.fields), history_values)) - else: - statements.append((_no_op_sql(target), ())) - affected_statement_index = len(statements) - 1 + publication = _publication_statements( + target, + temp_table, + schema.fields, + history_values, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, + has_rows=has_rows, + ) + affected_statement_index = len(statements) + publication.affected_statement_index + statements.extend(publication.statements) statements.append((_history_insert_sql(target), (*history_values, *history_values))) return RedshiftPublication( statements=tuple(statements), @@ -349,6 +458,241 @@ def _publication_plan( ) +def _publication_statements( + target: WriteTarget, + temporary: str, + fields: Sequence[CanonicalField], + history_values: tuple[object, ...], + *, + mode: WriteMode, + cursor_field: str | None, + snapshot_field: str | None, + has_rows: bool, +) -> RedshiftPublication: + if not has_rows: + statement = _replace_delete_sql(target) if mode is WriteMode.REPLACE else _no_op_sql(target) + parameters = history_values if mode is WriteMode.REPLACE else () + return RedshiftPublication(((statement, parameters),), 0) + if mode is WriteMode.REPLACE: + statements = _replace_statements(target, temporary, fields, history_values) + return RedshiftPublication(statements, len(statements) - 1) + if mode is WriteMode.SNAPSHOT: + assert snapshot_field is not None + return RedshiftPublication( + ((_snapshot_sql(target, temporary, fields, snapshot_field), history_values),), + 0, + ) + if mode is WriteMode.SCD2: + statements = _scd2_statements(target, temporary, fields, history_values) + return RedshiftPublication(statements, len(statements) - 1) + if mode is WriteMode.INCREMENTAL: + assert cursor_field is not None + statements = ( + (_incremental_prune_sql(target, temporary, cursor_field), ()), + ( + _merge_sql(target, temporary, fields, cursor_field=cursor_field), + history_values, + ), + ) + return RedshiftPublication(statements, len(statements) - 1) + return RedshiftPublication( + ( + ( + _merge_sql( + target, + temporary, + fields, + cursor_field=None, + ), + history_values, + ), + ), + 0, + ) + + +def _deduplicated_source( + target: WriteTarget, + temporary: str, + fields: Sequence[CanonicalField], + *, + cursor_field: str | None = None, +) -> str: + columns = ", ".join(_quote(field.name) for field in fields) + keys = ", ".join(_quote(key) for key in target.business_key) + ordering = ( + f"{_quote(cursor_field)} DESC NULLS LAST, {_quote(_ORDINAL)} DESC" + if cursor_field is not None + else f"{_quote(_ORDINAL)} DESC" + ) + return ( + f"SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {keys} " + f"ORDER BY {ordering}) AS {_quote('_dander_rank')} FROM {_quote(temporary)}) ranked " + f"WHERE {_quote('_dander_rank')} = 1" + ) + + +def _replace_statements( + target: WriteTarget, + temporary: str, + fields: Sequence[CanonicalField], + history_values: tuple[object, ...], +) -> tuple[tuple[str, Sequence[object]], ...]: + columns = ", ".join(_quote(field.name) for field in fields) + guard = _history_guard(target) + return ( + (_replace_delete_sql(target), history_values), + ( + f"INSERT INTO {_target(target)} ({columns}) SELECT {columns} " + f"FROM {_quote(temporary)} WHERE {guard}", + history_values, + ), + ) + + +def _replace_delete_sql(target: WriteTarget) -> str: + return f"DELETE FROM {_target(target)} WHERE {_history_guard(target)}" + + +def _snapshot_sql( + target: WriteTarget, + temporary: str, + fields: Sequence[CanonicalField], + snapshot_field: str, +) -> str: + columns = ", ".join(_quote(field.name) for field in fields) + selected = ", ".join(f"incoming.{_quote(field.name)}" for field in fields) + identical = " AND ".join( + _null_safe_equal(f"existing.{_quote(field.name)}", f"incoming.{_quote(field.name)}") + for field in fields + ) + return ( + f"INSERT INTO {_target(target)} ({columns}) SELECT DISTINCT {selected} " + f"FROM {_quote(temporary)} AS incoming WHERE " + f"incoming.{_quote(snapshot_field)} IS NOT NULL AND {_history_guard(target)} " + f"AND NOT EXISTS (SELECT 1 FROM {_target(target)} AS existing WHERE {identical})" + ) + + +def _incremental_prune_sql( + target: WriteTarget, + temporary: str, + cursor_field: str, +) -> str: + temporary_name = _quote(temporary) + target_name = _quote(target.relation_ref.name) + match = " AND ".join( + f"{temporary_name}.{_quote(key)} = {target_name}.{_quote(key)}" + for key in target.business_key + ) + return ( + f"DELETE FROM {temporary_name} USING {_target(target)} " + f"WHERE {match} AND {temporary_name}.{_quote(cursor_field)} " + f"< {target_name}.{_quote(cursor_field)}" + ) + + +def _scd2_statements( + target: WriteTarget, + temporary: str, + fields: Sequence[CanonicalField], + history_values: tuple[object, ...], +) -> tuple[tuple[str, Sequence[object]], ...]: + names = tuple(field.name for field in fields) + incoming = _deduplicated_source(target, temporary, fields) + match = " AND ".join( + f"current.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + ) + mutable = tuple(name for name in names if name not in target.business_key) + changed = ( + " OR ".join( + _null_safe_different(f"current.{_quote(name)}", f"incoming.{_quote(name)}") + for name in mutable + ) + or "FALSE" + ) + close = ( + f"UPDATE {_target(target)} AS current SET {_quote(_SCD2_VALID_TO)} = SYSDATE, " + f"{_quote(_SCD2_IS_CURRENT)} = FALSE FROM ({incoming}) AS incoming " + f"WHERE {match} AND current.{_quote(_SCD2_IS_CURRENT)} = TRUE AND ({changed}) " + f"AND {_history_guard(target)}" + ) + columns = ", ".join( + [ + *(_quote(name) for name in names), + _quote(_SCD2_VALID_FROM), + _quote(_SCD2_VALID_TO), + _quote(_SCD2_IS_CURRENT), + ] + ) + values = ", ".join( + [ + *(f"incoming.{_quote(name)}" for name in names), + "SYSDATE", + "NULL", + "TRUE", + ] + ) + insert = ( + f"INSERT INTO {_target(target)} ({columns}) SELECT {values} FROM ({incoming}) AS incoming " + f"WHERE {_history_guard(target)} AND NOT EXISTS " + f"(SELECT 1 FROM {_target(target)} AS current " + f"WHERE {match} AND current.{_quote(_SCD2_IS_CURRENT)} = TRUE)" + ) + return ((close, history_values), (insert, history_values)) + + +def _history_guard(target: WriteTarget) -> str: + return ( + f"NOT EXISTS (SELECT 1 FROM {_history(target)} WHERE " + '"target_id" = %s AND "pipeline_id" = %s AND "run_id" = %s ' + 'AND "schema_fingerprint" = %s AND "manifest_digest" = %s)' + ) + + +def _null_safe_equal(left: str, right: str) -> str: + return f"({left} = {right} OR ({left} IS NULL AND {right} IS NULL))" + + +def _null_safe_different(left: str, right: str) -> str: + return ( + f"({left} <> {right} OR ({left} IS NULL AND {right} IS NOT NULL) " + f"OR ({left} IS NOT NULL AND {right} IS NULL))" + ) + + +def _target_schema_for_mode(schema: RelationSchema, mode: WriteMode) -> RelationSchema: + if mode is not WriteMode.SCD2: + return schema + return RelationSchema( + fields=( + *schema.fields, + CanonicalField( + name=_SCD2_VALID_FROM, + data_type=CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + fractional_second_precision=6, + with_timezone=False, + ), + cardinality=FieldCardinality.REQUIRED, + ), + CanonicalField( + name=_SCD2_VALID_TO, + data_type=CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + fractional_second_precision=6, + with_timezone=False, + ), + ), + CanonicalField( + name=_SCD2_IS_CURRENT, + data_type=CanonicalType(kind=LogicalTypeKind.BOOLEAN), + cardinality=FieldCardinality.REQUIRED, + ), + ) + ) + + def _schema_changes( connection: RedshiftConnection, target: WriteTarget, @@ -439,23 +783,33 @@ def _copy_sql(name: str, *, bucket: str, manifest_key: str, role_arn: str) -> st ) -def _merge_sql(target: WriteTarget, temporary: str, fields: Sequence[CanonicalField]) -> str: +def _merge_sql( + target: WriteTarget, + temporary: str, + fields: Sequence[CanonicalField], + *, + cursor_field: str | None, +) -> str: names = tuple(field.name for field in fields) + target_name = _quote(target.relation_ref.name) columns = ", ".join(_quote(name) for name in names) - keys = ", ".join(_quote(key) for key in target.business_key) match = " AND ".join( - f"target.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + f"{target_name}.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key ) mutable = tuple(name for name in names if name not in target.business_key) updates = ", ".join(f"{_quote(name)} = incoming.{_quote(name)}" for name in mutable) selected = ", ".join(f"incoming.{_quote(name)}" for name in names) history = _history(target) matched = f" WHEN MATCHED THEN UPDATE SET {updates}" if updates else "" + source = _deduplicated_source( + target, + temporary, + fields, + cursor_field=cursor_field, + ) return ( - f"MERGE INTO {_target(target)} AS target USING (SELECT {columns} FROM " - f"(SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {keys} ORDER BY " - f"{_quote(_ORDINAL)} DESC) AS {_quote('_dander_rank')} FROM {_quote(temporary)}) ranked " - f"WHERE {_quote('_dander_rank')} = 1 AND NOT EXISTS (SELECT 1 FROM {history} WHERE " + f"MERGE INTO {_target(target)} USING (SELECT {columns} FROM ({source}) source " + f"WHERE NOT EXISTS (SELECT 1 FROM {history} WHERE " '"target_id" = %s AND "pipeline_id" = %s AND "run_id" = %s ' 'AND "schema_fingerprint" = %s AND "manifest_digest" = %s)) AS incoming ' f"ON {match}{matched} WHEN NOT MATCHED THEN " @@ -487,8 +841,8 @@ def _history(target: WriteTarget) -> str: def _no_op_sql(target: WriteTarget) -> str: - key = _quote(target.business_key[0]) - return f"UPDATE {_target(target)} SET {key} = {key} WHERE FALSE" + column = _quote(target.canonical_schema.fields[0].name) + return f"UPDATE {_target(target)} SET {column} = {column} WHERE FALSE" def _set_query_group( @@ -633,6 +987,7 @@ def _normalize_deployed_type( __all__ = [ "RedshiftS3Client", "RedshiftScd1Writer", + "RedshiftStagedWriter", "RedshiftStagingSettings", "RedshiftWriteError", "default_staging_settings", diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 842e8f11..078a4197 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -77,7 +77,7 @@ }, { "provider": "redshift", - "write_modes": ["scd1"], + "write_modes": ["incremental", "replace", "scd1", "scd2", "snapshot"], "transports": ["copy"], "logical_types": ["binary", "boolean", "date", "decimal", "float", "integer", "string", "time", "timestamp"], "max_decimal_precision": 38, @@ -85,7 +85,7 @@ "supports_transforms": true, "supports_graphs": false, "supports_target_fencing": true, - "limitations": ["graph execution is not available", "only scd1 ingestion is implemented", "semi-structured types require a future explicit SUPER mapping"] + "limitations": ["graph execution is not available", "only COPY transport is implemented", "semi-structured types require a future explicit SUPER mapping"] }, { "provider": "snowflake", diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index c02c957a..3ad6588e 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -164,6 +164,8 @@ def execute(self, command: str, args: Sequence[object] | None = None) -> Self: elif compact.startswith("INSERT INTO") and "dander_stage_loads" in compact: self.rowcount = 0 if parameters[:5] in self.backend.history else 1 self.backend.history.add(parameters[:5]) + elif len(parameters) == 5 and compact.startswith(("DELETE FROM", "INSERT INTO", "UPDATE")): + self.rowcount = 0 if parameters in self.backend.history else self.backend.merge_rowcount elif compact.startswith("MERGE INTO"): self.rowcount = ( 0 if parameters[:5] in self.backend.history else self.backend.merge_rowcount @@ -338,7 +340,7 @@ def test_redshift_runtime_validates_connection_and_exposes_fenced_transforms( relation = RelationRef(catalog="analytics", namespace="raw", name="records") assert runtime.relation_codec.render(relation) == '"analytics"."raw"."records"' - assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) + assert runtime.capabilities.write_modes == frozenset(WriteMode) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) assert runtime.capabilities.supports_transforms is True assert isinstance( @@ -702,6 +704,199 @@ def test_redshift_stages_bounded_parts_merges_deterministically_and_cleans( assert sql[touch_indexes[0] - 1].startswith("LOCK ") +@pytest.mark.parametrize( + ("mode", "field_name", "expected_sql"), + [ + ( + WriteMode.INCREMENTAL, + "updated_at", + '"updated_at" < "records"."updated_at"', + ), + (WriteMode.SNAPSHOT, "snapshot_at", "SELECT DISTINCT"), + (WriteMode.SCD2, None, '"valid_to" = SYSDATE'), + (WriteMode.REPLACE, None, 'DELETE FROM "raw"."records" WHERE NOT EXISTS'), + ], +) +def test_redshift_factory_reaches_each_additional_fenced_write_mode( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], + mode: WriteMode, + field_name: str | None, + expected_sql: str, +) -> None: + runtime, backend, _s3, staging_root = redshift_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=field_name if mode is WriteMode.INCREMENTAL else None, + snapshot_field=field_name if mode is WriteMode.SNAPSHOT else None, + ) + assert writer.mode is mode + assert writer.supports_batched_writes is (mode in {WriteMode.INCREMENTAL, WriteMode.SNAPSHOT}) + assert writer.accepts_streaming_input is (mode in {WriteMode.SCD2, WriteMode.REPLACE}) + relation = RelationRef(catalog="analytics", namespace="raw", name="records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=f"redshift_{mode.value}", + run_id=f"run-{mode.value}", + token=5, + authority_id="postgresql:test-state", + ), + ) + schema = [ + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ] + record: dict[str, object] = {"id": "one", "label": "first"} + if field_name is not None: + schema.append(WriteField(name=field_name, data_type="INT64")) + record[field_name] = 2 + backend.schema_rows.append((field_name, "bigint", None, None, None, "YES")) + if mode is WriteMode.SCD2: + backend.schema_rows = [] + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=tuple(schema), + publication_fence=publication, + ) + + assert writer.write((record,), target) == backend.merge_rowcount + + sql = [statement for _, statement, _ in backend.statements] + assert any(expected_sql in statement for statement in sql) + if mode is WriteMode.INCREMENTAL: + merge = next(statement for statement in sql if statement.startswith("MERGE INTO")) + prune = next(statement for statement in sql if statement.startswith("DELETE FROM")) + assert " USING " in prune + assert "WHEN MATCHED AND" not in merge + assert 'ORDER BY "updated_at" DESC NULLS LAST, "_dander_ordinal" DESC' in merge + if mode is WriteMode.SNAPSHOT: + snapshot = next( + statement + for statement in sql + if statement.startswith('INSERT INTO "raw"."records"') + and "SELECT DISTINCT" in statement + ) + assert 'existing."label" IS NULL AND incoming."label" IS NULL' in snapshot + if mode is WriteMode.SCD2: + create = next( + statement + for statement in sql + if statement.startswith('CREATE TABLE IF NOT EXISTS "raw"."records"') + ) + assert '"valid_from" TIMESTAMP NOT NULL' in create + assert '"valid_to" TIMESTAMP' in create + assert '"is_current" BOOLEAN NOT NULL' in create + close = next( + statement + for statement in sql + if statement.startswith('UPDATE "raw"."records" AS current') + ) + insert = next( + statement + for statement in sql + if statement.startswith('INSERT INTO "raw"."records"') and '"valid_from"' in statement + ) + assert "SYSDATE" in close and "SYSDATE" in insert + assert 'ORDER BY "_dander_ordinal" DESC' in close + assert not tuple(staging_root.iterdir()) + + +def test_redshift_replace_replay_is_guarded_by_the_complete_manifest( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=WriteMode.REPLACE, + ) + target = _target(runtime, run_id="run-replace-replay") + original = ({"id": "one", "label": "first"},) + + assert writer.write(original, target) == 1 + assert writer.write(original, target) == 0 + assert writer.write(({"id": "two", "label": "changed"},), target) == 1 + + mutations = [ + (statement, parameters) + for _, statement, parameters in backend.statements + if statement.startswith(("DELETE FROM", "INSERT INTO")) + and "dander_stage_loads" not in statement + and len(parameters) == 5 + ] + assert all("NOT EXISTS" in statement for statement, _ in mutations) + assert all(len(parameters) == 5 for _, parameters in mutations) + history_identities = [ + parameters[:5] + for _, statement, parameters in backend.statements + if statement.startswith("INSERT INTO") and "dander_stage_loads" in statement + ] + assert history_identities[0] == history_identities[1] + assert history_identities[-1] != history_identities[1] + + +def test_redshift_empty_replace_is_fenced_and_replay_safe( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, s3, _root = redshift_runtime + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=WriteMode.REPLACE, + ) + target = _target(runtime, run_id="run-empty-replace") + + assert writer.write((), target) == 1 + assert writer.write((), target) == 0 + assert not s3.uploads and not s3.puts + delete = next( + (statement, parameters) + for _, statement, parameters in backend.statements + if statement.startswith('DELETE FROM "raw"."records"') + ) + assert "NOT EXISTS" in delete[0] + assert len(delete[1]) == 5 + + +def test_redshift_mode_specific_fields_fail_before_staging( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, s3, _root = redshift_runtime + connections = backend.connections + with pytest.raises(ValueError, match="incremental writes require cursor_field"): + runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=WriteMode.INCREMENTAL, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=WriteMode.SNAPSHOT, + snapshot_field="snapshot_at", + ) + target = _target(runtime, run_id="run-null-snapshot") + target = WriteTarget( + relation=target.relation_ref, + business_key=target.business_key, + schema=(*target.schema, WriteField(name="snapshot_at", data_type="INT64")), + publication_fence=target.publication_fence, + ) + with pytest.raises(RedshiftWriteError, match="null snapshot value"): + writer.write(({"id": "one", "label": "first", "snapshot_at": None},), target) + assert not s3.uploads and not s3.puts + assert backend.connections == connections + 1 # target-fence claim only + + def test_redshift_retry_identity_excludes_random_local_staging_id( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], ) -> None: From 918f4af09a52015f276056794cf3c0b7591aec07 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 05:54:23 -0400 Subject: [PATCH 096/163] Add explicit Redshift SUPER support --- HANDOFF.md | 42 ++- docs/compatibility-matrix.md | 2 +- docs/decisions.md | 10 + docs/known-limitations.md | 7 +- docs/redshift.md | 24 +- src/dander/providers/redshift/runtime.py | 12 +- src/dander/providers/redshift/transform.py | 10 +- src/dander/providers/redshift/writer.py | 223 ++++++++++++-- src/dander/runtime-compatibility.json | 2 +- .../test_redshift_warehouse_runtime.py | 281 +++++++++++++++++- tests/warehouse/test_schema_support.py | 2 +- 11 files changed, 552 insertions(+), 63 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 81c3f6ff..622b67ba 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,13 +2,15 @@ ## Finished -- Added replace, SCD2, snapshot, and incremental ingestion to Redshift's existing bounded COPY - writer while preserving the SCD1 constructor. -- Kept every target mutation, complete-manifest replay record, and destination-fence touch in one - Redshift transaction. -- Preserved whole-stream semantics for replace and SCD2 and deterministic cross-batch behavior for - SCD1, snapshot, and incremental writes. -- Updated the exact runtime capability report and experimental Redshift documentation. +- Added explicit canonical JSON-to-Redshift `SUPER` mapping through + `redshift/fallback=super` without widening canonical schema v1. +- Staged strict deterministic JSON as `VARBYTE(16777216)` and applied `JSON_PARSE` inside every + fenced ingestion-mode publication. +- Rejected bare JSON, ARRAY/RECORD, invalid extensions, non-finite JSON, non-string object keys, and + SUPER keys/cursors/snapshot fields before upload. +- Enforced Redshift's 4 MB COPY row limit per normalized row, including multi-row artifacts. +- Allowed explicitly declared native SUPER outputs in fenced Redshift table/incremental models. +- Updated compatibility and experimental-provider documentation without promoting support. ## Try It @@ -16,29 +18,25 @@ Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py tests/t ## Checks -- Focused Redshift and compatibility tests passed: 33 tests. -- Ruff passed across 328 files, strict mypy passed across 304 source/test files, and full pytest - passed with 1,133 tests and 13 environment-dependent skips. -- Wheel/sdist inspection and source-free installation, generation, and validation passed outside - the checkout for both artifacts. -- Terraform format/init/validation, bootstrap and Fargate Terraform tests, Helm lint/render, Phase - 1B validation, and generated-project Terraform validation passed. -- Independent adversarial completion review passed with no material findings. -- Protected Linux CI still needs to repeat PostgreSQL integration, container, and security checks. +- Ruff formatting and lint passed across 328 files; strict mypy passed across 304 files. +- Full pytest passed: 1,145 tests with 13 intentional skips. +- Wheel and source archive passed distribution inspection, installed outside the checkout, and each + generated and validated a source-free project pinned to `0.8.0rc8`. ## Decisions -- All five modes reuse one bounded Parquet/S3 manifest/COPY path instead of adding a transport. -- Replace and SCD2 consume a complete logical stream; the other modes safely publish bounded batches. -- This expands implemented capability only; Redshift remains experimental and unqualified live. +- SUPER requires an exact field extension and is never inferred from bare JSON. +- Binary staging avoids Redshift's stored-VARCHAR limit; strict JSON validation prevents remote-only + parsing failures and silent key coercion. +- The existing 4 MB staged-row guard remains stricter than Redshift's 16 MB SUPER boundary. ## Remaining -- Push the focused PR, require protected CI, and merge only if clean. -- Keep direct transport, `SUPER`, graphs, views, telemetry, and paid live proof in separate slices. +- Push a focused PR, require protected CI, and merge only if clean. +- Keep direct transport, graphs, views, telemetry, and paid live proof in separate slices. ## Review First - `src/dander/providers/redshift/writer.py` - `src/dander/providers/redshift/runtime.py` -- `tests/providers/test_redshift_warehouse_runtime.py` +- `src/dander/providers/redshift/transform.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index c642869d..b67dd08d 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -40,7 +40,7 @@ their named live profiles pass. |---|---|---|---|---|---|---| | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | | PostgreSQL | SCD1 | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | -| Redshift | all five | COPY | scalar types; decimal 38, time 6; no SUPER fallback | yes | no | yes | +| Redshift | all five | COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | no | yes | | Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | Portable-provider schema validation uses these declarations before source extraction, staging diff --git a/docs/decisions.md b/docs/decisions.md index df375383..91e98ee7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1025,3 +1025,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc complete logical stream so batch boundaries cannot change their semantics. - **Boundary:** This is local conformance evidence, not a support promotion. Direct transport, `SUPER`, graphs, views, infrastructure, and live Redshift qualification remain separate gates. + +## 2026-08-10 — Redshift SUPER is an explicit strict-JSON fallback + +- **Selection:** Only canonical JSON carrying `redshift/fallback=super` maps to `SUPER`; bare JSON, + ARRAY/RECORD, unrelated Redshift extensions, and SUPER keys/cursors/snapshot fields fail closed. +- **Loading:** Dander validates strict JSON locally, serializes deterministic UTF-8 bytes into an + explicitly sized VARBYTE Parquet column, and applies `JSON_PARSE` within fenced publication DML. + This avoids the 65,535-byte stored-VARCHAR boundary. +- **Boundary:** The current 4 MB staged-row guard remains stricter than Redshift's 16 MB SUPER + service limit. This is local conformance behavior, not live qualification or support promotion. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 00c3cad1..1f7cd243 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -20,9 +20,10 @@ evaluate these limits before using it for an unattended system containing busine Redshift has an experimental, locally conformance-tested scalar adapter with all five writer modes using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. The ordinary hosted source runner - still selects SCD1. Redshift has no live qualification, views, executable graphs, or - semi-structured `SUPER` mapping. PostgreSQL state and warehouse execution are implemented and - locally conformance-tested. A packaged Helm + still selects SCD1. Explicit JSON-to-`SUPER` fields use strict UTF-8 serialization, bounded + VARBYTE staging, and `JSON_PARSE`; ARRAY/RECORD fallbacks remain unavailable. Redshift has no + live qualification, views, or executable graphs. PostgreSQL state and warehouse execution are + implemented and locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/redshift.md b/docs/redshift.md index 3c2c8410..ea69ed11 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -49,6 +49,9 @@ COPY role must read the dedicated same-region staging prefix. are also fenced and replay-safe. - SCD2 uses transaction-stable `SYSDATE` values for `valid_from`/`valid_to`, and snapshot mode appends only distinct full rows for a non-null declared snapshot field. +- A canonical JSON field maps to `SUPER` only with the exact `redshift/fallback=super` extension. + Dander rejects non-finite numbers and non-string object keys locally, stages deterministic UTF-8 + JSON as `VARBYTE(16777216)`, and calls `JSON_PARSE` inside the fenced publication. - Only declared nullable ingestion columns evolve automatically; drift fails closed. - Portable or Redshift-authored table models use fenced `DELETE`/`INSERT` replacement. - Incremental models collapse duplicate keys deterministically and reject cursor regression. @@ -57,7 +60,20 @@ COPY role must read the dedicated same-region staging prefix. ## Deliberate limits -Only scalar COPY ingestion plus table and incremental model materializations are implemented. The -ordinary hosted source runner still selects SCD1. Views, graphs, `SUPER`, live concurrency proof, -infrastructure provisioning, and support promotion remain separate work. Use -`catalog.provider: none` for this experimental path. +Scalar fields and explicit JSON-to-`SUPER` fallback are implemented for COPY ingestion and +table/incremental models. `SUPER` fields cannot be business keys, cursors, or snapshot fields. +ARRAY/RECORD fallbacks remain unavailable, and the current staged-row guard is 4 MB even though the +declared VARBYTE/SUPER boundary is larger. The ordinary hosted source runner still selects SCD1. +Views, graphs, live concurrency proof, infrastructure provisioning, and support promotion remain +separate work. Use `catalog.provider: none` for this experimental path. + +Declare the fallback on the field; bare canonical JSON continues to fail closed: + +```yaml +- name: payload + type: JSON + extensions: + - provider: redshift + name: fallback + value: super +``` diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index 0deb720b..85bfef9a 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -20,7 +20,9 @@ RedshiftScd1Writer, RedshiftStagedWriter, RedshiftStagingSettings, + RedshiftWriteError, default_staging_settings, + validate_redshift_schema, ) from dander.providers.registry import ( PROVIDER_API_VERSION, @@ -34,6 +36,7 @@ WarehouseCapabilities, WarehouseRuntime, WarehouseSchemaSupport, + WarehouseSchemaSupportError, ) from dander.writer import SchemaEvolution, WriteField, WriteMode, WritePattern, WriteTransport @@ -90,12 +93,17 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: canonical.append(field) else: raise TypeError("Redshift schema mapper received an unsupported field") - return REDSHIFT_SCHEMA_SUPPORT.require(RelationSchema(fields=tuple(canonical))) + schema = RelationSchema(fields=tuple(canonical)) + try: + validate_redshift_schema(schema) + except RedshiftWriteError as error: + raise WarehouseSchemaSupportError(str(error)) from error + return schema @dataclass(frozen=True, slots=True) class RedshiftWriterFactory: - """Construct Redshift's bounded Parquet/COPY SCD1 writer.""" + """Construct Redshift's bounded Parquet/COPY ingestion writer.""" database: str connection_factory: RedshiftConnectionFactory diff --git a/src/dander/providers/redshift/transform.py b/src/dander/providers/redshift/transform.py index 82b4481d..f09feea0 100644 --- a/src/dander/providers/redshift/transform.py +++ b/src/dander/providers/redshift/transform.py @@ -20,9 +20,10 @@ _create_target_sql, _qualified, _quote, - _redshift_type, + _redshift_field_type, _schema_changes, _set_query_group, + _validate_super_roles, ) from dander.transform import ( SqlDialect, @@ -244,7 +245,12 @@ def _model_plan(project: TransformProject, model: TransformModel) -> _RedshiftMo ) validate_redshift_relation(target.relation_ref) for field in target.canonical_schema.fields: - _redshift_type(field.data_type) + _redshift_field_type(field) + _validate_super_roles( + target, + cursor_field=model.metadata.incremental_cursor, + snapshot_field=None, + ) return _RedshiftModelPlan(model=model, target=target, query=project.compile(model)) diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py index 7eb499d4..e0572b29 100644 --- a/src/dander/providers/redshift/writer.py +++ b/src/dander/providers/redshift/writer.py @@ -29,6 +29,8 @@ ParquetStagingSession, RelationSchema, StagingManifest, + normalize_staging_record, + staging_logical_size, ) from dander.writer import SchemaEvolution, WriteMode, WritePattern, WriteTarget @@ -44,6 +46,7 @@ _SCD2_IS_CURRENT = "is_current" _SCD2_SYSTEM_FIELDS = frozenset({_SCD2_VALID_FROM, _SCD2_VALID_TO, _SCD2_IS_CURRENT}) _MAX_PARQUET_ROW_BYTES = 4 * 1_024 * 1_024 +_MAX_SUPER_BYTES = 16 * 1_024 * 1_024 class RedshiftWriteError(ValueError): @@ -129,7 +132,12 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> publication = target.publication_fence assert publication is not None schema = target.canonical_schema - _validate_schema(schema) + validate_redshift_schema(schema) + _validate_super_roles( + target, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) reserved = {_ORDINAL} if self.mode is WriteMode.SCD2: reserved.update(_SCD2_SYSTEM_FIELDS) @@ -137,7 +145,7 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> raise RedshiftWriteError(f"Declared schema reserves Dander field {collision[0]!r}") staged_schema = RelationSchema( fields=( - *schema.fields, + *(_staging_field(field) for field in schema.fields), CanonicalField( name=_ORDINAL, data_type=CanonicalType(kind=LogicalTypeKind.INTEGER, bit_width=64), @@ -154,15 +162,17 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> compression=self._staging.compression, ) as local: manifest = local.stage( - _with_ordinals( - records, - business_key=target.business_key, - cursor_field=self._cursor_field, - snapshot_field=self._snapshot_field, + _validated_staging_rows( + _with_ordinals( + _serialize_super_records(records, schema), + business_key=target.business_key, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ), + staged_schema, ), staged_schema, ) - _validate_manifest(manifest) return self._publish(target, publication, schema, staged_schema, manifest) def _publish( @@ -370,19 +380,153 @@ def _validate_target( raise RedshiftWriteError("Redshift destination target fence does not match the target") -def _validate_schema(schema: RelationSchema) -> None: +def validate_redshift_schema(schema: RelationSchema) -> None: + """Validate scalar mappings plus the explicit JSON-to-SUPER fallback.""" for field in schema.fields: if len(field.name.encode()) > 127: raise RedshiftWriteError("Redshift column identifiers cannot exceed 127 bytes") - _redshift_type(field.data_type) + _redshift_field_type(field) + + +def _validate_super_roles( + target: WriteTarget, + *, + cursor_field: str | None, + snapshot_field: str | None, +) -> None: + super_fields = { + field.name for field in target.canonical_schema.fields if _is_super_fallback(field) + } + restricted = {*target.business_key} + if cursor_field is not None: + restricted.add(cursor_field) + if snapshot_field is not None: + restricted.add(snapshot_field) + if collision := sorted(super_fields & restricted): + raise RedshiftWriteError( + f"Redshift SUPER field {collision[0]!r} cannot be a key, cursor, or snapshot field" + ) + +def _staging_field(field: CanonicalField) -> CanonicalField: + if _is_super_fallback(field): + return field.model_copy(update={"data_type": CanonicalType(kind=LogicalTypeKind.BINARY)}) + return field -def _validate_manifest(manifest: StagingManifest) -> None: - if any( - artifact.rows == 1 and artifact.logical_bytes > _MAX_PARQUET_ROW_BYTES - for artifact in manifest.artifacts - ): - raise RedshiftWriteError("Redshift Parquet row exceeds the 4 MB COPY limit") + +def _is_super_fallback(field: CanonicalField) -> bool: + redshift_extensions = tuple( + extension for extension in field.extensions if extension.provider == "redshift" + ) + expected = _has_super_extension(field) + if field.data_type.kind is LogicalTypeKind.JSON: + if not expected: + raise RedshiftWriteError("Redshift JSON fields require redshift/fallback=super") + return True + if field.data_type.kind in {LogicalTypeKind.ARRAY, LogicalTypeKind.RECORD}: + raise RedshiftWriteError( + "Redshift ARRAY and RECORD fields have no canonical fallback in this slice" + ) + if redshift_extensions: + raise RedshiftWriteError("Redshift field extension is unsupported for this type") + return False + + +def _has_super_extension(field: CanonicalField) -> bool: + redshift_extensions = tuple( + extension for extension in field.extensions if extension.provider == "redshift" + ) + return ( + len(redshift_extensions) == 1 + and redshift_extensions[0].name == "fallback" + and redshift_extensions[0].value == "super" + ) + + +def _redshift_field_type(field: CanonicalField) -> str: + if _is_super_fallback(field): + return "SUPER" + return _redshift_type(field.data_type) + + +def _serialize_super_records( + records: Iterable[Mapping[str, object]], + schema: RelationSchema, +) -> Iterable[Mapping[str, object]]: + super_fields = tuple(field.name for field in schema.fields if _is_super_fallback(field)) + for row_index, record in enumerate(records): + prepared = dict(record) + for field_name in super_fields: + value = prepared.get(field_name) + if value is None: + continue + _validate_json_keys(value, row_index=row_index, field_name=field_name) + try: + encoded = json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as error: + raise RedshiftWriteError( + f"Record {row_index} has invalid JSON in field {field_name!r}" + ) from error + if len(encoded) > _MAX_SUPER_BYTES: + raise RedshiftWriteError( + f"Record {row_index} JSON field {field_name!r} exceeds Redshift SUPER size" + ) + prepared[field_name] = encoded + yield prepared + + +def _validate_json_keys( + value: object, + *, + row_index: int, + field_name: str, + active: set[int] | None = None, +) -> None: + active = set() if active is None else active + if isinstance(value, (dict, list, tuple)): + identity = id(value) + if identity in active: + raise RedshiftWriteError(f"Record {row_index} has cyclic JSON in field {field_name!r}") + active.add(identity) + if isinstance(value, dict): + if any(not isinstance(key, str) for key in value): + raise RedshiftWriteError( + f"Record {row_index} has a non-string JSON key in field {field_name!r}" + ) + for nested in value.values(): + _validate_json_keys( + nested, + row_index=row_index, + field_name=field_name, + active=active, + ) + elif isinstance(value, (list, tuple)): + for nested in value: + _validate_json_keys( + nested, + row_index=row_index, + field_name=field_name, + active=active, + ) + if isinstance(value, (dict, list, tuple)): + active.remove(id(value)) + + +def _validated_staging_rows( + records: Iterable[Mapping[str, object]], + schema: RelationSchema, +) -> Iterable[Mapping[str, object]]: + for row_index, record in enumerate(records): + normalized = normalize_staging_record(record, schema, row_index=row_index) + if staging_logical_size(normalized) > _MAX_PARQUET_ROW_BYTES: + raise RedshiftWriteError("Redshift Parquet row exceeds the 4 MB COPY limit") + yield record def _validate_bucket_region(client: RedshiftS3Client, bucket: str, expected: str) -> None: @@ -525,13 +669,32 @@ def _deduplicated_source( if cursor_field is not None else f"{_quote(_ORDINAL)} DESC" ) + source = _projected_staging_source(temporary, fields, include_ordinal=True) return ( f"SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {keys} " - f"ORDER BY {ordering}) AS {_quote('_dander_rank')} FROM {_quote(temporary)}) ranked " + f"ORDER BY {ordering}) AS {_quote('_dander_rank')} FROM {source} AS normalized) ranked " f"WHERE {_quote('_dander_rank')} = 1" ) +def _projected_staging_source( + temporary: str, + fields: Sequence[CanonicalField], + *, + include_ordinal: bool = False, +) -> str: + projections = [_staging_projection(field) for field in fields] + if include_ordinal: + projections.append(f"staged.{_quote(_ORDINAL)} AS {_quote(_ORDINAL)}") + return f"(SELECT {', '.join(projections)} FROM {_quote(temporary)} AS staged)" + + +def _staging_projection(field: CanonicalField) -> str: + reference = f"staged.{_quote(field.name)}" + value = f"JSON_PARSE({reference})" if _is_super_fallback(field) else reference + return f"{value} AS {_quote(field.name)}" + + def _replace_statements( target: WriteTarget, temporary: str, @@ -539,12 +702,14 @@ def _replace_statements( history_values: tuple[object, ...], ) -> tuple[tuple[str, Sequence[object]], ...]: columns = ", ".join(_quote(field.name) for field in fields) + selected = ", ".join(f"incoming.{_quote(field.name)}" for field in fields) + source = _projected_staging_source(temporary, fields) guard = _history_guard(target) return ( (_replace_delete_sql(target), history_values), ( - f"INSERT INTO {_target(target)} ({columns}) SELECT {columns} " - f"FROM {_quote(temporary)} WHERE {guard}", + f"INSERT INTO {_target(target)} ({columns}) SELECT {selected} " + f"FROM {source} AS incoming WHERE {guard}", history_values, ), ) @@ -566,9 +731,10 @@ def _snapshot_sql( _null_safe_equal(f"existing.{_quote(field.name)}", f"incoming.{_quote(field.name)}") for field in fields ) + source = _projected_staging_source(temporary, fields) return ( f"INSERT INTO {_target(target)} ({columns}) SELECT DISTINCT {selected} " - f"FROM {_quote(temporary)} AS incoming WHERE " + f"FROM {source} AS incoming WHERE " f"incoming.{_quote(snapshot_field)} IS NOT NULL AND {_history_guard(target)} " f"AND NOT EXISTS (SELECT 1 FROM {_target(target)} AS existing WHERE {identical})" ) @@ -727,13 +893,13 @@ def _schema_changes( changes.append( ( f"ALTER TABLE {_target(target)} ADD COLUMN {_quote(field.name)} " - f"{_redshift_type(field.data_type)}", + f"{_redshift_field_type(field)}", (), ) ) continue deployed_type, nullable = current - if deployed_type.casefold() != _redshift_type(field.data_type).casefold(): + if deployed_type.casefold() != _redshift_field_type(field).casefold(): raise RedshiftWriteError(f"Redshift type drift for column {field.name!r}") if nullable == required: raise RedshiftWriteError(f"Redshift nullability drift for column {field.name!r}") @@ -762,20 +928,24 @@ def _create_target_sql(target: WriteTarget, schema: RelationSchema) -> str: def _target_column(field: CanonicalField, target: WriteTarget) -> str: required = field.cardinality is FieldCardinality.REQUIRED or field.name in target.business_key - return ( - f"{_quote(field.name)} {_redshift_type(field.data_type)}{' NOT NULL' if required else ''}" - ) + return f"{_quote(field.name)} {_redshift_field_type(field)}{' NOT NULL' if required else ''}" def _temporary_table_sql(name: str, schema: RelationSchema) -> str: fields = ", ".join( - f"{_quote(field.name)} {_redshift_type(field.data_type)}" + f"{_quote(field.name)} {_redshift_staging_type(field)}" f"{' NOT NULL' if field.cardinality is FieldCardinality.REQUIRED else ''}" for field in schema.fields ) return f"CREATE TEMP TABLE {_quote(name)} ({fields})" +def _redshift_staging_type(field: CanonicalField) -> str: + if _has_super_extension(field): + return f"VARBYTE({_MAX_SUPER_BYTES})" + return _redshift_type(field.data_type) + + def _copy_sql(name: str, *, bucket: str, manifest_key: str, role_arn: str) -> str: return ( f"COPY {_quote(name)} FROM {_literal(f's3://{bucket}/{manifest_key}')} " @@ -991,4 +1161,5 @@ def _normalize_deployed_type( "RedshiftStagingSettings", "RedshiftWriteError", "default_staging_settings", + "validate_redshift_schema", ] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 078a4197..8e63868a 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -85,7 +85,7 @@ "supports_transforms": true, "supports_graphs": false, "supports_target_fencing": true, - "limitations": ["graph execution is not available", "only COPY transport is implemented", "semi-structured types require a future explicit SUPER mapping"] + "limitations": ["JSON requires the explicit redshift/fallback=super field extension", "graph execution is not available", "only COPY transport is implemented"] }, { "provider": "snowflake", diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index 3ad6588e..b9ee1915 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -18,7 +18,12 @@ from dander.providers.redshift.transform import RedshiftTransformRunner from dander.providers.redshift.writer import RedshiftWriteError, _delete_owned from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError -from dander.warehouse import RelationRef, WarehouseRuntime +from dander.warehouse import ( + ProviderExtension, + RelationRef, + WarehouseRuntime, + WarehouseSchemaSupportError, +) from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport if TYPE_CHECKING: @@ -313,6 +318,28 @@ def test_redshift_registration_is_lazy_and_uses_native_coordinates() -> None: assert "password" not in config.model_dump_json() +def test_redshift_schema_mapper_requires_explicit_json_super_without_io( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + before = list(backend.statements) + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + + schema = runtime.schema_mapper.canonical_schema( + (WriteField(name="payload", data_type="JSON", extensions=(fallback,)),) + ) + + assert fallback in schema.fields[0].extensions + assert backend.statements == before + with pytest.raises(WarehouseSchemaSupportError, match="require redshift/fallback=super"): + runtime.schema_mapper.canonical_schema((WriteField(name="payload", data_type="JSON"),)) + with pytest.raises(WarehouseSchemaSupportError, match="unsupported for this type"): + runtime.schema_mapper.canonical_schema( + (WriteField(name="payload", data_type="STRING", extensions=(fallback,)),) + ) + assert backend.statements == before + + def test_redshift_serverless_uses_an_aws_derived_database_user() -> None: registry = default_provider_registry() raw = _config() @@ -593,6 +620,32 @@ def _write_redshift_model( ) +def _write_redshift_super_model(root: Path) -> None: + (root / "super_model.sql").write_text("SELECT id, payload FROM {{ ref('raw_super_records') }}") + (root / "super_model.yml").write_text( + "model: super_model\n" + "description: Portable Redshift SUPER fixture.\n" + "owner: data-eng\n" + "dialect: portable\n" + "materialization: table\n" + "dataset: analytics\n" + "source_system: fixture\n" + "sensitivity: internal\n" + "columns:\n" + " - name: id\n" + " type: STRING\n" + " description: Stable fixture identifier.\n" + " - name: payload\n" + " type: JSON\n" + " description: Explicit Redshift SUPER payload.\n" + " extensions:\n" + " - provider: redshift\n" + " name: fallback\n" + " value: super\n" + "tests: []\n" + ) + + def test_redshift_claim_serializes_and_rejects_a_stale_token( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], ) -> None: @@ -806,6 +859,189 @@ def test_redshift_factory_reaches_each_additional_fenced_write_mode( assert not tuple(staging_root.iterdir()) +@pytest.mark.parametrize("mode", tuple(WriteMode)) +def test_redshift_all_write_modes_parse_explicit_super_from_varbyte_staging( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], + mode: WriteMode, +) -> None: + runtime, backend, _s3, staging_root = redshift_runtime + cursor_field = "updated_at" if mode is WriteMode.INCREMENTAL else None + snapshot_field = "snapshot_at" if mode is WriteMode.SNAPSHOT else None + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, + ) + relation = RelationRef(catalog="analytics", namespace="raw", name="super_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=f"redshift_super_{mode.value}", + run_id=f"run-super-{mode.value}", + token=11, + authority_id="postgresql:test-state", + ), + ) + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + schema = [ + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(fallback,)), + ] + record: dict[str, object] = { + "id": "one", + "payload": {"nested": {"ready": True}}, + } + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "NO"), + ("payload", "super", None, None, None, "YES"), + ] + if cursor_field is not None: + schema.append(WriteField(name=cursor_field, data_type="INT64")) + record[cursor_field] = 2 + backend.schema_rows.append((cursor_field, "bigint", None, None, None, "YES")) + if snapshot_field is not None: + schema.append(WriteField(name=snapshot_field, data_type="INT64")) + record[snapshot_field] = 2 + backend.schema_rows.append((snapshot_field, "bigint", None, None, None, "YES")) + if mode is WriteMode.SCD2: + backend.schema_rows = [] + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=tuple(schema), + publication_fence=publication, + ) + + assert writer.write((record,), target) == backend.merge_rowcount + + sql = [statement for _, statement, _ in backend.statements] + temporary = next(statement for statement in sql if statement.startswith("CREATE TEMP TABLE")) + target_ddl = next( + statement + for statement in sql + if statement.startswith('CREATE TABLE IF NOT EXISTS "raw"."super_records"') + ) + assert '"payload" VARBYTE(16777216)' in temporary + assert '"payload" SUPER' in target_ddl + assert any('JSON_PARSE(staged."payload") AS "payload"' in statement for statement in sql) + assert not tuple(staging_root.iterdir()) + + +def test_redshift_transform_accepts_explicit_native_super_column( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + _write_redshift_super_model(tmp_path) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "YES"), + ("payload", "super", None, None, None, "YES"), + ] + backend.transform_source_rows = [{"id": "one", "payload": {"nested": True}}] + runner = runtime.transforms.build_transform_runner( + graph_plan=None, + build_models=True, + raw_namespace="raw", + ) + assert isinstance(runner, RedshiftTransformRunner) + + result = runner.build(tmp_path, ownership=_ownership("run-super-model", 12)) + + assert result.models == ("super_model",) + target_ddl = next( + statement + for _, statement, _ in backend.statements + if statement.startswith('CREATE TABLE IF NOT EXISTS "analytics"."super_model"') + ) + assert '"payload" SUPER' in target_ddl + + +def test_redshift_super_staging_accepts_json_larger_than_varchar_limit( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, s3, _root = redshift_runtime + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "NO"), + ("payload", "super", None, None, None, "YES"), + ] + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + target = _target(runtime, run_id="run-super-boundary") + target = WriteTarget( + relation=target.relation_ref, + business_key=target.business_key, + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(fallback,)), + ), + publication_fence=target.publication_fence, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + assert writer.write(({"id": "one", "payload": {"blob": "x" * 70_000}},), target) == 1 + assert s3.uploads + + +@pytest.mark.parametrize("payload", [float("nan"), {1: "coerced-key"}]) +def test_redshift_super_rejects_non_strict_json_before_upload( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], + payload: object, +) -> None: + runtime, _backend, s3, _root = redshift_runtime + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + target = _target(runtime, run_id="run-invalid-super") + target = WriteTarget( + relation=target.relation_ref, + business_key=target.business_key, + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(fallback,)), + ), + publication_fence=target.publication_fence, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + with pytest.raises(RedshiftWriteError, match="invalid JSON|non-string JSON key"): + writer.write(({"id": "one", "payload": payload},), target) + + assert not s3.uploads and not s3.puts + + +def test_redshift_super_cannot_be_a_business_key( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, _backend, s3, _root = redshift_runtime + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + target = _target(runtime, run_id="run-super-key") + target = WriteTarget( + relation=target.relation_ref, + business_key=("payload",), + schema=(WriteField(name="payload", data_type="JSON", extensions=(fallback,)),), + publication_fence=target.publication_fence, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + with pytest.raises(RedshiftWriteError, match="cannot be a key, cursor, or snapshot"): + writer.write(({"payload": {"id": "one"}},), target) + + assert not s3.uploads and not s3.puts + + def test_redshift_replace_replay_is_guarded_by_the_complete_manifest( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], ) -> None: @@ -995,6 +1231,49 @@ def test_redshift_rejects_oversized_single_row_before_s3_or_database_mutation( assert backend.connections == connections_before +def test_redshift_rejects_oversized_row_inside_multirow_artifact_before_mutation( + tmp_path: Path, +) -> None: + backend = _FakeRedshift() + s3 = _FakeS3() + raw_config = _config() + raw_config["max_logical_bytes_per_file"] = 8 * 1_024 * 1_024 + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, raw_config) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "analytics", + "connection_factory": backend.connect, + "s3_client": s3, + "staging_root": tmp_path, + }, + ) + assert isinstance(runtime, WarehouseRuntime) + target = _target(runtime, run_id="run-large-multirow") + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + connections_before = backend.connections + + with pytest.raises(RedshiftWriteError, match="4 MB COPY limit"): + writer.write( + ( + {"id": "small", "label": "ok"}, + {"id": "large", "label": "x" * 4_300_000}, + ), + target, + ) + + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts + assert backend.connections == connections_before + assert not tuple(tmp_path.iterdir()) + + def test_redshift_connection_validation_errors_are_sanitized(tmp_path: Path) -> None: class _BrokenConnectionFactory: def __call__(self) -> object: diff --git a/tests/warehouse/test_schema_support.py b/tests/warehouse/test_schema_support.py index 51c7b140..3b99ca59 100644 --- a/tests/warehouse/test_schema_support.py +++ b/tests/warehouse/test_schema_support.py @@ -35,7 +35,7 @@ def test_redshift_mapper_rejects_semistructured_fields_before_provider_io() -> N with pytest.raises( WarehouseSchemaSupportError, - match="does not support canonical type 'json' at field 'payload'", + match="require redshift/fallback=super", ): RedshiftSchemaMapper().canonical_schema([field]) From 440af354def4202802f119309fe90050771f1ba8 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 06:13:21 -0400 Subject: [PATCH 097/163] Execute portable graphs on Redshift --- HANDOFF.md | 37 ++- docs/compatibility-matrix.md | 2 +- docs/decisions.md | 11 + docs/known-limitations.md | 7 +- docs/redshift.md | 11 +- src/dander/pipeline/compiler.py | 6 +- src/dander/providers/redshift/runtime.py | 20 +- src/dander/providers/redshift/transform.py | 225 +++++++++++---- src/dander/runtime-compatibility.json | 4 +- tests/pipeline/test_compiler.py | 3 +- .../test_redshift_warehouse_runtime.py | 266 +++++++++++++++++- 11 files changed, 494 insertions(+), 98 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 622b67ba..e59c0410 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,15 +2,12 @@ ## Finished -- Added explicit canonical JSON-to-Redshift `SUPER` mapping through - `redshift/fallback=super` without widening canonical schema v1. -- Staged strict deterministic JSON as `VARBYTE(16777216)` and applied `JSON_PARSE` inside every - fenced ingestion-mode publication. -- Rejected bare JSON, ARRAY/RECORD, invalid extensions, non-finite JSON, non-string object keys, and - SUPER keys/cursors/snapshot fields before upload. -- Enforced Redshift's 4 MB COPY row limit per normalized row, including multi-row artifacts. -- Allowed explicitly declared native SUPER outputs in fenced Redshift table/incremental models. -- Updated compatibility and experimental-provider documentation without promoting support. +- Added `RedshiftGraphRunner` for the existing provider-neutral `GraphExecutionPlan`. +- Rendered canonical graph ASTs as Redshift SQL without adding another graph schema. +- Reused Redshift's run-scoped CTAS, exact target-fence transaction, stable table replacement, and + cleanup path. +- Preflighted every selected target before the first fence claim or provider connection. +- Updated the packaged capability report and experimental-provider documentation. ## Try It @@ -19,24 +16,24 @@ Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py tests/t ## Checks - Ruff formatting and lint passed across 328 files; strict mypy passed across 304 files. -- Full pytest passed: 1,145 tests with 13 intentional skips. -- Wheel and source archive passed distribution inspection, installed outside the checkout, and each - generated and validated a source-free project pinned to `0.8.0rc8`. +- Full pytest passed: 1,150 tests with 13 intentional skips. +- Wheel and source archive passed distribution inspection; the wheel installed outside the + checkout and generated and validated a source-free project pinned to `0.8.0rc8`. +- Independent review's safe-cast blocker was corrected with a pre-provider-I/O regression. ## Decisions -- SUPER requires an exact field extension and is never inferred from bare JSON. -- Binary staging avoids Redshift's stored-VARCHAR limit; strict JSON validation prevents remote-only - parsing failures and silent key coercion. -- The existing 4 MB staged-row guard remains stricter than Redshift's 16 MB SUPER boundary. +- Redshift consumes the canonical compiled graph rather than defining a provider graph model. +- Graph targets remain replace-only and publish through the proven fenced table path. +- Views, telemetry expansion, and live AWS qualification remain separate Phase 5 work. ## Remaining -- Push a focused PR, require protected CI, and merge only if clean. -- Keep direct transport, graphs, views, telemetry, and paid live proof in separate slices. +- Push a focused PR and require all protected CI checks before merge. +- Continue Redshift telemetry and live-profile work without beginning Azure. ## Review First -- `src/dander/providers/redshift/writer.py` -- `src/dander/providers/redshift/runtime.py` - `src/dander/providers/redshift/transform.py` +- `src/dander/providers/redshift/runtime.py` +- `tests/providers/test_redshift_warehouse_runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index b67dd08d..10f59b8e 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -40,7 +40,7 @@ their named live profiles pass. |---|---|---|---|---|---|---| | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | | PostgreSQL | SCD1 | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | -| Redshift | all five | COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | no | yes | +| Redshift | all five | COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | yes | yes | | Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | Portable-provider schema validation uses these declarations before source extraction, staging diff --git a/docs/decisions.md b/docs/decisions.md index 91e98ee7..538fa260 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1035,3 +1035,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc This avoids the 65,535-byte stored-VARCHAR boundary. - **Boundary:** The current 4 MB staged-row guard remains stricter than Redshift's 16 MB SUPER service limit. This is local conformance behavior, not live qualification or support promotion. + +## 2026-08-10 — Redshift graphs reuse the canonical AST and fenced table path + +- **Planning:** Redshift consumes the existing `GraphExecutionPlan` and renders each compiled + relational AST with the Redshift dialect; it does not introduce a provider-specific graph model. +- **Publication:** Every selected replace target preflights before provider I/O, then uses the same + run-scoped CTAS staging, exact destination-fence transaction, stable target, and cleanup path as + Redshift table models. +- **Boundary:** Graphs remain single-connector and replace-only. Redshift safe casts fail preflight + until an exact lowering exists. Views, telemetry expansion, live AWS proof, and support promotion + remain separate Phase 5 slices. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 1f7cd243..daa8c203 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -21,9 +21,10 @@ evaluate these limits before using it for an unattended system containing busine modes using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. The ordinary hosted source runner still selects SCD1. Explicit JSON-to-`SUPER` fields use strict UTF-8 serialization, bounded - VARBYTE staging, and `JSON_PARSE`; ARRAY/RECORD fallbacks remain unavailable. Redshift has no - live qualification, views, or executable graphs. PostgreSQL state and warehouse execution are - implemented and locally conformance-tested. A packaged Helm + VARBYTE staging, and `JSON_PARSE`; ARRAY/RECORD fallbacks remain unavailable. Replace-mode graph + targets use the provider-neutral relational AST and fenced table path; graph safe casts fail + preflight. Redshift has no live qualification or views. PostgreSQL state and warehouse execution + are implemented and locally conformance-tested. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/redshift.md b/docs/redshift.md index ea69ed11..87c62b73 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -2,7 +2,8 @@ Redshift is an experimental warehouse adapter, not a supported Dander profile. The current slice proves native database/schema coordinates, bounded IAM-only bulk loading, all five fenced ingestion -write modes, and fenced table and incremental models before a live support claim. +write modes, fenced table and incremental models, and replace-mode graph targets before a live +support claim. ## Configuration @@ -57,6 +58,8 @@ COPY role must read the dedicated same-region staging prefix. - Incremental models collapse duplicate keys deterministically and reject cursor regression. - Generic not-null, uniqueness, accepted-value, and relationship assertions run after publication. - The complete selected transform DAG preflights before any provider connection or mutation. +- Executable graph targets render the existing provider-neutral relational AST as Redshift SQL, + preflight as one set, and publish through the same fenced table-replacement path. ## Deliberate limits @@ -64,8 +67,10 @@ Scalar fields and explicit JSON-to-`SUPER` fallback are implemented for COPY ing table/incremental models. `SUPER` fields cannot be business keys, cursors, or snapshot fields. ARRAY/RECORD fallbacks remain unavailable, and the current staged-row guard is 4 MB even though the declared VARBYTE/SUPER boundary is larger. The ordinary hosted source runner still selects SCD1. -Views, graphs, live concurrency proof, infrastructure provisioning, and support promotion remain -separate work. Use `catalog.provider: none` for this experimental path. +Graph execution retains the shared single-connector, replace-target-only boundary; safe casts fail +preflight because Redshift cannot yet preserve their canonical semantics. Views, live concurrency +proof, infrastructure provisioning, and support promotion remain separate work. Use +`catalog.provider: none` for this experimental path. Declare the fallback on the field; bare canonical JSON continues to fail closed: diff --git a/src/dander/pipeline/compiler.py b/src/dander/pipeline/compiler.py index 12257949..0c423e98 100644 --- a/src/dander/pipeline/compiler.py +++ b/src/dander/pipeline/compiler.py @@ -204,7 +204,11 @@ def render_graph_query( raise PipelineCompileError(f"Unknown graph SQL target: {target_dialect}") from error if target is SqlDialect.PORTABLE: raise PipelineCompileError("portable is a graph contract, not a render target") - if target in {SqlDialect.SNOWFLAKE, SqlDialect.POSTGRES} and expression.find(exp.TryCast): + if target in { + SqlDialect.SNOWFLAKE, + SqlDialect.REDSHIFT, + SqlDialect.POSTGRES, + } and expression.find(exp.TryCast): raise PipelineCompileError( f"Graph safe-cast semantics are not yet exact for {target.value}" ) diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index 85bfef9a..ca22ec28 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -14,7 +14,7 @@ execute, open_connection, ) -from dander.providers.redshift.transform import RedshiftTransformRunner +from dander.providers.redshift.transform import RedshiftGraphRunner, RedshiftTransformRunner from dander.providers.redshift.writer import ( RedshiftS3Client, RedshiftScd1Writer, @@ -158,7 +158,7 @@ def build_ingestion_writer( @dataclass(frozen=True, slots=True) class RedshiftTransformFactory: - """Construct fenced Redshift model execution while rejecting graph plans.""" + """Construct fenced Redshift model or provider-neutral graph execution.""" database: str connection_factory: RedshiftConnectionFactory @@ -171,9 +171,19 @@ def build_transform_runner( graph_plan: object | None, build_models: bool, raw_namespace: str = "raw", - ) -> RedshiftTransformRunner | None: + ) -> RedshiftGraphRunner | RedshiftTransformRunner | None: if graph_plan is not None: - raise ValueError("Redshift graph execution is not available in this experimental slice") + from dander.pipeline.runtime import GraphExecutionPlan + + if not isinstance(graph_plan, GraphExecutionPlan): + raise TypeError("Redshift graph plan has the wrong type") + return RedshiftGraphRunner( + plan=graph_plan, + database=self.database, + connection_factory=self.connection_factory, + target_fence=self.target_fence, + statement_timeout_ms=self.statement_timeout_ms, + ) if not build_models: return None return RedshiftTransformRunner( @@ -216,7 +226,7 @@ def operation( write_modes=frozenset(WriteMode), transports=frozenset({WriteTransport.COPY}), supports_transforms=True, - supports_graphs=False, + supports_graphs=True, supports_target_fencing=True, schema_support=REDSHIFT_SCHEMA_SUPPORT, ) diff --git a/src/dander/providers/redshift/transform.py b/src/dander/providers/redshift/transform.py index f09feea0..7cfc7598 100644 --- a/src/dander/providers/redshift/transform.py +++ b/src/dander/providers/redshift/transform.py @@ -8,6 +8,8 @@ from typing import TYPE_CHECKING from dander.concurrency import TargetFenceLostError +from dander.pipeline.compiler import CompiledTarget, PipelineCompileError +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError from dander.providers.redshift.config import validate_redshift_relation from dander.providers.redshift.session import ( RedshiftConnection, @@ -24,6 +26,7 @@ _schema_changes, _set_query_group, _validate_super_roles, + validate_redshift_schema, ) from dander.transform import ( SqlDialect, @@ -34,7 +37,7 @@ TransformRunResult, ) from dander.transform.model import Materialization -from dander.writer import SchemaEvolution, WriteField, WriteTarget +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -48,9 +51,12 @@ @dataclass(frozen=True, slots=True) class _RedshiftModelPlan: - model: TransformModel + name: str target: WriteTarget query: str + materialization: Materialization + unique_key: tuple[str, ...] = () + incremental_cursor: str | None = None @dataclass(frozen=True, slots=True) @@ -95,7 +101,13 @@ def build( ownership.verify() publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) ownership.verify() - self._publish(plan, publication) + _publish_plan( + plan, + publication, + connection_factory=self._connection_factory, + target_fence=self._target_fence, + statement_timeout_ms=self._statement_timeout_ms, + ) self._run_assertions(assertions, ownership=ownership) return TransformRunResult( models=tuple(model.name for model in models), @@ -141,53 +153,6 @@ def _preflight( ) return project, models, plans, assertions - def _publish(self, plan: _RedshiftModelPlan, publication: TargetFence) -> None: - target = WriteTarget( - relation=plan.target.relation_ref, - business_key=plan.target.business_key, - schema=plan.target.schema, - declared_schema=plan.target.canonical_schema, - publication_fence=publication, - ) - temporary = _temporary_name(target.relation_ref, publication) - cleanup_started = False - with open_connection(self._connection_factory) as connection: - try: - _set_query_group( - connection, - publication, - statement_timeout_ms=self._statement_timeout_ms, - ) - cleanup_started = True - execute(connection, _create_temporary_sql(plan, temporary)) - # Redshift temp tables survive commit. End CTAS and schema-inspection snapshots - # before the single destination-fenced publication transaction begins. - connection.commit() - schema_changes = _schema_changes( - connection, - target, - target.canonical_schema, - evolution=SchemaEvolution.STRICT, - ) - connection.commit() - statements = ( - (_create_target_sql(target, target.canonical_schema), ()), - *schema_changes, - *_publication_statements(plan, temporary), - ) - self._target_fence.execute_statements(connection, statements, publication) - except (TargetFenceLostError, RedshiftWriteError, TransformRunError): - raise - except Exception as error: - raise TransformRunError("Redshift transform publication failed") from error - finally: - if cleanup_started: - _drop_temporary( - connection, - temporary, - suppress_failure=sys.exc_info()[0] is not None, - ) - def _run_assertions( self, assertions: Sequence[_RedshiftAssertion], @@ -217,6 +182,121 @@ def _run_assertions( raise TransformRunError(f"Data tests failed: {', '.join(failures)}") +class RedshiftGraphRunner: + """Publish provider-neutral graph targets through Redshift's fenced DML path.""" + + target_dialect = SqlDialect.REDSHIFT + + def __init__( + self, + *, + plan: GraphExecutionPlan, + database: str, + connection_factory: RedshiftConnectionFactory, + target_fence: RedshiftTargetFence, + statement_timeout_ms: int, + ) -> None: + self._plan = plan + self._database = database + self._connection_factory = connection_factory + self._target_fence = target_fence + self._statement_timeout_ms = statement_timeout_ms + + def build( + self, + _models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Preflight every selected target, then publish each target exactly once.""" + if ownership is None or ownership.fence is None: + raise GraphRuntimeError("Redshift graph execution requires active lease ownership") + selected_ids = set(selected) if selected is not None else None + known = {target.node_id for target in self._plan.targets} + if selected_ids is not None and (unknown := sorted(selected_ids - known)): + raise GraphRuntimeError(f"Unknown graph target: {unknown[0]!r}") + targets = tuple( + target + for target in self._plan.targets + if selected_ids is None or target.node_id in selected_ids + ) + if not targets: + raise GraphRuntimeError("Graph execution selected no targets") + + plans = tuple(_graph_plan(target, database=self._database) for target in targets) + for plan in plans: + ownership.verify() + publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) + ownership.verify() + _publish_plan( + plan, + publication, + connection_factory=self._connection_factory, + target_fence=self._target_fence, + statement_timeout_ms=self._statement_timeout_ms, + ) + return TransformRunResult( + models=tuple(plan.name for plan in plans), + assertions=0, + ) + + +def _publish_plan( + plan: _RedshiftModelPlan, + publication: TargetFence, + *, + connection_factory: RedshiftConnectionFactory, + target_fence: RedshiftTargetFence, + statement_timeout_ms: int, +) -> None: + target = WriteTarget( + relation=plan.target.relation_ref, + business_key=plan.target.business_key, + schema=plan.target.schema, + declared_schema=plan.target.canonical_schema, + publication_fence=publication, + ) + temporary = _temporary_name(target.relation_ref, publication) + cleanup_started = False + with open_connection(connection_factory) as connection: + try: + _set_query_group( + connection, + publication, + statement_timeout_ms=statement_timeout_ms, + ) + cleanup_started = True + execute(connection, _create_temporary_sql(plan, temporary)) + # Redshift temp tables survive commit. End CTAS and schema-inspection snapshots + # before the single destination-fenced publication transaction begins. + connection.commit() + schema_changes = _schema_changes( + connection, + target, + target.canonical_schema, + evolution=SchemaEvolution.STRICT, + ) + connection.commit() + statements = ( + (_create_target_sql(target, target.canonical_schema), ()), + *schema_changes, + *_publication_statements(plan, temporary), + ) + target_fence.execute_statements(connection, statements, publication) + except (TargetFenceLostError, RedshiftWriteError, TransformRunError): + raise + except Exception as error: + raise TransformRunError("Redshift transform publication failed") from error + finally: + if cleanup_started: + _drop_temporary( + connection, + temporary, + suppress_failure=sys.exc_info()[0] is not None, + ) + + def _model_plan(project: TransformProject, model: TransformModel) -> _RedshiftModelPlan: if model.metadata.materialization is Materialization.VIEW: raise TransformProjectError( @@ -251,19 +331,48 @@ def _model_plan(project: TransformProject, model: TransformModel) -> _RedshiftMo cursor_field=model.metadata.incremental_cursor, snapshot_field=None, ) - return _RedshiftModelPlan(model=model, target=target, query=project.compile(model)) + return _RedshiftModelPlan( + name=model.name, + target=target, + query=project.compile(model), + materialization=model.metadata.materialization, + unique_key=tuple(model.metadata.unique_key), + incremental_cursor=model.metadata.incremental_cursor, + ) + + +def _graph_plan(compiled: CompiledTarget, *, database: str) -> _RedshiftModelPlan: + if compiled.write_mode is not WriteMode.REPLACE: + raise GraphRuntimeError(f"Redshift graph target {compiled.node_id!r} requires replace mode") + if compiled.target.relation_ref.catalog != database: + raise GraphRuntimeError( + f"Redshift graph target {compiled.node_id!r} belongs to another database" + ) + try: + validate_redshift_relation(compiled.target.relation_ref) + validate_redshift_schema(compiled.target.canonical_schema) + _validate_super_roles(compiled.target, cursor_field=None, snapshot_field=None) + query = compiled.render(SqlDialect.REDSHIFT) + except (PipelineCompileError, RedshiftWriteError, ValueError) as error: + raise GraphRuntimeError(str(error)) from error + return _RedshiftModelPlan( + name=compiled.node_id, + target=compiled.target, + query=query, + materialization=Materialization.TABLE, + ) def _create_temporary_sql(plan: _RedshiftModelPlan, temporary: str) -> str: names = tuple(field.name for field in plan.target.canonical_schema.fields) selected = ", ".join(f"source.{_quote(name)}" for name in names) - if plan.model.metadata.materialization is Materialization.TABLE: + if plan.materialization is Materialization.TABLE: return ( f"CREATE TEMP TABLE {_quote(temporary)} AS SELECT {selected} " f"FROM ({plan.query}) AS source" ) - keys = tuple(plan.model.metadata.unique_key) - cursor = plan.model.metadata.incremental_cursor + keys = plan.unique_key + cursor = plan.incremental_cursor assert keys and cursor is not None partition = ", ".join(f"source.{_quote(key)}" for key in keys) tie_breakers = tuple(name for name in sorted(names) if name not in {*keys, cursor}) @@ -287,13 +396,13 @@ def _publication_statements( names = tuple(field.name for field in plan.target.canonical_schema.fields) columns = ", ".join(_quote(name) for name in names) incoming = ", ".join(f"incoming.{_quote(name)}" for name in names) - if plan.model.metadata.materialization is Materialization.TABLE: + if plan.materialization is Materialization.TABLE: return ( (f"DELETE FROM {target}", ()), (f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {staged}", ()), ) - keys = tuple(plan.model.metadata.unique_key) - cursor = plan.model.metadata.incremental_cursor + keys = plan.unique_key + cursor = plan.incremental_cursor assert keys and cursor is not None match = " AND ".join(f"target.{_quote(key)} = incoming.{_quote(key)}" for key in keys) mutable = tuple(name for name in names if name not in keys) @@ -430,4 +539,4 @@ def _failure_count(row: object | None, assertion: str) -> int: return value -__all__ = ["RedshiftTransformRunner"] +__all__ = ["RedshiftGraphRunner", "RedshiftTransformRunner"] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 8e63868a..0f8ccd3a 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -83,9 +83,9 @@ "max_decimal_precision": 38, "max_temporal_precision": 6, "supports_transforms": true, - "supports_graphs": false, + "supports_graphs": true, "supports_target_fencing": true, - "limitations": ["JSON requires the explicit redshift/fallback=super field extension", "graph execution is not available", "only COPY transport is implemented"] + "limitations": ["JSON requires the explicit redshift/fallback=super field extension", "graph safe-cast semantics are not available", "only COPY transport is implemented"] }, { "provider": "snowflake", diff --git a/tests/pipeline/test_compiler.py b/tests/pipeline/test_compiler.py index e7251f77..7ddae57f 100644 --- a/tests/pipeline/test_compiler.py +++ b/tests/pipeline/test_compiler.py @@ -226,7 +226,6 @@ def test_graph_compiles_to_one_provider_neutral_relational_ast() -> None: assert len(tuple(compiled.query_ast.find_all(exp.CTE))) == 3 assert len(tuple(compiled.query_ast.find_all(exp.TryCast))) == 2 assert compiled.render("bigquery") == compiled.query - assert 'FROM "dander-test"."raw"."people"' in compiled.render("redshift") def test_graph_target_preserves_provider_extension_in_canonical_schema() -> None: @@ -279,7 +278,7 @@ def test_returned_graph_ast_is_an_isolated_copy() -> None: assert compiled.render("bigquery") == compiled.query -@pytest.mark.parametrize("target", ["snowflake", "postgres"]) +@pytest.mark.parametrize("target", ["snowflake", "redshift", "postgres"]) def test_graph_render_fails_when_target_cannot_preserve_safe_cast(target: str) -> None: compiled = compile_target( _linear_graph(), diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index b9ee1915..351dbed2 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -6,16 +6,19 @@ import json import sys -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import TYPE_CHECKING, Self import pytest from dander.concurrency import FencingToken, TargetFenceLostError +from dander.ingestion import Endpoint, RawField, SourceConfig +from dander.pipeline.graph import PipelineGraph +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError, plan_graph_execution from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry from dander.providers.redshift import RedshiftWarehouseConfig -from dander.providers.redshift.transform import RedshiftTransformRunner +from dander.providers.redshift.transform import RedshiftGraphRunner, RedshiftTransformRunner from dander.providers.redshift.writer import RedshiftWriteError, _delete_owned from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError from dander.warehouse import ( @@ -370,6 +373,7 @@ def test_redshift_runtime_validates_connection_and_exposes_fenced_transforms( assert runtime.capabilities.write_modes == frozenset(WriteMode) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) assert runtime.capabilities.supports_transforms is True + assert runtime.capabilities.supports_graphs is True assert isinstance( runtime.transforms.build_transform_runner(graph_plan=None, build_models=True), RedshiftTransformRunner, @@ -385,6 +389,129 @@ def verify(self) -> None: self.verifications += 1 +def _redshift_graph_plan(*, include_unsupported_target: bool = False) -> GraphExecutionPlan: + source = SourceConfig( + name="fixture", + base_url="https://example.test", + auth_strategy="none", + endpoints=[ + Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING"), + RawField(name="label", data_type="STRING"), + ], + ) + ], + ) + targets: list[dict[str, object]] = [ + { + "id": "target", + "type": "target", + "name": "Target", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": "analytics", + "table": "graph_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + } + ] + edges: list[dict[str, object]] = [ + { + "from": "records", + "to": "target", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ] + if include_unsupported_target: + targets.append( + { + "id": "unsupported", + "type": "target", + "name": "Unsupported", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": "analytics", + "table": "unsupported_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING"}, + { + "name": "label", + "type": "STRING", + "cast_to": "JSON", + "extensions": [ + { + "provider": "redshift", + "name": "fallback", + "value": "super", + } + ], + }, + ], + } + ) + edges.append( + { + "from": "records", + "to": "unsupported", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ) + graph = PipelineGraph.model_validate( + { + "name": "redshift_graph", + "nodes": [ + { + "id": "records", + "type": "source", + "name": "Records", + "config": {"connector": "fixture", "endpoint": "records"}, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + *targets, + ], + "edges": edges, + } + ) + return plan_graph_execution( + graph, + source, + endpoint_relations={ + "records": RelationRef( + catalog="analytics", + namespace="raw", + name="fixture_records", + ) + }, + ) + + def test_redshift_builds_fenced_table_and_incremental_models( tmp_path: Path, redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], @@ -514,11 +641,144 @@ def test_redshift_preflights_the_dag_and_preserves_canonical_coordinates( "good_model", ) assert '"analytics"."raw_data"."records"' in project.compile(model) - with pytest.raises(ValueError, match="graph execution is not available"): + with pytest.raises(TypeError, match="graph plan has the wrong type"): runtime.transforms.build_transform_runner(graph_plan=object(), build_models=False) assert runtime.transforms.build_transform_runner(graph_plan=None, build_models=False) is None +def test_redshift_graph_uses_canonical_plan_fencing_and_cleanup( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=_redshift_graph_plan(), + build_models=True, + ) + assert isinstance(runner, RedshiftGraphRunner) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "YES"), + ("label", "character varying", 65_535, None, None, "YES"), + ] + backend.transform_source_rows = [{"id": "fresh", "label": "from-graph"}] + ownership = _ownership("run-graph", 5) + + result = runner.build(tmp_path, ownership=ownership) + + assert result.models == ("target",) + assert result.assertions == 0 + sql = [statement for _, statement, _ in backend.statements] + staged = next( + statement + for statement in sql + if statement.startswith("CREATE TEMP TABLE") and " AS SELECT" in statement + ) + assert 'FROM "analytics"."raw"."fixture_records"' in staged + assert "`" not in staged + assert any( + statement.startswith("DELETE FROM") and '"analytics"."graph_records"' in statement + for statement in sql + ) + assert any( + statement.startswith("INSERT INTO") and '"analytics"."graph_records"' in statement + for statement in sql + ) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in sql) + assert ownership.verifications == 2 + assert backend.transform_rows == {"fresh": {"id": "fresh", "label": "from-graph"}} + + +def test_redshift_graph_preflights_every_selected_target_before_mutation( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=_redshift_graph_plan(include_unsupported_target=True), + build_models=True, + ) + assert isinstance(runner, RedshiftGraphRunner) + before = list(backend.statements) + + with pytest.raises(GraphRuntimeError, match="safe-cast semantics"): + runner.build(tmp_path, ownership=_ownership("run-preflight-graph", 6)) + + assert backend.statements == before + + +def test_redshift_graph_rejects_invalid_target_contracts_before_mutation( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + plan = _redshift_graph_plan() + compiled = plan.targets[0] + other_target = WriteTarget( + relation=RelationRef(catalog="other", namespace="analytics", name="graph_records"), + business_key=compiled.target.business_key, + schema=compiled.target.schema, + ) + invalid_plans = ( + ( + replace(plan, targets=(replace(compiled, write_mode=WriteMode.SCD1),)), + "requires replace mode", + ), + ( + replace(plan, targets=(replace(compiled, target=other_target),)), + "belongs to another database", + ), + ) + before = list(backend.statements) + + for invalid, message in invalid_plans: + runner = runtime.transforms.build_transform_runner( + graph_plan=invalid, + build_models=True, + ) + assert isinstance(runner, RedshiftGraphRunner) + with pytest.raises(GraphRuntimeError, match=message): + runner.build(tmp_path, ownership=_ownership("run-invalid-graph", 7)) + + assert backend.statements == before + + +def test_redshift_graph_selection_and_stale_fence_fail_closed( + tmp_path: Path, + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + runner = runtime.transforms.build_transform_runner( + graph_plan=_redshift_graph_plan(), + build_models=True, + ) + assert isinstance(runner, RedshiftGraphRunner) + before = list(backend.statements) + with pytest.raises(GraphRuntimeError, match="Unknown graph target"): + runner.build(tmp_path, selected=["missing"], ownership=_ownership("run-missing", 8)) + with pytest.raises(GraphRuntimeError, match="selected no targets"): + runner.build(tmp_path, selected=[], ownership=_ownership("run-empty", 9)) + assert backend.statements == before + + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "YES"), + ("label", "character varying", 65_535, None, None, "YES"), + ] + backend.transform_source_rows = [{"id": "blocked", "label": "stale"}] + backend.fence_touch_rowcount = 0 + with pytest.raises(TargetFenceLostError, match="lost before publication"): + runner.build(tmp_path, ownership=_ownership("run-stale-graph", 10)) + + attempted = [statement for _, statement, _ in backend.statements[len(before) :]] + assert any(statement.startswith("CREATE TEMP TABLE") for statement in attempted) + assert not any(statement.startswith("DELETE FROM") for statement in attempted) + assert not any( + statement.startswith("INSERT INTO") and "graph_records" in statement + for statement in attempted + ) + assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in attempted) + assert backend.transform_rows == {} + + def test_redshift_transform_requires_ownership_and_fails_closed_after_staging( tmp_path: Path, redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], From 5c68129d2710fb3f50d124fcc365152dc0b0d199 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 06:38:31 -0400 Subject: [PATCH 098/163] Add safe Redshift operation telemetry --- HANDOFF.md | 39 ++-- docs/decisions.md | 11 ++ docs/redshift.md | 8 + src/dander/providers/redshift/session.py | 176 +++++++++++++++++- src/dander/providers/redshift/transform.py | 123 +++++++++--- src/dander/providers/redshift/writer.py | 102 +++++++++- .../test_redshift_warehouse_runtime.py | 152 ++++++++++++++- 7 files changed, 559 insertions(+), 52 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index e59c0410..2fddb635 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,39 @@ ## Finished -- Added `RedshiftGraphRunner` for the existing provider-neutral `GraphExecutionPlan`. -- Rendered canonical graph ASTs as Redshift SQL without adding another graph schema. -- Reused Redshift's run-scoped CTAS, exact target-fence transaction, stable table replacement, and - cleanup path. -- Preflighted every selected target before the first fence claim or provider connection. -- Updated the packaged capability report and experimental-provider documentation. +- Added Redshift COPY, transform, graph, and assertion operation telemetry. +- Enriched committed COPY/CTAS IDs from bounded same-session Redshift system-view queries. +- Kept fenced multi-statement publications unattributed rather than reporting a misleading ID. +- Made history denial, delay, or malformed rows best-effort and rollback-safe before cleanup. +- Documented the experimental telemetry contract and decision boundary. ## Try It -Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py tests/test_compatibility.py`. +Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py`. ## Checks -- Ruff formatting and lint passed across 328 files; strict mypy passed across 304 files. -- Full pytest passed: 1,150 tests with 13 intentional skips. -- Wheel and source archive passed distribution inspection; the wheel installed outside the - checkout and generated and validated a source-free project pinned to `0.8.0rc8`. -- Independent review's safe-cast blocker was corrected with a pre-provider-I/O regression. +- Ruff passed across 328 files; strict mypy passed across 304 files. +- Full pytest passed: 1,153 tests with 13 intentional skips. +- Terraform format, GCP/AWS validation, and AWS module tests passed without an apply. +- Wheel/sdist inspection, source-free installation/generation, and dependency audit passed. +- Local container and PostgreSQL-service checks remain for protected Linux CI. ## Decisions -- Redshift consumes the canonical compiled graph rather than defining a provider graph model. -- Graph targets remain replace-only and publish through the proven fenced table path. -- Views, telemetry expansion, and live AWS qualification remain separate Phase 5 work. +- Capture Redshift query IDs only after committed COPY and CTAS operations. +- Read only numeric/system metadata; never query SQL text, errors, S3 sources, or record data. +- Roll back every telemetry-only transaction so observability cannot poison cleanup. ## Remaining -- Push a focused PR and require all protected CI checks before merge. -- Continue Redshift telemetry and live-profile work without beginning Azure. +- Push a focused PR and require protected CI before merge. +- Continue Phase 5 with direct Redshift transport and live-profile work after this slice lands. +- Keep paid AWS mutation gated by separate explicit approval. +- Perform Phase 5.5 only after the full Snowflake/Redshift gate closes; do not begin Azure. ## Review First +- `src/dander/providers/redshift/session.py` +- `src/dander/providers/redshift/writer.py` - `src/dander/providers/redshift/transform.py` -- `src/dander/providers/redshift/runtime.py` -- `tests/providers/test_redshift_warehouse_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 538fa260..785148c2 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1046,3 +1046,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Graphs remain single-connector and replace-only. Redshift safe casts fail preflight until an exact lowering exists. Views, telemetry expansion, live AWS proof, and support promotion remain separate Phase 5 slices. + +## 2026-08-10 — Redshift telemetry never competes with correctness transactions + +- **Attribution:** Dander captures `PG_LAST_QUERY_ID()` only after committed COPY and CTAS work. + Multi-statement fenced publication and assertion operations retain local counters without a + potentially misleading query ID. +- **Enrichment:** One bounded same-session lookup reads numeric counters from `SYS_QUERY_HISTORY`, + step-level `SYS_QUERY_DETAIL`, and `SYS_LOAD_HISTORY`; SQL text, errors, and S3 data sources are + never selected. +- **Failure boundary:** Every telemetry-only read rolls back its own transaction, including denied + or malformed history, so observability cannot poison target cleanup or fail completed work. diff --git a/docs/redshift.md b/docs/redshift.md index 87c62b73..f504e16c 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -60,6 +60,9 @@ COPY role must read the dedicated same-region staging prefix. - The complete selected transform DAG preflights before any provider connection or mutation. - Executable graph targets render the existing provider-neutral relational AST as Redshift SQL, preflight as one set, and publish through the same fenced table-replacement path. +- Completed COPY and CTAS operations emit provider-neutral telemetry. A bounded, same-session + lookup enriches those operations with Redshift queue/execution time, bytes, loaded rows, spill + blocks, service class, compute type, and COPY job ID when the system views are available. ## Deliberate limits @@ -72,6 +75,11 @@ preflight because Redshift cannot yet preserve their canonical semantics. Views, proof, infrastructure provisioning, and support promotion remain separate work. Use `catalog.provider: none` for this experimental path. +Telemetry is best-effort and never carries SQL text, error text, S3 locations, or record data. +Multi-statement fenced publication and assertion operations report local duration and affected-row +counters without claiming a potentially incorrect Redshift query ID. Missing or delayed system +history leaves those base counters unchanged. + Declare the fallback on the field; bare canonical JSON continues to fail closed: ```yaml diff --git a/src/dander/providers/redshift/session.py b/src/dander/providers/redshift/session.py index a3e2e4c4..7c1f11fe 100644 --- a/src/dander/providers/redshift/session.py +++ b/src/dander/providers/redshift/session.py @@ -3,12 +3,17 @@ from __future__ import annotations from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Literal, Protocol if TYPE_CHECKING: from collections.abc import Iterator, Sequence + from dander.telemetry import OperationTelemetry + +_QUERY_HISTORY_ID_LIMIT = 1_000 +_REDSHIFT_BLOCK_BYTES = 1_024 * 1_024 + class RedshiftCursor(Protocol): rowcount: int @@ -37,6 +42,7 @@ class RedshiftStatementResult: rowcount: int row: object | None = None rows: tuple[object, ...] = () + query_id: str | None = None @contextmanager @@ -66,10 +72,178 @@ def execute( cursor.close() +def capture_last_query_id(connection: RedshiftConnection) -> str | None: + """Capture a completed compute query ID after its transaction is already committed.""" + try: + row = execute(connection, "SELECT pg_last_query_id()", fetch="one").row + value = row[0] if isinstance(row, (tuple, list)) and row else None + valid = isinstance(value, int) and not isinstance(value, bool) and value >= 0 + return str(value) if valid else None + except Exception: + return None + finally: + # The lookup itself is telemetry-only. End it even when a denied lookup aborted the + # transaction so later correctness or cleanup SQL starts from a usable session. + _rollback_telemetry_transaction(connection) + + +def enrich_operation_telemetry( + connection: RedshiftConnection, + operations: Sequence[OperationTelemetry], +) -> tuple[OperationTelemetry, ...]: + """Best-effort enrichment for already captured COPY or CTAS query IDs.""" + requested = _recent_query_ids(operations) + if not requested: + return tuple(operations) + placeholders = ", ".join("%s" for _query_id in requested) + statement = ( + "WITH recent AS (SELECT query_id, queue_time, execution_time, " + "TRIM(service_class_name) AS service_class_name, " + "TRIM(compute_type) AS compute_type FROM sys_query_history " + f"WHERE status = 'success' AND query_id IN ({placeholders})), " + "detail AS (SELECT query_id, SUM(input_bytes) AS input_bytes, " + "SUM(spilled_block_local_disk) AS local_spill_blocks, " + "SUM(spilled_block_remote_disk) AS remote_spill_blocks FROM sys_query_detail " + "WHERE metrics_level = 'Step' AND query_id IN (SELECT query_id FROM recent) " + "GROUP BY query_id), loads AS (SELECT query_id, loaded_rows, loaded_bytes, " + "source_file_bytes, copy_job_id FROM sys_load_history " + "WHERE status = 'completed' AND query_id IN (SELECT query_id FROM recent)) " + "SELECT recent.query_id, recent.queue_time, recent.execution_time, " + "recent.service_class_name, recent.compute_type, detail.input_bytes, " + "detail.local_spill_blocks, detail.remote_spill_blocks, " + "loads.loaded_rows, loads.loaded_bytes, loads.source_file_bytes, loads.copy_job_id " + "FROM recent LEFT JOIN detail USING (query_id) LEFT JOIN loads USING (query_id)" + ) + try: + rows = execute(connection, statement, requested, fetch="all").rows + metrics = { + parsed.query_id: parsed for row in rows if (parsed := _query_metrics(row)) is not None + } + return tuple(_enrich_operation(operation, metrics) for operation in operations) + except Exception: + return tuple(operations) + finally: + # A read failure aborts Redshift's current transaction. Always clear that telemetry-only + # transaction before the caller performs temporary-table cleanup. + _rollback_telemetry_transaction(connection) + + +@dataclass(frozen=True, slots=True) +class _RedshiftQueryMetrics: + query_id: str + queue_duration_ms: int + execution_duration_ms: int + resource_name: str | None + resource_size: str | None + bytes_processed: int + spill_bytes: int + loaded_rows: int + loaded_bytes: int + source_file_bytes: int + copy_job_id: str | None + + +def _recent_query_ids(operations: Sequence[OperationTelemetry]) -> tuple[int, ...]: + requested: list[int] = [] + seen: set[int] = set() + for operation in reversed(operations): + raw = operation.query_id + if raw is None: + continue + try: + query_id = int(raw) + except (TypeError, ValueError): + continue + if query_id < 0 or query_id in seen: + continue + seen.add(query_id) + requested.append(query_id) + if len(requested) == _QUERY_HISTORY_ID_LIMIT: + break + requested.reverse() + return tuple(requested) + + +def _query_metrics(row: object) -> _RedshiftQueryMetrics | None: + if not isinstance(row, (tuple, list)) or len(row) != 12: + return None + query_id = _nonnegative_integer(row[0]) + queue_us = _nonnegative_integer(row[1]) + execution_us = _nonnegative_integer(row[2]) + if query_id is None or queue_us is None or execution_us is None: + return None + input_bytes = _optional_counter(row[5]) + local_spill = _optional_counter(row[6]) + remote_spill = _optional_counter(row[7]) + copy_job = _optional_counter(row[11]) + return _RedshiftQueryMetrics( + query_id=str(query_id), + queue_duration_ms=_microseconds_to_milliseconds(queue_us), + execution_duration_ms=_microseconds_to_milliseconds(execution_us), + resource_name=_optional_text(row[3]), + resource_size=_optional_text(row[4]), + bytes_processed=input_bytes, + spill_bytes=(local_spill + remote_spill) * _REDSHIFT_BLOCK_BYTES, + loaded_rows=_optional_counter(row[8]), + loaded_bytes=_optional_counter(row[9]), + source_file_bytes=_optional_counter(row[10]), + copy_job_id=str(copy_job) if copy_job > 0 else None, + ) + + +def _enrich_operation( + operation: OperationTelemetry, + metrics: dict[str, _RedshiftQueryMetrics], +) -> OperationTelemetry: + if operation.query_id is None or (query := metrics.get(operation.query_id)) is None: + return operation + return replace( + operation, + rows_written=query.loaded_rows or operation.rows_written, + bytes_read=query.source_file_bytes or operation.bytes_read, + bytes_written=query.loaded_bytes or operation.bytes_written, + bytes_processed=query.bytes_processed, + job_id=query.copy_job_id or operation.job_id, + queue_duration_ms=query.queue_duration_ms, + execution_duration_ms=query.execution_duration_ms, + spill_bytes=query.spill_bytes, + resource_name=query.resource_name or operation.resource_name, + resource_size=query.resource_size or operation.resource_size, + ) + + +def _nonnegative_integer(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _optional_counter(value: object) -> int: + return _nonnegative_integer(value) or 0 + + +def _optional_text(value: object) -> str | None: + return value.strip() if isinstance(value, str) and value.strip() else None + + +def _microseconds_to_milliseconds(value: int) -> int: + return (value + 999) // 1_000 + + +def _rollback_telemetry_transaction(connection: RedshiftConnection) -> None: + try: + connection.rollback() + except Exception: + # Telemetry is best-effort and must never obscure a completed warehouse operation. + return + + __all__ = [ "RedshiftConnection", "RedshiftConnectionFactory", "RedshiftStatementResult", + "capture_last_query_id", + "enrich_operation_telemetry", "execute", "open_connection", ] diff --git a/src/dander/providers/redshift/transform.py b/src/dander/providers/redshift/transform.py index 7cfc7598..b5286658 100644 --- a/src/dander/providers/redshift/transform.py +++ b/src/dander/providers/redshift/transform.py @@ -4,7 +4,8 @@ import hashlib import sys -from dataclasses import dataclass +from dataclasses import dataclass, replace +from time import perf_counter_ns from typing import TYPE_CHECKING from dander.concurrency import TargetFenceLostError @@ -14,6 +15,9 @@ from dander.providers.redshift.session import ( RedshiftConnection, RedshiftConnectionFactory, + RedshiftStatementResult, + capture_last_query_id, + enrich_operation_telemetry, execute, open_connection, ) @@ -28,6 +32,7 @@ _validate_super_roles, validate_redshift_schema, ) +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import ( SqlDialect, TransformModel, @@ -40,7 +45,7 @@ from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Callable, Iterable, Sequence from pathlib import Path from dander.concurrency import OwnershipGuard, TargetFence @@ -97,21 +102,25 @@ def build( if ownership is None or ownership.fence is None: raise TransformRunError("Redshift hosted transforms require active lease ownership") _project, models, plans, assertions = self._preflight(models_dir, selected=selected) + telemetry: list[OperationTelemetry] = [] for plan in plans: ownership.verify() publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) ownership.verify() - _publish_plan( - plan, - publication, - connection_factory=self._connection_factory, - target_fence=self._target_fence, - statement_timeout_ms=self._statement_timeout_ms, + telemetry.extend( + _publish_plan( + plan, + publication, + connection_factory=self._connection_factory, + target_fence=self._target_fence, + statement_timeout_ms=self._statement_timeout_ms, + ) ) - self._run_assertions(assertions, ownership=ownership) + telemetry.extend(self._run_assertions(assertions, ownership=ownership)) return TransformRunResult( models=tuple(model.name for model in models), assertions=len(assertions), + telemetry=tuple(telemetry), ) def test( @@ -122,10 +131,11 @@ def test( ) -> TransformRunResult: """Run assertions against already materialized Redshift relations.""" _project, models, _plans, assertions = self._preflight(models_dir, selected=selected) - self._run_assertions(assertions) + telemetry = self._run_assertions(assertions) return TransformRunResult( models=tuple(model.name for model in models), assertions=len(assertions), + telemetry=telemetry, ) def _preflight( @@ -158,28 +168,38 @@ def _run_assertions( assertions: Sequence[_RedshiftAssertion], *, ownership: OwnershipGuard | None = None, - ) -> None: + ) -> tuple[OperationTelemetry, ...]: failures: list[str] = [] + telemetry: list[OperationTelemetry] = [] with open_connection(self._connection_factory) as connection: for assertion in assertions: if ownership is not None: ownership.verify() try: - row = execute( + result, duration_ms = _timed_call( + execute, connection, assertion.statement, assertion.parameters, fetch="one", - ).row + ) except Exception as error: raise TransformRunError( f"Redshift assertion execution failed: {assertion.name}" ) from error - if _failure_count(row, assertion.name) > 0: + telemetry.append( + _operation_telemetry( + result, + operation=TelemetryOperation.TEST, + duration_ms=duration_ms, + ) + ) + if _failure_count(result.row, assertion.name) > 0: failures.append(assertion.name) connection.commit() if failures: raise TransformRunError(f"Data tests failed: {', '.join(failures)}") + return tuple(telemetry) class RedshiftGraphRunner: @@ -225,20 +245,24 @@ def build( raise GraphRuntimeError("Graph execution selected no targets") plans = tuple(_graph_plan(target, database=self._database) for target in targets) + telemetry: list[OperationTelemetry] = [] for plan in plans: ownership.verify() publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) ownership.verify() - _publish_plan( - plan, - publication, - connection_factory=self._connection_factory, - target_fence=self._target_fence, - statement_timeout_ms=self._statement_timeout_ms, + telemetry.extend( + _publish_plan( + plan, + publication, + connection_factory=self._connection_factory, + target_fence=self._target_fence, + statement_timeout_ms=self._statement_timeout_ms, + ) ) return TransformRunResult( models=tuple(plan.name for plan in plans), assertions=0, + telemetry=tuple(telemetry), ) @@ -249,7 +273,7 @@ def _publish_plan( connection_factory: RedshiftConnectionFactory, target_fence: RedshiftTargetFence, statement_timeout_ms: int, -) -> None: +) -> tuple[OperationTelemetry, ...]: target = WriteTarget( relation=plan.target.relation_ref, business_key=plan.target.business_key, @@ -259,6 +283,7 @@ def _publish_plan( ) temporary = _temporary_name(target.relation_ref, publication) cleanup_started = False + telemetry: list[OperationTelemetry] = [] with open_connection(connection_factory) as connection: try: _set_query_group( @@ -267,10 +292,22 @@ def _publish_plan( statement_timeout_ms=statement_timeout_ms, ) cleanup_started = True - execute(connection, _create_temporary_sql(plan, temporary)) + staged, duration_ms = _timed_call( + execute, + connection, + _create_temporary_sql(plan, temporary), + ) # Redshift temp tables survive commit. End CTAS and schema-inspection snapshots # before the single destination-fenced publication transaction begins. connection.commit() + staged = replace(staged, query_id=capture_last_query_id(connection)) + telemetry.append( + _operation_telemetry( + staged, + operation=TelemetryOperation.TRANSFORM, + duration_ms=duration_ms, + ) + ) schema_changes = _schema_changes( connection, target, @@ -283,7 +320,17 @@ def _publish_plan( *schema_changes, *_publication_statements(plan, temporary), ) - target_fence.execute_statements(connection, statements, publication) + started = perf_counter_ns() + results = target_fence.execute_statements(connection, statements, publication) + publication_duration_ms = _elapsed_milliseconds(started) + telemetry.append( + _operation_telemetry( + results[-1], + operation=TelemetryOperation.TRANSFORM, + duration_ms=publication_duration_ms, + ) + ) + completed = enrich_operation_telemetry(connection, telemetry) except (TargetFenceLostError, RedshiftWriteError, TransformRunError): raise except Exception as error: @@ -295,6 +342,7 @@ def _publish_plan( temporary, suppress_failure=sys.exc_info()[0] is not None, ) + return completed def _model_plan(project: TransformProject, model: TransformModel) -> _RedshiftModelPlan: @@ -539,4 +587,33 @@ def _failure_count(row: object | None, assertion: str) -> int: return value +def _timed_call( + function: Callable[..., RedshiftStatementResult], + *arguments: object, + **keywords: object, +) -> tuple[RedshiftStatementResult, int]: + started = perf_counter_ns() + result = function(*arguments, **keywords) + return result, _elapsed_milliseconds(started) + + +def _elapsed_milliseconds(started: int) -> int: + return max((perf_counter_ns() - started) // 1_000_000, 0) + + +def _operation_telemetry( + result: RedshiftStatementResult, + *, + operation: TelemetryOperation, + duration_ms: int, +) -> OperationTelemetry: + return OperationTelemetry( + provider="redshift", + operation=operation, + duration_ms=duration_ms, + rows_affected=max(result.rowcount, 0), + query_id=result.query_id, + ) + + __all__ = ["RedshiftGraphRunner", "RedshiftTransformRunner"] diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py index e0572b29..edc8a31b 100644 --- a/src/dander/providers/redshift/writer.py +++ b/src/dander/providers/redshift/writer.py @@ -7,9 +7,10 @@ import hashlib import json import tempfile -from collections.abc import Mapping -from dataclasses import dataclass +from collections.abc import Callable, Mapping +from dataclasses import dataclass, replace from pathlib import Path +from time import perf_counter_ns from typing import TYPE_CHECKING, Protocol from uuid import uuid4 @@ -18,9 +19,13 @@ from dander.providers.redshift.session import ( RedshiftConnection, RedshiftConnectionFactory, + RedshiftStatementResult, + capture_last_query_id, + enrich_operation_telemetry, execute, open_connection, ) +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.warehouse import ( CanonicalField, CanonicalType, @@ -32,7 +37,13 @@ normalize_staging_record, staging_logical_size, ) -from dander.writer import SchemaEvolution, WriteMode, WritePattern, WriteTarget +from dander.writer import ( + SchemaEvolution, + WriteMode, + WritePattern, + WriteTarget, + WriteTransport, +) if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -87,6 +98,12 @@ class RedshiftPublication: affected_statement_index: int +@dataclass(frozen=True, slots=True) +class _LoadOutcome: + affected: int + operations: tuple[OperationTelemetry, ...] + + class RedshiftStagedWriter(WritePattern): """Upload bounded Parquet parts and publish one selected logical write mode.""" @@ -120,8 +137,16 @@ def __init__( WriteMode.SNAPSHOT, } self.accepts_streaming_input = not self.supports_batched_writes + self._telemetry: list[OperationTelemetry] = [] + + def drain_telemetry(self) -> tuple[OperationTelemetry, ...]: + """Return completed COPY/publication operations exactly once.""" + operations = tuple(self._telemetry) + self._telemetry.clear() + return operations def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: + self._telemetry.clear() _validate_target( target, database=self._database, @@ -173,7 +198,9 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> ), staged_schema, ) - return self._publish(target, publication, schema, staged_schema, manifest) + outcome = self._publish(target, publication, schema, staged_schema, manifest) + self._telemetry.extend(outcome.operations) + return outcome.affected def _publish( self, @@ -182,7 +209,7 @@ def _publish( target_schema: RelationSchema, staged_schema: RelationSchema, manifest: StagingManifest, - ) -> int: + ) -> _LoadOutcome: _validate_bucket_region(self._s3, self._staging.bucket, self._staging.region) remote_id = hashlib.sha256( f"{publication.target_id}:{publication.run_id}:{uuid4().hex}".encode() @@ -190,6 +217,9 @@ def _publish( remote_prefix = f"{self._staging.prefix}/{remote_id}" temp_table = f"dander_stage_{remote_id}" uploaded: list[str] = [] + operations: list[OperationTelemetry] = [] + loaded: RedshiftStatementResult | None = None + load_duration_ms = 0 with open_connection(self._connection_factory) as connection: publication_failed = True try: @@ -211,7 +241,8 @@ def _publish( Body=_copy_manifest(self._staging.bucket, remote_prefix, manifest), ) execute(connection, _temporary_table_sql(temp_table, staged_schema)) - execute( + loaded, load_duration_ms = _timed_call( + execute, connection, _copy_sql( temp_table, @@ -223,6 +254,20 @@ def _publish( # COPY and SET open an implicit transaction. Close it while retaining the # session-scoped temp table so the fenced publication owns one transaction. connection.commit() + if manifest.rows: + assert loaded is not None + loaded = replace(loaded, query_id=capture_last_query_id(connection)) + operations.append( + _operation_telemetry( + loaded, + operation=TelemetryOperation.LOAD, + duration_ms=load_duration_ms, + rows_written=manifest.rows, + bytes_written=sum( + artifact.compressed_bytes for artifact in manifest.artifacts + ), + ) + ) plan = _publication_plan( connection, target, @@ -238,16 +283,26 @@ def _publish( # The schema read also opens an implicit transaction. End that read-only # snapshot before beginning the one fenced publication transaction. connection.commit() + started = perf_counter_ns() results = self._target_fence.execute_statements( connection, plan.statements, publication, ) + publication_duration_ms = _elapsed_milliseconds(started) affected_rows = results[plan.affected_statement_index].rowcount if affected_rows < 0: raise RedshiftWriteError("Redshift publication did not report affected rows") + operations.append( + _operation_telemetry( + results[plan.affected_statement_index], + operation=TelemetryOperation.QUERY, + duration_ms=publication_duration_ms, + ) + ) + completed = enrich_operation_telemetry(connection, operations) publication_failed = False - return affected_rows + return _LoadOutcome(affected_rows, completed) except (RedshiftWriteError, TargetFenceLostError): raise except Exception as error: @@ -318,6 +373,39 @@ def default_staging_settings( ) +def _timed_call( + function: Callable[..., RedshiftStatementResult], + *arguments: object, +) -> tuple[RedshiftStatementResult, int]: + started = perf_counter_ns() + result = function(*arguments) + return result, _elapsed_milliseconds(started) + + +def _elapsed_milliseconds(started: int) -> int: + return max((perf_counter_ns() - started) // 1_000_000, 0) + + +def _operation_telemetry( + result: RedshiftStatementResult, + *, + operation: TelemetryOperation, + duration_ms: int, + rows_written: int = 0, + bytes_written: int = 0, +) -> OperationTelemetry: + return OperationTelemetry( + provider="redshift", + operation=operation, + duration_ms=duration_ms, + rows_written=rows_written, + rows_affected=max(result.rowcount, 0), + bytes_written=bytes_written, + query_id=result.query_id, + transport=WriteTransport.COPY, + ) + + def _with_ordinals( records: Iterable[Mapping[str, object]], *, diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index 351dbed2..e2d1c4dc 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -18,8 +18,10 @@ from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError, plan_graph_execution from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry from dander.providers.redshift import RedshiftWarehouseConfig +from dander.providers.redshift.session import enrich_operation_telemetry from dander.providers.redshift.transform import RedshiftGraphRunner, RedshiftTransformRunner from dander.providers.redshift.writer import RedshiftWriteError, _delete_owned +from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.transform import SqlDialect, TransformProject, TransformProjectError, TransformRunError from dander.warehouse import ( ProviderExtension, @@ -53,6 +55,9 @@ class _FakeRedshift: transform_source_rows: list[dict[str, object]] | None = None transform_rows: dict[str, dict[str, object]] = field(default_factory=dict) assertion_failure_count: int = 0 + query_counter: int = 100 + telemetry_rows: list[object] = field(default_factory=list) + telemetry_error: bool = False def connect(self) -> _FakeConnection: self.connections += 1 @@ -65,6 +70,8 @@ def __init__(self, backend: _FakeRedshift, connection_id: int) -> None: self.connection_id = connection_id self.autocommit = True self.in_transaction = False + self.aborted = False + self.last_query_id: int | None = None self.temporary_rows: list[dict[str, object]] = [] def cursor(self) -> _FakeCursor: @@ -72,10 +79,12 @@ def cursor(self) -> _FakeCursor: def commit(self) -> None: self.in_transaction = False + self.aborted = False self.backend.commits += 1 def rollback(self) -> None: self.in_transaction = False + self.aborted = False self.backend.rollbacks += 1 def close(self) -> None: @@ -91,14 +100,28 @@ def __init__(self, connection: _FakeConnection) -> None: self._rows: list[object] = [] def execute(self, command: str, args: Sequence[object] | None = None) -> Self: + if self.connection.aborted: + raise RuntimeError("Redshift transaction is aborted until rollback") parameters = tuple(args or ()) compact = " ".join(command.split()) if compact == "BEGIN" and self.connection.in_transaction: raise AssertionError("Redshift writer attempted to nest a transaction") self.connection.in_transaction = True self.backend.statements.append((self.connection.connection_id, compact, parameters)) + if compact.startswith("COPY ") or ( + compact.startswith("CREATE TEMP TABLE") and " AS SELECT" in compact + ): + self.backend.query_counter += 1 + self.connection.last_query_id = self.backend.query_counter if compact == "SELECT current_database(), current_user": self._row = ("analytics", "dander_user") + elif compact == "SELECT pg_last_query_id()": + self._row = (self.connection.last_query_id,) + elif compact.startswith("WITH recent AS (SELECT query_id"): + if self.backend.telemetry_error: + self.connection.aborted = True + raise RuntimeError("system view denied") + self._rows = list(self.backend.telemetry_rows) elif compact.startswith('SELECT "authority_id"'): self._row = self.backend.claim elif compact.startswith("INSERT INTO") and "dander_target_commits" in compact: @@ -540,6 +563,16 @@ def test_redshift_builds_fenced_table_and_incremental_models( assert result.models == ("table_model",) assert result.assertions == 4 + assert [operation.operation for operation in result.telemetry] == [ + TelemetryOperation.TRANSFORM, + TelemetryOperation.TRANSFORM, + TelemetryOperation.TEST, + TelemetryOperation.TEST, + TelemetryOperation.TEST, + TelemetryOperation.TEST, + ] + assert result.telemetry[0].query_id == "101" + assert all(operation.query_id is None for operation in result.telemetry[1:]) assert backend.transform_rows == {"fresh": {"id": "fresh", "label": "newer", "updated_at": 1}} sql = [statement for _, statement, _ in backend.statements] assert any( @@ -667,6 +700,12 @@ def test_redshift_graph_uses_canonical_plan_fencing_and_cleanup( assert result.models == ("target",) assert result.assertions == 0 + assert [operation.operation for operation in result.telemetry] == [ + TelemetryOperation.TRANSFORM, + TelemetryOperation.TRANSFORM, + ] + assert result.telemetry[0].query_id is not None + assert result.telemetry[1].query_id is None sql = [statement for _, statement, _ in backend.statements] staged = next( statement @@ -812,7 +851,8 @@ def test_redshift_transform_requires_ownership_and_fails_closed_after_staging( for statement in attempted ) assert any(statement.startswith("DROP TABLE IF EXISTS") for statement in attempted) - assert backend.rollbacks == rollbacks + 1 + # Query-ID capture ends its telemetry-only transaction before the failed fence rolls back. + assert backend.rollbacks == rollbacks + 2 assert backend.transform_rows == {} @@ -1017,6 +1057,113 @@ def test_redshift_stages_bounded_parts_merges_deterministically_and_cleans( assert sql[touch_indexes[0] - 1].startswith("LOCK ") +def test_redshift_copy_telemetry_is_enriched_without_sensitive_history( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + backend.telemetry_rows = [ + (101, 1_500, 2_001, "service-class-5", "primary", 1_000, 1, 2, 3, 222, 333, 55) + ] + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=3, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + assert ( + writer.write( + ( + {"id": "one", "label": "first"}, + {"id": "two", "label": "second"}, + {"id": "three", "label": "third"}, + ), + _target(runtime), + ) + == 1 + ) + + operations = writer.drain_telemetry() + assert [operation.operation for operation in operations] == [ + TelemetryOperation.LOAD, + TelemetryOperation.QUERY, + ] + loaded, published = operations + assert loaded.query_id == "101" + assert loaded.rows_written == 3 + assert loaded.bytes_read == 333 + assert loaded.bytes_written == 222 + assert loaded.bytes_processed == 1_000 + assert loaded.queue_duration_ms == 2 + assert loaded.execution_duration_ms == 3 + assert loaded.spill_bytes == 3 * 1_024 * 1_024 + assert loaded.job_id == "55" + assert loaded.resource_name == "service-class-5" + assert loaded.resource_size == "primary" + assert loaded.transport is WriteTransport.COPY + assert published.query_id is None + assert published.rows_affected == 1 + assert writer.drain_telemetry() == () + history_statement, history_parameters = next( + (statement, parameters) + for _, statement, parameters in backend.statements + if statement.startswith("WITH recent AS (SELECT query_id") + ) + assert history_parameters == (101,) + assert "metrics_level = 'Step'" in history_statement + assert "query_text" not in history_statement.casefold() + assert "error" not in history_statement.casefold() + assert "data_source" not in history_statement.casefold() + + +def test_redshift_telemetry_failure_rolls_back_before_staging_cleanup( + redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], +) -> None: + runtime, backend, _s3, _root = redshift_runtime + backend.telemetry_error = True + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + assert writer.write(({"id": "one", "label": "first"},), _target(runtime)) == 1 + + assert [operation.operation for operation in writer.drain_telemetry()] == [ + TelemetryOperation.LOAD, + TelemetryOperation.QUERY, + ] + assert backend.rollbacks >= 2 + assert any( + statement.startswith("DROP TABLE IF EXISTS") for _, statement, _ in backend.statements + ) + + +def test_redshift_history_enrichment_is_bounded_and_ignores_malformed_metrics() -> None: + backend = _FakeRedshift(telemetry_rows=[("bad", -1)]) + connection = backend.connect() + operations = tuple( + OperationTelemetry( + provider="redshift", + operation=TelemetryOperation.QUERY, + query_id=str(query_id), + ) + for query_id in range(1_002) + ) + + assert enrich_operation_telemetry(connection, operations) == operations + + history_statement, history_parameters = next( + (statement, parameters) + for _, statement, parameters in backend.statements + if statement.startswith("WITH recent AS (SELECT query_id") + ) + assert len(history_parameters) == 1_000 + assert history_parameters[0] == 2 + assert history_parameters[-1] == 1_001 + assert "metrics_level = 'Step'" in history_statement + assert backend.rollbacks == 1 + + @pytest.mark.parametrize( ("mode", "field_name", "expected_sql"), [ @@ -1434,7 +1581,8 @@ def test_redshift_lost_publication_rolls_back_and_removes_owned_staging( with pytest.raises(TargetFenceLostError, match="lost before publication"): writer.write(({"id": "one", "label": "blocked"},), target) - assert backend.rollbacks == rollback_before + 1 + # Query-ID capture ends its telemetry-only transaction before the failed fence rolls back. + assert backend.rollbacks == rollback_before + 2 assert s3.deleted and len(s3.deleted[-1]) == 2 assert not tuple(staging_root.iterdir()) assert not any(statement.startswith("MERGE INTO") for _, statement, _ in backend.statements) From 30aeb74e5d95cd42d212b0f2ca6ecf393e9b7e57 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 07:07:49 -0400 Subject: [PATCH 099/163] Add bounded direct Redshift staging --- HANDOFF.md | 36 +- docs/decisions.md | 10 + docs/redshift.md | 24 +- src/dander/providers/redshift/config.py | 10 + src/dander/providers/redshift/runtime.py | 8 +- src/dander/providers/redshift/session.py | 17 +- src/dander/providers/redshift/writer.py | 217 +++++++++-- src/dander/runtime-compatibility.json | 4 +- .../test_redshift_warehouse_runtime.py | 342 +++++++++++++++++- 9 files changed, 608 insertions(+), 60 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 2fddb635..137be4f8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Added Redshift COPY, transform, graph, and assertion operation telemetry. -- Enriched committed COPY/CTAS IDs from bounded same-session Redshift system-view queries. -- Kept fenced multi-statement publications unattributed rather than reporting a misleading ID. -- Made history denial, delay, or malformed rows best-effort and rollback-safe before cleanup. -- Documented the experimental telemetry contract and decision boundary. +- Added opt-in bounded Redshift direct staging with paired row/byte thresholds. +- Preserved zero-default COPY behavior and whole-endpoint fallback without losing the inspected prefix. +- Reused all five fenced write modes, replay identity, schema checks, SUPER parsing, and cleanup. +- Added unattributed direct telemetry from exact local counters; direct loads never contact S3. +- Updated Redshift configuration, compatibility output, decisions, and operator documentation. ## Try It -Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py`. +Set both `direct_max_rows` and `direct_max_logical_bytes` in a Redshift warehouse profile, then run +`uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py`. ## Checks - Ruff passed across 328 files; strict mypy passed across 304 files. -- Full pytest passed: 1,153 tests with 13 intentional skips. -- Terraform format, GCP/AWS validation, and AWS module tests passed without an apply. -- Wheel/sdist inspection, source-free installation/generation, and dependency audit passed. -- Local container and PostgreSQL-service checks remain for protected Linux CI. +- Full pytest passed: 1,165 tests with 13 intentional skips. +- GCP/AWS Terraform roots validated; AWS bootstrap and Fargate module tests passed without apply. +- Wheel/sdist inspection, source-free installs, runtime-all import, and generated Terraform validation passed. +- Dependency audit found no known vulnerabilities; container, secret, and PostgreSQL-service checks await protected CI. ## Decisions -- Capture Redshift query IDs only after committed COPY and CTAS operations. -- Read only numeric/system metadata; never query SQL text, errors, S3 sources, or record data. -- Roll back every telemetry-only transaction so observability cannot poison cleanup. +- COPY remains default; both direct thresholds must be positive to enable endpoint-wide selection. +- Threshold overflow falls back once to COPY; executor batching is disabled only while direct selection is enabled. +- Direct `executemany` telemetry has no query ID because Redshift exposes no honest whole-batch ID. ## Remaining - Push a focused PR and require protected CI before merge. -- Continue Phase 5 with direct Redshift transport and live-profile work after this slice lands. -- Keep paid AWS mutation gated by separate explicit approval. -- Perform Phase 5.5 only after the full Snowflake/Redshift gate closes; do not begin Azure. +- Run live Redshift/Glue acceptance only after a separately approved paid AWS plan/apply. +- Complete Snowflake and cross-warehouse live proof before Phase 5.5; do not begin Azure. ## Review First -- `src/dander/providers/redshift/session.py` - `src/dander/providers/redshift/writer.py` -- `src/dander/providers/redshift/transform.py` +- `tests/providers/test_redshift_warehouse_runtime.py` +- `docs/redshift.md` diff --git a/docs/decisions.md b/docs/decisions.md index 785148c2..746f617d 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1057,3 +1057,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc never selected. - **Failure boundary:** Every telemetry-only read rolls back its own transaction, including denied or malformed history, so observability cannot poison target cleanup or fail completed work. + +## 2026-08-10 — Redshift direct staging is explicit, bounded, and endpoint-wide + +- **Selection:** COPY remains the zero-configuration default. Direct staging is enabled only by + paired row and logical-byte limits and sees the complete endpoint stream before selecting a + transport; an overflow falls back once to COPY with the inspected prefix preserved. +- **Publication:** Parameterized direct inserts feed the same session-temporary table, deterministic + replay identity, five write modes, schema checks, and fenced publication transaction as COPY. +- **Telemetry:** Direct loads report exact local rows, logical bytes, and duration without a query + ID or system-history enrichment because `executemany` has no whole-batch Redshift query identity. diff --git a/docs/redshift.md b/docs/redshift.md index f504e16c..ceb350b5 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -24,6 +24,8 @@ platforms: cluster_identifier: dander-test copy_role_arn: arn:aws:iam::123456789012:role/DanderRedshiftCopy staging_bucket: dander-redshift-staging + direct_max_rows: 0 + direct_max_logical_bytes: 0 state: provider: postgresql authority_id: postgresql:redshift-test @@ -37,11 +39,20 @@ Serverless uses `deployment: serverless` and `workgroup_name` instead of `cluste `db_user`. Dander obtains temporary IAM credentials from the ordinary AWS SDK chain. The configured COPY role must read the dedicated same-region staging prefix. +COPY remains the default. Set both direct limits to positive values to opt small complete endpoint +streams into bounded parameterized inserts; for example, `direct_max_rows: 1000` and +`direct_max_logical_bytes: 1048576`. Dander selects direct loading only after the entire stream +fits both limits. Otherwise it falls back once to the existing Parquet/S3/COPY path without losing +the inspected prefix. The S3 bucket and COPY role remain required because they handle that fallback. + ## Current contract - `database + schema + relation` become one canonical `RelationRef`; GCP aliases are not used. -- Bounded, checksummed Parquet parts load through a mandatory same-region S3 manifest and `COPY`. -- Replace, SCD1, SCD2, snapshot, and incremental publication reuse the same bounded COPY path. +- Bounded, checksummed Parquet parts load through a mandatory same-region S3 manifest and `COPY` by + default. An explicit paired threshold can select bounded parameterized inserts for a complete + small endpoint without contacting S3. +- Replace, SCD1, SCD2, snapshot, and incremental publication reuse the same fenced publication path + after either direct or COPY staging. - Every mode's target mutation, replay history, and exact destination fencing-token touch commit together. - SCD1 and incremental input is deterministically de-duplicated by business key; incremental writes @@ -66,16 +77,19 @@ COPY role must read the dedicated same-region staging prefix. ## Deliberate limits -Scalar fields and explicit JSON-to-`SUPER` fallback are implemented for COPY ingestion and +Scalar fields and explicit JSON-to-`SUPER` fallback are implemented for direct/COPY ingestion and table/incremental models. `SUPER` fields cannot be business keys, cursors, or snapshot fields. -ARRAY/RECORD fallbacks remain unavailable, and the current staged-row guard is 4 MB even though the -declared VARBYTE/SUPER boundary is larger. The ordinary hosted source runner still selects SCD1. +ARRAY/RECORD fallbacks remain unavailable. COPY retains its 4 MB staged-row guard, while an +explicitly bounded direct load may use the declared 16 MB VARBYTE/SUPER boundary. The ordinary +hosted source runner still selects SCD1. Graph execution retains the shared single-connector, replace-target-only boundary; safe casts fail preflight because Redshift cannot yet preserve their canonical semantics. Views, live concurrency proof, infrastructure provisioning, and support promotion remain separate work. Use `catalog.provider: none` for this experimental path. Telemetry is best-effort and never carries SQL text, error text, S3 locations, or record data. +Direct-load telemetry uses exact local row/byte/duration counters and deliberately has no query ID; +DB-API `executemany` does not provide one honest Redshift query ID for the complete batch. Multi-statement fenced publication and assertion operations report local duration and affected-row counters without claiming a potentially incorrect Redshift query ID. Missing or delayed system history leaves those base counters unchanged. diff --git a/src/dander/providers/redshift/config.py b/src/dander/providers/redshift/config.py index 0056dbf4..68a85033 100644 --- a/src/dander/providers/redshift/config.py +++ b/src/dander/providers/redshift/config.py @@ -49,9 +49,19 @@ class RedshiftWarehouseConfig(BaseModel): le=1_073_741_824, ) compression: Literal["snappy", "zstd"] = "zstd" + direct_max_rows: int = Field(default=0, ge=0, le=10_000) + direct_max_logical_bytes: int = Field( + default=0, + ge=0, + le=16 * 1_024 * 1_024, + ) @model_validator(mode="after") def validate_settings(self) -> Self: + if (self.direct_max_rows == 0) != (self.direct_max_logical_bytes == 0): + raise ValueError( + "direct_max_rows and direct_max_logical_bytes must both be zero or positive" + ) for field_name, value in ( ("database", self.database), ("schema", self.schema_name), diff --git a/src/dander/providers/redshift/runtime.py b/src/dander/providers/redshift/runtime.py index ca22ec28..30cd55de 100644 --- a/src/dander/providers/redshift/runtime.py +++ b/src/dander/providers/redshift/runtime.py @@ -103,7 +103,7 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: @dataclass(frozen=True, slots=True) class RedshiftWriterFactory: - """Construct Redshift's bounded Parquet/COPY ingestion writer.""" + """Construct Redshift's bounded direct-or-Parquet/COPY ingestion writer.""" database: str connection_factory: RedshiftConnectionFactory @@ -224,7 +224,7 @@ def operation( provider_id="redshift", schema_contract_version=1, write_modes=frozenset(WriteMode), - transports=frozenset({WriteTransport.COPY}), + transports=frozenset({WriteTransport.COPY, WriteTransport.DIRECT}), supports_transforms=True, supports_graphs=True, supports_target_fencing=True, @@ -287,6 +287,8 @@ def build_redshift_warehouse( max_logical_bytes_per_file=config.max_logical_bytes_per_file, compression=config.compression, statement_timeout_ms=config.statement_timeout_ms, + direct_max_rows=config.direct_max_rows, + direct_max_logical_bytes=config.direct_max_logical_bytes, ) supplied_root = context.get("staging_root") if supplied_root is not None: @@ -302,6 +304,8 @@ def build_redshift_warehouse( max_logical_bytes_per_file=staging.max_logical_bytes_per_file, compression=staging.compression, statement_timeout_ms=staging.statement_timeout_ms, + direct_max_rows=staging.direct_max_rows, + direct_max_logical_bytes=staging.direct_max_logical_bytes, ) schema_mapper = RedshiftSchemaMapper() return WarehouseRuntime( diff --git a/src/dander/providers/redshift/session.py b/src/dander/providers/redshift/session.py index 7c1f11fe..c155c090 100644 --- a/src/dander/providers/redshift/session.py +++ b/src/dander/providers/redshift/session.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Literal, Protocol if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Iterable, Iterator, Sequence from dander.telemetry import OperationTelemetry @@ -19,6 +19,7 @@ class RedshiftCursor(Protocol): rowcount: int def execute(self, operation: str, args: Sequence[object] | None = None) -> object: ... + def executemany(self, operation: str, args: Iterable[Sequence[object]]) -> object: ... def fetchone(self) -> object | None: ... def fetchall(self) -> list[object]: ... def close(self) -> None: ... @@ -72,6 +73,20 @@ def execute( cursor.close() +def execute_many( + connection: RedshiftConnection, + statement: str, + parameters: Iterable[Sequence[object]], +) -> RedshiftStatementResult: + """Execute one bounded parameter batch without retaining provider response data.""" + cursor = connection.cursor() + try: + cursor.executemany(statement, parameters) + return RedshiftStatementResult(rowcount=cursor.rowcount) + finally: + cursor.close() + + def capture_last_query_id(connection: RedshiftConnection) -> str | None: """Capture a completed compute query ID after its transaction is already committed.""" try: diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py index edc8a31b..e86d0ff4 100644 --- a/src/dander/providers/redshift/writer.py +++ b/src/dander/providers/redshift/writer.py @@ -1,4 +1,4 @@ -"""Bounded S3/Parquet COPY and transactionally fenced Redshift writes.""" +"""Bounded direct or S3/Parquet COPY staging with fenced Redshift publication.""" # ruff: noqa: N803 -- boto3's public S3 API uses capitalized parameter names. @@ -7,8 +7,12 @@ import hashlib import json import tempfile +from base64 import b64encode from collections.abc import Callable, Mapping from dataclasses import dataclass, replace +from datetime import date, datetime, time +from decimal import Decimal +from itertools import chain from pathlib import Path from time import perf_counter_ns from typing import TYPE_CHECKING, Protocol @@ -23,6 +27,7 @@ capture_last_query_id, enrich_operation_telemetry, execute, + execute_many, open_connection, ) from dander.telemetry import OperationTelemetry, TelemetryOperation @@ -90,6 +95,16 @@ class RedshiftStagingSettings: max_logical_bytes_per_file: int compression: str statement_timeout_ms: int + direct_max_rows: int = 0 + direct_max_logical_bytes: int = 0 + + +@dataclass(frozen=True, slots=True) +class _DirectBatch: + rows: tuple[dict[str, object], ...] + logical_bytes: int + schema_fingerprint: str + sha256: str @dataclass(frozen=True, slots=True) @@ -105,7 +120,7 @@ class _LoadOutcome: class RedshiftStagedWriter(WritePattern): - """Upload bounded Parquet parts and publish one selected logical write mode.""" + """Select bounded direct/COPY staging and publish one logical write mode.""" requires_publication_fence = True @@ -131,7 +146,10 @@ def __init__( self.mode = mode self._cursor_field = cursor_field self._snapshot_field = snapshot_field - self.supports_batched_writes = mode in { + direct_enabled = staging.direct_max_rows > 0 + # Direct/COPY selection must see the complete endpoint. When direct is disabled, + # preserve the established executor batching behavior exactly. + self.supports_batched_writes = not direct_enabled and mode in { WriteMode.SCD1, WriteMode.INCREMENTAL, WriteMode.SNAPSHOT, @@ -178,6 +196,32 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> ), ) ) + prepared = iter( + _with_ordinals( + _serialize_super_records(records, schema), + business_key=target.business_key, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) + ) + direct, remaining = _select_direct_batch( + prepared, + staged_schema, + max_rows=self._staging.direct_max_rows, + max_logical_bytes=self._staging.direct_max_logical_bytes, + ) + if direct is not None: + outcome = self._load_and_publish( + target, + publication, + schema, + staged_schema, + transport=WriteTransport.DIRECT, + direct=direct, + ) + self._telemetry.extend(outcome.operations) + return outcome.affected + local_id = f"redshift-{uuid4().hex}" with ParquetStagingSession( self._staging.root, @@ -187,30 +231,47 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> compression=self._staging.compression, ) as local: manifest = local.stage( - _validated_staging_rows( - _with_ordinals( - _serialize_super_records(records, schema), - business_key=target.business_key, - cursor_field=self._cursor_field, - snapshot_field=self._snapshot_field, - ), - staged_schema, - ), + _validated_staging_rows(remaining, staged_schema), + staged_schema, + ) + outcome = self._load_and_publish( + target, + publication, + schema, staged_schema, + transport=WriteTransport.COPY, + manifest=manifest, ) - outcome = self._publish(target, publication, schema, staged_schema, manifest) self._telemetry.extend(outcome.operations) return outcome.affected - def _publish( + def _load_and_publish( self, target: WriteTarget, publication: TargetFence, target_schema: RelationSchema, staged_schema: RelationSchema, - manifest: StagingManifest, + *, + transport: WriteTransport, + manifest: StagingManifest | None = None, + direct: _DirectBatch | None = None, ) -> _LoadOutcome: - _validate_bucket_region(self._s3, self._staging.bucket, self._staging.region) + if (manifest is None) == (direct is None): + raise TypeError("Redshift writer requires exactly one staged load input") + if manifest is not None: + _validate_bucket_region(self._s3, self._staging.bucket, self._staging.region) + rows = manifest.rows + logical_or_compressed_bytes = sum( + artifact.compressed_bytes for artifact in manifest.artifacts + ) + schema_fingerprint = manifest.schema_fingerprint + load_digest = _manifest_digest(manifest) + else: + assert direct is not None + rows = len(direct.rows) + logical_or_compressed_bytes = direct.logical_bytes + schema_fingerprint = direct.schema_fingerprint + load_digest = direct.sha256 remote_id = hashlib.sha256( f"{publication.target_id}:{publication.run_id}:{uuid4().hex}".encode() ).hexdigest()[:24] @@ -228,7 +289,7 @@ def _publish( publication, statement_timeout_ms=self._staging.statement_timeout_ms, ) - if manifest.rows: + if rows and manifest is not None: for artifact in manifest.artifacts: key = f"{remote_prefix}/{artifact.path.name}" uploaded.append(key) @@ -251,21 +312,29 @@ def _publish( role_arn=self._staging.copy_role_arn, ), ) + elif rows: + assert direct is not None + execute(connection, _temporary_table_sql(temp_table, staged_schema)) + loaded, load_duration_ms = _timed_call( + execute_many, + connection, + *_direct_insert(temp_table, staged_schema, direct.rows), + ) # COPY and SET open an implicit transaction. Close it while retaining the # session-scoped temp table so the fenced publication owns one transaction. connection.commit() - if manifest.rows: + if rows: assert loaded is not None - loaded = replace(loaded, query_id=capture_last_query_id(connection)) + if transport is WriteTransport.COPY: + loaded = replace(loaded, query_id=capture_last_query_id(connection)) operations.append( _operation_telemetry( loaded, operation=TelemetryOperation.LOAD, duration_ms=load_duration_ms, - rows_written=manifest.rows, - bytes_written=sum( - artifact.compressed_bytes for artifact in manifest.artifacts - ), + rows_written=rows, + bytes_written=logical_or_compressed_bytes, + transport=transport, ) ) plan = _publication_plan( @@ -273,9 +342,10 @@ def _publish( target, temp_table, target_schema, - manifest, + schema_fingerprint=schema_fingerprint, + load_digest=load_digest, evolution=self._evolution, - has_rows=bool(manifest.rows), + has_rows=bool(rows), mode=self.mode, cursor_field=self._cursor_field, snapshot_field=self._snapshot_field, @@ -298,6 +368,7 @@ def _publish( results[plan.affected_statement_index], operation=TelemetryOperation.QUERY, duration_ms=publication_duration_ms, + transport=transport, ) ) completed = enrich_operation_telemetry(connection, operations) @@ -359,6 +430,8 @@ def default_staging_settings( max_logical_bytes_per_file: int, compression: str, statement_timeout_ms: int, + direct_max_rows: int = 0, + direct_max_logical_bytes: int = 0, ) -> RedshiftStagingSettings: return RedshiftStagingSettings( root=Path(tempfile.gettempdir()) / "dander-redshift-staging", @@ -370,6 +443,8 @@ def default_staging_settings( max_logical_bytes_per_file=max_logical_bytes_per_file, compression=compression, statement_timeout_ms=statement_timeout_ms, + direct_max_rows=direct_max_rows, + direct_max_logical_bytes=direct_max_logical_bytes, ) @@ -393,6 +468,7 @@ def _operation_telemetry( duration_ms: int, rows_written: int = 0, bytes_written: int = 0, + transport: WriteTransport = WriteTransport.COPY, ) -> OperationTelemetry: return OperationTelemetry( provider="redshift", @@ -402,10 +478,91 @@ def _operation_telemetry( rows_affected=max(result.rowcount, 0), bytes_written=bytes_written, query_id=result.query_id, - transport=WriteTransport.COPY, + transport=transport, ) +def _select_direct_batch( + records: Iterable[Mapping[str, object]], + schema: RelationSchema, + *, + max_rows: int, + max_logical_bytes: int, +) -> tuple[_DirectBatch | None, Iterable[Mapping[str, object]]]: + """Choose direct only after the complete endpoint fits both bounded limits.""" + if max_rows == 0 or max_logical_bytes == 0: + return None, records + iterator = iter(records) + raw_prefix: list[dict[str, object]] = [] + normalized_prefix: list[dict[str, object]] = [] + logical_bytes = 0 + for row_index, record in enumerate(iterator): + raw = dict(record) + normalized = normalize_staging_record(raw, schema, row_index=row_index) + raw_prefix.append(raw) + normalized_prefix.append(normalized) + logical_bytes += staging_logical_size(normalized) + if len(raw_prefix) > max_rows or logical_bytes > max_logical_bytes: + return None, chain(raw_prefix, iterator) + rows = tuple(normalized_prefix) + schema_fingerprint = _schema_fingerprint(schema) + return ( + _DirectBatch( + rows=rows, + logical_bytes=logical_bytes, + schema_fingerprint=schema_fingerprint, + sha256=_direct_checksum(schema, rows), + ), + (), + ) + + +def _schema_fingerprint(schema: RelationSchema) -> str: + return hashlib.sha256(schema.model_dump_json(by_alias=True).encode("utf-8")).hexdigest() + + +def _direct_checksum( + schema: RelationSchema, + rows: tuple[dict[str, object], ...], +) -> str: + payload = { + "schema": schema.model_dump(mode="json", by_alias=True), + "rows": [[_checksum_value(row[field.name]) for field in schema.fields] for row in rows], + } + encoded = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _checksum_value(value: object) -> object: + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, float): + return {"float": value.hex()} + if isinstance(value, Decimal): + return {"decimal": str(value)} + if isinstance(value, bytes): + return {"binary": b64encode(value).decode("ascii")} + if isinstance(value, datetime): + return {"datetime": value.isoformat()} + if isinstance(value, date): + return {"date": value.isoformat()} + if isinstance(value, time): + return {"time": value.isoformat()} + raise RedshiftWriteError("Redshift direct staging received an unsupported scalar value") + + +def _direct_insert( + temporary: str, + schema: RelationSchema, + rows: tuple[dict[str, object], ...], +) -> tuple[str, tuple[tuple[object, ...], ...]]: + columns = ", ".join(_quote(field.name) for field in schema.fields) + placeholders = ", ".join("%s" for _field in schema.fields) + statement = f"INSERT INTO {_quote(temporary)} ({columns}) VALUES ({placeholders})" + parameters = tuple(tuple(row[field.name] for field in schema.fields) for row in rows) + return statement, parameters + + def _with_ordinals( records: Iterable[Mapping[str, object]], *, @@ -649,8 +806,9 @@ def _publication_plan( target: WriteTarget, temp_table: str, schema: RelationSchema, - manifest: StagingManifest, *, + schema_fingerprint: str, + load_digest: str, evolution: SchemaEvolution, has_rows: bool, mode: WriteMode, @@ -663,13 +821,12 @@ def _publication_plan( (_history_table_sql(target), ()), ] statements.extend(_schema_changes(connection, target, deployed_schema, evolution=evolution)) - digest = _manifest_digest(manifest) history_values: tuple[object, ...] = ( ".".join(target.relation_ref.coordinates), target.publication_fence.pipeline_id if target.publication_fence else "", target.publication_fence.run_id if target.publication_fence else "", - manifest.schema_fingerprint, - digest, + schema_fingerprint, + load_digest, ) publication = _publication_statements( target, diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 0f8ccd3a..90fc3d7a 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -78,14 +78,14 @@ { "provider": "redshift", "write_modes": ["incremental", "replace", "scd1", "scd2", "snapshot"], - "transports": ["copy"], + "transports": ["copy", "direct"], "logical_types": ["binary", "boolean", "date", "decimal", "float", "integer", "string", "time", "timestamp"], "max_decimal_precision": 38, "max_temporal_precision": 6, "supports_transforms": true, "supports_graphs": true, "supports_target_fencing": true, - "limitations": ["JSON requires the explicit redshift/fallback=super field extension", "graph safe-cast semantics are not available", "only COPY transport is implemented"] + "limitations": ["JSON requires the explicit redshift/fallback=super field extension", "direct thresholds default to disabled until the live crossover is measured", "graph safe-cast semantics are not available"] }, { "provider": "snowflake", diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index e2d1c4dc..18a4b83e 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -32,7 +32,7 @@ from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport if TYPE_CHECKING: - from collections.abc import Mapping, Sequence + from collections.abc import Iterable, Mapping, Sequence @dataclass @@ -58,6 +58,7 @@ class _FakeRedshift: query_counter: int = 100 telemetry_rows: list[object] = field(default_factory=list) telemetry_error: bool = False + direct_error: bool = False def connect(self) -> _FakeConnection: self.connections += 1 @@ -205,6 +206,19 @@ def execute(self, command: str, args: Sequence[object] | None = None) -> Self: self._row = (self.backend.assertion_failure_count,) return self + def executemany(self, command: str, args: Iterable[Sequence[object]]) -> Self: + if self.connection.aborted: + raise RuntimeError("Redshift transaction is aborted until rollback") + rows = tuple(tuple(row) for row in args) + compact = " ".join(command.split()) + self.connection.in_transaction = True + self.backend.statements.append((self.connection.connection_id, compact, rows)) + if self.backend.direct_error: + self.connection.aborted = True + raise RuntimeError("private Redshift direct-load response") + self.rowcount = len(rows) + return self + def fetchone(self) -> object | None: return self._row @@ -303,6 +317,33 @@ def redshift_runtime( return runtime, backend, s3, tmp_path +def _direct_redshift_runtime( + tmp_path: Path, + *, + max_rows: int = 2, + max_logical_bytes: int = 4_096, +) -> tuple[WarehouseRuntime, _FakeRedshift, _FakeS3]: + backend = _FakeRedshift() + s3 = _FakeS3() + registry = default_provider_registry() + raw = _config() + raw["direct_max_rows"] = max_rows + raw["direct_max_logical_bytes"] = max_logical_bytes + config = registry.parse(ProviderKind.WAREHOUSE, raw) + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={ + "catalog": "analytics", + "connection_factory": backend.connect, + "s3_client": s3, + "staging_root": tmp_path, + }, + ) + assert isinstance(runtime, WarehouseRuntime) + return runtime, backend, s3 + + def _target(runtime: WarehouseRuntime, *, run_id: str = "run-one") -> WriteTarget: relation = RelationRef(catalog="analytics", namespace="raw", name="records") publication = runtime.target_fence.claim( @@ -342,6 +383,16 @@ def test_redshift_registration_is_lazy_and_uses_native_coordinates() -> None: ) assert relation == RelationRef(catalog="analytics", namespace="raw", name="records") assert "password" not in config.model_dump_json() + assert config.direct_max_rows == 0 + assert config.direct_max_logical_bytes == 0 + + +def test_redshift_direct_thresholds_are_explicit_and_paired() -> None: + raw = _config() + raw["direct_max_rows"] = 10 + + with pytest.raises(ValueError, match="must both be zero or positive"): + RedshiftWarehouseConfig.model_validate(raw) def test_redshift_schema_mapper_requires_explicit_json_super_without_io( @@ -394,7 +445,9 @@ def test_redshift_runtime_validates_connection_and_exposes_fenced_transforms( assert runtime.relation_codec.render(relation) == '"analytics"."raw"."records"' assert runtime.capabilities.write_modes == frozenset(WriteMode) - assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) + assert runtime.capabilities.transports == frozenset( + {WriteTransport.COPY, WriteTransport.DIRECT} + ) assert runtime.capabilities.supports_transforms is True assert runtime.capabilities.supports_graphs is True assert isinstance( @@ -1115,6 +1168,229 @@ def test_redshift_copy_telemetry_is_enriched_without_sensitive_history( assert "data_source" not in history_statement.casefold() +def test_redshift_direct_load_parses_explicit_super_without_s3_or_query_attribution( + tmp_path: Path, +) -> None: + runtime, backend, s3 = _direct_redshift_runtime(tmp_path) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "NO"), + ("payload", "super", None, None, None, "YES"), + ] + relation = RelationRef(catalog="analytics", namespace="raw", name="direct_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="redshift_direct_records", + run_id="run-direct", + token=7, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(fallback,)), + ), + publication_fence=publication, + ) + + assert writer.supports_batched_writes is False + assert writer.accepts_streaming_input is True + assert writer.write(({"id": "one", "payload": {"ready": True}},), target) == 1 + + direct_statement, direct_parameters = next( + (statement, parameters) + for _, statement, parameters in backend.statements + if statement.startswith('INSERT INTO "dander_stage_') + and "dander_stage_loads" not in statement + ) + assert '"payload" VARBYTE(16777216)' in next( + statement + for _, statement, _ in backend.statements + if statement.startswith("CREATE TEMP TABLE") + ) + assert direct_statement.endswith("VALUES (%s, %s, %s)") + assert direct_parameters == (("one", b'{"ready":true}', 0),) + assert 'JSON_PARSE(staged."payload") AS "payload"' in next( + statement for _, statement, _ in backend.statements if statement.startswith("MERGE INTO") + ) + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts and not s3.deleted + operations = writer.drain_telemetry() + assert [operation.operation for operation in operations] == [ + TelemetryOperation.LOAD, + TelemetryOperation.QUERY, + ] + assert all(operation.transport is WriteTransport.DIRECT for operation in operations) + assert operations[0].rows_written == 1 + assert operations[0].query_id is None + assert not any( + statement.startswith("WITH recent AS (SELECT query_id") + for _, statement, _ in backend.statements + ) + history_before = set(backend.history) + assert writer.write(({"id": "one", "payload": {"ready": True}},), target) == 0 + assert backend.history == history_before + assert not tuple(tmp_path.iterdir()) + + +def test_redshift_direct_load_accepts_a_valid_super_row_above_the_copy_limit( + tmp_path: Path, +) -> None: + runtime, backend, s3 = _direct_redshift_runtime( + tmp_path, + max_rows=1, + max_logical_bytes=8 * 1_024 * 1_024, + ) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "NO"), + ("payload", "super", None, None, None, "YES"), + ] + relation = RelationRef(catalog="analytics", namespace="raw", name="large_direct_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="redshift_large_direct", + run_id="run-large-direct", + token=8, + authority_id="postgresql:test-state", + ), + ) + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(fallback,)), + ), + publication_fence=publication, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + assert ( + writer.write( + ({"id": "one", "payload": {"value": "x" * 4_300_000}},), + target, + ) + == backend.merge_rowcount + ) + + direct_parameters = next( + parameters + for _, statement, parameters in backend.statements + if statement.startswith('INSERT INTO "dander_stage_') + and "dander_stage_loads" not in statement + ) + direct_row = direct_parameters[0] + assert isinstance(direct_row, (tuple, list)) + assert isinstance(direct_row[1], bytes) + assert len(direct_row[1]) > 4 * 1_024 * 1_024 + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts and not s3.deleted + assert not tuple(tmp_path.iterdir()) + + +@pytest.mark.parametrize( + ("max_rows", "max_logical_bytes"), + ((2, 4_096), (10, 1)), +) +def test_redshift_direct_threshold_is_decided_for_the_complete_endpoint( + tmp_path: Path, + max_rows: int, + max_logical_bytes: int, +) -> None: + runtime, backend, s3 = _direct_redshift_runtime( + tmp_path, + max_rows=max_rows, + max_logical_bytes=max_logical_bytes, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + consumed = 0 + + def records() -> Iterable[dict[str, object]]: + nonlocal consumed + for index in range(3): + consumed += 1 + yield {"id": str(index), "label": f"record-{index}"} + + assert writer.supports_batched_writes is False + assert writer.accepts_streaming_input is True + assert writer.write(records(), _target(runtime, run_id="run-threshold")) == 1 + + assert consumed == 3 + assert s3.region_checks == 1 + assert s3.uploads and s3.puts and s3.deleted + sql = [statement for _, statement, _ in backend.statements] + assert any(statement.startswith("COPY ") for statement in sql) + assert not any( + statement.startswith('INSERT INTO "dander_stage_') and "dander_stage_loads" not in statement + for statement in sql + ) + assert all(operation.transport is WriteTransport.COPY for operation in writer.drain_telemetry()) + assert not tuple(tmp_path.iterdir()) + + +def test_redshift_direct_empty_stream_skips_s3_and_parameter_inserts(tmp_path: Path) -> None: + runtime, backend, s3 = _direct_redshift_runtime(tmp_path) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + assert writer.write((), _target(runtime, run_id="run-empty-direct")) == 0 + + sql = [statement for _, statement, _ in backend.statements] + assert not any( + statement.startswith(("COPY ", 'INSERT INTO "dander_stage_')) for statement in sql + ) + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts and not s3.deleted + operations = writer.drain_telemetry() + assert len(operations) == 1 + assert operations[0].transport is WriteTransport.DIRECT + assert not tuple(tmp_path.iterdir()) + + +def test_redshift_direct_failure_is_sanitized_and_uses_no_s3(tmp_path: Path) -> None: + runtime, backend, s3 = _direct_redshift_runtime(tmp_path) + backend.direct_error = True + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + + with pytest.raises(RedshiftWriteError, match="staged SCD1 write failed") as raised: + writer.write(({"id": "one", "label": "first"},), _target(runtime, run_id="run-bad")) + + assert "private Redshift" not in str(raised.value) + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts and not s3.deleted + assert writer.drain_telemetry() == () + assert backend.rollbacks >= 1 + assert not tuple(tmp_path.iterdir()) + + def test_redshift_telemetry_failure_rolls_back_before_staging_cleanup( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], ) -> None: @@ -1266,6 +1542,68 @@ def test_redshift_factory_reaches_each_additional_fenced_write_mode( assert not tuple(staging_root.iterdir()) +@pytest.mark.parametrize("mode", tuple(WriteMode)) +def test_redshift_direct_transport_reaches_every_fenced_write_mode( + tmp_path: Path, + mode: WriteMode, +) -> None: + runtime, backend, s3 = _direct_redshift_runtime(tmp_path) + cursor_field = "updated_at" if mode is WriteMode.INCREMENTAL else None + snapshot_field = "snapshot_at" if mode is WriteMode.SNAPSHOT else None + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, + ) + relation = RelationRef(catalog="analytics", namespace="raw", name=f"direct_{mode.value}") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=f"redshift_direct_{mode.value}", + run_id=f"run-direct-{mode.value}", + token=9, + authority_id="postgresql:test-state", + ), + ) + schema = [ + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ] + record: dict[str, object] = {"id": "one", "label": "first"} + if cursor_field is not None: + schema.append(WriteField(name=cursor_field, data_type="INT64")) + record[cursor_field] = 2 + backend.schema_rows.append((cursor_field, "bigint", None, None, None, "YES")) + if snapshot_field is not None: + schema.append(WriteField(name=snapshot_field, data_type="INT64")) + record[snapshot_field] = 2 + backend.schema_rows.append((snapshot_field, "bigint", None, None, None, "YES")) + if mode is WriteMode.SCD2: + backend.schema_rows = [] + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=tuple(schema), + publication_fence=publication, + ) + + assert writer.write((record,), target) == backend.merge_rowcount + + sql = [statement for _, statement, _ in backend.statements] + assert any(statement.startswith('INSERT INTO "dander_stage_') for statement in sql) + assert not any(statement.startswith("COPY ") for statement in sql) + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts and not s3.deleted + assert all( + operation.transport is WriteTransport.DIRECT for operation in writer.drain_telemetry() + ) + assert not tuple(tmp_path.iterdir()) + + @pytest.mark.parametrize("mode", tuple(WriteMode)) def test_redshift_all_write_modes_parse_explicit_super_from_varbyte_staging( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], From 00eaee5911942a7639f060e481b66377a650197b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 07:49:42 -0400 Subject: [PATCH 100/163] Add PostgreSQL writer modes --- HANDOFF.md | 42 +- docs/compatibility-matrix.md | 2 +- docs/decisions.md | 10 + docs/known-limitations.md | 3 +- docs/platform-profiles.md | 19 +- src/dander/providers/postgresql/runtime.py | 15 +- src/dander/providers/postgresql/writer.py | 416 ++++++++++++++--- src/dander/runtime-compatibility.json | 4 +- .../test_postgresql_warehouse_runtime.py | 428 +++++++++++++++++- 9 files changed, 839 insertions(+), 100 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 137be4f8..cec2ae3d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Added opt-in bounded Redshift direct staging with paired row/byte thresholds. -- Preserved zero-default COPY behavior and whole-endpoint fallback without losing the inspected prefix. -- Reused all five fenced write modes, replay identity, schema checks, SUPER parsing, and cleanup. -- Added unattributed direct telemetry from exact local counters; direct loads never contact S3. -- Updated Redshift configuration, compatibility output, decisions, and operator documentation. +- Added bounded COPY-backed replace, SCD1, SCD2, snapshot, and incremental PostgreSQL writes. +- Kept replace/SCD2 endpoint-wide while preserving batching for the three batch-safe modes. +- Added cursor ordering, replay-safe snapshots, SCD2 history, and transactional fencing tests. +- Preserved full-row replace semantics and the pre-upgrade SCD1 unique-index identity. +- Updated the machine-readable capabilities and focused operator documentation. ## Try It -Set both `direct_max_rows` and `direct_max_logical_bytes` in a Redshift warehouse profile, then run -`uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py`. +Set `DANDER_TEST_POSTGRES_DSN` to PostgreSQL 15+ and run +`uv run pytest -q tests/providers/test_postgresql_warehouse_runtime.py`. ## Checks -- Ruff passed across 328 files; strict mypy passed across 304 files. -- Full pytest passed: 1,165 tests with 13 intentional skips. -- GCP/AWS Terraform roots validated; AWS bootstrap and Fargate module tests passed without apply. -- Wheel/sdist inspection, source-free installs, runtime-all import, and generated Terraform validation passed. -- Dependency audit found no known vulnerabilities; container, secret, and PostgreSQL-service checks await protected CI. +- Ruff passed across the repository; strict mypy passed across 304 source files. +- All 16 focused tests and the complete 1,189-test suite passed against PostgreSQL 15. +- Terraform roots/modules/tests and Helm lint/template validation passed. +- Wheel/sdist inspection, source-free install/generation, and dependency audit passed. +- Container conformance passed; retained GCP stage-zero and platform plans reported no changes. ## Decisions -- COPY remains default; both direct thresholds must be positive to enable endpoint-wide selection. -- Threshold overflow falls back once to COPY; executor batching is disabled only while direct selection is enabled. -- Direct `executemany` telemetry has no query ID because Redshift exposes no honest whole-batch ID. +- Replace and SCD2 stream a complete endpoint through COPY; they never use executor batches. +- Incremental de-duplication ranks cursor descending, then source ordinal for equal cursors. +- PostgreSQL remains experimental until graph and live Kubernetes qualification are complete. ## Remaining -- Push a focused PR and require protected CI before merge. -- Run live Redshift/Glue acceptance only after a separately approved paid AWS plan/apply. -- Complete Snowflake and cross-warehouse live proof before Phase 5.5; do not begin Azure. +- Let protected CI repeat Linux PostgreSQL, image, and secret checks on PR #175. +- Merge through protected main only when every required check is green. +- Continue PostgreSQL graph work separately; do not mix it into this writer PR. ## Review First -- `src/dander/providers/redshift/writer.py` -- `tests/providers/test_redshift_warehouse_runtime.py` -- `docs/redshift.md` +- `src/dander/providers/postgresql/writer.py` +- `tests/providers/test_postgresql_warehouse_runtime.py` +- `src/dander/providers/postgresql/runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 10f59b8e..09c0ee11 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -39,7 +39,7 @@ their named live profiles pass. | Warehouse | Ingestion modes | Transport | Canonical schema | Models | Graphs | Fence | |---|---|---|---|---|---|---| | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | -| PostgreSQL | SCD1 | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | +| PostgreSQL | all five | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | | Redshift | all five | COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | yes | yes | | Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | diff --git a/docs/decisions.md b/docs/decisions.md index 746f617d..4c18e08f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-10 — PostgreSQL write modes preserve endpoint and batch semantics + +- **Modes:** One COPY-backed writer now publishes replace, SCD1, SCD2, snapshot, and incremental + data through the existing destination-side PostgreSQL fence. +- **Boundaries:** Replace and SCD2 receive one streamed endpoint; SCD1, incremental, and snapshot + remain executor-batched. Changing `batch_rows` therefore cannot truncate replace output or + change SCD2 history. +- **Safety:** Incremental input ranks cursor then ordinal and rejects target regression; snapshot + comparison is null-safe; SCD2 uses transaction-stable timestamps and one-current-row indexes. + ## 2026-08-10 — Fargate lifecycle evidence does not skip profile qualification - **Accepted slice:** Public `0.8.0rc8` passed manual and scheduled execution, replay, diff --git a/docs/known-limitations.md b/docs/known-limitations.md index daa8c203..bab2862c 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -24,7 +24,8 @@ evaluate these limits before using it for an unattended system containing busine VARBYTE staging, and `JSON_PARSE`; ARRAY/RECORD fallbacks remain unavailable. Replace-mode graph targets use the provider-neutral relational AST and fenced table path; graph safe casts fail preflight. Redshift has no live qualification or views. PostgreSQL state and warehouse execution - are implemented and locally conformance-tested. A packaged Helm + are implemented and locally conformance-tested; all five ingestion modes use bounded `COPY` and + destination fencing, while the ordinary hosted source runner still selects SCD1. A packaged Helm chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 69c92f5d..729dd13e 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -115,18 +115,23 @@ migration: keep schedules paused until the destination-fence and cutover workflo ## PostgreSQL warehouse adapter The PostgreSQL warehouse adapter is selectable but not yet a supported hosted profile. It accepts -declared schemas, creates database-local schemas and relations, streams each -bounded Dander batch through PostgreSQL `COPY`, and performs deterministic last-record-wins SCD1 -publication inside a transactionally verified destination fence. Temporary staging uses -`ON COMMIT DROP`; nullable top-level columns may be added when additive evolution is selected. +declared schemas, creates database-local schemas and relations, and streams records through +PostgreSQL `COPY`. Replace and SCD2 consume one streamed endpoint so executor batch size cannot +change their meaning; SCD1, incremental, and snapshot retain bounded Dander batches. Every mode +publishes inside a transactionally verified destination fence. Temporary staging uses `ON COMMIT +DROP`; nullable top-level columns may be added when additive evolution is selected. The configured database must already exist. Runtime-created connections require TLS, and the DSN is supplied only through the configured environment variable. The adapter supports PostgreSQL -15+, SCD1 ingestion, canonical scalar/array/JSON mappings, portable or PostgreSQL-exact model SQL, -table/view/incremental materialization, and the four generic assertions. Every materialization is +15+, replace, SCD1, SCD2, snapshot, and incremental ingestion, canonical scalar/array/JSON +mappings, portable or PostgreSQL-exact model SQL, table/view/incremental materialization, and the +four generic assertions. SCD1 and incremental use deterministic business-key de-duplication; +incremental publication rejects cursor regression. Snapshot comparison is null-safe, replace is +empty-source safe, and SCD2 uses transaction-stable validity timestamps. Every materialization is claimed and published through its destination target fence. The local native-profile proof covers bounded ingestion, replay, transforms, assertions, metadata, run history, watermarks, and leases. -Graph execution remains follow-up work. The packaged existing-cluster Kubernetes/Helm launcher can +The ordinary hosted source runner still selects SCD1, and graph execution remains follow-up work. +The packaged existing-cluster Kubernetes/Helm launcher can render and verify this profile without cloud-specific identity assumptions, but no live cluster qualification has passed. Until that acceptance does, this adapter must not be represented as a supported hosted PostgreSQL deployment. See [Kubernetes existing-cluster launcher](kubernetes.md). diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py index 8f4d584e..42d14de9 100644 --- a/src/dander/providers/postgresql/runtime.py +++ b/src/dander/providers/postgresql/runtime.py @@ -14,7 +14,7 @@ from dander.providers.postgresql.config import PostgreSQLWarehouseConfig from dander.providers.postgresql.fence import PostgreSQLTargetFence from dander.providers.postgresql.transform import PostgreSQLTransformRunner -from dander.providers.postgresql.writer import PostgreSQLScd1Writer, PostgreSQLTimeouts +from dander.providers.postgresql.writer import PostgreSQLCopyWriter, PostgreSQLTimeouts from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.telemetry import OperationTelemetry, TelemetryOperation from dander.warehouse import CanonicalField, LogicalTypeKind, RelationRef, RelationSchema @@ -79,7 +79,7 @@ def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: @dataclass(frozen=True, slots=True) class PostgreSQLWriterFactory: - """Construct PostgreSQL's first bounded hosted ingestion writer.""" + """Construct a bounded COPY-backed PostgreSQL ingestion writer.""" database: str pool: PostgreSQLPool @@ -97,18 +97,17 @@ def build_ingestion_writer( snapshot_field: str | None = None, ) -> WritePattern: del batch_rows - del cursor_field - del snapshot_field if sandbox: raise ValueError("PostgreSQL warehouse does not use Dander's BigQuery sandbox mode") - if mode is not WriteMode.SCD1: - raise ValueError(f"PostgreSQL warehouse does not support {mode.value} writes") - return PostgreSQLScd1Writer( + return PostgreSQLCopyWriter( database=self.database, pool=self.pool, target_fence=self.target_fence, schema_evolution=schema_evolution, timeouts=self.timeouts, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, ) @@ -167,7 +166,7 @@ def operation( POSTGRESQL_CAPABILITIES = WarehouseCapabilities( provider_id="postgresql", schema_contract_version=1, - write_modes=frozenset({WriteMode.SCD1}), + write_modes=frozenset(WriteMode), transports=frozenset({WriteTransport.COPY}), supports_transforms=True, supports_graphs=False, diff --git a/src/dander/providers/postgresql/writer.py b/src/dander/providers/postgresql/writer.py index 3a3de60a..9d681049 100644 --- a/src/dander/providers/postgresql/writer.py +++ b/src/dander/providers/postgresql/writer.py @@ -1,4 +1,4 @@ -"""PostgreSQL schema management and bounded COPY-backed SCD1 writes.""" +"""PostgreSQL schema management and bounded, fenced COPY-backed writes.""" from __future__ import annotations @@ -30,6 +30,12 @@ PostgreSQLRow = dict[str, object] PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] +_ORDINAL = "_dander_ordinal" +_SCD2_VALID_FROM = "valid_from" +_SCD2_VALID_TO = "valid_to" +_SCD2_IS_CURRENT = "is_current" +_SCD2_SYSTEM_FIELDS = frozenset({_SCD2_VALID_FROM, _SCD2_VALID_TO, _SCD2_IS_CURRENT}) + class PostgreSQLWriteError(ValueError): """Raised when records or a deployed relation violate the PostgreSQL contract.""" @@ -44,11 +50,9 @@ class PostgreSQLTimeouts: idle_transaction_ms: int -class PostgreSQLScd1Writer(WritePattern): - """Stream records through COPY and publish keyed rows transactionally.""" +class PostgreSQLCopyWriter(WritePattern): + """Stream records through COPY and publish one logical mode transactionally.""" - mode = WriteMode.SCD1 - supports_batched_writes = True requires_publication_fence = True def __init__( @@ -59,33 +63,47 @@ def __init__( target_fence: PostgreSQLTargetFence, schema_evolution: SchemaEvolution = SchemaEvolution.STRICT, timeouts: PostgreSQLTimeouts, + mode: WriteMode, + cursor_field: str | None = None, + snapshot_field: str | None = None, ) -> None: self._database = database self._pool = pool self._target_fence = target_fence self._schema_evolution = schema_evolution self._timeouts = timeouts + self.mode = mode + self._cursor_field = cursor_field + self._snapshot_field = snapshot_field + self.supports_batched_writes = mode in { + WriteMode.SCD1, + WriteMode.INCREMENTAL, + WriteMode.SNAPSHOT, + } + self.accepts_streaming_input = not self.supports_batched_writes def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> int: - """COPY one bounded batch and upsert its deterministic last record per key.""" - _validate_target(target, database=self._database) + """COPY one batch or streamed endpoint and publish it behind the target fence.""" + _validate_target( + target, + database=self._database, + mode=self.mode, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) publication_fence = target.publication_fence assert publication_fence is not None schema = target.canonical_schema - fields = {field.name: field for field in schema.fields} - missing_keys = sorted(set(target.business_key) - set(fields)) - if missing_keys: - raise PostgreSQLWriteError( - f"Business-key column {missing_keys[0]!r} is absent from the declared schema" - ) + target_schema = _target_schema_for_mode(schema, self.mode) with self._pool.connection() as connection, connection.transaction(): _set_timeouts(connection, self._timeouts) _ensure_target( connection, target, - schema, + target_schema, evolution=self._schema_evolution, + mode=self.mode, ) staging = f"dander_stage_{uuid4().hex}" _create_staging(connection, staging, schema, target.business_key) @@ -95,38 +113,86 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> records, schema.fields, target.business_key, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, ) - if written == 0: - no_op = sql.SQL("UPDATE {} SET {} = {} WHERE FALSE").format( - _target_identifier(target), - sql.Identifier(target.business_key[0]), - sql.Identifier(target.business_key[0]), - ) - self._target_fence.execute_dml( - connection, - no_op.as_string(connection), - publication_fence, - ) - return 0 - finalizer = _scd1_sql(target, staging, schema.fields) - cursor = self._target_fence.execute_dml( + statements = _publication_statements( + target, + staging, + schema.fields, + mode=self.mode, + cursor_field=self._cursor_field, + has_rows=written > 0, + ) + cursor = self._target_fence.execute_statements( connection, - finalizer.as_string(connection), + statements, publication_fence, ) return max(cursor.rowcount, 0) -def _validate_target(target: WriteTarget, *, database: str) -> None: +class PostgreSQLScd1Writer(PostgreSQLCopyWriter): + """Compatibility name for the original PostgreSQL SCD1 writer.""" + + def __init__( + self, + *, + database: str, + pool: PostgreSQLPool, + target_fence: PostgreSQLTargetFence, + schema_evolution: SchemaEvolution = SchemaEvolution.STRICT, + timeouts: PostgreSQLTimeouts, + ) -> None: + super().__init__( + database=database, + pool=pool, + target_fence=target_fence, + schema_evolution=schema_evolution, + timeouts=timeouts, + mode=WriteMode.SCD1, + ) + + +def _validate_target( + target: WriteTarget, + *, + database: str, + mode: WriteMode, + cursor_field: str | None, + snapshot_field: str | None, +) -> None: relation = target.relation_ref if relation.catalog != database: raise PostgreSQLWriteError( f"Writer database {database!r} does not match target catalog {relation.catalog!r}" ) - if not target.business_key: - raise PostgreSQLWriteError("PostgreSQL SCD1 writes require a business key") - if not target.schema: + if not target.canonical_schema.fields: raise PostgreSQLWriteError("PostgreSQL writes require a declared schema") + fields = {field.name for field in target.canonical_schema.fields} + missing_keys = sorted(set(target.business_key) - fields) + if missing_keys: + raise PostgreSQLWriteError( + f"Business-key column {missing_keys[0]!r} is absent from the declared schema" + ) + if mode in {WriteMode.SCD1, WriteMode.SCD2, WriteMode.INCREMENTAL} and not target.business_key: + raise PostgreSQLWriteError(f"PostgreSQL {mode.value.upper()} writes require a business key") + if mode is WriteMode.INCREMENTAL: + if cursor_field is None: + raise PostgreSQLWriteError("PostgreSQL incremental writes require cursor_field") + if cursor_field not in fields: + raise PostgreSQLWriteError("PostgreSQL incremental cursor field is undeclared") + elif cursor_field is not None: + raise PostgreSQLWriteError("cursor_field is valid only for PostgreSQL incremental writes") + if mode is WriteMode.SNAPSHOT: + if snapshot_field is None: + raise PostgreSQLWriteError("PostgreSQL snapshot writes require snapshot_field") + if snapshot_field not in fields: + raise PostgreSQLWriteError("PostgreSQL snapshot field is undeclared") + elif snapshot_field is not None: + raise PostgreSQLWriteError("snapshot_field is valid only for PostgreSQL snapshot writes") + if mode is WriteMode.SCD2 and (collision := sorted(fields & _SCD2_SYSTEM_FIELDS)): + raise PostgreSQLWriteError(f"Declared schema reserves Dander field {collision[0]!r}") publication_fence = target.publication_fence if publication_fence is None: raise PostgreSQLWriteError("PostgreSQL hosted writes require a destination target fence") @@ -157,6 +223,7 @@ def _ensure_target( schema: RelationSchema, *, evolution: SchemaEvolution, + mode: WriteMode, ) -> None: connection.execute( sql.SQL("CREATE SCHEMA IF NOT EXISTS {}").format( @@ -171,11 +238,17 @@ def _ensure_target( ) for field in schema.fields ] + constraints: list[sql.SQL | sql.Composed] = list(definitions) + if mode in {WriteMode.SCD1, WriteMode.INCREMENTAL}: + constraints.append( + sql.SQL("PRIMARY KEY ({})").format( + sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key) + ) + ) connection.execute( - sql.SQL("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))").format( + sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format( _target_identifier(target), - sql.SQL(", ").join(definitions), - sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key), + sql.SQL(", ").join(constraints), ) ) deployed = _deployed_columns(connection, target) @@ -208,14 +281,20 @@ def _ensure_target( if deployed_required != expected_required: raise PostgreSQLWriteError(f"PostgreSQL nullability drift for column {field.name!r}") - index_name = _unique_index_name(target) - connection.execute( - sql.SQL("CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} ({})").format( - sql.Identifier(index_name), - _target_identifier(target), - sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key), + if mode in {WriteMode.SCD1, WriteMode.INCREMENTAL, WriteMode.SCD2}: + where = ( + sql.SQL(" WHERE {} IS TRUE").format(sql.Identifier(_SCD2_IS_CURRENT)) + if mode is WriteMode.SCD2 + else sql.SQL("") + ) + connection.execute( + sql.SQL("CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} ({}){}").format( + sql.Identifier(_unique_index_name(target, mode=mode)), + _target_identifier(target), + sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key), + where, + ) ) - ) def _deployed_columns( @@ -279,6 +358,9 @@ def _copy_records( records: Iterable[Mapping[str, object]], fields: Sequence[CanonicalField], business_key: Sequence[str], + *, + cursor_field: str | None, + snapshot_field: str | None, ) -> int: names = tuple(field.name for field in fields) expected = set(names) @@ -295,6 +377,10 @@ def _copy_records( ) if any(record[key] is None for key in business_key): raise PostgreSQLWriteError(f"Record {index} has a null business-key value") + if cursor_field is not None and record[cursor_field] is None: + raise PostgreSQLWriteError(f"Record {index} has a null incremental cursor value") + if snapshot_field is not None and record[snapshot_field] is None: + raise PostgreSQLWriteError(f"Record {index} has a null snapshot value") copy.write_row( tuple(_postgresql_value(record[field.name], field.data_type) for field in fields) ) @@ -302,15 +388,42 @@ def _copy_records( return copied -def _scd1_sql( +def _publication_statements( target: WriteTarget, staging: str, fields: Sequence[CanonicalField], + *, + mode: WriteMode, + cursor_field: str | None, + has_rows: bool, +) -> tuple[sql.SQL | sql.Composed, ...]: + if not has_rows and mode is not WriteMode.REPLACE: + return (_no_op_sql(target),) + if mode is WriteMode.SCD1: + return (_merge_sql(target, staging, fields, cursor_field=None),) + if mode is WriteMode.INCREMENTAL: + assert cursor_field is not None + return (_merge_sql(target, staging, fields, cursor_field=cursor_field),) + if mode is WriteMode.SNAPSHOT: + return (_snapshot_sql(target, staging, fields),) + if mode is WriteMode.SCD2: + return _scd2_sql(target, staging, fields) + if mode is WriteMode.REPLACE: + return _replace_sql(target, staging, fields) + raise AssertionError("Unhandled PostgreSQL write mode") + + +def _merge_sql( + target: WriteTarget, + staging: str, + fields: Sequence[CanonicalField], + *, + cursor_field: str | None, ) -> sql.Composed: names = tuple(field.name for field in fields) mutable = tuple(name for name in names if name not in target.business_key) conflict = ( - sql.SQL("DO UPDATE SET {} ").format( + sql.SQL("DO UPDATE SET {}").format( sql.SQL(", ").join( sql.SQL("{} = EXCLUDED.{}").format( sql.Identifier(name), @@ -320,28 +433,215 @@ def _scd1_sql( ) ) if mutable - else sql.SQL("DO NOTHING ") + else sql.SQL("DO NOTHING") ) + if cursor_field is not None and mutable: + conflict = sql.SQL("{} WHERE current.{} IS NULL OR EXCLUDED.{} >= current.{}").format( + conflict, + sql.Identifier(cursor_field), + sql.Identifier(cursor_field), + sql.Identifier(cursor_field), + ) selected = sql.SQL(", ").join(sql.Identifier(name) for name in names) keys = sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key) - ordering = sql.SQL(", ").join( - [*(sql.Identifier(key) for key in target.business_key), sql.Identifier("_dander_ordinal")] + source = _deduplicated_source( + target, + staging, + fields, + cursor_field=cursor_field, ) return sql.SQL( - "INSERT INTO {} ({}) SELECT {} FROM (" - "SELECT DISTINCT ON ({}) {}, {} FROM {} ORDER BY {} DESC" - ") AS incoming ON CONFLICT ({}) {}RETURNING 1" + "INSERT INTO {} AS current ({}) SELECT {} FROM ({}) AS incoming " + "ON CONFLICT ({}) {} RETURNING 1" ).format( _target_identifier(target), selected, selected, + source, + keys, + conflict, + ) + + +def _deduplicated_source( + target: WriteTarget, + staging: str, + fields: Sequence[CanonicalField], + *, + cursor_field: str | None = None, +) -> sql.Composed: + selected = sql.SQL(", ").join(sql.Identifier(field.name) for field in fields) + keys = sql.SQL(", ").join(sql.Identifier(key) for key in target.business_key) + ordering: list[sql.SQL | sql.Identifier | sql.Composed] = [ + *(sql.Identifier(key) for key in target.business_key) + ] + if cursor_field is not None: + ordering.append(sql.SQL("{} DESC").format(sql.Identifier(cursor_field))) + ordering.append(sql.SQL("{} DESC").format(sql.Identifier(_ORDINAL))) + return sql.SQL("SELECT DISTINCT ON ({}) {} FROM {} ORDER BY {}").format( keys, selected, - sql.Identifier("_dander_ordinal"), sql.Identifier(staging), - ordering, - keys, - conflict, + sql.SQL(", ").join(ordering), + ) + + +def _snapshot_sql( + target: WriteTarget, + staging: str, + fields: Sequence[CanonicalField], +) -> sql.Composed: + selected = sql.SQL(", ").join(sql.Identifier(field.name) for field in fields) + incoming = sql.SQL(", ").join( + sql.SQL("incoming.{}").format(sql.Identifier(field.name)) for field in fields + ) + match = sql.SQL(" AND ").join( + sql.SQL("current.{} IS NOT DISTINCT FROM incoming.{}").format( + sql.Identifier(field.name), + sql.Identifier(field.name), + ) + for field in fields + ) + return sql.SQL( + "INSERT INTO {} ({}) SELECT {} FROM (SELECT DISTINCT {} FROM {}) AS incoming " + "WHERE NOT EXISTS (SELECT 1 FROM {} AS current WHERE {}) RETURNING 1" + ).format( + _target_identifier(target), + selected, + incoming, + selected, + sql.Identifier(staging), + _target_identifier(target), + match, + ) + + +def _scd2_sql( + target: WriteTarget, + staging: str, + fields: Sequence[CanonicalField], +) -> tuple[sql.Composed, sql.Composed]: + names = tuple(field.name for field in fields) + source = _deduplicated_source(target, staging, fields) + match = sql.SQL(" AND ").join( + sql.SQL("current.{} = incoming.{}").format( + sql.Identifier(key), + sql.Identifier(key), + ) + for key in target.business_key + ) + mutable = tuple(name for name in names if name not in target.business_key) + changed = ( + sql.SQL(" OR ").join( + sql.SQL("current.{} IS DISTINCT FROM incoming.{}").format( + sql.Identifier(name), + sql.Identifier(name), + ) + for name in mutable + ) + if mutable + else sql.SQL("FALSE") + ) + close = sql.SQL( + "UPDATE {} AS current SET {} = transaction_timestamp(), {} = FALSE " + "FROM ({}) AS incoming WHERE {} AND current.{} IS TRUE AND ({})" + ).format( + _target_identifier(target), + sql.Identifier(_SCD2_VALID_TO), + sql.Identifier(_SCD2_IS_CURRENT), + source, + match, + sql.Identifier(_SCD2_IS_CURRENT), + changed, + ) + incoming = sql.SQL(", ").join( + sql.SQL("incoming.{}").format(sql.Identifier(name)) for name in names + ) + columns = sql.SQL(", ").join( + [ + *(sql.Identifier(name) for name in names), + sql.Identifier(_SCD2_VALID_FROM), + sql.Identifier(_SCD2_VALID_TO), + sql.Identifier(_SCD2_IS_CURRENT), + ] + ) + values = sql.SQL(", ").join( + [ + incoming, + sql.SQL("transaction_timestamp()"), + sql.SQL("NULL"), + sql.SQL("TRUE"), + ] + ) + insert = sql.SQL( + "INSERT INTO {} ({}) SELECT {} FROM ({}) AS incoming WHERE NOT EXISTS (" + "SELECT 1 FROM {} AS current WHERE {} AND current.{} IS TRUE) RETURNING 1" + ).format( + _target_identifier(target), + columns, + values, + source, + _target_identifier(target), + match, + sql.Identifier(_SCD2_IS_CURRENT), + ) + return close, insert + + +def _replace_sql( + target: WriteTarget, + staging: str, + fields: Sequence[CanonicalField], +) -> tuple[sql.Composed, sql.Composed]: + selected = sql.SQL(", ").join(sql.Identifier(field.name) for field in fields) + source = sql.SQL("SELECT {} FROM {} ORDER BY {}").format( + selected, + sql.Identifier(staging), + sql.Identifier(_ORDINAL), + ) + return ( + sql.SQL("DELETE FROM {}").format(_target_identifier(target)), + sql.SQL("INSERT INTO {} ({}) SELECT {} FROM ({}) AS incoming RETURNING 1").format( + _target_identifier(target), + selected, + selected, + source, + ), + ) + + +def _no_op_sql(target: WriteTarget) -> sql.Composed: + field = target.canonical_schema.fields[0].name + return sql.SQL("UPDATE {} SET {} = {} WHERE FALSE").format( + _target_identifier(target), + sql.Identifier(field), + sql.Identifier(field), + ) + + +def _target_schema_for_mode(schema: RelationSchema, mode: WriteMode) -> RelationSchema: + if mode is not WriteMode.SCD2: + return schema + timestamp_type = CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + fractional_second_precision=6, + with_timezone=True, + ) + return RelationSchema( + fields=( + *schema.fields, + CanonicalField( + name=_SCD2_VALID_FROM, + data_type=timestamp_type, + cardinality=FieldCardinality.REQUIRED, + ), + CanonicalField(name=_SCD2_VALID_TO, data_type=timestamp_type), + CanonicalField( + name=_SCD2_IS_CURRENT, + data_type=CanonicalType(kind=LogicalTypeKind.BOOLEAN), + cardinality=FieldCardinality.REQUIRED, + ), + ) ) @@ -349,11 +649,12 @@ def _target_identifier(target: WriteTarget) -> sql.Identifier: return sql.Identifier(target.relation_ref.namespace, target.relation_ref.name) -def _unique_index_name(target: WriteTarget) -> str: +def _unique_index_name(target: WriteTarget, *, mode: WriteMode) -> str: + mode_identity = ":scd2" if mode is WriteMode.SCD2 else "" digest = hashlib.sha256( ( f"{target.relation_ref.namespace}.{target.relation_ref.name}:" - f"{','.join(target.business_key)}" + f"{','.join(target.business_key)}{mode_identity}" ).encode() ).hexdigest()[:10] prefix = f"dander_uq_{target.relation_ref.name}"[:51] @@ -417,6 +718,7 @@ def _postgresql_value(value: object, data_type: CanonicalType) -> object: __all__ = [ + "PostgreSQLCopyWriter", "PostgreSQLScd1Writer", "PostgreSQLTimeouts", "PostgreSQLWriteError", diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 90fc3d7a..94d00b67 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -65,7 +65,7 @@ }, { "provider": "postgresql", - "write_modes": ["scd1"], + "write_modes": ["incremental", "replace", "scd1", "scd2", "snapshot"], "transports": ["copy"], "logical_types": ["array", "binary", "boolean", "date", "decimal", "float", "integer", "json", "record", "string", "time", "timestamp"], "max_decimal_precision": 1000, @@ -73,7 +73,7 @@ "supports_transforms": true, "supports_graphs": false, "supports_target_fencing": true, - "limitations": ["graph execution is not available", "only scd1 ingestion is implemented"] + "limitations": ["graph execution is not available", "hosted source execution selects scd1"] }, { "provider": "redshift", diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py index 1f384892..26c28d1b 100644 --- a/tests/providers/test_postgresql_warehouse_runtime.py +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -1,7 +1,8 @@ -"""PostgreSQL warehouse registration and live SCD1 conformance.""" +"""PostgreSQL warehouse registration and live writer conformance.""" from __future__ import annotations +import hashlib import os import sys import uuid @@ -15,20 +16,76 @@ from psycopg_pool import ConnectionPool from dander.concurrency import FencingToken +from dander.ingestion.source import Endpoint, RawField, Source, SourceConfig from dander.providers import ProviderKind, default_provider_registry from dander.providers.postgresql.config import PostgreSQLWarehouseConfig from dander.providers.postgresql.writer import PostgreSQLWriteError +from dander.runtime import PipelineRunner +from dander.state import WatermarkStore from dander.telemetry import TelemetryOperation from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Iterator, Mapping PostgreSQLRow = dict[str, Any] PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] +class _StaticSource(Source): + def __init__(self, *, name: str, records: list[Mapping[str, object]]) -> None: + super().__init__( + SourceConfig( + name=name, + base_url="https://example.invalid", + auth_strategy="none", + endpoints=[ + Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING", mode="REQUIRED"), + RawField(name="label", data_type="STRING"), + ], + ) + ], + ) + ) + self._records = records + + def discover(self) -> Mapping[str, object]: + return {} + + def extract( + self, + endpoint: str, + *, + since: str | None = None, + ) -> Iterator[Mapping[str, object]]: + assert endpoint == "records" + assert since is None + yield from self._records + + +class _NoWatermarks(WatermarkStore): + def get(self, source: str, entity: str) -> str | None: + del source, entity + return None + + def set(self, source: str, entity: str, cursor: str) -> None: + raise AssertionError((source, entity, cursor)) + + +class _Ownership: + def __init__(self, fence: FencingToken) -> None: + self.fence = fence + + def verify(self) -> None: + return None + + def test_postgresql_warehouse_registration_is_lazy_and_explicit() -> None: module_name = "dander.providers.postgresql.runtime" sys.modules.pop(module_name, None) @@ -157,12 +214,199 @@ class _Cursor: assert schema.fields[0].data_type.kind is LogicalTypeKind.INTEGER assert schema.fields[1].data_type.kind is LogicalTypeKind.JSON assert schema.fields[2].data_type.kind is LogicalTypeKind.ARRAY - assert runtime.capabilities.write_modes == frozenset({WriteMode.SCD1}) + assert runtime.capabilities.write_modes == frozenset(WriteMode) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) assert runtime.capabilities.supports_transforms is True assert telemetry.rows_affected == 9 +@pytest.mark.parametrize( + ("mode", "batched", "streaming"), + [ + (WriteMode.SCD1, True, False), + (WriteMode.INCREMENTAL, True, False), + (WriteMode.SNAPSHOT, True, False), + (WriteMode.SCD2, False, True), + (WriteMode.REPLACE, False, True), + ], +) +def test_postgresql_writer_modes_expose_safe_executor_boundaries( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], + mode: WriteMode, + batched: bool, + streaming: bool, +) -> None: + runtime, _pool, _database, _schema_name = postgresql_warehouse + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.STRICT, + mode=mode, + cursor_field="updated_at" if mode is WriteMode.INCREMENTAL else None, + snapshot_field="snapshot_at" if mode is WriteMode.SNAPSHOT else None, + ) + + assert writer.supports_batched_writes is batched + assert writer.accepts_streaming_input is streaming + + +@pytest.mark.parametrize("mode", [WriteMode.REPLACE, WriteMode.SCD2]) +def test_postgresql_whole_endpoint_modes_cross_runner_batch_boundary( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], + mode: WriteMode, +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.STRICT, + mode=mode, + ) + relation = RelationRef( + catalog=database, + namespace=schema_name, + name=f"whole_{mode.value}", + ) + fence = FencingToken( + lease_table=None, + pipeline_id=f"postgresql_{mode.value}", + run_id="run-whole", + token=1, + authority_id="postgresql:test-state", + ) + source = _StaticSource( + name=f"postgresql_{mode.value}", + records=[ + {"id": "one", "label": "old"}, + {"id": "one", "label": "new"}, + {"id": "two", "label": "second"}, + ], + ) + runner = PipelineRunner( + source=source, + writer=writer, + watermarks=_NoWatermarks(), + endpoint_relations={"records": relation}, + batch_rows=1, + target_fence=runtime.target_fence, + schema_mapper=runtime.ingestion_schema_mapper, + ) + + result = runner.run(run_id=fence.run_id, ownership=_Ownership(fence)) + + assert result.endpoints[0].extracted == 3 + assert result.endpoints[0].affected == (3 if mode is WriteMode.REPLACE else 2) + with pool.connection() as connection: + if mode is WriteMode.SCD2: + rows = connection.execute( + sql.SQL("SELECT id, label, is_current FROM {} ORDER BY id, label").format( + sql.Identifier(schema_name, relation.name) + ) + ).fetchall() + assert rows == [ + {"id": "one", "label": "new", "is_current": True}, + {"id": "two", "label": "second", "is_current": True}, + ] + else: + rows = connection.execute( + sql.SQL("SELECT id, label FROM {} ORDER BY id, label").format( + sql.Identifier(schema_name, relation.name) + ) + ).fetchall() + assert rows == [ + {"id": "one", "label": "new"}, + {"id": "one", "label": "old"}, + {"id": "two", "label": "second"}, + ] + + if mode is WriteMode.REPLACE: + empty_fence = FencingToken( + lease_table=None, + pipeline_id="postgresql_replace", + run_id="run-empty", + token=2, + authority_id="postgresql:test-state", + ) + empty_runner = PipelineRunner( + source=_StaticSource(name="postgresql_replace", records=[]), + writer=writer, + watermarks=_NoWatermarks(), + endpoint_relations={"records": relation}, + batch_rows=1, + target_fence=runtime.target_fence, + schema_mapper=runtime.ingestion_schema_mapper, + ) + + empty_result = empty_runner.run( + run_id=empty_fence.run_id, + ownership=_Ownership(empty_fence), + ) + + assert empty_result.endpoints[0].affected == 0 + with pool.connection() as connection: + remaining = connection.execute( + sql.SQL("SELECT COUNT(*) AS count FROM {}").format( + sql.Identifier(schema_name, relation.name) + ) + ).fetchone() + assert remaining == {"count": 0} + + +def test_postgresql_scd1_reuses_pre_upgrade_unique_index( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + relation = RelationRef(catalog=database, namespace=schema_name, name="legacy_scd1") + legacy_digest = hashlib.sha256(f"{schema_name}.legacy_scd1:id".encode()).hexdigest()[:10] + legacy_index = f"dander_uq_legacy_scd1_{legacy_digest}" + with pool.connection() as connection: + connection.execute( + sql.SQL("CREATE TABLE {} (id text NOT NULL PRIMARY KEY, label text)").format( + sql.Identifier(schema_name, relation.name) + ) + ) + connection.execute( + sql.SQL("CREATE UNIQUE INDEX {} ON {} (id)").format( + sql.Identifier(legacy_index), + sql.Identifier(schema_name, relation.name), + ) + ) + fence = FencingToken( + lease_table=None, + pipeline_id="postgresql_legacy_scd1", + run_id="run-legacy-index", + token=1, + authority_id="postgresql:test-state", + ) + publication = runtime.target_fence.claim(relation, fence) + target = WriteTarget( + project=database, + dataset=schema_name, + table=relation.name, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING", mode="REQUIRED"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=publication, + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.STRICT, + ) + + writer.write([{"id": "one", "label": "value"}], target) + + with pool.connection() as connection: + indexes = connection.execute( + "SELECT indexname FROM pg_indexes WHERE schemaname = %s AND tablename = %s " + "AND indexname LIKE 'dander_uq_%%' ORDER BY indexname", + (schema_name, relation.name), + ).fetchall() + assert indexes == [{"indexname": legacy_index}] + + def test_postgresql_scd1_streams_copy_replays_and_evolves_nullable_columns( postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], ) -> None: @@ -327,3 +571,181 @@ def initial_records() -> Iterator[dict[str, object]]: } assert ledger == {"status": "committed", "run_id": "run-one", "fencing_token": 1} assert staging == {"count": 0} + + +def test_postgresql_incremental_keeps_newest_cursor_and_rejects_regression( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=10, + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.INCREMENTAL, + cursor_field="updated_at", + ) + relation = RelationRef(catalog=database, namespace=schema_name, name="incremental_records") + fence = FencingToken( + lease_table=None, + pipeline_id="postgresql_incremental", + run_id="run-incremental", + token=1, + authority_id="postgresql:test-state", + ) + publication = runtime.target_fence.claim(relation, fence) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + WriteField(name="updated_at", data_type="TIMESTAMP"), + ), + publication_fence=publication, + ) + newest = datetime(2026, 8, 10, 12, 0, tzinfo=UTC) + older = datetime(2026, 8, 10, 11, 0, tzinfo=UTC) + later = datetime(2026, 8, 10, 13, 0, tzinfo=UTC) + + assert ( + writer.write( + [ + {"id": "one", "label": "newest", "updated_at": newest}, + {"id": "one", "label": "older-last", "updated_at": older}, + ], + target, + ) + == 1 + ) + assert ( + writer.write( + [{"id": "one", "label": "must-not-regress", "updated_at": older}], + target, + ) + == 0 + ) + assert ( + writer.write( + [{"id": "one", "label": "later", "updated_at": later}], + target, + ) + == 1 + ) + + with pool.connection() as connection: + row = connection.execute( + sql.SQL("SELECT id, label, updated_at FROM {}").format( + sql.Identifier(schema_name, relation.name) + ) + ).fetchone() + assert row == {"id": "one", "label": "later", "updated_at": later} + + +def test_postgresql_snapshot_is_null_safe_and_replay_safe( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=10, + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.SNAPSHOT, + snapshot_field="snapshot_at", + ) + relation = RelationRef(catalog=database, namespace=schema_name, name="snapshot_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="postgresql_snapshot", + run_id="run-snapshot", + token=1, + authority_id="postgresql:test-state", + ), + ) + target = WriteTarget( + relation=relation, + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + WriteField(name="snapshot_at", data_type="TIMESTAMP"), + ), + publication_fence=publication, + ) + first = datetime(2026, 8, 10, 12, 0, tzinfo=UTC) + second = datetime(2026, 8, 10, 13, 0, tzinfo=UTC) + record = {"id": "one", "label": None, "snapshot_at": first} + + assert writer.write([record, record], target) == 1 + assert writer.write([record], target) == 0 + assert writer.write([{"id": "one", "label": None, "snapshot_at": second}], target) == 1 + + with pool.connection() as connection: + rows = connection.execute( + sql.SQL("SELECT id, label, snapshot_at FROM {} ORDER BY snapshot_at").format( + sql.Identifier(schema_name, relation.name) + ) + ).fetchall() + assert rows == [ + {"id": "one", "label": None, "snapshot_at": first}, + {"id": "one", "label": None, "snapshot_at": second}, + ] + + +def test_postgresql_scd2_closes_changed_rows_and_replays_without_new_versions( + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.SCD2, + ) + relation = RelationRef(catalog=database, namespace=schema_name, name="scd2_records") + first_fence = FencingToken( + lease_table=None, + pipeline_id="postgresql_scd2", + run_id="run-scd2-one", + token=1, + authority_id="postgresql:test-state", + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ), + publication_fence=runtime.target_fence.claim(relation, first_fence), + ) + assert writer.write([{"id": "one", "label": "first"}], target) == 1 + + second_fence = FencingToken( + lease_table=None, + pipeline_id="postgresql_scd2", + run_id="run-scd2-two", + token=2, + authority_id="postgresql:test-state", + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=target.schema, + publication_fence=runtime.target_fence.claim(relation, second_fence), + ) + changed = [{"id": "one", "label": "second"}] + assert writer.write(changed, target) == 1 + assert writer.write(changed, target) == 0 + + with pool.connection() as connection: + rows = connection.execute( + sql.SQL( + "SELECT id, label, is_current, valid_to IS NOT NULL AS closed " + "FROM {} ORDER BY valid_from" + ).format(sql.Identifier(schema_name, relation.name)) + ).fetchall() + assert rows == [ + {"id": "one", "label": "first", "is_current": False, "closed": True}, + {"id": "one", "label": "second", "is_current": True, "closed": False}, + ] From 6c7e6a18f5f46965c82b8865a26a53c573489948 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 08:29:47 -0400 Subject: [PATCH 101/163] Execute portable graphs on PostgreSQL --- HANDOFF.md | 34 +- docs/compatibility-matrix.md | 2 +- docs/decisions.md | 11 + docs/known-limitations.md | 8 +- docs/platform-profiles.md | 6 +- src/dander/providers/postgresql/runtime.py | 32 +- src/dander/providers/postgresql/schema.py | 14 + src/dander/providers/postgresql/transform.py | 185 ++++++++- src/dander/runtime-compatibility.json | 7 +- .../test_postgresql_warehouse_runtime.py | 353 +++++++++++++++++- 10 files changed, 607 insertions(+), 45 deletions(-) create mode 100644 src/dander/providers/postgresql/schema.py diff --git a/HANDOFF.md b/HANDOFF.md index cec2ae3d..0285d7d0 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,11 +2,11 @@ ## Finished -- Added bounded COPY-backed replace, SCD1, SCD2, snapshot, and incremental PostgreSQL writes. -- Kept replace/SCD2 endpoint-wide while preserving batching for the three batch-safe modes. -- Added cursor ordering, replay-safe snapshots, SCD2 history, and transactional fencing tests. -- Preserved full-row replace semantics and the pre-upgrade SCD1 unique-index identity. -- Updated the machine-readable capabilities and focused operator documentation. +- Added PostgreSQL execution for the existing provider-neutral `GraphExecutionPlan`. +- Published replace-mode graph targets through PostgreSQL's transactional destination fence. +- Added whole-plan preflight for dialect support and database-local source/target coordinates. +- Added live PostgreSQL replay, selection, ownership, and stale-fence rollback coverage. +- Updated the packaged capability contract and focused PostgreSQL documentation. ## Try It @@ -15,26 +15,26 @@ Set `DANDER_TEST_POSTGRES_DSN` to PostgreSQL 15+ and run ## Checks -- Ruff passed across the repository; strict mypy passed across 304 source files. -- All 16 focused tests and the complete 1,189-test suite passed against PostgreSQL 15. -- Terraform roots/modules/tests and Helm lint/template validation passed. -- Wheel/sdist inspection, source-free install/generation, and dependency audit passed. -- Container conformance passed; retained GCP stage-zero and platform plans reported no changes. +- Ruff and strict mypy passed across the repository's 305 source files. +- All 20 focused PostgreSQL tests passed against PostgreSQL 15. +- The complete 1,193-test suite and dependency audit passed. +- Wheel/sdist inspection, source-free installs, and container conformance passed. +- Terraform roots/tests and Helm lint/template validation passed. +- Retained GCP stage-zero and platform plans each reported exactly `No changes.` ## Decisions -- Replace and SCD2 stream a complete endpoint through COPY; they never use executor batches. -- Incremental de-duplication ranks cursor descending, then source ordinal for equal cursors. -- PostgreSQL remains experimental until graph and live Kubernetes qualification are complete. +- Reuse the canonical graph AST; PostgreSQL receives no second graph abstraction. +- Reject foreign source catalogs before rendering because PostgreSQL SQL is database-local. +- Keep graph execution replace-only and PostgreSQL experimental pending live Kubernetes proof. ## Remaining -- Let protected CI repeat Linux PostgreSQL, image, and secret checks on PR #175. -- Merge through protected main only when every required check is green. -- Continue PostgreSQL graph work separately; do not mix it into this writer PR. +- Let protected CI repeat Linux PostgreSQL, image, and secret checks before merge. +- Open and review the focused PR; do not promote PostgreSQL support yet. ## Review First -- `src/dander/providers/postgresql/writer.py` +- `src/dander/providers/postgresql/transform.py` - `tests/providers/test_postgresql_warehouse_runtime.py` - `src/dander/providers/postgresql/runtime.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 09c0ee11..d261b0c1 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -39,7 +39,7 @@ their named live profiles pass. | Warehouse | Ingestion modes | Transport | Canonical schema | Models | Graphs | Fence | |---|---|---|---|---|---|---| | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | -| PostgreSQL | all five | COPY | all v1 types; decimal 1000, time 6 | yes | no | yes | +| PostgreSQL | all five | COPY | all v1 types; decimal 1000, time 6 | yes | yes | yes | | Redshift | all five | COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | yes | yes | | Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | diff --git a/docs/decisions.md b/docs/decisions.md index 4c18e08f..1b88aa11 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,16 @@ # Engineering Decisions +## 2026-08-10 — PostgreSQL graphs reuse canonical plans and database-local fencing + +- **Planning:** PostgreSQL consumes the existing `GraphExecutionPlan` and renders its relational + AST with the PostgreSQL dialect; no provider-specific graph representation is introduced. +- **Publication:** Every selected replace target preflights before provider I/O, then creates a + transaction-local CTAS stage and replaces the stable target behind the existing destination + fence. A lost fence rolls back target creation, staging, and publication together. +- **Boundary:** Source and target catalogs must equal the configured database because PostgreSQL + rendering is database-local. Graph safe casts, other write modes, live Kubernetes proof, and + support promotion remain separate work. + ## 2026-08-10 — PostgreSQL write modes preserve endpoint and batch semantics - **Modes:** One COPY-backed writer now publishes replace, SCD1, SCD2, snapshot, and incremental diff --git a/docs/known-limitations.md b/docs/known-limitations.md index bab2862c..5e5c5dfc 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -25,9 +25,11 @@ evaluate these limits before using it for an unattended system containing busine targets use the provider-neutral relational AST and fenced table path; graph safe casts fail preflight. Redshift has no live qualification or views. PostgreSQL state and warehouse execution are implemented and locally conformance-tested; all five ingestion modes use bounded `COPY` and - destination fencing, while the ordinary hosted source runner still selects SCD1. A packaged Helm - chart now renders the Kubernetes launcher against an existing cluster, but no Kubernetes live - profile is qualified yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse + destination fencing. Replace-mode graph targets use the provider-neutral relational AST and + fenced table path; graph safe casts and cross-database relations fail preflight. The ordinary + hosted source runner still selects SCD1. A packaged Helm chart now renders the Kubernetes + launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander + does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The package publishes this pair matrix and each warehouse's exact implemented modes, transports, schema limits, transforms, graphs, and fencing through `dander runtime compatibility`, while the diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 729dd13e..788b03ba 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -130,8 +130,10 @@ incremental publication rejects cursor regression. Snapshot comparison is null-s empty-source safe, and SCD2 uses transaction-stable validity timestamps. Every materialization is claimed and published through its destination target fence. The local native-profile proof covers bounded ingestion, replay, transforms, assertions, metadata, run history, watermarks, and leases. -The ordinary hosted source runner still selects SCD1, and graph execution remains follow-up work. -The packaged existing-cluster Kubernetes/Helm launcher can +Replace-mode graphs render the provider-neutral relational AST as PostgreSQL SQL and publish stable +targets inside the same destination-fence transaction. Cross-database source or target coordinates +and graph safe casts fail before provider mutation. The ordinary hosted source runner still selects +SCD1. The packaged existing-cluster Kubernetes/Helm launcher can render and verify this profile without cloud-specific identity assumptions, but no live cluster qualification has passed. Until that acceptance does, this adapter must not be represented as a supported hosted PostgreSQL deployment. See [Kubernetes existing-cluster launcher](kubernetes.md). diff --git a/src/dander/providers/postgresql/runtime.py b/src/dander/providers/postgresql/runtime.py index 42d14de9..fb13dcfd 100644 --- a/src/dander/providers/postgresql/runtime.py +++ b/src/dander/providers/postgresql/runtime.py @@ -13,15 +13,15 @@ from dander.providers.postgresql.config import PostgreSQLWarehouseConfig from dander.providers.postgresql.fence import PostgreSQLTargetFence -from dander.providers.postgresql.transform import PostgreSQLTransformRunner +from dander.providers.postgresql.schema import POSTGRESQL_SCHEMA_SUPPORT +from dander.providers.postgresql.transform import PostgreSQLGraphRunner, PostgreSQLTransformRunner from dander.providers.postgresql.writer import PostgreSQLCopyWriter, PostgreSQLTimeouts from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.telemetry import OperationTelemetry, TelemetryOperation -from dander.warehouse import CanonicalField, LogicalTypeKind, RelationRef, RelationSchema +from dander.warehouse import CanonicalField, RelationRef, RelationSchema from dander.warehouse.runtime import ( WarehouseCapabilities, WarehouseRuntime, - WarehouseSchemaSupport, ) from dander.writer import ( SchemaEvolution, @@ -39,14 +39,6 @@ PostgreSQLRow = dict[str, object] PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] -POSTGRESQL_SCHEMA_SUPPORT = WarehouseSchemaSupport( - provider_id="postgresql", - logical_types=frozenset(LogicalTypeKind), - max_decimal_precision=1_000, - max_temporal_precision=6, - supports_nested_arrays=True, -) - @dataclass(frozen=True, slots=True) class PostgreSQLRelationCodec: @@ -113,7 +105,7 @@ def build_ingestion_writer( @dataclass(frozen=True, slots=True) class PostgreSQLTransformFactory: - """Construct PostgreSQL model execution while rejecting graph plans.""" + """Construct fenced PostgreSQL model or provider-neutral graph execution.""" database: str pool: PostgreSQLPool @@ -126,9 +118,19 @@ def build_transform_runner( graph_plan: object | None, build_models: bool, raw_namespace: str = "raw", - ) -> PostgreSQLTransformRunner | None: + ) -> PostgreSQLGraphRunner | PostgreSQLTransformRunner | None: if graph_plan is not None: - raise ValueError("PostgreSQL graph execution is not available") + from dander.pipeline.runtime import GraphExecutionPlan + + if not isinstance(graph_plan, GraphExecutionPlan): + raise TypeError("PostgreSQL graph plan has the wrong type") + return PostgreSQLGraphRunner( + plan=graph_plan, + database=self.database, + pool=self.pool, + target_fence=self.target_fence, + timeouts=self.timeouts, + ) if not build_models: return None return PostgreSQLTransformRunner( @@ -169,7 +171,7 @@ def operation( write_modes=frozenset(WriteMode), transports=frozenset({WriteTransport.COPY}), supports_transforms=True, - supports_graphs=False, + supports_graphs=True, supports_target_fencing=True, schema_support=POSTGRESQL_SCHEMA_SUPPORT, ) diff --git a/src/dander/providers/postgresql/schema.py b/src/dander/providers/postgresql/schema.py new file mode 100644 index 00000000..12ae1583 --- /dev/null +++ b/src/dander/providers/postgresql/schema.py @@ -0,0 +1,14 @@ +"""Canonical schema support advertised and enforced by PostgreSQL runtimes.""" + +from dander.warehouse import LogicalTypeKind +from dander.warehouse.runtime import WarehouseSchemaSupport + +POSTGRESQL_SCHEMA_SUPPORT = WarehouseSchemaSupport( + provider_id="postgresql", + logical_types=frozenset(LogicalTypeKind), + max_decimal_precision=1_000, + max_temporal_precision=6, + supports_nested_arrays=True, +) + +__all__ = ["POSTGRESQL_SCHEMA_SUPPORT"] diff --git a/src/dander/providers/postgresql/transform.py b/src/dander/providers/postgresql/transform.py index 6373ae59..7e2e8357 100644 --- a/src/dander/providers/postgresql/transform.py +++ b/src/dander/providers/postgresql/transform.py @@ -5,11 +5,22 @@ import hashlib from dataclasses import dataclass from typing import TYPE_CHECKING +from uuid import uuid4 from psycopg import Connection, sql from psycopg_pool import ConnectionPool -from dander.providers.postgresql.writer import PostgreSQLTimeouts, _set_timeouts +from dander.concurrency import TargetFenceLostError +from dander.pipeline.compiler import CompiledTarget, PipelineCompileError +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError +from dander.providers.postgresql.schema import POSTGRESQL_SCHEMA_SUPPORT +from dander.providers.postgresql.writer import ( + PostgreSQLTimeouts, + PostgreSQLWriteError, + _ensure_target, + _postgresql_type, + _set_timeouts, +) from dander.transform import ( SqlDialect, TransformModel, @@ -19,6 +30,7 @@ TransformRunResult, ) from dander.transform.model import Materialization +from dander.writer import SchemaEvolution, WriteMode, WriteTarget if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -40,6 +52,13 @@ class _PostgreSQLAssertion: parameters: tuple[object, ...] = () +@dataclass(frozen=True, slots=True) +class _PostgreSQLGraphPlan: + name: str + target: WriteTarget + query: str + + class PostgreSQLTransformRunner: """Build provider-exact or portable models in PostgreSQL 15 or newer.""" @@ -154,6 +173,168 @@ def _run_assertions( raise TransformRunError(f"Data tests failed: {', '.join(failures)}") +class PostgreSQLGraphRunner: + """Publish provider-neutral graph targets through PostgreSQL's fenced DML path.""" + + target_dialect = SqlDialect.POSTGRES + + def __init__( + self, + *, + plan: GraphExecutionPlan, + database: str, + pool: PostgreSQLPool, + target_fence: PostgreSQLTargetFence, + timeouts: PostgreSQLTimeouts, + ) -> None: + self._plan = plan + self._database = database + self._pool = pool + self._target_fence = target_fence + self._timeouts = timeouts + + def build( + self, + _models_dir: Path, + *, + selected: Iterable[str] | None = None, + ownership: OwnershipGuard | None = None, + ) -> TransformRunResult: + """Preflight selected targets, then replace each behind its destination fence.""" + if ownership is None or ownership.fence is None: + raise GraphRuntimeError("PostgreSQL graph execution requires active lease ownership") + selected_ids = set(selected) if selected is not None else None + known = {target.node_id for target in self._plan.targets} + if selected_ids is not None and (unknown := sorted(selected_ids - known)): + raise GraphRuntimeError(f"Unknown graph target: {unknown[0]!r}") + targets = tuple( + target + for target in self._plan.targets + if selected_ids is None or target.node_id in selected_ids + ) + if not targets: + raise GraphRuntimeError("Graph execution selected no targets") + if foreign := sorted( + { + relation.catalog + for relation in self._plan.bindings.source_relations.values() + if relation.catalog != self._database + } + ): + raise GraphRuntimeError( + f"PostgreSQL graph source belongs to another database: {foreign[0]!r}" + ) + + plans = tuple(_graph_plan(target, database=self._database) for target in targets) + for plan in plans: + ownership.verify() + publication = self._target_fence.claim(plan.target.relation_ref, ownership.fence) + ownership.verify() + _publish_graph_plan( + plan, + publication, + pool=self._pool, + target_fence=self._target_fence, + timeouts=self._timeouts, + ) + return TransformRunResult( + models=tuple(plan.name for plan in plans), + assertions=0, + ) + + +def _graph_plan(compiled: CompiledTarget, *, database: str) -> _PostgreSQLGraphPlan: + if compiled.write_mode is not WriteMode.REPLACE: + raise GraphRuntimeError( + f"PostgreSQL graph target {compiled.node_id!r} requires replace mode" + ) + if compiled.target.relation_ref.catalog != database: + raise GraphRuntimeError( + f"PostgreSQL graph target {compiled.node_id!r} belongs to another database" + ) + try: + POSTGRESQL_SCHEMA_SUPPORT.require(compiled.target.canonical_schema) + for field in compiled.target.canonical_schema.fields: + _postgresql_type(field.data_type) + query = compiled.render(SqlDialect.POSTGRES) + except (PipelineCompileError, PostgreSQLWriteError, ValueError) as error: + raise GraphRuntimeError(str(error)) from error + return _PostgreSQLGraphPlan( + name=compiled.node_id, + target=compiled.target, + query=query, + ) + + +def _publish_graph_plan( + plan: _PostgreSQLGraphPlan, + publication: TargetFence, + *, + pool: PostgreSQLPool, + target_fence: PostgreSQLTargetFence, + timeouts: PostgreSQLTimeouts, +) -> None: + target = WriteTarget( + relation=plan.target.relation_ref, + business_key=plan.target.business_key, + schema=plan.target.schema, + declared_schema=plan.target.canonical_schema, + publication_fence=publication, + ) + staging = f"dander_graph_{uuid4().hex}" + try: + with pool.connection() as connection, connection.transaction(): + _set_timeouts(connection, timeouts) + _ensure_target( + connection, + target, + target.canonical_schema, + evolution=SchemaEvolution.STRICT, + mode=WriteMode.REPLACE, + ) + connection.execute(_graph_staging_sql(plan, staging)) + target_fence.execute_statements( + connection, + _graph_publication_statements(target, staging), + publication, + ) + except (TargetFenceLostError, PostgreSQLWriteError, GraphRuntimeError, TransformRunError): + raise + except Exception as error: + raise TransformRunError("PostgreSQL graph publication failed") from error + + +def _graph_staging_sql(plan: _PostgreSQLGraphPlan, staging: str) -> sql.Composed: + source_columns = sql.SQL(", ").join( + sql.SQL("source.{}").format(sql.Identifier(field.name)) + for field in plan.target.canonical_schema.fields + ) + return sql.SQL("CREATE TEMP TABLE {} ON COMMIT DROP AS SELECT {} FROM ({}) AS source").format( + sql.Identifier(staging), + source_columns, + sql.SQL(plan.query), + ) + + +def _graph_publication_statements( + target: WriteTarget, + staging: str, +) -> tuple[sql.Composed, sql.Composed]: + columns = sql.SQL(", ").join( + sql.Identifier(field.name) for field in target.canonical_schema.fields + ) + relation = sql.Identifier(target.relation_ref.namespace, target.relation_ref.name) + return ( + sql.SQL("DELETE FROM {}").format(relation), + sql.SQL("INSERT INTO {} ({}) SELECT {} FROM {}").format( + relation, + columns, + columns, + sql.Identifier(staging), + ), + ) + + def _materialization_statements( model: TransformModel, query: str, @@ -363,4 +544,4 @@ def _unique_index_name(model: TransformModel) -> str: return f"dander_uq_{model.name}"[:51] + f"_{digest}" -__all__ = ["PostgreSQLTransformRunner"] +__all__ = ["PostgreSQLGraphRunner", "PostgreSQLTransformRunner"] diff --git a/src/dander/runtime-compatibility.json b/src/dander/runtime-compatibility.json index 94d00b67..60392d15 100644 --- a/src/dander/runtime-compatibility.json +++ b/src/dander/runtime-compatibility.json @@ -71,9 +71,12 @@ "max_decimal_precision": 1000, "max_temporal_precision": 6, "supports_transforms": true, - "supports_graphs": false, + "supports_graphs": true, "supports_target_fencing": true, - "limitations": ["graph execution is not available", "hosted source execution selects scd1"] + "limitations": [ + "graph safe-cast semantics are not available", + "hosted source execution selects scd1" + ] }, { "provider": "redshift", diff --git a/tests/providers/test_postgresql_warehouse_runtime.py b/tests/providers/test_postgresql_warehouse_runtime.py index 26c28d1b..a27da17e 100644 --- a/tests/providers/test_postgresql_warehouse_runtime.py +++ b/tests/providers/test_postgresql_warehouse_runtime.py @@ -6,6 +6,7 @@ import os import sys import uuid +from dataclasses import replace from datetime import UTC, datetime from decimal import Decimal from typing import TYPE_CHECKING, Any, cast @@ -15,19 +16,29 @@ from psycopg.rows import dict_row from psycopg_pool import ConnectionPool -from dander.concurrency import FencingToken +from dander.concurrency import FencingToken, TargetFenceLostError from dander.ingestion.source import Endpoint, RawField, Source, SourceConfig +from dander.pipeline.graph import PipelineGraph +from dander.pipeline.runtime import GraphExecutionPlan, GraphRuntimeError, plan_graph_execution from dander.providers import ProviderKind, default_provider_registry from dander.providers.postgresql.config import PostgreSQLWarehouseConfig +from dander.providers.postgresql.transform import PostgreSQLGraphRunner from dander.providers.postgresql.writer import PostgreSQLWriteError from dander.runtime import PipelineRunner from dander.state import WatermarkStore from dander.telemetry import TelemetryOperation -from dander.warehouse import LogicalTypeKind, RelationRef, WarehouseRuntime +from dander.warehouse import ( + CanonicalType, + LogicalTypeKind, + RelationRef, + RelationSchema, + WarehouseRuntime, +) from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport if TYPE_CHECKING: from collections.abc import Iterator, Mapping + from pathlib import Path PostgreSQLRow = dict[str, Any] PostgreSQLPool = ConnectionPool[Connection[PostgreSQLRow]] @@ -81,9 +92,10 @@ def set(self, source: str, entity: str, cursor: str) -> None: class _Ownership: def __init__(self, fence: FencingToken) -> None: self.fence = fence + self.verifications = 0 def verify(self) -> None: - return None + self.verifications += 1 def test_postgresql_warehouse_registration_is_lazy_and_explicit() -> None: @@ -217,6 +229,7 @@ class _Cursor: assert runtime.capabilities.write_modes == frozenset(WriteMode) assert runtime.capabilities.transports == frozenset({WriteTransport.COPY}) assert runtime.capabilities.supports_transforms is True + assert runtime.capabilities.supports_graphs is True assert telemetry.rows_affected == 9 @@ -749,3 +762,337 @@ def test_postgresql_scd2_closes_changed_rows_and_replays_without_new_versions( {"id": "one", "label": "first", "is_current": False, "closed": True}, {"id": "one", "label": "second", "is_current": True, "closed": False}, ] + + +def _postgresql_graph_plan( + database: str, + schema_name: str, + *, + include_unsupported_target: bool = False, +) -> GraphExecutionPlan: + source = SourceConfig( + name="fixture", + base_url="https://example.test", + auth_strategy="none", + endpoints=[ + Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING"), + RawField(name="label", data_type="STRING"), + ], + ) + ], + ) + targets: list[dict[str, object]] = [ + { + "id": "target", + "type": "target", + "name": "Target", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": schema_name, + "table": "graph_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + } + ] + edges: list[dict[str, object]] = [ + { + "from": "records", + "to": "target", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ] + if include_unsupported_target: + targets.append( + { + "id": "unsupported", + "type": "target", + "name": "Unsupported", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": schema_name, + "table": "unsupported_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING", "cast_to": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + } + ) + edges.append( + { + "from": "records", + "to": "unsupported", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ) + graph = PipelineGraph.model_validate( + { + "name": "postgresql_graph", + "nodes": [ + { + "id": "records", + "type": "source", + "name": "Records", + "config": {"connector": "fixture", "endpoint": "records"}, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + *targets, + ], + "edges": edges, + } + ) + return plan_graph_execution( + graph, + source, + endpoint_relations={ + "records": RelationRef( + catalog=database, + namespace=schema_name, + name="fixture_records", + ) + }, + ) + + +def _graph_fence(run_id: str, token: int) -> FencingToken: + return FencingToken( + lease_table=None, + pipeline_id="postgresql_graph", + run_id=run_id, + token=token, + authority_id="postgresql:test-state", + ) + + +def _create_graph_source(pool: PostgreSQLPool, schema_name: str) -> None: + with pool.connection() as connection: + connection.execute( + sql.SQL("CREATE TABLE {} (id TEXT, label TEXT)").format( + sql.Identifier(schema_name, "fixture_records") + ) + ) + connection.execute( + sql.SQL("INSERT INTO {} (id, label) VALUES (%s, %s), (%s, %s)").format( + sql.Identifier(schema_name, "fixture_records") + ), + ("one", "first", "two", "second"), + ) + + +def test_postgresql_graph_replaces_and_replays_canonical_plan( + tmp_path: Path, + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + _create_graph_source(pool, schema_name) + runner = runtime.transforms.build_transform_runner( + graph_plan=_postgresql_graph_plan(database, schema_name), + build_models=True, + ) + assert isinstance(runner, PostgreSQLGraphRunner) + ownership = _Ownership(_graph_fence("run-graph-one", 1)) + + result = runner.build(tmp_path, ownership=ownership) + + assert result.models == ("target",) + assert result.assertions == 0 + assert ownership.verifications == 2 + with pool.connection() as connection: + first = connection.execute( + sql.SQL("SELECT id, label FROM {} ORDER BY id").format( + sql.Identifier(schema_name, "graph_records") + ) + ).fetchall() + connection.execute( + sql.SQL("DELETE FROM {}").format(sql.Identifier(schema_name, "fixture_records")) + ) + connection.execute( + sql.SQL("INSERT INTO {} (id, label) VALUES (%s, %s)").format( + sql.Identifier(schema_name, "fixture_records") + ), + ("two", "replayed"), + ) + assert first == [ + {"id": "one", "label": "first"}, + {"id": "two", "label": "second"}, + ] + + replay = _Ownership(_graph_fence("run-graph-two", 2)) + assert runner.build(tmp_path, ownership=replay).models == ("target",) + with pool.connection() as connection: + rows = connection.execute( + sql.SQL("SELECT id, label FROM {} ORDER BY id").format( + sql.Identifier(schema_name, "graph_records") + ) + ).fetchall() + assert rows == [{"id": "two", "label": "replayed"}] + + +def test_postgresql_graph_preflights_every_target_and_source_catalog_before_mutation( + tmp_path: Path, + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + unsupported = runtime.transforms.build_transform_runner( + graph_plan=_postgresql_graph_plan( + database, + schema_name, + include_unsupported_target=True, + ), + build_models=True, + ) + assert isinstance(unsupported, PostgreSQLGraphRunner) + with pytest.raises(GraphRuntimeError, match="safe-cast semantics"): + unsupported.build(tmp_path, ownership=_Ownership(_graph_fence("run-preflight", 3))) + + plan = _postgresql_graph_plan(database, schema_name) + compiled = plan.targets[0] + precision_schema = RelationSchema( + fields=( + compiled.target.canonical_schema.fields[0].model_copy( + update={ + "data_type": CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=True, + fractional_second_precision=7, + ) + } + ), + *compiled.target.canonical_schema.fields[1:], + ) + ) + precision_target = WriteTarget( + relation=compiled.target.relation_ref, + business_key=compiled.target.business_key, + schema=compiled.target.schema, + declared_schema=precision_schema, + ) + precision_plan = replace( + plan, + targets=(replace(compiled, target=precision_target),), + ) + precision_runner = runtime.transforms.build_transform_runner( + graph_plan=precision_plan, + build_models=True, + ) + assert isinstance(precision_runner, PostgreSQLGraphRunner) + with pytest.raises(GraphRuntimeError, match="temporal precision up to 6"): + precision_runner.build( + tmp_path, + ownership=_Ownership(_graph_fence("run-precision", 4)), + ) + + foreign = replace( + plan, + bindings=replace( + plan.bindings, + source_relations={ + "records": RelationRef( + catalog="other_database", + namespace=schema_name, + name="fixture_records", + ) + }, + ), + ) + runner = runtime.transforms.build_transform_runner(graph_plan=foreign, build_models=True) + assert isinstance(runner, PostgreSQLGraphRunner) + with pytest.raises(GraphRuntimeError, match="source belongs to another database"): + runner.build(tmp_path, ownership=_Ownership(_graph_fence("run-foreign", 5))) + + with pool.connection() as connection: + rows = connection.execute( + "SELECT relname FROM pg_catalog.pg_class AS relation " + "JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace " + "WHERE namespace.nspname = %s AND relation.relname IN " + "('graph_records', 'unsupported_records', 'dander_target_commits')", + (schema_name,), + ).fetchall() + assert rows == [] + + +def test_postgresql_graph_selection_ownership_and_factory_fail_closed( + tmp_path: Path, + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, _pool, database, schema_name = postgresql_warehouse + with pytest.raises(TypeError, match="graph plan has the wrong type"): + runtime.transforms.build_transform_runner(graph_plan=object(), build_models=False) + assert runtime.transforms.build_transform_runner(graph_plan=None, build_models=False) is None + runner = runtime.transforms.build_transform_runner( + graph_plan=_postgresql_graph_plan(database, schema_name), + build_models=False, + ) + assert isinstance(runner, PostgreSQLGraphRunner) + ownership = _Ownership(_graph_fence("run-selection", 5)) + with pytest.raises(GraphRuntimeError, match="Unknown graph target"): + runner.build(tmp_path, selected=["missing"], ownership=ownership) + with pytest.raises(GraphRuntimeError, match="selected no targets"): + runner.build(tmp_path, selected=[], ownership=ownership) + with pytest.raises(GraphRuntimeError, match="requires active lease ownership"): + runner.build(tmp_path) + + +def test_postgresql_graph_stale_fence_rolls_back_target_publication( + tmp_path: Path, + postgresql_warehouse: tuple[WarehouseRuntime, PostgreSQLPool, str, str], +) -> None: + runtime, pool, database, schema_name = postgresql_warehouse + _create_graph_source(pool, schema_name) + plan = _postgresql_graph_plan(database, schema_name) + runner = runtime.transforms.build_transform_runner(graph_plan=plan, build_models=True) + assert isinstance(runner, PostgreSQLGraphRunner) + + class _SupersededOwnership: + def __init__(self) -> None: + self.fence = _graph_fence("run-stale", 6) + self.verifications = 0 + + def verify(self) -> None: + self.verifications += 1 + if self.verifications == 2: + runtime.target_fence.claim( + plan.targets[0].target.relation_ref, + _graph_fence("run-newer", 7), + ) + + with pytest.raises(TargetFenceLostError, match="lost before publication"): + runner.build(tmp_path, ownership=_SupersededOwnership()) + + with pool.connection() as connection: + target = connection.execute( + "SELECT to_regclass(%s) AS relation", + (f"{schema_name}.graph_records",), + ).fetchone() + temporary = connection.execute( + "SELECT count(*) AS count FROM pg_catalog.pg_class WHERE relname LIKE 'dander_graph_%'", + ).fetchone() + assert target == {"relation": None} + assert temporary == {"count": 0} From a9804ac0f8198923af9bb50512f688034e4501ee Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 08:58:32 -0400 Subject: [PATCH 102/163] Support immutable images from registries with ports --- infra/kubernetes/chart/dander/values.schema.json | 2 +- src/dander/deployment/projection.py | 4 +++- tests/deployment/test_execution_projection.py | 8 ++++++++ tests/infra/test_kubernetes_chart.py | 13 +++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/infra/kubernetes/chart/dander/values.schema.json b/infra/kubernetes/chart/dander/values.schema.json index 330c08a8..eeaa1583 100644 --- a/infra/kubernetes/chart/dander/values.schema.json +++ b/infra/kubernetes/chart/dander/values.schema.json @@ -13,7 +13,7 @@ "properties": { "reference": { "type": "string", - "pattern": "^[a-z0-9.-]+/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" + "pattern": "^[a-z0-9.-]+(?::[0-9]{1,5})?/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" } } }, diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index 3efbcb4b..45e12f50 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -14,7 +14,9 @@ from dander.project import DanderProject EXECUTION_PROJECTION_SCHEMA = "io.dander.execution/v1" -_IMMUTABLE_IMAGE = re.compile(r"^[a-z0-9.-]+/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$") +_IMMUTABLE_IMAGE = re.compile( + r"^[a-z0-9.-]+(?::[0-9]{1,5})?/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" +) _IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") _PROFILE = re.compile(r"^[a-z][a-z0-9_-]{0,62}$") _ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") diff --git a/tests/deployment/test_execution_projection.py b/tests/deployment/test_execution_projection.py index 663f2e21..3eba0eb3 100644 --- a/tests/deployment/test_execution_projection.py +++ b/tests/deployment/test_execution_projection.py @@ -142,6 +142,14 @@ def test_projection_rejects_mutable_image_reference() -> None: ) +def test_projection_accepts_immutable_image_from_registry_with_port() -> None: + template = _template(Path(__file__).parents[2]) + + local_image = "localhost:5001/dander/runtime@sha256:" + "c" * 64 + + assert replace(template, image=local_image).image == local_image + + def test_graph_projection_does_not_request_model_catalog_output() -> None: templates = build_gcp_v1_execution_templates( load_project_config(Path(__file__).parents[2] / "dander.yaml"), diff --git a/tests/infra/test_kubernetes_chart.py b/tests/infra/test_kubernetes_chart.py index c6b8af83..1e3cae2d 100644 --- a/tests/infra/test_kubernetes_chart.py +++ b/tests/infra/test_kubernetes_chart.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re import shutil import subprocess from pathlib import Path @@ -18,6 +19,7 @@ ROOT = Path(__file__).parents[2] IMAGE = "ghcr.io/example/dander@sha256:" + "b" * 64 +LOCAL_IMAGE = "localhost:5001/dander@sha256:" + "c" * 64 if TYPE_CHECKING: from collections.abc import Sequence @@ -190,6 +192,17 @@ def test_plan_saves_non_secret_values_and_rendered_manifests(tmp_path: Path) -> ] +def test_chart_accepts_immutable_local_registry_image() -> None: + schema = json.loads( + (ROOT / "infra" / "kubernetes" / "chart" / "dander" / "values.schema.json").read_text( + encoding="utf-8" + ) + ) + pattern = schema["properties"]["image"]["properties"]["reference"]["pattern"] + + assert re.fullmatch(pattern, LOCAL_IMAGE) + + def test_verify_is_read_only_and_checks_schedule_pause_image_and_overlap(tmp_path: Path) -> None: project, platforms = _project(tmp_path) cronjob = { From 74afe6b46a727993be937eda4681f2a02ee54470 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 09:21:40 -0400 Subject: [PATCH 103/163] Record PostgreSQL Kubernetes lifecycle acceptance --- HANDOFF.md | 40 ++++++------- ...bility-postgresql-kubernetes-acceptance.md | 57 +++++++++++++++++++ src/dander/deployment/projection.py | 4 +- 3 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 docs/cloud-portability-postgresql-kubernetes-acceptance.md diff --git a/HANDOFF.md b/HANDOFF.md index 0285d7d0..35ad532c 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Added PostgreSQL execution for the existing provider-neutral `GraphExecutionPlan`. -- Published replace-mode graph targets through PostgreSQL's transactional destination fence. -- Added whole-plan preflight for dialect support and database-local source/target coordinates. -- Added live PostgreSQL replay, selection, ownership, and stale-fence rollback coverage. -- Updated the packaged capability contract and focused PostgreSQL documentation. +- Accepted the experimental PostgreSQL/Kubernetes profile through a complete local existing-cluster lifecycle. +- Added immutable image validation for local registries with numeric ports. +- Proved source-free installation, non-root execution, replay, updates, models, assertions, metadata, and fencing. +- Proved reviewed schedule change, Helm rollback, read-only verification, and complete local cleanup. +- Recorded exact artifact digests, environment versions, results, exclusions, and retained GCP no-drift evidence. ## Try It -Set `DANDER_TEST_POSTGRES_DSN` to PostgreSQL 15+ and run -`uv run pytest -q tests/providers/test_postgresql_warehouse_runtime.py`. +Run `uv run pytest -q tests/deployment/test_execution_projection.py tests/infra/test_kubernetes_chart.py`. +The bounded lifecycle evidence is in `docs/cloud-portability-postgresql-kubernetes-acceptance.md`. ## Checks -- Ruff and strict mypy passed across the repository's 305 source files. -- All 20 focused PostgreSQL tests passed against PostgreSQL 15. -- The complete 1,193-test suite and dependency audit passed. -- Wheel/sdist inspection, source-free installs, and container conformance passed. +- Ruff, formatting, strict mypy, and all 1,195 tests passed; PostgreSQL integration used PostgreSQL 15. +- Dependency audit, wheel/sdist inspection, source-free installs, and runtime-all import passed. +- Non-root read-only container conformance and packaged proof-asset checks passed. - Terraform roots/tests and Helm lint/template validation passed. -- Retained GCP stage-zero and platform plans each reported exactly `No changes.` +- Fresh retained GCP stage-zero and platform plans each reported exactly `No changes.` ## Decisions -- Reuse the canonical graph AST; PostgreSQL receives no second graph abstraction. -- Reject foreign source catalogs before rendering because PostgreSQL SQL is database-local. -- Keep graph execution replace-only and PostgreSQL experimental pending live Kubernetes proof. +- Treat kind as valid existing-cluster lifecycle evidence, not hosted-provider or scale qualification. +- Preserve TLS-required PostgreSQL behavior; fix the disposable fixture rather than weakening Dander. +- Keep PostgreSQL/Kubernetes experimental and explicitly leave overlap, interruption, and alerts unevaluated. ## Remaining -- Let protected CI repeat Linux PostgreSQL, image, and secret checks before merge. -- Open and review the focused PR; do not promote PostgreSQL support yet. +- Let protected CI repeat Linux PostgreSQL, packaging, image, secret, Terraform, and Helm checks. +- Merge only if completion review and required checks remain clean. +- Begin the Snowflake/Redshift phase only after this gate closes. ## Review First -- `src/dander/providers/postgresql/transform.py` -- `tests/providers/test_postgresql_warehouse_runtime.py` -- `src/dander/providers/postgresql/runtime.py` +- `src/dander/deployment/projection.py` +- `infra/kubernetes/chart/dander/values.schema.json` +- `docs/cloud-portability-postgresql-kubernetes-acceptance.md` diff --git a/docs/cloud-portability-postgresql-kubernetes-acceptance.md b/docs/cloud-portability-postgresql-kubernetes-acceptance.md new file mode 100644 index 00000000..74e009b8 --- /dev/null +++ b/docs/cloud-portability-postgresql-kubernetes-acceptance.md @@ -0,0 +1,57 @@ +# PostgreSQL on Kubernetes lifecycle acceptance + +Accepted locally on 2026-08-10 against commit +`a9804ac0f8198923af9bb50512f688034e4501ee`. This is existing-cluster lifecycle evidence for +the experimental PostgreSQL/Kubernetes profile. It is not hosted-provider, scale, reliability, +alerting, or support qualification. + +## Artifact and environment + +- A source-free image was built from an unpublished local `dander-platform==0.8.0rc8` wheel. The + wheel SHA-256 was `8ac41a105a7539bf3feda117a533a4a2495b43c9b446f64c479943ea3e27c3f2`; + the image contained generated project files and no `src/` directory. +- The non-root image user was `65532:65532`. The local-registry image digest was + `sha256:9e7a88784ecb160cff1158409c6b3f4feb5ae8f88c7baf71b7f7e8cd53a09539`, + and its revision label matched the tested commit. +- The disposable environment used kind 0.32.0, Kubernetes client/server 1.32.2, Helm 4.2.3, + Docker 28.3.0, and PostgreSQL 15.18 at + `sha256:3d0f7584ed7d04e27fa050d6683a74746608faf21f202be78460d679cc56461f`. +- PostgreSQL used an ephemeral self-signed TLS certificate. Dander rejected the initial + unencrypted fixture, preserving its TLS-required provider boundary. + +## Plan and execution + +- `dander kubernetes plan` linted the packaged chart and saved reviewed values and manifests. + The final image-only revision changed exactly the image reference and projected digest. +- Helm installed one paused CronJob plus its ServiceAccount and ConfigMap. The operator-managed + PostgreSQL DSN Secret remained outside Helm ownership. `dander kubernetes verify` matched the + context, namespace, pipeline, schedule, and immutable image before and after rollback. +- The initial run and inclusive replay each extracted two rows, built one model, passed two + assertions, and published one metadata asset without duplicates or cursor regression. +- An update run captured a changed row and a third row. The final committed-image smoke retained + three distinct raw rows and three distinct model rows, passed both assertions, and advanced the + watermark to `2026-08-10T12:03:00+00:00`. +- PostgreSQL retained four successful run-ledger records. The lease released after every accepted + run, its fencing token reached 4, raw and model target commits were committed, and no staging + relation remained. + +## Schedule, rollback, and cleanup + +- A reviewed Helm revision changed the paused `0 9 * * *` schedule to enabled `30 23 * * *`. + `helm rollback` restored the original paused schedule, and the final image revision kept it + paused. Installed values were semantically identical to the final reviewed plan. +- Helm uninstall removed all Helm-owned Dander resources. The fixture namespace, PostgreSQL, + local registry, and kind cluster were then deleted; `kind get clusters` returned none. +- No GCP project, retained deployment, public package, or other cloud resource was changed. + +## Result + +The local existing-cluster lifecycle gate passes for Kubernetes plus PostgreSQL state, ingestion, +transforms, assertions, metadata, replay, fencing, scheduling, rollback, and cleanup. Controlled +overlap, process interruption, alerts, hosted Kubernetes, and scale qualification were not +evaluated here and remain separate gates. PostgreSQL/Kubernetes remains experimental. + +Before review, fresh read-only plans against the retained GCP deployment's authoritative +source-free bundle reported exactly `No changes.` for both stage zero and the platform. An older, +superseded retained-project copy was not changed or used to manufacture that result. No Terraform +apply or cloud mutation occurred. diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index 45e12f50..25835290 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -14,9 +14,7 @@ from dander.project import DanderProject EXECUTION_PROJECTION_SCHEMA = "io.dander.execution/v1" -_IMMUTABLE_IMAGE = re.compile( - r"^[a-z0-9.-]+(?::[0-9]{1,5})?/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" -) +_IMMUTABLE_IMAGE = re.compile(r"^[a-z0-9.-]+(?::[0-9]{1,5})?/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$") _IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") _PROFILE = re.compile(r"^[a-z][a-z0-9_-]{0,62}$") _ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") From cc8625ffbcf19fd1a58e19116cb21355dbc84289 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 10:13:41 -0400 Subject: [PATCH 104/163] Qualify Snowflake warehouse runtime --- HANDOFF.md | 36 +- docs/decisions.md | 11 + docs/snowflake.md | 28 + scripts/benchmarks/snowflake.py | 860 ++++++++++++++++++ src/dander/providers/snowflake/writer.py | 32 + .../test_snowflake_qualification.py | 142 +++ .../test_snowflake_warehouse_runtime.py | 66 ++ tickets/DANDER-104-snowflake-warehouse.md | 15 + 8 files changed, 1173 insertions(+), 17 deletions(-) create mode 100644 scripts/benchmarks/snowflake.py create mode 100644 tests/portability/test_snowflake_qualification.py diff --git a/HANDOFF.md b/HANDOFF.md index 35ad532c..4e043842 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,20 +2,21 @@ ## Finished -- Accepted the experimental PostgreSQL/Kubernetes profile through a complete local existing-cluster lifecycle. -- Added immutable image validation for local registries with numeric ports. -- Proved source-free installation, non-root execution, replay, updates, models, assertions, metadata, and fencing. -- Proved reviewed schedule change, Helm rollback, read-only verification, and complete local cleanup. -- Recorded exact artifact digests, environment versions, results, exclusions, and retained GCP no-drift evidence. +- Guaranteed a current Snowflake schema before connector-managed direct qmark binding. +- Rejected JSON-to-`VARIANT` fields as keys, incremental cursors, or snapshot identity. +- Added one opt-in disposable-schema Snowflake warehouse qualification harness. +- Covered direct/COPY threshold crossing, all writer modes, replay, cursor safety, fencing, graph execution, readback, and cleanup. +- Documented the paid-test boundary and kept Snowflake explicitly experimental. ## Try It -Run `uv run pytest -q tests/deployment/test_execution_projection.py tests/infra/test_kubernetes_chart.py`. -The bounded lifecycle evidence is in `docs/cloud-portability-postgresql-kubernetes-acceptance.md`. +Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py +tests/portability/test_snowflake_qualification.py`. The live command is in `docs/snowflake.md` and +requires a separately approved Snowflake test account and credit ceiling. ## Checks -- Ruff, formatting, strict mypy, and all 1,195 tests passed; PostgreSQL integration used PostgreSQL 15. +- Ruff, formatting, strict mypy, and all 1,205 tests passed; PostgreSQL integration used PostgreSQL 15. - Dependency audit, wheel/sdist inspection, source-free installs, and runtime-all import passed. - Non-root read-only container conformance and packaged proof-asset checks passed. - Terraform roots/tests and Helm lint/template validation passed. @@ -23,18 +24,19 @@ The bounded lifecycle evidence is in `docs/cloud-portability-postgresql-kubernet ## Decisions -- Treat kind as valid existing-cluster lifecycle evidence, not hosted-provider or scale qualification. -- Preserve TLS-required PostgreSQL behavior; fix the disposable fixture rather than weakening Dander. -- Keep PostgreSQL/Kubernetes experimental and explicitly leave overlap, interruption, and alerts unevaluated. +- Use existing provider/runtime contracts rather than creating a benchmark framework. +- Mutate and remove one random `DANDER_QUAL_*` schema; never provision account-level resources. +- Keep direct thresholds at zero by default until real-account timing evidence exists. ## Remaining -- Let protected CI repeat Linux PostgreSQL, packaging, image, secret, Terraform, and Helm checks. -- Merge only if completion review and required checks remain clean. -- Begin the Snowflake/Redshift phase only after this gate closes. +- Let protected CI repeat Linux packaging, container, security, Terraform, and Helm checks. +- Configure non-interactive access for the signed-in Snowflake trial account. +- Approve a trial-credit ceiling, run the live harness, and preserve its sanitized report. +- Compare normalized results with the shared cross-warehouse conformance fixture. ## Review First -- `src/dander/deployment/projection.py` -- `infra/kubernetes/chart/dander/values.schema.json` -- `docs/cloud-portability-postgresql-kubernetes-acceptance.md` +- `src/dander/providers/snowflake/writer.py` +- `scripts/benchmarks/snowflake.py` +- `tests/portability/test_snowflake_qualification.py` diff --git a/docs/decisions.md b/docs/decisions.md index 1b88aa11..be03042b 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1088,3 +1088,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc replay identity, five write modes, schema checks, and fenced publication transaction as COPY. - **Telemetry:** Direct loads report exact local rows, logical bytes, and duration without a query ID or system-history enrichment because `executemany` has no whole-batch Redshift query identity. + +## 2026-08-10 — Snowflake qualification mutates one disposable schema only + +- **Correctness:** Dander selects the fully qualified target schema immediately before direct qmark + binding because the Snowflake connector may use its current schema for `SYSTEM$BIND` staging. + JSON-to-`VARIANT` fields cannot define business keys, incremental cursors, or snapshot identity. +- **Proof boundary:** One opt-in harness forces direct and multi-part `COPY` paths, all five writer + modes, replay, monotonic cursors, concurrent stale-fence rejection, graph execution, readback, and + cleanup in a random `DANDER_QUAL_*` schema. +- **Exclusions:** The harness never provisions an account, database, warehouse, role, or monitor; + it makes no performance, cost, support, or release claim before a separately approved live run. diff --git a/docs/snowflake.md b/docs/snowflake.md index c2c114ae..ebfff6aa 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -92,6 +92,34 @@ The Snowflake role needs usage on its database and warehouse plus permission to schemas, tables, and temporary stages in the selected namespace. Use a dedicated disposable role until a live least-privilege profile is qualified. +## Opt-in warehouse qualification + +The repository contains a bounded live harness for the warehouse adapter. It does not create an +account, database, warehouse, role, network policy, or resource monitor. Running it may resume the +selected warehouse and consume Snowflake credits, so agree on a paid-test ceiling first. + +The role must have warehouse usage plus database usage and permission to create and drop schemas in +the selected database. Supply only a credential reference; never put the private key or OAuth token +on the command line. For key-pair authentication: + +```bash +export DANDER_SNOWFLAKE_PRIVATE_KEY_FILE=/secure/path/rsa_key.p8 +uv run python -m scripts.benchmarks.snowflake \ + --account myorg-myaccount \ + --user DANDER_USER \ + --database DANDER_TEST \ + --warehouse DANDER_WH \ + --role DANDER_ROLE +``` + +The harness creates one random `DANDER_QUAL_*` schema and removes that exact schema in `finally`. +It forces both bounded direct binding and multi-part Parquet `COPY`, crosses the configured direct +threshold, exercises all five write modes, replays data, rejects cursor regression and a stale +concurrent publisher, executes one provider-neutral graph target, reads every result back, and +checks for staging residue. Its JSON report contains bounded query IDs and timings but no account, +user, credential reference, SQL, row value, or provider response. It deliberately reports cost as +`not_measured` and support as `experimental`. + ## Deliberate limits All five scalar write patterns are reachable through the warehouse writer capability, while the diff --git a/scripts/benchmarks/snowflake.py b/scripts/benchmarks/snowflake.py new file mode 100644 index 00000000..50230bc7 --- /dev/null +++ b/scripts/benchmarks/snowflake.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +"""Opt-in Snowflake warehouse qualification using one disposable schema.""" + +from __future__ import annotations + +import argparse +import json +import platform +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import suppress +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from dander.concurrency import FencingToken, TargetFenceLostError +from dander.ingestion import Endpoint, RawField, SourceConfig +from dander.pipeline.graph import PipelineGraph +from dander.pipeline.runtime import GraphExecutionPlan, plan_graph_execution +from dander.providers import ProviderKind, default_provider_registry +from dander.providers.snowflake.session import execute, open_connection +from dander.warehouse import RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Sequence + + from dander.providers.snowflake.fence import SnowflakeTargetFence + from dander.providers.snowflake.session import SnowflakeConnectionFactory + from dander.telemetry import OperationTelemetry + from dander.writer import WritePattern + + +_REPORT_SCHEMA = "io.dander.benchmark.snowflake/v1" +_AUTHORITY_ID = "snowflake:qualification" + + +class SnowflakeQualificationError(RuntimeError): + """Raised with a sanitized qualification failure summary.""" + + +@dataclass(frozen=True, slots=True) +class SnowflakeQualificationConfig: + """Non-secret live-profile coordinates and bounded workload controls.""" + + account: str + user: str + database: str + warehouse: str + role: str | None = None + auth_method: str = "key_pair" + token_env: str = "DANDER_SNOWFLAKE_OAUTH_TOKEN" + private_key_file_env: str = "DANDER_SNOWFLAKE_PRIVATE_KEY_FILE" + private_key_password_env: str | None = None + direct_max_rows: int = 2 + direct_max_logical_bytes: int = 1_048_576 + copy_part_rows: int = 2 + copy_part_logical_bytes: int = 1_048_576 + + def __post_init__(self) -> None: + if self.auth_method not in {"key_pair", "oauth"}: + raise ValueError("auth_method must be key_pair or oauth") + for name in ( + "direct_max_rows", + "direct_max_logical_bytes", + "copy_part_rows", + "copy_part_logical_bytes", + ): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + # Reuse the actual provider parser so identifiers and credential references fail before I/O. + default_provider_registry().parse( + ProviderKind.WAREHOUSE, + _provider_values(self, direct=True, schema_name="DANDER_QUALIFICATION_CHECK"), + ) + + +@dataclass(frozen=True, slots=True) +class SnowflakeQualificationReport: + """Sanitized correctness and bounded-work evidence from one disposable schema.""" + + schema: str + provider: str + provider_version: str + python_version: str + warehouse: str + disposable_schema: str + duration_seconds: float + direct_rows: int + copy_rows: int + write_modes: tuple[str, ...] + graph_rows: int + replay_duplicate_free: bool + cursor_monotonic: bool + stale_publication_rejected: bool + concurrent_claim_attempts: int + direct_operations: int + copy_operations: int + query_ids: tuple[str, ...] + staging_tables: int + staging_stages: int + schema_cleanup_verified: bool + qualification_status: str = "passed" + support_status: str = "experimental" + cost_status: str = "not_measured" + + def to_json(self) -> str: + return json.dumps(asdict(self), separators=(",", ":"), sort_keys=True) + + +@dataclass(slots=True) +class _Ownership: + fence: FencingToken + verifications: int = 0 + + def verify(self) -> None: + self.verifications += 1 + + +def run_snowflake_qualification( + config: SnowflakeQualificationConfig, +) -> SnowflakeQualificationReport: + """Exercise direct/COPY writes, replay, fencing, graph execution, and cleanup.""" + suffix = uuid.uuid4().hex[:12].upper() + schema_name = f"DANDER_QUAL_{suffix}" + direct = _warehouse_runtime(config, schema_name=schema_name, direct=True) + copy = _warehouse_runtime(config, schema_name=schema_name, direct=False) + started = time.perf_counter() + try: + report = _run_profile(config, direct=direct, copy=copy, schema_name=schema_name) + except (TargetFenceLostError, SnowflakeQualificationError): + raise + except Exception as error: + raise SnowflakeQualificationError( + f"Snowflake qualification failed in disposable schema {schema_name}" + ) from error + finally: + _drop_schema(direct, config.database, schema_name) + if _schema_exists(direct, config.database, schema_name): + raise SnowflakeQualificationError( + f"Snowflake qualification cleanup left disposable schema {schema_name}" + ) + return replace( + report, + duration_seconds=round(max(time.perf_counter() - started, 1e-9), 6), + schema_cleanup_verified=True, + ) + + +def _run_profile( + config: SnowflakeQualificationConfig, + *, + direct: WarehouseRuntime, + copy: WarehouseRuntime, + schema_name: str, +) -> SnowflakeQualificationReport: + database = config.database + all_operations: list[OperationTelemetry] = [] + + direct_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="direct_records", + ) + direct_writer, direct_target = _writer_target( + direct, + relation=direct_relation, + mode=WriteMode.SCD1, + pipeline_id="snowflake_qualification_direct", + run_id="direct-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + direct_rows = ( + {"id": "one", "label": "first"}, + {"id": "two", "label": "second"}, + ) + direct_writer.write(direct_rows, direct_target) + direct_operations = direct_writer.drain_telemetry() + _require_transport(direct_operations, WriteTransport.DIRECT) + all_operations.extend(direct_operations) + expected_direct = (("one", "first"), ("two", "second")) + _require_rows(direct, direct_relation, ("id", "label"), expected_direct) + direct_writer.write(direct_rows, direct_target) + all_operations.extend(direct_writer.drain_telemetry()) + _require_rows(direct, direct_relation, ("id", "label"), expected_direct) + + overflow_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="threshold_fallback_records", + ) + overflow_writer, overflow_target = _writer_target( + direct, + relation=overflow_relation, + mode=WriteMode.SCD1, + pipeline_id="snowflake_qualification_threshold", + run_id="threshold-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + overflow_writer.write( + ( + {"id": "one", "label": "first"}, + {"id": "two", "label": "second"}, + {"id": "three", "label": "third"}, + ), + overflow_target, + ) + overflow_operations = overflow_writer.drain_telemetry() + _require_transport(overflow_operations, WriteTransport.COPY) + all_operations.extend(overflow_operations) + + copy_relation = RelationRef(catalog=database, namespace=schema_name, name="scd1_records") + copy_writer, copy_target = _writer_target( + copy, + relation=copy_relation, + mode=WriteMode.SCD1, + pipeline_id="snowflake_qualification_scd1", + run_id="scd1-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + copy_writer.write( + ( + {"id": "one", "label": "older"}, + {"id": "two", "label": "second"}, + {"id": "one", "label": "newer"}, + ), + copy_target, + ) + copy_operations = copy_writer.drain_telemetry() + _require_transport(copy_operations, WriteTransport.COPY) + all_operations.extend(copy_operations) + _require_rows(copy, copy_relation, ("id", "label"), (("one", "newer"), ("two", "second"))) + + incremental_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="incremental_records", + ) + incremental_fields = (*_base_fields(), WriteField(name="updated_at", data_type="INT64")) + incremental_writer, incremental_target = _writer_target( + copy, + relation=incremental_relation, + mode=WriteMode.INCREMENTAL, + pipeline_id="snowflake_qualification_incremental", + run_id="incremental-1", + token=1, + business_key=("id",), + fields=incremental_fields, + cursor_field="updated_at", + ) + incremental_writer.write( + ( + {"id": "one", "label": "newest", "updated_at": 2}, + {"id": "one", "label": "stale", "updated_at": 1}, + ), + incremental_target, + ) + all_operations.extend(incremental_writer.drain_telemetry()) + _require_rows( + copy, + incremental_relation, + ("id", "label", "updated_at"), + (("one", "newest", 2),), + ) + regression_writer, regression_target = _writer_target( + copy, + relation=incremental_relation, + mode=WriteMode.INCREMENTAL, + pipeline_id="snowflake_qualification_incremental", + run_id="incremental-2", + token=2, + business_key=("id",), + fields=incremental_fields, + cursor_field="updated_at", + ) + regression_writer.write( + ({"id": "one", "label": "regressed", "updated_at": 1},), + regression_target, + ) + all_operations.extend(regression_writer.drain_telemetry()) + _require_rows( + copy, + incremental_relation, + ("id", "label", "updated_at"), + (("one", "newest", 2),), + ) + + snapshot_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="snapshot_records", + ) + snapshot_fields = (*_base_fields(), WriteField(name="snapshot_at", data_type="STRING")) + snapshot_writer, snapshot_target = _writer_target( + copy, + relation=snapshot_relation, + mode=WriteMode.SNAPSHOT, + pipeline_id="snowflake_qualification_snapshot", + run_id="snapshot-1", + token=1, + fields=snapshot_fields, + snapshot_field="snapshot_at", + ) + snapshot_rows = ( + {"id": "one", "label": "first", "snapshot_at": "2026-08-10"}, + {"id": "two", "label": "second", "snapshot_at": "2026-08-10"}, + ) + snapshot_writer.write(snapshot_rows, snapshot_target) + all_operations.extend(snapshot_writer.drain_telemetry()) + snapshot_writer.write(snapshot_rows, snapshot_target) + all_operations.extend(snapshot_writer.drain_telemetry()) + _require_count(copy, snapshot_relation, 2) + + scd2_relation = RelationRef(catalog=database, namespace=schema_name, name="scd2_records") + scd2_writer, scd2_target = _writer_target( + copy, + relation=scd2_relation, + mode=WriteMode.SCD2, + pipeline_id="snowflake_qualification_scd2", + run_id="scd2-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + scd2_writer.write(({"id": "one", "label": "first"},), scd2_target) + all_operations.extend(scd2_writer.drain_telemetry()) + scd2_update, scd2_update_target = _writer_target( + copy, + relation=scd2_relation, + mode=WriteMode.SCD2, + pipeline_id="snowflake_qualification_scd2", + run_id="scd2-2", + token=2, + business_key=("id",), + fields=_base_fields(), + ) + scd2_update.write(({"id": "one", "label": "second"},), scd2_update_target) + all_operations.extend(scd2_update.drain_telemetry()) + scd2_rows = _select( + copy, + scd2_relation, + ("label", "is_current"), + order_by=("valid_from",), + ) + if len(scd2_rows) != 2 or sum(bool(row[1]) for row in scd2_rows) != 1: + raise SnowflakeQualificationError("Snowflake SCD2 readback did not retain one current row") + + replace_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="replace_records", + ) + replace_writer, replace_target = _writer_target( + copy, + relation=replace_relation, + mode=WriteMode.REPLACE, + pipeline_id="snowflake_qualification_replace", + run_id="replace-1", + token=1, + fields=_base_fields(), + ) + replace_writer.write(direct_rows, replace_target) + all_operations.extend(replace_writer.drain_telemetry()) + replace_writer.write(direct_rows, replace_target) + all_operations.extend(replace_writer.drain_telemetry()) + replacement_writer, replacement_target = _writer_target( + copy, + relation=replace_relation, + mode=WriteMode.REPLACE, + pipeline_id="snowflake_qualification_replace", + run_id="replace-2", + token=2, + fields=_base_fields(), + ) + replacement_writer.write( + ({"id": "three", "label": "replacement"},), + replacement_target, + ) + all_operations.extend(replacement_writer.drain_telemetry()) + _require_rows(copy, replace_relation, ("id", "label"), (("three", "replacement"),)) + + stale_rejected, concurrent_attempts = _exercise_concurrent_fence(copy, schema_name) + graph_rows, graph_operations = _exercise_graph(copy, schema_name, source=direct_relation) + all_operations.extend(graph_operations) + staging_tables, staging_stages = _staging_residue(copy, database, schema_name) + if staging_tables or staging_stages: + raise SnowflakeQualificationError("Snowflake qualification left run-scoped staging objects") + + provider_version = _scalar(copy, "SELECT CURRENT_VERSION()") + query_ids = tuple( + dict.fromkeys( + operation.query_id + for operation in all_operations + if isinstance(operation.query_id, str) and operation.query_id + ) + )[:100] + copy_count = sum(operation.transport is WriteTransport.COPY for operation in all_operations) + direct_count = sum(operation.transport is WriteTransport.DIRECT for operation in all_operations) + return SnowflakeQualificationReport( + schema=_REPORT_SCHEMA, + provider="snowflake", + provider_version=str(provider_version), + python_version=platform.python_version(), + warehouse=config.warehouse, + disposable_schema=schema_name, + duration_seconds=0.0, + direct_rows=len(expected_direct), + copy_rows=2, + write_modes=tuple(mode.value for mode in WriteMode), + graph_rows=graph_rows, + replay_duplicate_free=True, + cursor_monotonic=True, + stale_publication_rejected=stale_rejected, + concurrent_claim_attempts=concurrent_attempts, + direct_operations=direct_count, + copy_operations=copy_count, + query_ids=query_ids, + staging_tables=staging_tables, + staging_stages=staging_stages, + schema_cleanup_verified=False, + ) + + +def _writer_target( + runtime: WarehouseRuntime, + *, + relation: RelationRef, + mode: WriteMode, + pipeline_id: str, + run_id: str, + token: int, + fields: Sequence[WriteField], + business_key: tuple[str, ...] = (), + cursor_field: str | None = None, + snapshot_field: str | None = None, +) -> tuple[WritePattern, WriteTarget]: + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=pipeline_id, + run_id=run_id, + token=token, + authority_id=_AUTHORITY_ID, + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, + ) + return writer, WriteTarget( + relation=relation, + business_key=business_key, + schema=tuple(fields), + publication_fence=publication, + ) + + +def _exercise_concurrent_fence( + runtime: WarehouseRuntime, + schema_name: str, +) -> tuple[bool, int]: + target_fence = cast("SnowflakeTargetFence", runtime.target_fence) + relation = RelationRef( + catalog=target_fence.database, + namespace=schema_name, + name="concurrent_records", + ) + old_token = FencingToken( + lease_table=None, + pipeline_id="snowflake_qualification_concurrency", + run_id="concurrent-old", + token=20, + authority_id=_AUTHORITY_ID, + ) + old_publication = target_fence.claim(relation, old_token) + newer_token = replace(old_token, run_id="concurrent-new", token=21) + with ThreadPoolExecutor(max_workers=2) as executor: + futures = ( + executor.submit(target_fence.claim, relation, old_token), + executor.submit(target_fence.claim, relation, newer_token), + ) + for future in futures: + with suppress(TargetFenceLostError): + future.result() + target_fence.claim(relation, newer_token) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + try: + writer.write( + ({"id": "stale", "label": "must-not-publish"},), + WriteTarget( + relation=relation, + business_key=("id",), + schema=_base_fields(), + publication_fence=old_publication, + ), + ) + except TargetFenceLostError: + _require_count(runtime, relation, 0) + return True, len(futures) + raise SnowflakeQualificationError("Snowflake accepted a stale concurrent publication") + + +def _exercise_graph( + runtime: WarehouseRuntime, + schema_name: str, + *, + source: RelationRef, +) -> tuple[int, tuple[OperationTelemetry, ...]]: + plan = _graph_plan(source=source, target_schema=schema_name) + runner = runtime.transforms.build_transform_runner(graph_plan=plan, build_models=True) + if runner is None: + raise SnowflakeQualificationError("Snowflake graph runner is unavailable") + ownership = _Ownership( + FencingToken( + lease_table=None, + pipeline_id="snowflake_qualification_graph", + run_id="graph-1", + token=1, + authority_id=_AUTHORITY_ID, + ) + ) + result = runner.build(Path("."), ownership=ownership) + if result.models != ("target",) or ownership.verifications < 2: + raise SnowflakeQualificationError("Snowflake graph ownership verification was incomplete") + target = RelationRef( + catalog=source.catalog, + namespace=schema_name, + name="graph_records", + ) + rows = _select(runtime, target, ("id", "label"), order_by=("id",)) + if rows != (("one", "first"), ("two", "second")): + raise SnowflakeQualificationError("Snowflake graph result differs from its source fixture") + return len(rows), result.telemetry + + +def _graph_plan(*, source: RelationRef, target_schema: str) -> GraphExecutionPlan: + source_config = SourceConfig( + name="snowflake_qualification", + base_url="https://qualification.invalid", + auth_strategy="none", + endpoints=[ + Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING"), + RawField(name="label", data_type="STRING"), + ], + ) + ], + ) + graph = PipelineGraph.model_validate( + { + "name": "snowflake_qualification_graph", + "nodes": [ + { + "id": "records", + "type": "source", + "name": "Records", + "config": {"connector": "snowflake_qualification", "endpoint": "records"}, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + { + "id": "target", + "type": "target", + "name": "Target", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": target_schema, + "table": "graph_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + ], + "edges": [ + { + "from": "records", + "to": "target", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ], + } + ) + return plan_graph_execution(graph, source_config, endpoint_relations={"records": source}) + + +def _warehouse_runtime( + config: SnowflakeQualificationConfig, + *, + schema_name: str, + direct: bool, +) -> WarehouseRuntime: + registry = default_provider_registry() + parsed = registry.parse( + ProviderKind.WAREHOUSE, + _provider_values(config, direct=direct, schema_name=schema_name), + ) + runtime = registry.build( + ProviderKind.WAREHOUSE, + parsed, + context={"catalog": config.database}, + ) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("Snowflake provider returned an invalid warehouse runtime") + return runtime + + +def _provider_values( + config: SnowflakeQualificationConfig, + *, + direct: bool, + schema_name: str, +) -> dict[str, object]: + auth: dict[str, object] + if config.auth_method == "oauth": + auth = {"method": "oauth", "token_env": config.token_env} + else: + auth = { + "method": "key_pair", + "private_key_file_env": config.private_key_file_env, + } + if config.private_key_password_env is not None: + auth["private_key_password_env"] = config.private_key_password_env + return { + "provider": "snowflake", + "account": config.account, + "user": config.user, + "database": config.database, + "schema": schema_name, + "warehouse": config.warehouse, + "role": config.role, + "auth": auth, + "max_rows_per_file": config.copy_part_rows, + "max_logical_bytes_per_file": config.copy_part_logical_bytes, + "direct_max_rows": config.direct_max_rows if direct else 0, + "direct_max_logical_bytes": config.direct_max_logical_bytes if direct else 0, + } + + +def _base_fields() -> tuple[WriteField, ...]: + return ( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ) + + +def _require_transport( + operations: Sequence[OperationTelemetry], + transport: WriteTransport, +) -> None: + if not operations or any(operation.transport is not transport for operation in operations): + raise SnowflakeQualificationError( + f"Snowflake qualification did not use required {transport.value} transport" + ) + + +def _connection_factory(runtime: WarehouseRuntime) -> SnowflakeConnectionFactory: + target_fence = cast("SnowflakeTargetFence", runtime.target_fence) + return target_fence.connection_factory + + +def _select( + runtime: WarehouseRuntime, + relation: RelationRef, + fields: Sequence[str], + *, + order_by: Sequence[str] = (), +) -> tuple[tuple[object, ...], ...]: + columns = ", ".join(_quote(field) for field in fields) + ordering = f" ORDER BY {', '.join(_quote(field) for field in order_by)}" if order_by else "" + statement = f"SELECT {columns} FROM {_relation(relation)}{ordering}" + with open_connection(_connection_factory(runtime)) as connection: + rows = execute(connection, statement, fetch="all").rows + return tuple(tuple(row) for row in rows if isinstance(row, (tuple, list))) + + +def _require_rows( + runtime: WarehouseRuntime, + relation: RelationRef, + fields: Sequence[str], + expected: tuple[tuple[object, ...], ...], +) -> None: + actual = _select(runtime, relation, fields, order_by=(fields[0],)) + if actual != expected: + raise SnowflakeQualificationError( + f"Snowflake readback mismatch for qualification relation {relation.name}" + ) + + +def _require_count(runtime: WarehouseRuntime, relation: RelationRef, expected: int) -> None: + with open_connection(_connection_factory(runtime)) as connection: + row = execute(connection, f"SELECT COUNT(*) FROM {_relation(relation)}", fetch="one").row + value = row[0] if isinstance(row, (tuple, list)) and row else None + if value != expected: + raise SnowflakeQualificationError( + f"Snowflake row-count mismatch for qualification relation {relation.name}" + ) + + +def _scalar(runtime: WarehouseRuntime, statement: str) -> object: + with open_connection(_connection_factory(runtime)) as connection: + row = execute(connection, statement, fetch="one").row + if not isinstance(row, (tuple, list)) or not row: + raise SnowflakeQualificationError("Snowflake qualification metadata query was malformed") + return row[0] + + +def _staging_residue( + runtime: WarehouseRuntime, + database: str, + schema_name: str, +) -> tuple[int, int]: + with open_connection(_connection_factory(runtime)) as connection: + tables = execute( + connection, + f"SELECT COUNT(*) FROM {_qualified(database, 'INFORMATION_SCHEMA', 'TABLES')} " + "WHERE TABLE_SCHEMA = ? " + "AND REGEXP_LIKE(TABLE_NAME, '^dander_stage_[0-9a-f]{20}$')", + (schema_name,), + fetch="one", + ).row + stages = execute( + connection, + f"SELECT COUNT(*) FROM {_qualified(database, 'INFORMATION_SCHEMA', 'STAGES')} " + "WHERE STAGE_SCHEMA = ? AND STAGE_NAME ILIKE 'DANDER_FILES_%'", + (schema_name,), + fetch="one", + ).row + return _count(tables), _count(stages) + + +def _drop_schema(runtime: WarehouseRuntime, database: str, schema_name: str) -> None: + try: + with open_connection(_connection_factory(runtime)) as connection: + execute( + connection, + f"DROP SCHEMA IF EXISTS {_qualified(database, schema_name)} CASCADE", + ) + connection.commit() + except Exception as error: + raise SnowflakeQualificationError( + f"Snowflake qualification could not remove disposable schema {schema_name}" + ) from error + + +def _schema_exists(runtime: WarehouseRuntime, database: str, schema_name: str) -> bool: + with open_connection(_connection_factory(runtime)) as connection: + row = execute( + connection, + f"SELECT COUNT(*) FROM {_qualified(database, 'INFORMATION_SCHEMA', 'SCHEMATA')} " + "WHERE SCHEMA_NAME = ?", + (schema_name,), + fetch="one", + ).row + return _count(row) != 0 + + +def _count(row: object) -> int: + if not isinstance(row, (tuple, list)) or not row: + raise SnowflakeQualificationError("Snowflake qualification count query was malformed") + value = row[0] + if isinstance(value, bool) or not isinstance(value, int): + try: + return int(value) + except (TypeError, ValueError) as error: + raise SnowflakeQualificationError( + "Snowflake qualification count query returned a non-integer" + ) from error + return value + + +def _relation(relation: RelationRef) -> str: + return _qualified(*relation.coordinates) + + +def _qualified(*parts: str) -> str: + return ".".join(_quote(part) for part in parts) + + +def _quote(value: str) -> str: + return f'"{value.replace(chr(34), chr(34) * 2)}"' + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--account", required=True) + parser.add_argument("--user", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--warehouse", required=True) + parser.add_argument("--role") + parser.add_argument("--auth-method", choices=("key_pair", "oauth"), default="key_pair") + parser.add_argument("--token-env", default="DANDER_SNOWFLAKE_OAUTH_TOKEN") + parser.add_argument("--private-key-file-env", default="DANDER_SNOWFLAKE_PRIVATE_KEY_FILE") + parser.add_argument("--private-key-password-env") + parser.add_argument("--direct-max-rows", type=int, default=2) + parser.add_argument("--direct-max-logical-bytes", type=int, default=1_048_576) + parser.add_argument("--copy-part-rows", type=int, default=2) + parser.add_argument("--copy-part-logical-bytes", type=int, default=1_048_576) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + try: + report = run_snowflake_qualification(SnowflakeQualificationConfig(**vars(arguments))) + except (ValueError, SnowflakeQualificationError): + print( + json.dumps( + { + "schema": _REPORT_SCHEMA, + "provider": "snowflake", + "qualification_status": "failed", + "summary": ( + "Snowflake qualification failed; inspect provider logs and rerun after " + "cleanup." + ), + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 1 + print(report.to_json()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/dander/providers/snowflake/writer.py b/src/dander/providers/snowflake/writer.py index a53120f7..1711833a 100644 --- a/src/dander/providers/snowflake/writer.py +++ b/src/dander/providers/snowflake/writer.py @@ -148,6 +148,11 @@ def write(self, records: Iterable[Mapping[str, object]], target: WriteTarget) -> ): raise SnowflakeWriteError(f"Declared schema reserves Dander field {collision[0]!r}") validate_snowflake_schema(target_schema) + _validate_variant_roles( + target, + cursor_field=self._cursor_field, + snapshot_field=self._snapshot_field, + ) normalization_schema = RelationSchema( fields=( *target_schema.fields, @@ -352,6 +357,13 @@ def _load_and_publish( ) else: assert direct is not None + # The connector may switch larger qmark batches to its temporary + # SYSTEM$BIND stage. That path requires an explicit current schema even + # though every Dander relation remains fully qualified. + execute( + connection, + f"USE SCHEMA {_qualified(relation.catalog, relation.namespace)}", + ) load, duration_ms = _timed_call( execute_many, connection, @@ -680,6 +692,26 @@ def validate_snowflake_schema(schema: RelationSchema) -> None: _snowflake_field_type(field) +def _validate_variant_roles( + target: WriteTarget, + *, + cursor_field: str | None, + snapshot_field: str | None, +) -> None: + variant_fields = { + field.name for field in target.canonical_schema.fields if _is_variant_fallback(field) + } + restricted = {*target.business_key} + if cursor_field is not None: + restricted.add(cursor_field) + if snapshot_field is not None: + restricted.add(snapshot_field) + if collision := sorted(variant_fields & restricted): + raise SnowflakeWriteError( + f"Snowflake VARIANT field {collision[0]!r} cannot be a key, cursor, or snapshot field" + ) + + def _staging_field(field: CanonicalField) -> CanonicalField: if _is_variant_fallback(field): return field.model_copy( diff --git a/tests/portability/test_snowflake_qualification.py b/tests/portability/test_snowflake_qualification.py new file mode 100644 index 00000000..e954fa04 --- /dev/null +++ b/tests/portability/test_snowflake_qualification.py @@ -0,0 +1,142 @@ +"""Credential-free checks for the opt-in Snowflake qualification harness.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import cast + +import pytest +from scripts.benchmarks import snowflake + +from dander.providers.snowflake.session import SnowflakeStatementResult +from dander.warehouse import RelationRef, WarehouseRuntime + + +def _config(**overrides: object) -> snowflake.SnowflakeQualificationConfig: + values: dict[str, object] = { + "account": "org-account", + "user": "DANDER_USER", + "database": "DANDER_TEST", + "warehouse": "DANDER_WH", + "role": "DANDER_ROLE", + } + values.update(overrides) + return snowflake.SnowflakeQualificationConfig(**values) # type: ignore[arg-type] + + +def test_qualification_config_reuses_provider_validation_and_only_stores_references() -> None: + config = _config( + auth_method="oauth", + token_env="DANDER_TEST_SNOWFLAKE_TOKEN", + ) + + values = snowflake._provider_values( + config, + direct=True, + schema_name="DANDER_QUAL_TEST", + ) + + assert values["auth"] == { + "method": "oauth", + "token_env": "DANDER_TEST_SNOWFLAKE_TOKEN", + } + assert values["direct_max_rows"] == 2 + assert values["schema"] == "DANDER_QUAL_TEST" + assert "token" not in values + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"auth_method": "password"}, "auth_method"), + ({"copy_part_rows": 0}, "copy_part_rows"), + ({"token_env": "not-an-env", "auth_method": "oauth"}, "token_env"), + ], +) +def test_qualification_config_fails_before_provider_io( + overrides: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + _config(**overrides) + + +def test_graph_fixture_preserves_native_database_and_schema_coordinates() -> None: + source = RelationRef( + catalog="DANDER_TEST", + namespace="DANDER_QUAL_TEST", + name="direct_records", + ) + + plan = snowflake._graph_plan(source=source, target_schema="DANDER_QUAL_TEST") + + assert plan.bindings.source_relations == {"records": source} + assert plan.targets[0].target.relation_ref == RelationRef( + catalog="DANDER_TEST", + namespace="DANDER_QUAL_TEST", + name="graph_records", + ) + + +def test_staging_residue_query_excludes_durable_load_history( + monkeypatch: pytest.MonkeyPatch, +) -> None: + statements: list[str] = [] + + class _Connection: + def close(self) -> None: + return None + + def fake_execute( + _connection: object, + statement: str, + _parameters: object = (), + *, + fetch: str = "none", + ) -> SnowflakeStatementResult: + statements.append(statement) + assert fetch == "one" + return SnowflakeStatementResult(rowcount=1, query_id=None, row=(0,)) + + runtime = cast( + "WarehouseRuntime", + SimpleNamespace( + target_fence=SimpleNamespace(connection_factory=lambda: _Connection()), + ), + ) + monkeypatch.setattr(snowflake, "execute", fake_execute) + + assert snowflake._staging_residue(runtime, "DANDER_TEST", "DANDER_QUAL_TEST") == (0, 0) + assert "REGEXP_LIKE(TABLE_NAME, '^dander_stage_[0-9a-f]{20}$')" in statements[0] + assert "DANDER_STAGE_LOADS" not in statements[0] + + +def test_cli_failure_record_never_exposes_provider_error( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + secret = "provider-secret-response" + + def fail(_config: snowflake.SnowflakeQualificationConfig) -> None: + raise snowflake.SnowflakeQualificationError(secret) + + monkeypatch.setattr(snowflake, "run_snowflake_qualification", fail) + + exit_code = snowflake.main( + [ + "--account", + "org-account", + "--user", + "DANDER_USER", + "--database", + "DANDER_TEST", + "--warehouse", + "DANDER_WH", + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["qualification_status"] == "failed" + assert secret not in json.dumps(payload) diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index 7c6733e3..a78b7bba 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -467,6 +467,9 @@ def test_snowflake_direct_load_parses_explicit_variant_and_emits_transport( if statement.startswith('INSERT INTO "DANDER_TEST"."raw"."dander_stage_') and "dander_stage_loads" not in statement ) + use_schema_index = sql.index('USE SCHEMA "DANDER_TEST"."raw"') + direct_insert_index = sql.index(direct[0]) + assert use_schema_index < direct_insert_index assert '"payload" VARCHAR' in next( statement for statement in sql if statement.startswith("CREATE OR REPLACE TEMPORARY TABLE") ) @@ -1636,6 +1639,69 @@ class _Result: assert telemetry.resource_name == "DANDER_WH" +@pytest.mark.parametrize( + ("mode", "business_key", "cursor_field", "snapshot_field"), + [ + (WriteMode.SCD1, ("payload",), None, None), + (WriteMode.INCREMENTAL, ("id",), "payload", None), + (WriteMode.SNAPSHOT, (), None, "payload"), + ], +) +def test_snowflake_variant_cannot_define_identity_or_ordering( + snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], + mode: WriteMode, + business_key: tuple[str, ...], + cursor_field: str | None, + snapshot_field: str | None, +) -> None: + runtime, backend, _staging_root = snowflake_runtime + fallback = ProviderExtension(provider="snowflake", name="fallback", value="variant") + relation = RelationRef( + catalog="DANDER_TEST", + namespace="raw", + name=f"variant_{mode.value}_role", + ) + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=f"snowflake_variant_{mode.value}_role", + run_id=f"run-variant-{mode.value}-role", + token=12, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, + ) + before = len(backend.statements) + + with pytest.raises(SnowflakeWriteError, match="cannot be a key, cursor, or snapshot"): + writer.write( + ({"id": "one", "payload": {"rank": 1}},), + WriteTarget( + relation=relation, + business_key=business_key, + schema=( + WriteField(name="id", data_type="STRING"), + WriteField( + name="payload", + data_type="JSON", + extensions=(fallback,), + ), + ), + publication_fence=publication, + ), + ) + + assert len(backend.statements) == before + + def test_snowflake_copy_parses_explicit_variant_before_publication( snowflake_runtime: tuple[WarehouseRuntime, _FakeSnowflake, Path], ) -> None: diff --git a/tickets/DANDER-104-snowflake-warehouse.md b/tickets/DANDER-104-snowflake-warehouse.md index 52be129d..a23e6282 100644 --- a/tickets/DANDER-104-snowflake-warehouse.md +++ b/tickets/DANDER-104-snowflake-warehouse.md @@ -31,3 +31,18 @@ support. This ticket adds no Snowflake transforms, graphs, semi-structured fields, other write modes, Terraform, live deployment, package publication, or support-status promotion. + +## Subsequent qualification slices + +Later focused PRs added all five scalar writer modes, explicit JSON-to-`VARIANT`, fenced portable +models and graphs, bounded direct/COPY selection, and operation telemetry. Before live execution, +the qualification slice also: + +- [x] selects the target schema explicitly before connector-managed direct binding; +- [x] rejects `VARIANT` fields as keys, cursors, or snapshot identity; +- [x] provides a disposable-schema live harness with sanitized output and exact cleanup; +- [ ] records a real Snowflake execution under an approved paid-test ceiling; and +- [ ] compares its normalized results with the shared cross-warehouse conformance dataset. + +Until those final two items pass, Snowflake remains experimental and direct thresholds retain their +zero defaults. From 4dd80b29b3dcb94866fb32e7ab952bfe9572fd58 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 11:04:45 -0400 Subject: [PATCH 105/163] Qualify experimental Redshift runtime --- HANDOFF.md | 49 +- docs/compatibility-matrix.md | 2 +- docs/decisions.md | 16 +- docs/redshift.md | 44 +- scripts/benchmarks/redshift.py | 1221 +++++++++++++++++ src/dander/providers/redshift/config.py | 3 +- src/dander/providers/redshift/session.py | 2 +- .../test_redshift_qualification.py | 283 ++++ .../test_redshift_warehouse_runtime.py | 31 +- 9 files changed, 1605 insertions(+), 46 deletions(-) create mode 100644 scripts/benchmarks/redshift.py create mode 100644 tests/portability/test_redshift_qualification.py diff --git a/HANDOFF.md b/HANDOFF.md index 35ad532c..0a1ae206 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,46 @@ ## Finished -- Accepted the experimental PostgreSQL/Kubernetes profile through a complete local existing-cluster lifecycle. -- Added immutable image validation for local registries with numeric ports. -- Proved source-free installation, non-root execution, replay, updates, models, assertions, metadata, and fencing. -- Proved reviewed schedule change, Helm rollback, read-only verification, and complete local cleanup. -- Recorded exact artifact digests, environment versions, results, exclusions, and retained GCP no-drift evidence. +- Capped Redshift direct ingestion at its documented 1 MiB non-file limit. +- Correlated Redshift telemetry with `LAST_USER_QUERY_ID()` and `SYS_QUERY_HISTORY`. +- Added an opt-in existing-profile Redshift qualification harness with no provisioning behavior. +- Covered all writers, `SUPER`, models, graph execution, replay, stale fencing, readback, and cleanup. +- Kept Redshift experimental and documented the paid-test boundary. ## Try It -Run `uv run pytest -q tests/deployment/test_execution_projection.py tests/infra/test_kubernetes_chart.py`. -The bounded lifecycle evidence is in `docs/cloud-portability-postgresql-kubernetes-acceptance.md`. +Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py +tests/portability/test_redshift_qualification.py`. The opt-in live command is in +`docs/redshift.md` and requires an existing profile plus a separately approved cost ceiling. ## Checks -- Ruff, formatting, strict mypy, and all 1,195 tests passed; PostgreSQL integration used PostgreSQL 15. -- Dependency audit, wheel/sdist inspection, source-free installs, and runtime-all import passed. -- Non-root read-only container conformance and packaged proof-asset checks passed. -- Terraform roots/tests and Helm lint/template validation passed. -- Fresh retained GCP stage-zero and platform plans each reported exactly `No changes.` +- Ruff, formatting, strict mypy, dependency audit, and all 1,207 tests passed with PostgreSQL 15. +- Wheel, sdist, source-free install, runtime-all install, and generated-project validation passed. +- Non-root/read-only container conformance, Terraform validation/tests, and Helm checks passed. +- Retained stage-zero plan reported `No changes.`; no apply or live mutation occurred. +- Public source-free `dander-platform==0.7.0` with the retained deployment's existing `600s` + compatibility input produced an exact retained platform `No changes.` plan. +- The candidate plan is 0 add/5 change/0 destroy: only five job version labels (`0.7.0` to + `0.8.0rc8`) and declared timeouts (`600s` to `300s`) differ. +- A detached `origin/main` plan reproduced those same five changes; this branch contains no GCP, + Terraform, manifest, or deployment-file changes. +- Final adversarial review's prefix-normalization and partial-delete findings were fixed and covered. ## Decisions -- Treat kind as valid existing-cluster lifecycle evidence, not hosted-provider or scale qualification. -- Preserve TLS-required PostgreSQL behavior; fix the disposable fixture rather than weakening Dander. -- Keep PostgreSQL/Kubernetes experimental and explicitly leave overlap, interruption, and alerts unevaluated. +- Use the existing Redshift provider/runtime contracts, not a benchmark framework. +- Mutate only one random schema and S3 prefix and remove both on every handled exit. +- Keep both provisioned and Serverless selectable without provisioning either one. ## Remaining -- Let protected CI repeat Linux PostgreSQL, packaging, image, secret, Terraform, and Helm checks. -- Merge only if completion review and required checks remain clean. -- Begin the Snowflake/Redshift phase only after this gate closes. +- Let protected CI repeat Linux checks plus unavailable local Trivy and Gitleaks scans. +- Provision or select a separately approved disposable Redshift profile for the live run. +- Preserve a sanitized live report before any support-status change. ## Review First -- `src/dander/deployment/projection.py` -- `infra/kubernetes/chart/dander/values.schema.json` -- `docs/cloud-portability-postgresql-kubernetes-acceptance.md` +- `src/dander/providers/redshift/config.py` +- `src/dander/providers/redshift/session.py` +- `scripts/benchmarks/redshift.py` diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index d261b0c1..d0a7f241 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -40,7 +40,7 @@ their named live profiles pass. |---|---|---|---|---|---|---| | BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | | PostgreSQL | all five | COPY | all v1 types; decimal 1000, time 6 | yes | yes | yes | -| Redshift | all five | COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | yes | yes | +| Redshift | all five | direct, COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | yes | yes | | Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | Portable-provider schema validation uses these declarations before source extraction, staging diff --git a/docs/decisions.md b/docs/decisions.md index 1b88aa11..4e937f2f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1070,9 +1070,9 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc ## 2026-08-10 — Redshift telemetry never competes with correctness transactions -- **Attribution:** Dander captures `PG_LAST_QUERY_ID()` only after committed COPY and CTAS work. - Multi-statement fenced publication and assertion operations retain local counters without a - potentially misleading query ID. +- **Attribution:** Dander captures `LAST_USER_QUERY_ID()` only after committed COPY and CTAS work, + matching the user-query IDs exposed by `SYS_QUERY_HISTORY`. Multi-statement fenced publication + and assertion operations retain local counters without a potentially misleading query ID. - **Enrichment:** One bounded same-session lookup reads numeric counters from `SYS_QUERY_HISTORY`, step-level `SYS_QUERY_DETAIL`, and `SYS_LOAD_HISTORY`; SQL text, errors, and S3 data sources are never selected. @@ -1088,3 +1088,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc replay identity, five write modes, schema checks, and fenced publication transaction as COPY. - **Telemetry:** Direct loads report exact local rows, logical bytes, and duration without a query ID or system-history enrichment because `executemany` has no whole-batch Redshift query identity. + +## 2026-08-10 — Redshift qualification owns one schema and one S3 prefix + +- **Proof:** One opt-in harness forces direct and multi-part `COPY`, all five write modes, `SUPER`, + table/incremental models, replay, cursor safety, two-session stale fencing, graphs, telemetry, + readback, and residue checks against an existing Redshift profile. +- **Ownership:** The harness creates and removes one random `dander_qual_*` schema and one random + child of the configured staging prefix. It provisions no Redshift, S3, IAM, or network resource. +- **Boundary:** Credential-free tests and a harness do not constitute live qualification, cost or + scale evidence, infrastructure support, or profile promotion. diff --git a/docs/redshift.md b/docs/redshift.md index ceb350b5..e1d36dcf 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -43,7 +43,9 @@ COPY remains the default. Set both direct limits to positive values to opt small streams into bounded parameterized inserts; for example, `direct_max_rows: 1000` and `direct_max_logical_bytes: 1048576`. Dander selects direct loading only after the entire stream fits both limits. Otherwise it falls back once to the existing Parquet/S3/COPY path without losing -the inspected prefix. The S3 bucket and COPY role remain required because they handle that fallback. +the inspected prefix. Redshift permits values larger than 1 MiB only through file ingestion, so +`direct_max_logical_bytes` cannot exceed 1 MiB. The S3 bucket and COPY role remain required because +they handle that fallback. ## Current contract @@ -75,13 +77,47 @@ the inspected prefix. The S3 bucket and COPY role remain required because they h lookup enriches those operations with Redshift queue/execution time, bytes, loaded rows, spill blocks, service class, compute type, and COPY job ID when the system views are available. +## Opt-in warehouse qualification + +The repository contains a bounded live harness for an existing Redshift cluster or Serverless +workgroup. It does not provision Redshift, S3, IAM, networking, or any other AWS resource. Running +it may resume compute and incur AWS charges, so agree on a paid-test ceiling first. + +The active AWS SDK identity must be able to obtain temporary Redshift credentials and list, write, +and delete only the chosen staging prefix. The configured `COPY` role must be usable by Redshift and +able to read that same-region prefix. For a provisioned cluster: + +```bash +uv run python -m scripts.benchmarks.redshift \ + --deployment provisioned \ + --host example.abc123.us-east-1.redshift.amazonaws.com \ + --database analytics \ + --region us-east-1 \ + --cluster-identifier dander-test \ + --db-user dander_user \ + --copy-role-arn arn:aws:iam::123456789012:role/DanderRedshiftCopy \ + --staging-bucket dander-redshift-staging +``` + +Serverless uses `--deployment serverless --workgroup-name NAME` and omits the cluster identifier +and database user. The harness relies only on ambient IAM credentials. Never pass AWS access keys, +database passwords, or session tokens on the command line. + +The harness creates one random `dander_qual_*` schema and one random child of the configured S3 +staging prefix. It forces bounded direct insertion and multi-part Parquet `COPY`, exercises all +five write modes, explicit `SUPER`, table and incremental models, replay, cursor regression, a +real two-session stale publisher, one provider-neutral graph, telemetry, and direct result +readback. It checks for run-scoped staging residue, then removes only its exact schema and S3 +prefix in `finally`. Its JSON report excludes host, database user, bucket, role, SQL, row values, +and provider responses. It reports cost as `not_measured` and support as `experimental`. + ## Deliberate limits Scalar fields and explicit JSON-to-`SUPER` fallback are implemented for direct/COPY ingestion and table/incremental models. `SUPER` fields cannot be business keys, cursors, or snapshot fields. -ARRAY/RECORD fallbacks remain unavailable. COPY retains its 4 MB staged-row guard, while an -explicitly bounded direct load may use the declared 16 MB VARBYTE/SUPER boundary. The ordinary -hosted source runner still selects SCD1. +ARRAY/RECORD fallbacks remain unavailable. COPY retains its 4 MB staged-row guard. Direct loading +is capped at 1 MiB and larger eligible rows fall back to file ingestion. The ordinary hosted source +runner still selects SCD1. Graph execution retains the shared single-connector, replace-target-only boundary; safe casts fail preflight because Redshift cannot yet preserve their canonical semantics. Views, live concurrency proof, infrastructure provisioning, and support promotion remain separate work. Use diff --git a/scripts/benchmarks/redshift.py b/scripts/benchmarks/redshift.py new file mode 100644 index 00000000..06222044 --- /dev/null +++ b/scripts/benchmarks/redshift.py @@ -0,0 +1,1221 @@ +#!/usr/bin/env python3 +"""Opt-in Redshift warehouse qualification using one schema and S3 prefix.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import platform +import time +import uuid +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import suppress +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING, Literal, Protocol, cast + +from dander.concurrency import FencingToken, TargetFenceLostError +from dander.ingestion import Endpoint, RawField, SourceConfig +from dander.pipeline.graph import PipelineGraph +from dander.pipeline.runtime import GraphExecutionPlan, plan_graph_execution +from dander.providers import ProviderKind, default_provider_registry +from dander.providers.redshift.session import execute, open_connection +from dander.warehouse import ProviderExtension, RelationRef, WarehouseRuntime +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Sequence + + from dander.providers.redshift.fence import RedshiftTargetFence + from dander.providers.redshift.session import RedshiftConnectionFactory + from dander.telemetry import OperationTelemetry + from dander.writer import WritePattern + + +_REPORT_SCHEMA = "io.dander.benchmark.redshift/v1" +_AUTHORITY_ID = "redshift:qualification" + + +class RedshiftQualificationError(RuntimeError): + """Raised with a sanitized qualification failure summary.""" + + +class _S3Inspector(Protocol): + def list_objects_v2(self, **kwargs: object) -> Mapping[str, object]: ... + def delete_objects(self, **kwargs: object) -> object: ... + + +@dataclass(frozen=True, slots=True) +class RedshiftQualificationConfig: + """Non-secret live-profile coordinates and bounded workload controls.""" + + deployment: Literal["provisioned", "serverless"] + host: str + database: str + region: str + copy_role_arn: str + staging_bucket: str + cluster_identifier: str | None = None + db_user: str | None = None + workgroup_name: str | None = None + port: int = 5439 + staging_prefix: str = "dander/staging" + direct_max_rows: int = 2 + direct_max_logical_bytes: int = 1_024 * 1_024 + copy_part_rows: int = 2 + copy_part_logical_bytes: int = 1_024 * 1_024 + + def __post_init__(self) -> None: + for name in ( + "direct_max_rows", + "direct_max_logical_bytes", + "copy_part_rows", + "copy_part_logical_bytes", + ): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + normalized_prefix = self.staging_prefix.strip("/") + if ( + not normalized_prefix + or ".." in normalized_prefix.split("/") + or any(ord(char) < 32 for char in normalized_prefix) + ): + raise ValueError("staging_prefix must be a safe non-empty S3 key prefix") + object.__setattr__(self, "staging_prefix", normalized_prefix) + default_provider_registry().parse( + ProviderKind.WAREHOUSE, + _provider_values( + self, + direct=True, + schema_name="dander_qualification_check", + staging_prefix=f"{self.staging_prefix}/qualification/check", + ), + ) + + +@dataclass(frozen=True, slots=True) +class RedshiftQualificationReport: + """Sanitized correctness and bounded-work evidence from one disposable schema.""" + + schema: str + provider: str + provider_version: str + python_version: str + deployment: str + disposable_schema: str + duration_seconds: float + direct_rows: int + copy_rows: int + write_modes: tuple[str, ...] + super_rows: int + table_model_rows: int + incremental_model_rows: int + graph_rows: int + replay_duplicate_free: bool + cursor_monotonic: bool + stale_publication_rejected: bool + concurrent_claim_attempts: int + direct_operations: int + copy_operations: int + query_ids: tuple[str, ...] + staging_tables: int + staging_objects: int + schema_cleanup_verified: bool + s3_cleanup_verified: bool + qualification_status: str = "passed" + support_status: str = "experimental" + cost_status: str = "not_measured" + + def to_json(self) -> str: + return json.dumps(asdict(self), separators=(",", ":"), sort_keys=True) + + +@dataclass(slots=True) +class _Ownership: + fence: FencingToken + verifications: int = 0 + + def verify(self) -> None: + self.verifications += 1 + + +def run_redshift_qualification( + config: RedshiftQualificationConfig, +) -> RedshiftQualificationReport: + """Exercise Redshift writes, transforms, fencing, readback, and cleanup.""" + suffix = uuid.uuid4().hex[:12] + schema_name = f"dander_qual_{suffix}" + staging_prefix = f"{config.staging_prefix}/qualification/{suffix}" + try: + direct = _warehouse_runtime( + config, + schema_name=schema_name, + staging_prefix=staging_prefix, + direct=True, + ) + copy = _warehouse_runtime( + config, + schema_name=schema_name, + staging_prefix=staging_prefix, + direct=False, + ) + except Exception as error: + raise RedshiftQualificationError( + "Redshift qualification connection validation failed" + ) from error + + started = time.perf_counter() + try: + report = _run_profile( + config, + direct=direct, + copy=copy, + schema_name=schema_name, + staging_prefix=staging_prefix, + ) + except (TargetFenceLostError, RedshiftQualificationError): + raise + except Exception as error: + raise RedshiftQualificationError( + f"Redshift qualification failed in disposable schema {schema_name}" + ) from error + finally: + cleanup_errors: list[Exception] = [] + try: + _drop_schema(direct, schema_name) + except Exception as error: + cleanup_errors.append(error) + try: + _delete_prefix(config, staging_prefix) + except Exception as error: + cleanup_errors.append(error) + if cleanup_errors: + raise RedshiftQualificationError( + "Redshift qualification could not remove all owned resources" + ) from ExceptionGroup("Redshift qualification cleanup failures", cleanup_errors) + + schema_exists = _schema_exists(direct, schema_name) + staging_objects = _prefix_object_count(config, staging_prefix) + if schema_exists or staging_objects: + raise RedshiftQualificationError("Redshift qualification cleanup left owned resources") + return replace( + report, + duration_seconds=round(max(time.perf_counter() - started, 1e-9), 6), + schema_cleanup_verified=True, + s3_cleanup_verified=True, + ) + + +def _run_profile( + config: RedshiftQualificationConfig, + *, + direct: WarehouseRuntime, + copy: WarehouseRuntime, + schema_name: str, + staging_prefix: str, +) -> RedshiftQualificationReport: + database = config.database + all_operations: list[OperationTelemetry] = [] + + direct_relation = RelationRef(catalog=database, namespace=schema_name, name="direct_records") + direct_writer, direct_target = _writer_target( + direct, + relation=direct_relation, + mode=WriteMode.SCD1, + pipeline_id="redshift_qualification_direct", + run_id="direct-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + direct_rows = ( + {"id": "one", "label": "first"}, + {"id": "two", "label": "second"}, + ) + direct_writer.write(direct_rows, direct_target) + direct_operations = direct_writer.drain_telemetry() + _require_transport(direct_operations, WriteTransport.DIRECT) + all_operations.extend(direct_operations) + expected_direct = (("one", "first"), ("two", "second")) + _require_rows(direct, direct_relation, ("id", "label"), expected_direct) + direct_writer.write(direct_rows, direct_target) + all_operations.extend(direct_writer.drain_telemetry()) + _require_rows(direct, direct_relation, ("id", "label"), expected_direct) + + copy_relation = RelationRef(catalog=database, namespace=schema_name, name="scd1_records") + copy_writer, copy_target = _writer_target( + copy, + relation=copy_relation, + mode=WriteMode.SCD1, + pipeline_id="redshift_qualification_scd1", + run_id="scd1-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + copy_writer.write( + ( + {"id": "one", "label": "older"}, + {"id": "two", "label": "second"}, + {"id": "three", "label": "third"}, + {"id": "four", "label": "fourth"}, + {"id": "one", "label": "newer"}, + ), + copy_target, + ) + copy_operations = copy_writer.drain_telemetry() + _require_transport(copy_operations, WriteTransport.COPY) + all_operations.extend(copy_operations) + expected_copy = ( + ("four", "fourth"), + ("one", "newer"), + ("three", "third"), + ("two", "second"), + ) + _require_rows(copy, copy_relation, ("id", "label"), expected_copy) + copy_writer.write( + ( + {"id": "one", "label": "older"}, + {"id": "two", "label": "second"}, + {"id": "three", "label": "third"}, + {"id": "four", "label": "fourth"}, + {"id": "one", "label": "newer"}, + ), + copy_target, + ) + all_operations.extend(copy_writer.drain_telemetry()) + _require_rows(copy, copy_relation, ("id", "label"), expected_copy) + + incremental_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="incremental_records", + ) + incremental_fields = (*_base_fields(), WriteField(name="updated_at", data_type="INT64")) + incremental_writer, incremental_target = _writer_target( + copy, + relation=incremental_relation, + mode=WriteMode.INCREMENTAL, + pipeline_id="redshift_qualification_incremental", + run_id="incremental-1", + token=1, + business_key=("id",), + fields=incremental_fields, + cursor_field="updated_at", + ) + incremental_writer.write( + ( + {"id": "one", "label": "newest", "updated_at": 2}, + {"id": "one", "label": "stale", "updated_at": 1}, + ), + incremental_target, + ) + all_operations.extend(incremental_writer.drain_telemetry()) + _require_rows( + copy, + incremental_relation, + ("id", "label", "updated_at"), + (("one", "newest", 2),), + ) + regression_writer, regression_target = _writer_target( + copy, + relation=incremental_relation, + mode=WriteMode.INCREMENTAL, + pipeline_id="redshift_qualification_incremental", + run_id="incremental-2", + token=2, + business_key=("id",), + fields=incremental_fields, + cursor_field="updated_at", + ) + regression_writer.write( + ({"id": "one", "label": "regressed", "updated_at": 1},), + regression_target, + ) + all_operations.extend(regression_writer.drain_telemetry()) + _require_rows( + copy, + incremental_relation, + ("id", "label", "updated_at"), + (("one", "newest", 2),), + ) + + snapshot_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="snapshot_records", + ) + snapshot_fields = (*_base_fields(), WriteField(name="snapshot_at", data_type="STRING")) + snapshot_writer, snapshot_target = _writer_target( + copy, + relation=snapshot_relation, + mode=WriteMode.SNAPSHOT, + pipeline_id="redshift_qualification_snapshot", + run_id="snapshot-1", + token=1, + fields=snapshot_fields, + snapshot_field="snapshot_at", + ) + snapshot_rows = ( + {"id": "one", "label": "first", "snapshot_at": "2026-08-10"}, + {"id": "two", "label": "second", "snapshot_at": "2026-08-10"}, + ) + snapshot_writer.write(snapshot_rows, snapshot_target) + all_operations.extend(snapshot_writer.drain_telemetry()) + snapshot_writer.write(snapshot_rows, snapshot_target) + all_operations.extend(snapshot_writer.drain_telemetry()) + _require_count(copy, snapshot_relation, 2) + + scd2_relation = RelationRef(catalog=database, namespace=schema_name, name="scd2_records") + scd2_writer, scd2_target = _writer_target( + copy, + relation=scd2_relation, + mode=WriteMode.SCD2, + pipeline_id="redshift_qualification_scd2", + run_id="scd2-1", + token=1, + business_key=("id",), + fields=_base_fields(), + ) + scd2_writer.write(({"id": "one", "label": "first"},), scd2_target) + all_operations.extend(scd2_writer.drain_telemetry()) + scd2_update, scd2_update_target = _writer_target( + copy, + relation=scd2_relation, + mode=WriteMode.SCD2, + pipeline_id="redshift_qualification_scd2", + run_id="scd2-2", + token=2, + business_key=("id",), + fields=_base_fields(), + ) + scd2_update.write(({"id": "one", "label": "second"},), scd2_update_target) + all_operations.extend(scd2_update.drain_telemetry()) + scd2_rows = _select(copy, scd2_relation, ("label", "is_current"), order_by=("valid_from",)) + if len(scd2_rows) != 2 or sum(bool(row[1]) for row in scd2_rows) != 1: + raise RedshiftQualificationError("Redshift SCD2 readback did not retain one current row") + + replace_relation = RelationRef(catalog=database, namespace=schema_name, name="replace_records") + replace_writer, replace_target = _writer_target( + copy, + relation=replace_relation, + mode=WriteMode.REPLACE, + pipeline_id="redshift_qualification_replace", + run_id="replace-1", + token=1, + fields=_base_fields(), + ) + replace_writer.write(direct_rows, replace_target) + all_operations.extend(replace_writer.drain_telemetry()) + replace_writer.write(direct_rows, replace_target) + all_operations.extend(replace_writer.drain_telemetry()) + replacement_writer, replacement_target = _writer_target( + copy, + relation=replace_relation, + mode=WriteMode.REPLACE, + pipeline_id="redshift_qualification_replace", + run_id="replace-2", + token=2, + fields=_base_fields(), + ) + replacement_writer.write(({"id": "three", "label": "replacement"},), replacement_target) + all_operations.extend(replacement_writer.drain_telemetry()) + _require_rows(copy, replace_relation, ("id", "label"), (("three", "replacement"),)) + + super_rows, super_operations = _exercise_super(copy, schema_name) + all_operations.extend(super_operations) + table_rows, incremental_model_rows, model_operations = _exercise_models( + copy, + schema_name, + ) + all_operations.extend(model_operations) + stale_rejected, concurrent_attempts = _exercise_concurrent_fence(copy, schema_name) + graph_rows, graph_operations = _exercise_graph(copy, schema_name, source=direct_relation) + all_operations.extend(graph_operations) + + staging_tables = _staging_table_count(copy, database) + staging_objects = _prefix_object_count(config, staging_prefix) + if staging_tables or staging_objects: + raise RedshiftQualificationError("Redshift qualification left run-scoped staging objects") + query_ids = tuple( + dict.fromkeys( + operation.query_id + for operation in all_operations + if isinstance(operation.query_id, str) and operation.query_id + ) + )[:100] + if not query_ids: + raise RedshiftQualificationError("Redshift qualification captured no warehouse query IDs") + return RedshiftQualificationReport( + schema=_REPORT_SCHEMA, + provider="redshift", + provider_version=str(_scalar(copy, "SELECT version()")), + python_version=platform.python_version(), + deployment=config.deployment, + disposable_schema=schema_name, + duration_seconds=0.0, + direct_rows=len(expected_direct), + copy_rows=len(expected_copy), + write_modes=tuple(mode.value for mode in WriteMode), + super_rows=super_rows, + table_model_rows=table_rows, + incremental_model_rows=incremental_model_rows, + graph_rows=graph_rows, + replay_duplicate_free=True, + cursor_monotonic=True, + stale_publication_rejected=stale_rejected, + concurrent_claim_attempts=concurrent_attempts, + direct_operations=sum( + operation.transport is WriteTransport.DIRECT for operation in all_operations + ), + copy_operations=sum( + operation.transport is WriteTransport.COPY for operation in all_operations + ), + query_ids=query_ids, + staging_tables=staging_tables, + staging_objects=staging_objects, + schema_cleanup_verified=False, + s3_cleanup_verified=False, + ) + + +def _exercise_super( + runtime: WarehouseRuntime, + schema_name: str, +) -> tuple[int, tuple[OperationTelemetry, ...]]: + relation = RelationRef( + catalog=_database(runtime), + namespace=schema_name, + name="super_records", + ) + fallback = ProviderExtension(provider="redshift", name="fallback", value="super") + writer, target = _writer_target( + runtime, + relation=relation, + mode=WriteMode.SCD1, + pipeline_id="redshift_qualification_super", + run_id="super-1", + token=1, + business_key=("id",), + fields=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="JSON", extensions=(fallback,)), + ), + ) + writer.write(({"id": "one", "payload": {"nested": {"ready": True}}},), target) + operations = writer.drain_telemetry() + _require_transport(operations, WriteTransport.COPY) + rows = _query( + runtime, + f'SELECT "id", JSON_SERIALIZE("payload") FROM {_relation(relation)}', + ) + if len(rows) != 1 or rows[0][0] != "one": + raise RedshiftQualificationError("Redshift SUPER readback returned the wrong row") + try: + payload = json.loads(str(rows[0][1])) + except (TypeError, ValueError) as error: + raise RedshiftQualificationError("Redshift SUPER readback was not valid JSON") from error + if payload != {"nested": {"ready": True}}: + raise RedshiftQualificationError("Redshift SUPER readback changed the JSON value") + return len(rows), operations + + +def _exercise_models( + runtime: WarehouseRuntime, + schema_name: str, +) -> tuple[int, int, tuple[OperationTelemetry, ...]]: + database = _database(runtime) + source = RelationRef(catalog=database, namespace=schema_name, name="raw_records") + source_fields = (*_base_fields(), WriteField(name="updated_at", data_type="INT64")) + source_writer, source_target = _writer_target( + runtime, + relation=source, + mode=WriteMode.SCD1, + pipeline_id="redshift_qualification_model_source", + run_id="source-1", + token=1, + business_key=("id",), + fields=source_fields, + ) + source_writer.write( + ( + {"id": "one", "label": "older", "updated_at": 1}, + {"id": "one", "label": "newer", "updated_at": 2}, + {"id": "two", "label": "second", "updated_at": 1}, + ), + source_target, + ) + operations: list[OperationTelemetry] = list(source_writer.drain_telemetry()) + with TemporaryDirectory(prefix="dander-redshift-models-") as directory: + models = Path(directory) + _write_model(models, "table_model", "table", schema_name, with_tests=True) + _write_model(models, "incremental_model", "incremental", schema_name) + runner = runtime.transforms.build_transform_runner( + graph_plan=None, + build_models=True, + raw_namespace=schema_name, + ) + if runner is None: + raise RedshiftQualificationError("Redshift model runner is unavailable") + table_result = runner.build( + models, + selected=("table_model",), + ownership=_ownership("redshift_qualification_table", "table-1", 1), + ) + operations.extend(table_result.telemetry) + table_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="table_model", + ) + _require_rows( + runtime, + table_relation, + ("id", "label", "updated_at"), + (("one", "newer", 2), ("two", "second", 1)), + ) + incremental_result = runner.build( + models, + selected=("incremental_model",), + ownership=_ownership("redshift_qualification_incremental_model", "model-1", 1), + ) + operations.extend(incremental_result.telemetry) + incremental_relation = RelationRef( + catalog=database, + namespace=schema_name, + name="incremental_model", + ) + _require_rows( + runtime, + incremental_relation, + ("id", "label", "updated_at"), + (("one", "newer", 2), ("two", "second", 1)), + ) + + update_writer, update_target = _writer_target( + runtime, + relation=source, + mode=WriteMode.SCD1, + pipeline_id="redshift_qualification_model_source", + run_id="source-2", + token=2, + business_key=("id",), + fields=source_fields, + ) + update_writer.write( + ({"id": "one", "label": "latest", "updated_at": 3},), + update_target, + ) + operations.extend(update_writer.drain_telemetry()) + update_result = runner.build( + models, + selected=("incremental_model",), + ownership=_ownership("redshift_qualification_incremental_model", "model-2", 2), + ) + operations.extend(update_result.telemetry) + + stale_writer, stale_target = _writer_target( + runtime, + relation=source, + mode=WriteMode.SCD1, + pipeline_id="redshift_qualification_model_source", + run_id="source-3", + token=3, + business_key=("id",), + fields=source_fields, + ) + stale_writer.write( + ({"id": "one", "label": "stale", "updated_at": 1},), + stale_target, + ) + operations.extend(stale_writer.drain_telemetry()) + replay_result = runner.build( + models, + selected=("incremental_model",), + ownership=_ownership("redshift_qualification_incremental_model", "model-3", 3), + ) + operations.extend(replay_result.telemetry) + _require_rows( + runtime, + incremental_relation, + ("id", "label", "updated_at"), + (("one", "latest", 3), ("two", "second", 1)), + ) + return ( + _count_relation(runtime, table_relation), + _count_relation(runtime, incremental_relation), + tuple(operations), + ) + + +def _exercise_concurrent_fence( + runtime: WarehouseRuntime, + schema_name: str, +) -> tuple[bool, int]: + target_fence = cast("RedshiftTargetFence", runtime.target_fence) + relation = RelationRef( + catalog=target_fence.database, + namespace=schema_name, + name="concurrent_records", + ) + old_token = FencingToken( + lease_table=None, + pipeline_id="redshift_qualification_concurrency", + run_id="concurrent-old", + token=20, + authority_id=_AUTHORITY_ID, + ) + old_publication = target_fence.claim(relation, old_token) + newer_token = replace(old_token, run_id="concurrent-new", token=21) + with ThreadPoolExecutor(max_workers=2) as executor: + futures = ( + executor.submit(target_fence.claim, relation, old_token), + executor.submit(target_fence.claim, relation, newer_token), + ) + for future in futures: + with suppress(TargetFenceLostError): + future.result() + current_publication = target_fence.claim(relation, newer_token) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + current_target = WriteTarget( + relation=relation, + business_key=("id",), + schema=_base_fields(), + publication_fence=current_publication, + ) + writer.write(({"id": "current", "label": "accepted"},), current_target) + try: + writer.write( + ({"id": "stale", "label": "must-not-publish"},), + WriteTarget( + relation=relation, + business_key=("id",), + schema=_base_fields(), + publication_fence=old_publication, + ), + ) + except TargetFenceLostError: + _require_rows(runtime, relation, ("id", "label"), (("current", "accepted"),)) + return True, len(futures) + raise RedshiftQualificationError("Redshift accepted a stale concurrent publication") + + +def _exercise_graph( + runtime: WarehouseRuntime, + schema_name: str, + *, + source: RelationRef, +) -> tuple[int, tuple[OperationTelemetry, ...]]: + runner = runtime.transforms.build_transform_runner( + graph_plan=_graph_plan(source=source, target_schema=schema_name), + build_models=True, + ) + if runner is None: + raise RedshiftQualificationError("Redshift graph runner is unavailable") + ownership = _ownership("redshift_qualification_graph", "graph-1", 1) + result = runner.build(Path("."), ownership=ownership) + if result.models != ("target",) or ownership.verifications < 2: + raise RedshiftQualificationError("Redshift graph ownership verification was incomplete") + target = RelationRef( + catalog=source.catalog, + namespace=schema_name, + name="graph_records", + ) + _require_rows( + runtime, + target, + ("id", "label"), + (("one", "first"), ("two", "second")), + ) + return _count_relation(runtime, target), result.telemetry + + +def _graph_plan(*, source: RelationRef, target_schema: str) -> GraphExecutionPlan: + source_config = SourceConfig( + name="redshift_qualification", + base_url="https://qualification.invalid", + auth_strategy="none", + endpoints=[ + Endpoint( + name="records", + path="/records", + primary_key=["id"], + raw_schema=[ + RawField(name="id", data_type="STRING"), + RawField(name="label", data_type="STRING"), + ], + ) + ], + ) + graph = PipelineGraph.model_validate( + { + "name": "redshift_qualification_graph", + "nodes": [ + { + "id": "records", + "type": "source", + "name": "Records", + "config": { + "connector": "redshift_qualification", + "endpoint": "records", + }, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + { + "id": "target", + "type": "target", + "name": "Target", + "config": { + "writer": { + "write_mode": "replace", + "destination": { + "dataset": target_schema, + "table": "graph_records", + "business_key": [], + }, + } + }, + "fields": [ + {"name": "id", "type": "STRING"}, + {"name": "label", "type": "STRING"}, + ], + }, + ], + "edges": [ + { + "from": "records", + "to": "target", + "mappings": [ + {"source": "id", "target": "id"}, + {"source": "label", "target": "label"}, + ], + } + ], + } + ) + return plan_graph_execution(graph, source_config, endpoint_relations={"records": source}) + + +def _warehouse_runtime( + config: RedshiftQualificationConfig, + *, + schema_name: str, + staging_prefix: str, + direct: bool, +) -> WarehouseRuntime: + registry = default_provider_registry() + parsed = registry.parse( + ProviderKind.WAREHOUSE, + _provider_values( + config, + direct=direct, + schema_name=schema_name, + staging_prefix=staging_prefix, + ), + ) + runtime = registry.build( + ProviderKind.WAREHOUSE, + parsed, + context={"catalog": config.database}, + ) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("Redshift provider returned an invalid warehouse runtime") + return runtime + + +def _provider_values( + config: RedshiftQualificationConfig, + *, + direct: bool, + schema_name: str, + staging_prefix: str, +) -> dict[str, object]: + values: dict[str, object] = { + "provider": "redshift", + "deployment": config.deployment, + "host": config.host, + "port": config.port, + "database": config.database, + "schema": schema_name, + "region": config.region, + "copy_role_arn": config.copy_role_arn, + "staging_bucket": config.staging_bucket, + "staging_prefix": staging_prefix, + "max_rows_per_file": config.copy_part_rows, + "max_logical_bytes_per_file": config.copy_part_logical_bytes, + "direct_max_rows": config.direct_max_rows if direct else 0, + "direct_max_logical_bytes": config.direct_max_logical_bytes if direct else 0, + } + if config.deployment == "provisioned": + values.update( + { + "cluster_identifier": config.cluster_identifier, + "db_user": config.db_user, + } + ) + else: + values["workgroup_name"] = config.workgroup_name + return values + + +def _writer_target( + runtime: WarehouseRuntime, + *, + relation: RelationRef, + mode: WriteMode, + pipeline_id: str, + run_id: str, + token: int, + fields: Sequence[WriteField], + business_key: tuple[str, ...] = (), + cursor_field: str | None = None, + snapshot_field: str | None = None, +) -> tuple[WritePattern, WriteTarget]: + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id=pipeline_id, + run_id=run_id, + token=token, + authority_id=_AUTHORITY_ID, + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=2, + schema_evolution=SchemaEvolution.ADDITIVE, + mode=mode, + cursor_field=cursor_field, + snapshot_field=snapshot_field, + ) + return writer, WriteTarget( + relation=relation, + business_key=business_key, + schema=tuple(fields), + publication_fence=publication, + ) + + +def _base_fields() -> tuple[WriteField, ...]: + return ( + WriteField(name="id", data_type="STRING"), + WriteField(name="label", data_type="STRING"), + ) + + +def _ownership(pipeline_id: str, run_id: str, token: int) -> _Ownership: + return _Ownership( + FencingToken( + lease_table=None, + pipeline_id=pipeline_id, + run_id=run_id, + token=token, + authority_id=_AUTHORITY_ID, + ) + ) + + +def _write_model( + root: Path, + name: str, + materialization: str, + namespace: str, + *, + with_tests: bool = False, +) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / f"{name}.sql").write_text("SELECT id, label, updated_at FROM {{ ref('raw_records') }}") + incremental = ( + "unique_key: [id]\nincremental_cursor: updated_at\n" + if materialization == "incremental" + else "" + ) + tests = ( + "tests:\n - column: id\n not_null: true\n unique: true\n" + if with_tests + else "tests: []\n" + ) + (root / f"{name}.yml").write_text( + f"model: {name}\n" + "description: Portable Redshift qualification model.\n" + "owner: data-eng\n" + "dialect: portable\n" + f"materialization: {materialization}\n" + f"dataset: {namespace}\n" + "source_system: qualification\n" + "sensitivity: public\n" + f"{incremental}" + "columns:\n" + " - name: id\n" + " type: STRING\n" + " description: Stable fixture identifier.\n" + " - name: label\n" + " type: STRING\n" + " description: Deterministic fixture value.\n" + " - name: updated_at\n" + " type: INT64\n" + " description: Monotonic fixture cursor.\n" + f"{tests}" + ) + + +def _require_transport( + operations: Sequence[OperationTelemetry], + transport: WriteTransport, +) -> None: + if not operations or any(operation.transport is not transport for operation in operations): + raise RedshiftQualificationError( + f"Redshift qualification did not use required {transport.value} transport" + ) + + +def _connection_factory(runtime: WarehouseRuntime) -> RedshiftConnectionFactory: + return cast("RedshiftTargetFence", runtime.target_fence).connection_factory + + +def _database(runtime: WarehouseRuntime) -> str: + return cast("RedshiftTargetFence", runtime.target_fence).database + + +def _query(runtime: WarehouseRuntime, statement: str) -> tuple[tuple[object, ...], ...]: + with open_connection(_connection_factory(runtime)) as connection: + rows = execute(connection, statement, fetch="all").rows + return tuple(tuple(row) for row in rows if isinstance(row, (tuple, list))) + + +def _select( + runtime: WarehouseRuntime, + relation: RelationRef, + fields: Sequence[str], + *, + order_by: Sequence[str] = (), +) -> tuple[tuple[object, ...], ...]: + columns = ", ".join(_quote(field) for field in fields) + ordering = f" ORDER BY {', '.join(_quote(field) for field in order_by)}" if order_by else "" + return _query(runtime, f"SELECT {columns} FROM {_relation(relation)}{ordering}") + + +def _require_rows( + runtime: WarehouseRuntime, + relation: RelationRef, + fields: Sequence[str], + expected: tuple[tuple[object, ...], ...], +) -> None: + actual = _select(runtime, relation, fields, order_by=(fields[0],)) + if actual != expected: + raise RedshiftQualificationError( + f"Redshift readback mismatch for qualification relation {relation.name}" + ) + + +def _count_relation(runtime: WarehouseRuntime, relation: RelationRef) -> int: + with open_connection(_connection_factory(runtime)) as connection: + row = execute(connection, f"SELECT COUNT(*) FROM {_relation(relation)}", fetch="one").row + return _count(row) + + +def _require_count(runtime: WarehouseRuntime, relation: RelationRef, expected: int) -> None: + if _count_relation(runtime, relation) != expected: + raise RedshiftQualificationError( + f"Redshift row-count mismatch for qualification relation {relation.name}" + ) + + +def _scalar(runtime: WarehouseRuntime, statement: str) -> object: + with open_connection(_connection_factory(runtime)) as connection: + row = execute(connection, statement, fetch="one").row + if not isinstance(row, (tuple, list)) or not row: + raise RedshiftQualificationError("Redshift qualification metadata query was malformed") + return row[0] + + +def _staging_table_count(runtime: WarehouseRuntime, database: str) -> int: + with open_connection(_connection_factory(runtime)) as connection: + row = execute( + connection, + "SELECT COUNT(*) FROM svv_tables WHERE database_name = %s " + "AND table_name ~ '^dander_stage_[0-9a-f]{24}$'", + (database,), + fetch="one", + ).row + return _count(row) + + +def _drop_schema(runtime: WarehouseRuntime, schema_name: str) -> None: + try: + with open_connection(_connection_factory(runtime)) as connection: + execute(connection, f"DROP SCHEMA IF EXISTS {_quote(schema_name)} CASCADE") + connection.commit() + except Exception as error: + raise RedshiftQualificationError( + f"Redshift qualification could not remove disposable schema {schema_name}" + ) from error + + +def _schema_exists(runtime: WarehouseRuntime, schema_name: str) -> bool: + with open_connection(_connection_factory(runtime)) as connection: + row = execute( + connection, + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = %s", + (schema_name,), + fetch="one", + ).row + return _count(row) != 0 + + +def _s3(config: RedshiftQualificationConfig) -> _S3Inspector: + try: + boto3 = importlib.import_module("boto3") + except ModuleNotFoundError as error: + raise RedshiftQualificationError( + "Redshift qualification requires the dander-platform[redshift] extra" + ) from error + return cast("_S3Inspector", boto3.client("s3", region_name=config.region)) + + +def _prefix_keys(config: RedshiftQualificationConfig, staging_prefix: str) -> tuple[str, ...]: + client = _s3(config) + keys: list[str] = [] + token: str | None = None + while True: + arguments: dict[str, object] = { + "Bucket": config.staging_bucket, + "Prefix": f"{staging_prefix}/", + } + if token is not None: + arguments["ContinuationToken"] = token + response = client.list_objects_v2(**arguments) + contents = response.get("Contents", []) + if not isinstance(contents, list): + raise RedshiftQualificationError("Redshift S3 cleanup returned malformed contents") + for item in contents: + if not isinstance(item, Mapping) or not isinstance(item.get("Key"), str): + raise RedshiftQualificationError("Redshift S3 cleanup returned a malformed key") + key = cast("str", item["Key"]) + if not key.startswith(f"{staging_prefix}/"): + raise RedshiftQualificationError("Redshift S3 cleanup escaped its owned prefix") + keys.append(key) + if response.get("IsTruncated") is not True: + break + next_token = response.get("NextContinuationToken") + if not isinstance(next_token, str) or not next_token: + raise RedshiftQualificationError("Redshift S3 cleanup pagination was malformed") + token = next_token + return tuple(keys) + + +def _prefix_object_count(config: RedshiftQualificationConfig, staging_prefix: str) -> int: + return len(_prefix_keys(config, staging_prefix)) + + +def _delete_prefix(config: RedshiftQualificationConfig, staging_prefix: str) -> None: + try: + client = _s3(config) + keys = _prefix_keys(config, staging_prefix) + for index in range(0, len(keys), 1_000): + response = client.delete_objects( + Bucket=config.staging_bucket, + Delete={"Objects": [{"Key": key} for key in keys[index : index + 1_000]]}, + ) + if not isinstance(response, Mapping): + raise RedshiftQualificationError( + "Redshift S3 cleanup returned a malformed delete response" + ) + errors = response.get("Errors", []) + if not isinstance(errors, list): + raise RedshiftQualificationError( + "Redshift S3 cleanup returned malformed delete errors" + ) + if errors: + raise RedshiftQualificationError( + "Redshift S3 cleanup reported one or more undeleted objects" + ) + except RedshiftQualificationError: + raise + except Exception as error: + raise RedshiftQualificationError( + "Redshift qualification could not clean its S3 staging prefix" + ) from error + + +def _count(row: object) -> int: + if not isinstance(row, (tuple, list)) or not row: + raise RedshiftQualificationError("Redshift qualification count query was malformed") + value = row[0] + if isinstance(value, bool): + raise RedshiftQualificationError("Redshift qualification count was not an integer") + try: + return int(value) + except (TypeError, ValueError) as error: + raise RedshiftQualificationError( + "Redshift qualification count was not an integer" + ) from error + + +def _relation(relation: RelationRef) -> str: + return ".".join(_quote(part) for part in relation.coordinates[1:]) + + +def _quote(value: str) -> str: + return f'"{value.replace(chr(34), chr(34) * 2)}"' + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--deployment", required=True, choices=("provisioned", "serverless")) + parser.add_argument("--host", required=True) + parser.add_argument("--port", type=int, default=5439) + parser.add_argument("--database", required=True) + parser.add_argument("--region", required=True) + parser.add_argument("--copy-role-arn", required=True) + parser.add_argument("--staging-bucket", required=True) + parser.add_argument("--staging-prefix", default="dander/staging") + parser.add_argument("--cluster-identifier") + parser.add_argument("--db-user") + parser.add_argument("--workgroup-name") + parser.add_argument("--direct-max-rows", type=int, default=2) + parser.add_argument("--direct-max-logical-bytes", type=int, default=1_024 * 1_024) + parser.add_argument("--copy-part-rows", type=int, default=2) + parser.add_argument("--copy-part-logical-bytes", type=int, default=1_024 * 1_024) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + try: + report = run_redshift_qualification(RedshiftQualificationConfig(**vars(arguments))) + except Exception: + print( + json.dumps( + { + "schema": _REPORT_SCHEMA, + "provider": "redshift", + "qualification_status": "failed", + "summary": ( + "Redshift qualification failed; inspect provider logs and rerun after " + "cleanup." + ), + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 1 + print(report.to_json()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/dander/providers/redshift/config.py b/src/dander/providers/redshift/config.py index 68a85033..d9222f78 100644 --- a/src/dander/providers/redshift/config.py +++ b/src/dander/providers/redshift/config.py @@ -15,6 +15,7 @@ _S3_BUCKET = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$") _IAM_ROLE = re.compile(r"^arn:(?:aws|aws-us-gov):iam::\d{12}:role/[A-Za-z0-9+=,.@_/-]+$") _RESOURCE_NAME = re.compile(r"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$") +_MAX_DIRECT_LOGICAL_BYTES = 1_024 * 1_024 class RedshiftWarehouseConfig(BaseModel): @@ -53,7 +54,7 @@ class RedshiftWarehouseConfig(BaseModel): direct_max_logical_bytes: int = Field( default=0, ge=0, - le=16 * 1_024 * 1_024, + le=_MAX_DIRECT_LOGICAL_BYTES, ) @model_validator(mode="after") diff --git a/src/dander/providers/redshift/session.py b/src/dander/providers/redshift/session.py index c155c090..248cfaaf 100644 --- a/src/dander/providers/redshift/session.py +++ b/src/dander/providers/redshift/session.py @@ -90,7 +90,7 @@ def execute_many( def capture_last_query_id(connection: RedshiftConnection) -> str | None: """Capture a completed compute query ID after its transaction is already committed.""" try: - row = execute(connection, "SELECT pg_last_query_id()", fetch="one").row + row = execute(connection, "SELECT last_user_query_id()", fetch="one").row value = row[0] if isinstance(row, (tuple, list)) and row else None valid = isinstance(value, int) and not isinstance(value, bool) and value >= 0 return str(value) if valid else None diff --git a/tests/portability/test_redshift_qualification.py b/tests/portability/test_redshift_qualification.py new file mode 100644 index 00000000..3a4837c6 --- /dev/null +++ b/tests/portability/test_redshift_qualification.py @@ -0,0 +1,283 @@ +"""Credential-free checks for the opt-in Redshift qualification harness.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import cast + +import pytest +from scripts.benchmarks import redshift + +from dander.providers.redshift.session import RedshiftStatementResult +from dander.warehouse import RelationRef, WarehouseRuntime + + +def _config(**overrides: object) -> redshift.RedshiftQualificationConfig: + values: dict[str, object] = { + "deployment": "provisioned", + "host": "example.abc123.us-east-1.redshift.amazonaws.com", + "database": "analytics", + "region": "us-east-1", + "copy_role_arn": "arn:aws:iam::123456789012:role/DanderRedshiftCopy", + "staging_bucket": "dander-redshift-staging", + "cluster_identifier": "dander-test", + "db_user": "dander_user", + } + values.update(overrides) + return redshift.RedshiftQualificationConfig(**values) # type: ignore[arg-type] + + +def test_qualification_config_reuses_provider_validation_and_stores_no_credentials() -> None: + config = _config() + + values = redshift._provider_values( + config, + direct=True, + schema_name="dander_qual_test", + staging_prefix="dander/staging/qualification/test", + ) + + assert values["deployment"] == "provisioned" + assert values["cluster_identifier"] == "dander-test" + assert values["direct_max_logical_bytes"] == 1_024 * 1_024 + assert values["schema"] == "dander_qual_test" + assert "password" not in values + assert "access_key" not in values + + +def test_serverless_qualification_uses_workgroup_without_database_user() -> None: + config = _config( + deployment="serverless", + workgroup_name="dander-test", + cluster_identifier=None, + db_user=None, + ) + + values = redshift._provider_values( + config, + direct=False, + schema_name="dander_qual_test", + staging_prefix="dander/staging/qualification/test", + ) + + assert values["workgroup_name"] == "dander-test" + assert values["direct_max_rows"] == 0 + assert "cluster_identifier" not in values + assert "db_user" not in values + + +def test_qualification_normalizes_its_owned_staging_prefix() -> None: + config = _config(staging_prefix="/dander/staging/") + + assert config.staging_prefix == "dander/staging" + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"db_user": None}, "check: "), + ({"direct_max_logical_bytes": 1_024 * 1_024 + 1}, "direct_max_logical_bytes"), + ({"copy_part_rows": 0}, "copy_part_rows"), + ], +) +def test_qualification_config_fails_before_provider_io( + overrides: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + _config(**overrides) + + +def test_graph_fixture_preserves_native_database_and_schema_coordinates() -> None: + source = RelationRef( + catalog="analytics", + namespace="dander_qual_test", + name="direct_records", + ) + + plan = redshift._graph_plan(source=source, target_schema="dander_qual_test") + + assert plan.bindings.source_relations == {"records": source} + assert plan.targets[0].target.relation_ref == RelationRef( + catalog="analytics", + namespace="dander_qual_test", + name="graph_records", + ) + + +def test_staging_residue_query_excludes_durable_load_history( + monkeypatch: pytest.MonkeyPatch, +) -> None: + statements: list[str] = [] + + class _Connection: + autocommit = True + + def close(self) -> None: + return None + + def fake_execute( + _connection: object, + statement: str, + _parameters: object = (), + *, + fetch: str = "none", + ) -> RedshiftStatementResult: + statements.append(statement) + assert fetch == "one" + return RedshiftStatementResult(rowcount=1, row=(0,)) + + runtime = cast( + "WarehouseRuntime", + SimpleNamespace( + target_fence=SimpleNamespace( + connection_factory=lambda: _Connection(), + database="analytics", + ), + ), + ) + monkeypatch.setattr(redshift, "execute", fake_execute) + + assert redshift._staging_table_count(runtime, "analytics") == 0 + assert "^dander_stage_[0-9a-f]{24}$" in statements[0] + assert "dander_stage_loads" not in statements[0] + + +def test_s3_cleanup_deletes_only_the_owned_paginated_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prefix = "dander/staging/qualification/abc123" + + class _S3: + def __init__(self) -> None: + self.pages = 0 + self.deleted: list[dict[str, object]] = [] + + def list_objects_v2(self, **kwargs: object) -> dict[str, object]: + self.pages += 1 + assert kwargs["Prefix"] == f"{prefix}/" + if self.pages == 1: + return { + "Contents": [{"Key": f"{prefix}/part-1.parquet"}], + "IsTruncated": True, + "NextContinuationToken": "next", + } + assert kwargs["ContinuationToken"] == "next" + return { + "Contents": [{"Key": f"{prefix}/manifest.json"}], + "IsTruncated": False, + } + + def delete_objects(self, **kwargs: object) -> dict[str, object]: + self.deleted.append(dict(kwargs)) + return {} + + client = _S3() + monkeypatch.setattr(redshift, "_s3", lambda _config: client) + + redshift._delete_prefix(_config(), prefix) + + assert client.deleted == [ + { + "Bucket": "dander-redshift-staging", + "Delete": { + "Objects": [ + {"Key": f"{prefix}/part-1.parquet"}, + {"Key": f"{prefix}/manifest.json"}, + ] + }, + } + ] + + +def test_s3_cleanup_rejects_partial_delete_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prefix = "dander/staging/qualification/abc123" + + class _S3: + def list_objects_v2(self, **_kwargs: object) -> dict[str, object]: + return { + "Contents": [{"Key": f"{prefix}/part-1.parquet"}], + "IsTruncated": False, + } + + def delete_objects(self, **_kwargs: object) -> dict[str, object]: + return {"Errors": [{"Key": f"{prefix}/part-1.parquet", "Code": "AccessDenied"}]} + + monkeypatch.setattr(redshift, "_s3", lambda _config: _S3()) + + with pytest.raises( + redshift.RedshiftQualificationError, + match="reported one or more undeleted objects", + ): + redshift._delete_prefix(_config(), prefix) + + +def test_failed_qualification_still_cleans_its_exact_owned_resources( + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime = cast("WarehouseRuntime", object()) + cleanup: list[tuple[str, str]] = [] + monkeypatch.setattr(redshift, "_warehouse_runtime", lambda *_args, **_kwargs: runtime) + + def fail(*_args: object, **_kwargs: object) -> None: + raise redshift.RedshiftQualificationError("sanitized failure") + + monkeypatch.setattr(redshift, "_run_profile", fail) + monkeypatch.setattr( + redshift, + "_drop_schema", + lambda _runtime, schema: cleanup.append(("schema", schema)), + ) + monkeypatch.setattr( + redshift, + "_delete_prefix", + lambda _config, prefix: cleanup.append(("prefix", prefix)), + ) + + with pytest.raises(redshift.RedshiftQualificationError, match="sanitized failure"): + redshift.run_redshift_qualification(_config()) + + assert [kind for kind, _value in cleanup] == ["schema", "prefix"] + assert cleanup[0][1].startswith("dander_qual_") + assert cleanup[1][1].startswith("dander/staging/qualification/") + + +def test_cli_failure_record_never_exposes_provider_error( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + secret = "provider-secret-response" + + def fail(_config: redshift.RedshiftQualificationConfig) -> None: + raise RuntimeError(secret) + + monkeypatch.setattr(redshift, "run_redshift_qualification", fail) + + exit_code = redshift.main( + [ + "--deployment", + "provisioned", + "--host", + "example.abc123.us-east-1.redshift.amazonaws.com", + "--database", + "analytics", + "--region", + "us-east-1", + "--copy-role-arn", + "arn:aws:iam::123456789012:role/DanderRedshiftCopy", + "--staging-bucket", + "dander-redshift-staging", + "--cluster-identifier", + "dander-test", + "--db-user", + "dander_user", + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["qualification_status"] == "failed" + assert secret not in json.dumps(payload) diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index 18a4b83e..47326966 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -116,7 +116,7 @@ def execute(self, command: str, args: Sequence[object] | None = None) -> Self: self.connection.last_query_id = self.backend.query_counter if compact == "SELECT current_database(), current_user": self._row = ("analytics", "dander_user") - elif compact == "SELECT pg_last_query_id()": + elif compact == "SELECT last_user_query_id()": self._row = (self.connection.last_query_id,) elif compact.startswith("WITH recent AS (SELECT query_id"): if self.backend.telemetry_error: @@ -394,6 +394,10 @@ def test_redshift_direct_thresholds_are_explicit_and_paired() -> None: with pytest.raises(ValueError, match="must both be zero or positive"): RedshiftWarehouseConfig.model_validate(raw) + raw["direct_max_logical_bytes"] = 1_024 * 1_024 + 1 + with pytest.raises(ValueError, match="less than or equal to 1048576"): + RedshiftWarehouseConfig.model_validate(raw) + def test_redshift_schema_mapper_requires_explicit_json_super_without_io( redshift_runtime: tuple[WarehouseRuntime, _FakeRedshift, _FakeS3, Path], @@ -1243,13 +1247,13 @@ def test_redshift_direct_load_parses_explicit_super_without_s3_or_query_attribut assert not tuple(tmp_path.iterdir()) -def test_redshift_direct_load_accepts_a_valid_super_row_above_the_copy_limit( +def test_redshift_direct_load_falls_back_to_copy_above_the_varbyte_limit( tmp_path: Path, ) -> None: runtime, backend, s3 = _direct_redshift_runtime( tmp_path, max_rows=1, - max_logical_bytes=8 * 1_024 * 1_024, + max_logical_bytes=1_024 * 1_024, ) backend.schema_rows = [ ("id", "character varying", 65_535, None, None, "NO"), @@ -1284,24 +1288,21 @@ def test_redshift_direct_load_accepts_a_valid_super_row_above_the_copy_limit( assert ( writer.write( - ({"id": "one", "payload": {"value": "x" * 4_300_000}},), + ({"id": "one", "payload": {"value": "x" * 1_100_000}},), target, ) == backend.merge_rowcount ) - direct_parameters = next( - parameters - for _, statement, parameters in backend.statements - if statement.startswith('INSERT INTO "dander_stage_') - and "dander_stage_loads" not in statement + sql = [statement for _, statement, _ in backend.statements] + assert any(statement.startswith("COPY ") for statement in sql) + assert not any( + statement.startswith('INSERT INTO "dander_stage_') and "dander_stage_loads" not in statement + for statement in sql ) - direct_row = direct_parameters[0] - assert isinstance(direct_row, (tuple, list)) - assert isinstance(direct_row[1], bytes) - assert len(direct_row[1]) > 4 * 1_024 * 1_024 - assert s3.region_checks == 0 - assert not s3.uploads and not s3.puts and not s3.deleted + assert s3.region_checks == 1 + assert s3.uploads and s3.puts and s3.deleted + assert all(operation.transport is WriteTransport.COPY for operation in writer.drain_telemetry()) assert not tuple(tmp_path.iterdir()) From 75ce0f3fd481747797cfed21d584b7713874b678 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 13:34:10 -0400 Subject: [PATCH 106/163] Fix Snowflake SQL found in live qualification --- HANDOFF.md | 41 +++++++++---------- ...oud-portability-snowflake-qualification.md | 34 +++++++++++++++ docs/known-limitations.md | 6 ++- docs/snowflake.md | 11 +++-- src/dander/providers/snowflake/fence.py | 25 +++++------ src/dander/providers/snowflake/writer.py | 15 +++---- .../test_snowflake_warehouse_runtime.py | 7 ++++ 7 files changed, 93 insertions(+), 46 deletions(-) create mode 100644 docs/cloud-portability-snowflake-qualification.md diff --git a/HANDOFF.md b/HANDOFF.md index 4e043842..9eea275d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,41 +2,40 @@ ## Finished -- Guaranteed a current Snowflake schema before connector-managed direct qmark binding. -- Rejected JSON-to-`VARIANT` fields as keys, incremental cursors, or snapshot identity. -- Added one opt-in disposable-schema Snowflake warehouse qualification harness. -- Covered direct/COPY threshold crossing, all writer modes, replay, cursor safety, fencing, graph execution, readback, and cleanup. -- Documented the paid-test boundary and kept Snowflake explicitly experimental. +- Passed the disposable-account Snowflake qualification across all five write modes. +- Proved bounded direct and Parquet paths, transforms, graph execution, replay, and fencing. +- Replaced the reserved `CURRENT` SQL alias with `target_row` and added regressions. +- Dropped every qualification object and deleted the temporary RSA key. +- Recorded sanitized acceptance evidence while keeping support experimental. ## Try It Run `uv run pytest -q tests/providers/test_snowflake_warehouse_runtime.py -tests/portability/test_snowflake_qualification.py`. The live command is in `docs/snowflake.md` and -requires a separately approved Snowflake test account and credit ceiling. +tests/portability/test_snowflake_qualification.py`. Live setup remains an explicitly approved, +operator-managed prerequisite described in `docs/snowflake.md`. ## Checks -- Ruff, formatting, strict mypy, and all 1,205 tests passed; PostgreSQL integration used PostgreSQL 15. -- Dependency audit, wheel/sdist inspection, source-free installs, and runtime-all import passed. -- Non-root read-only container conformance and packaged proof-asset checks passed. -- Terraform roots/tests and Helm lint/template validation passed. -- Fresh retained GCP stage-zero and platform plans each reported exactly `No changes.` +- Live qualification passed in 93.05163 seconds. +- All five qualification objects returned zero matches after teardown. +- Ruff, formatting, strict mypy, and all 1,205 tests passed with PostgreSQL 15. +- Wheel, sdist, source-free installs, runtime-all install, and generated-project validation passed. +- Terraform validation/tests, Helm checks, and non-root/read-only container conformance passed. +- The locked runtime dependency audit found no known vulnerabilities. ## Decisions -- Use existing provider/runtime contracts rather than creating a benchmark framework. -- Mutate and remove one random `DANDER_QUAL_*` schema; never provision account-level resources. -- Keep direct thresholds at zero by default until real-account timing evidence exists. +- Treat `target_row` as the generated SQL alias; `CURRENT` is reserved in Snowflake. +- Preserve experimental status until provisioning and remaining first-class gates pass. +- Record cost as `not_measured`; do not infer account-wide spend from same-session history. ## Remaining -- Let protected CI repeat Linux packaging, container, security, Terraform, and Helm checks. -- Configure non-interactive access for the signed-in Snowflake trial account. -- Approve a trial-credit ceiling, run the live harness, and preserve its sanitized report. -- Compare normalized results with the shared cross-warehouse conformance fixture. +- Push the branch and let protected CI repeat Linux and unavailable local security scans. +- Merge only through the normal protected-main review process. ## Review First +- `src/dander/providers/snowflake/fence.py` - `src/dander/providers/snowflake/writer.py` -- `scripts/benchmarks/snowflake.py` -- `tests/portability/test_snowflake_qualification.py` +- `docs/cloud-portability-snowflake-qualification.md` diff --git a/docs/cloud-portability-snowflake-qualification.md b/docs/cloud-portability-snowflake-qualification.md new file mode 100644 index 00000000..0caed28c --- /dev/null +++ b/docs/cloud-portability-snowflake-qualification.md @@ -0,0 +1,34 @@ +# Snowflake live qualification + +On 2026-08-10, Dander's experimental Snowflake warehouse adapter passed its bounded live +qualification in a disposable Snowflake trial account. This records runtime evidence; it does not +promote Snowflake to a supported profile. + +## Boundary + +- The qualification used a temporary service user, role, database, X-Small warehouse, resource + monitor, and RSA key pair. +- The resource monitor used Snowflake's minimum accepted one-credit quota. Exact account-level + credit attribution was not available during the run; the monitor did not suspend or abort the + 93-second qualification. +- The private key remained in a mode-`0600` operator directory, was never committed or printed, + and was deleted after teardown. + +## Result + +The sanitized `io.dander.benchmark.snowflake/v1` report recorded `passed` in 93.05163 seconds with +connector version `10.27.101`. Direct binding wrote two rows in three operations; Parquet `COPY` +wrote two rows in 22 operations. SCD1, SCD2, snapshot, incremental, and replace all passed. The +provider-neutral graph returned two rows. Replay remained duplicate-free, the cursor remained +monotonic, a stale publication was rejected, and two concurrent claim attempts were exercised. + +The run finished with zero staging stages and zero staging tables, and verified removal of its +random qualification schema. Postflight account checks returned zero matching qualification +users, roles, databases, warehouses, and resource monitors. + +## Findings + +Live execution exposed Snowflake's reserved `CURRENT` keyword in generated destination-fence and +SCD2 SQL. The aliases now use `target_row`, with regression coverage. All support claims remain +experimental; views, provider-managed infrastructure, performance crossover measurement, and +synchronous total-credit attribution remain outside this proof. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 5e5c5dfc..a4e2d413 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -13,8 +13,10 @@ evaluate these limits before using it for an unattended system containing busine Redshift, AWS, Azure, or OCI profiles. Snowflake has an experimental adapter with all five writer modes behind its provider capability, bounded direct/Parquet paths, temporary remote staging, schema checks, destination fencing, fenced table/incremental transforms, and fenced replace-mode - graph targets. The ordinary hosted source runner still selects SCD1, and Snowflake has no live - qualification, views, ARRAY/RECORD fallback, measured direct-write crossover, or synchronous + graph targets. Its disposable-account qualification covered all five modes, direct and Parquet + paths, transforms, graph execution, replay, stale fencing, concurrent claims, and cleanup. The + ordinary hosted source runner still selects SCD1, and Snowflake has no views, ARRAY/RECORD + fallback, measured direct-write crossover, infrastructure provisioning, or synchronous total-credit attribution. Same-session query-history enrichment is best-effort and bounded to the most recent 1,000 operation IDs per writer or transform session. Redshift has an experimental, locally conformance-tested scalar adapter with all five writer diff --git a/docs/snowflake.md b/docs/snowflake.md index ebfff6aa..a0b643a5 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -3,8 +3,8 @@ Snowflake is an experimental warehouse adapter, not a supported Dander profile. The current slice proves native database/schema coordinates, bounded direct and bulk paths, all five writer modes, an explicit JSON-to-`VARIANT` fallback, fenced table/incremental models, and fenced replace-mode -graph targets. Live qualification and the remaining first-class gates are still required before -support promotion. +graph targets. A bounded live qualification has passed; infrastructure provisioning and the +remaining first-class gates are still required before support promotion. ## Configuration @@ -120,6 +120,9 @@ checks for staging residue. Its JSON report contains bounded query IDs and timin user, credential reference, SQL, row value, or provider response. It deliberately reports cost as `not_measured` and support as `experimental`. +The sanitized disposable-account result is recorded in +[Snowflake live qualification](cloud-portability-snowflake-qualification.md). + ## Deliberate limits All five scalar write patterns are reachable through the warehouse writer capability, while the @@ -130,5 +133,5 @@ because Snowflake permanent DDL cannot share the destination-fence transaction. default to zero because no live crossover has been measured; do not claim a performance benefit until that qualification is recorded. Same-session history deliberately does not estimate total Snowflake credits or costs; account-level usage history is delayed and remains evidence-side work. -Live concurrency proof, infrastructure provisioning, and support promotion remain separate work. -Use `catalog.provider: none` for this experimental path. +Infrastructure provisioning and support promotion remain separate work. Use +`catalog.provider: none` for this experimental path. diff --git a/src/dander/providers/snowflake/fence.py b/src/dander/providers/snowflake/fence.py index 4cffc805..4e7a64db 100644 --- a/src/dander/providers/snowflake/fence.py +++ b/src/dander/providers/snowflake/fence.py @@ -161,19 +161,20 @@ def _target_fence_table_sql(table: str) -> str: def _target_claim_sql(table: str) -> str: return ( - f"MERGE INTO {table} AS current USING (SELECT ? AS target_id, ? AS pipeline_id, " + f"MERGE INTO {table} AS target_row USING (SELECT ? AS target_id, ? AS pipeline_id, " "? AS authority_id, ? AS authority_epoch, ? AS run_id, ? AS fencing_token) incoming " - 'ON current."target_id" = incoming.target_id ' - 'AND current."pipeline_id" = incoming.pipeline_id ' - 'WHEN MATCHED AND current."authority_id" = incoming.authority_id ' - 'AND current."authority_epoch" = incoming.authority_epoch ' - 'AND (incoming.fencing_token > current."fencing_token" OR ' - '(incoming.fencing_token = current."fencing_token" ' - 'AND incoming.run_id = current."run_id")) THEN UPDATE SET ' - 'current."run_id" = incoming.run_id, ' - 'current."fencing_token" = incoming.fencing_token, ' - 'current."status" = \'claimed\', current."claimed_at" = CURRENT_TIMESTAMP(), ' - 'current."committed_at" = NULL WHEN NOT MATCHED THEN INSERT ' + 'ON target_row."target_id" = incoming.target_id ' + 'AND target_row."pipeline_id" = incoming.pipeline_id ' + 'WHEN MATCHED AND target_row."authority_id" = incoming.authority_id ' + 'AND target_row."authority_epoch" = incoming.authority_epoch ' + 'AND (incoming.fencing_token > target_row."fencing_token" OR ' + '(incoming.fencing_token = target_row."fencing_token" ' + 'AND incoming.run_id = target_row."run_id")) THEN UPDATE SET ' + 'target_row."run_id" = incoming.run_id, ' + 'target_row."fencing_token" = incoming.fencing_token, ' + "target_row.\"status\" = 'claimed', " + 'target_row."claimed_at" = CURRENT_TIMESTAMP(), ' + 'target_row."committed_at" = NULL WHEN NOT MATCHED THEN INSERT ' '("target_id", "pipeline_id", "authority_id", "authority_epoch", "run_id", ' '"fencing_token", "status", "claimed_at", "committed_at") VALUES ' "(incoming.target_id, incoming.pipeline_id, incoming.authority_id, " diff --git a/src/dander/providers/snowflake/writer.py b/src/dander/providers/snowflake/writer.py index 1711833a..979d812d 100644 --- a/src/dander/providers/snowflake/writer.py +++ b/src/dander/providers/snowflake/writer.py @@ -1112,20 +1112,21 @@ def _scd2_statements( names = tuple(field.name for field in fields) incoming = _deduplicated_source(target, staging_relation, fields) match = " AND ".join( - f"current.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + f"target_row.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key ) mutable = tuple(name for name in names if name not in target.business_key) changed = ( " OR ".join( - f"NOT EQUAL_NULL(current.{_quote(name)}, incoming.{_quote(name)})" for name in mutable + f"NOT EQUAL_NULL(target_row.{_quote(name)}, incoming.{_quote(name)})" + for name in mutable ) or "FALSE" ) close = ( - f"UPDATE {_target(target)} AS current SET " + f"UPDATE {_target(target)} AS target_row SET " f"{_quote(_SCD2_VALID_TO)} = $DANDER_SCD2_EFFECTIVE_AT, " f"{_quote(_SCD2_IS_CURRENT)} = FALSE FROM ({incoming}) AS incoming WHERE {match} " - f"AND current.{_quote(_SCD2_IS_CURRENT)} = TRUE AND ({changed})" + f"AND target_row.{_quote(_SCD2_IS_CURRENT)} = TRUE AND ({changed})" ) columns = ", ".join( [ @@ -1144,12 +1145,12 @@ def _scd2_statements( ] ) current_match = " AND ".join( - f"current.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key + f"target_row.{_quote(key)} = incoming.{_quote(key)}" for key in target.business_key ) insert = ( f"INSERT INTO {_target(target)} ({columns}) SELECT {values} FROM ({incoming}) AS incoming " - f"WHERE NOT EXISTS (SELECT 1 FROM {_target(target)} AS current WHERE {current_match} " - f"AND current.{_quote(_SCD2_IS_CURRENT)} = TRUE)" + f"WHERE NOT EXISTS (SELECT 1 FROM {_target(target)} AS target_row " + f"WHERE {current_match} AND target_row.{_quote(_SCD2_IS_CURRENT)} = TRUE)" ) return ((close, ()), (insert, ())) diff --git a/tests/providers/test_snowflake_warehouse_runtime.py b/tests/providers/test_snowflake_warehouse_runtime.py index a78b7bba..d51051d7 100644 --- a/tests/providers/test_snowflake_warehouse_runtime.py +++ b/tests/providers/test_snowflake_warehouse_runtime.py @@ -350,6 +350,11 @@ def test_snowflake_claim_commits_when_account_autocommit_is_disabled( assert backend.committed_claims == 1 assert backend.discarded_claims == 0 assert backend.commits == 1 + claim_sql = next( + statement for statement, _parameters in backend.statements if statement.startswith("MERGE") + ) + assert " AS target_row USING " in claim_sql + assert " AS current " not in claim_sql and "current." not in claim_sql def test_snowflake_stages_bounded_parts_merges_last_record_and_cleans( @@ -725,6 +730,8 @@ def test_snowflake_factory_reaches_every_fenced_write_mode( elif mode is WriteMode.SCD2: assert sql.count("SET DANDER_SCD2_EFFECTIVE_AT = CURRENT_TIMESTAMP()") == 1 assert len([item for item in sql if "$DANDER_SCD2_EFFECTIVE_AT" in item]) == 2 + assert all(" AS current " not in item and "current." not in item for item in sql) + assert any(" AS target_row SET " in item for item in sql) assert any( '"valid_from" TIMESTAMP_TZ(9) NOT NULL' in item and '"is_current" BOOLEAN NOT NULL' in item From 44f474eafdcceae13227a774fb8b36ccecbf26ad Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 13:34:10 -0400 Subject: [PATCH 107/163] Fix Redshift contracts found in live qualification --- HANDOFF.md | 47 +++++++++---------- ...loud-portability-redshift-qualification.md | 35 ++++++++++++++ docs/known-limitations.md | 18 +++---- docs/redshift.md | 12 +++-- scripts/benchmarks/redshift.py | 6 ++- src/dander/providers/redshift/writer.py | 2 +- .../test_redshift_qualification.py | 27 ++++++++++- .../test_redshift_warehouse_runtime.py | 3 ++ 8 files changed, 108 insertions(+), 42 deletions(-) create mode 100644 docs/cloud-portability-redshift-qualification.md diff --git a/HANDOFF.md b/HANDOFF.md index 0a1ae206..ad549eae 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,46 +2,41 @@ ## Finished -- Capped Redshift direct ingestion at its documented 1 MiB non-file limit. -- Correlated Redshift telemetry with `LAST_USER_QUERY_ID()` and `SYS_QUERY_HISTORY`. -- Added an opt-in existing-profile Redshift qualification harness with no provisioning behavior. -- Covered all writers, `SUPER`, models, graph execution, replay, stale fencing, readback, and cleanup. -- Kept Redshift experimental and documented the paid-test boundary. +- Passed the disposable Redshift Serverless qualification across all five write modes. +- Proved direct and Parquet paths, `SUPER`, transforms, graph execution, replay, and fencing. +- Corrected live system-view and benchmark relation assumptions with regression coverage. +- Destroyed all ten Terraform resources and verified the account cleanup. +- Recorded sanitized acceptance and cost evidence while keeping support experimental. ## Try It Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py -tests/portability/test_redshift_qualification.py`. The opt-in live command is in -`docs/redshift.md` and requires an existing profile plus a separately approved cost ceiling. +tests/portability/test_redshift_qualification.py`. The opt-in live command and its existing-profile +boundary remain documented in `docs/redshift.md`. ## Checks -- Ruff, formatting, strict mypy, dependency audit, and all 1,207 tests passed with PostgreSQL 15. -- Wheel, sdist, source-free install, runtime-all install, and generated-project validation passed. -- Non-root/read-only container conformance, Terraform validation/tests, and Helm checks passed. -- Retained stage-zero plan reported `No changes.`; no apply or live mutation occurred. -- Public source-free `dander-platform==0.7.0` with the retained deployment's existing `600s` - compatibility input produced an exact retained platform `No changes.` plan. -- The candidate plan is 0 add/5 change/0 destroy: only five job version labels (`0.7.0` to - `0.8.0rc8`) and declared timeouts (`600s` to `300s`) differ. -- A detached `origin/main` plan reproduced those same five changes; this branch contains no GCP, - Terraform, manifest, or deployment-file changes. -- Final adversarial review's prefix-normalization and partial-delete findings were fixed and covered. +- Live qualification passed in 238.579339 seconds. +- AWS reported 4,080 charged RPU-seconds, approximately $0.425 of compute. +- Postflight Terraform state and all named resource lookups returned zero resources. +- Ruff, formatting, strict mypy, and all 1,208 tests passed with PostgreSQL 15. +- Wheel, sdist, source-free installs, runtime-all install, and generated-project validation passed. +- Terraform validation/tests, Helm checks, and non-root/read-only container conformance passed. +- The locked runtime dependency audit found no known vulnerabilities. ## Decisions -- Use the existing Redshift provider/runtime contracts, not a benchmark framework. -- Mutate only one random schema and S3 prefix and remove both on every handled exit. -- Keep both provisioned and Serverless selectable without provisioning either one. +- Match Redshift's actual `table_catalog` and `table_schema` system-view coordinates. +- Keep disposable cost enforcement operator-side; do not add provisioning to the harness. +- Preserve experimental status until provisioning and remaining first-class gates pass. ## Remaining -- Let protected CI repeat Linux checks plus unavailable local Trivy and Gitleaks scans. -- Provision or select a separately approved disposable Redshift profile for the live run. -- Preserve a sanitized live report before any support-status change. +- Push the branch and let protected CI repeat Linux and unavailable local security scans. +- Merge only through the normal protected-main review process. ## Review First -- `src/dander/providers/redshift/config.py` -- `src/dander/providers/redshift/session.py` +- `src/dander/providers/redshift/writer.py` - `scripts/benchmarks/redshift.py` +- `docs/cloud-portability-redshift-qualification.md` diff --git a/docs/cloud-portability-redshift-qualification.md b/docs/cloud-portability-redshift-qualification.md new file mode 100644 index 00000000..fc5c3965 --- /dev/null +++ b/docs/cloud-portability-redshift-qualification.md @@ -0,0 +1,35 @@ +# Redshift Serverless live qualification + +On 2026-08-10, Dander's experimental Redshift warehouse adapter passed its bounded live +qualification in a disposable AWS account. This records runtime evidence; it does not promote +Redshift to a supported profile. + +## Boundary + +- Terraform created exactly ten disposable resources: Redshift Serverless namespace and + workgroup, staging bucket, COPY role and policy, network resources, and their required bindings. +- A six RPU-hour operator guard capped Serverless compute below the approved $3 ceiling. AWS + reported 4,080 charged RPU-seconds (1.133333 RPU-hours), approximately $0.425 of compute at the + reviewed $0.375 per RPU-hour rate. +- The qualification mutated only its random database schema and S3 staging prefix. + +## Result + +The sanitized `io.dander.benchmark.redshift/v1` report recorded `passed` in 238.579339 seconds. +Direct insertion wrote two rows in four operations; Parquet `COPY` wrote four rows in 30 +operations. SCD1, SCD2, snapshot, incremental, and replace all passed. Explicit `SUPER`, table and +incremental models, and a provider-neutral graph returned their expected rows. Replay remained +duplicate-free, the cursor remained monotonic, a stale publication was rejected, and two +concurrent claim attempts were exercised. + +The run finished with zero staging tables and zero staging objects and verified removal of its +random schema and S3 prefix. Terraform then destroyed all ten resources. Postflight checks found +zero state resources, workgroups, namespaces, matching security groups, buckets, and IAM roles. + +## Findings + +Live execution exposed Redshift system-view coordinate names and an incorrect benchmark relation +name. The runtime now uses `table_catalog`/`table_schema`, and the qualification's physical raw +relation and telemetry lookup match Redshift's actual contracts, with regression coverage. +Support remains experimental; views, provider-managed reusable infrastructure, and support +promotion remain outside this proof. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 5e5c5dfc..948d3284 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -17,19 +17,21 @@ evaluate these limits before using it for an unattended system containing busine qualification, views, ARRAY/RECORD fallback, measured direct-write crossover, or synchronous total-credit attribution. Same-session query-history enrichment is best-effort and bounded to the most recent 1,000 operation IDs per writer or transform session. - Redshift has an experimental, locally conformance-tested scalar adapter with all five writer + Redshift has an experimental, live-qualified scalar adapter with all five writer modes using bounded Parquet parts, same-region S3 manifest `COPY`, IAM roles, replay history, destination fencing, and fenced table/incremental transforms. The ordinary hosted source runner still selects SCD1. Explicit JSON-to-`SUPER` fields use strict UTF-8 serialization, bounded VARBYTE staging, and `JSON_PARSE`; ARRAY/RECORD fallbacks remain unavailable. Replace-mode graph targets use the provider-neutral relational AST and fenced table path; graph safe casts fail - preflight. Redshift has no live qualification or views. PostgreSQL state and warehouse execution - are implemented and locally conformance-tested; all five ingestion modes use bounded `COPY` and - destination fencing. Replace-mode graph targets use the provider-neutral relational AST and - fenced table path; graph safe casts and cross-database relations fail preflight. The ordinary - hosted source runner still selects SCD1. A packaged Helm chart now renders the Kubernetes - launcher against an existing cluster, but no Kubernetes live profile is qualified yet and Dander - does not create clusters. PostgreSQL-state/BigQuery-warehouse + preflight. Its disposable Serverless qualification covered all five modes, direct and Parquet + paths, transforms, graph execution, replay, stale fencing, concurrent claims, and cleanup. + Redshift still has no views or provider-managed infrastructure. PostgreSQL state and warehouse + execution are implemented and locally conformance-tested; all five ingestion modes use bounded + `COPY` and destination fencing. Replace-mode graph targets use the provider-neutral relational + AST and fenced table path; graph safe casts and cross-database relations fail preflight. The + ordinary hosted source runner still selects SCD1. A packaged Helm chart now renders the + Kubernetes launcher against an existing cluster, but no Kubernetes live profile is qualified + yet and Dander does not create clusters. PostgreSQL-state/BigQuery-warehouse execution remains fail-closed until every BigQuery write mode uses destination-side fencing. The package publishes this pair matrix and each warehouse's exact implemented modes, transports, schema limits, transforms, graphs, and fencing through `dander runtime compatibility`, while the diff --git a/docs/redshift.md b/docs/redshift.md index e1d36dcf..e904033f 100644 --- a/docs/redshift.md +++ b/docs/redshift.md @@ -2,8 +2,9 @@ Redshift is an experimental warehouse adapter, not a supported Dander profile. The current slice proves native database/schema coordinates, bounded IAM-only bulk loading, all five fenced ingestion -write modes, fenced table and incremental models, and replace-mode graph targets before a live -support claim. +write modes, fenced table and incremental models, and replace-mode graph targets. A bounded +Serverless qualification has passed; provisioning and the remaining first-class gates are still +required before support promotion. ## Configuration @@ -111,6 +112,9 @@ readback. It checks for run-scoped staging residue, then removes only its exact prefix in `finally`. Its JSON report excludes host, database user, bucket, role, SQL, row values, and provider responses. It reports cost as `not_measured` and support as `experimental`. +The sanitized disposable-account result is recorded in +[Redshift Serverless live qualification](cloud-portability-redshift-qualification.md). + ## Deliberate limits Scalar fields and explicit JSON-to-`SUPER` fallback are implemented for direct/COPY ingestion and @@ -119,8 +123,8 @@ ARRAY/RECORD fallbacks remain unavailable. COPY retains its 4 MB staged-row guar is capped at 1 MiB and larger eligible rows fall back to file ingestion. The ordinary hosted source runner still selects SCD1. Graph execution retains the shared single-connector, replace-target-only boundary; safe casts fail -preflight because Redshift cannot yet preserve their canonical semantics. Views, live concurrency -proof, infrastructure provisioning, and support promotion remain separate work. Use +preflight because Redshift cannot yet preserve their canonical semantics. Views, infrastructure +provisioning, and support promotion remain separate work. Use `catalog.provider: none` for this experimental path. Telemetry is best-effort and never carries SQL text, error text, S3 locations, or record data. diff --git a/scripts/benchmarks/redshift.py b/scripts/benchmarks/redshift.py index 06222044..ad501ed1 100644 --- a/scripts/benchmarks/redshift.py +++ b/scripts/benchmarks/redshift.py @@ -529,7 +529,9 @@ def _exercise_models( schema_name: str, ) -> tuple[int, int, tuple[OperationTelemetry, ...]]: database = _database(runtime) - source = RelationRef(catalog=database, namespace=schema_name, name="raw_records") + # ``raw_records`` is the transform reference; the project resolver strips the + # conventional ``raw_`` prefix before selecting the physical source relation. + source = RelationRef(catalog=database, namespace=schema_name, name="records") source_fields = (*_base_fields(), WriteField(name="updated_at", data_type="INT64")) source_writer, source_target = _writer_target( runtime, @@ -1045,7 +1047,7 @@ def _staging_table_count(runtime: WarehouseRuntime, database: str) -> int: with open_connection(_connection_factory(runtime)) as connection: row = execute( connection, - "SELECT COUNT(*) FROM svv_tables WHERE database_name = %s " + "SELECT COUNT(*) FROM svv_tables WHERE table_catalog = %s " "AND table_name ~ '^dander_stage_[0-9a-f]{24}$'", (database,), fetch="one", diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py index e86d0ff4..5b53ad6d 100644 --- a/src/dander/providers/redshift/writer.py +++ b/src/dander/providers/redshift/writer.py @@ -1116,7 +1116,7 @@ def _schema_changes( connection, "SELECT column_name, data_type, character_maximum_length, numeric_precision, " "numeric_scale, is_nullable FROM svv_columns " - "WHERE database_name = %s AND schema_name = %s AND table_name = %s", + "WHERE table_catalog = %s AND table_schema = %s AND table_name = %s", relation.coordinates, fetch="all", ).rows diff --git a/tests/portability/test_redshift_qualification.py b/tests/portability/test_redshift_qualification.py index 3a4837c6..b3ad5e0d 100644 --- a/tests/portability/test_redshift_qualification.py +++ b/tests/portability/test_redshift_qualification.py @@ -4,14 +4,18 @@ import json from types import SimpleNamespace -from typing import cast +from typing import TYPE_CHECKING, cast import pytest from scripts.benchmarks import redshift from dander.providers.redshift.session import RedshiftStatementResult +from dander.transform import SqlDialect, TransformProject from dander.warehouse import RelationRef, WarehouseRuntime +if TYPE_CHECKING: + from pathlib import Path + def _config(**overrides: object) -> redshift.RedshiftQualificationConfig: values: dict[str, object] = { @@ -106,6 +110,25 @@ def test_graph_fixture_preserves_native_database_and_schema_coordinates() -> Non ) +def test_model_fixture_and_physical_source_use_the_same_relation(tmp_path: Path) -> None: + redshift._write_model( + tmp_path, + "table_model", + "table", + "dander_qual_test", + ) + + project = TransformProject.load( + tmp_path, + catalog="analytics", + raw_namespace="dander_qual_test", + target_dialect=SqlDialect.REDSHIFT, + ) + model = project.ordered(("table_model",))[0] + + assert '"analytics"."dander_qual_test"."records"' in project.compile(model) + + def test_staging_residue_query_excludes_durable_load_history( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -142,6 +165,8 @@ def fake_execute( assert redshift._staging_table_count(runtime, "analytics") == 0 assert "^dander_stage_[0-9a-f]{24}$" in statements[0] assert "dander_stage_loads" not in statements[0] + assert "table_catalog = %s" in statements[0] + assert "database_name" not in statements[0] def test_s3_cleanup_deletes_only_the_owned_paginated_prefix( diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index 47326966..f28ca67a 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -1099,6 +1099,9 @@ def test_redshift_stages_bounded_parts_merges_deterministically_and_cleans( if connection_id == writer_connection ] assert any(statement.startswith("CREATE TEMP TABLE") for statement in sql) + schema_query = next(statement for statement in sql if "FROM svv_columns" in statement) + assert "table_catalog = %s AND table_schema = %s AND table_name = %s" in schema_query + assert "database_name" not in schema_query and "schema_name" not in schema_query copy = next(statement for statement in sql if statement.startswith("COPY ")) assert "IAM_ROLE 'arn:aws:iam::123456789012:role/DanderRedshiftCopy'" in copy assert "ACCESS_KEY_ID" not in copy and "SECRET_ACCESS_KEY" not in copy From 2e546dd34e9e7476bba81f2fca17d5a889744316 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 14:54:38 -0400 Subject: [PATCH 108/163] Reconcile Phase 5 roadmap and evidence --- HANDOFF.md | 42 +- docs/cloud-portability-plan.md | 1299 +++++++++++++++++++++ docs/compatibility-matrix.md | 17 +- docs/decisions.md | 12 + docs/known-limitations.md | 5 + docs/warehouse-staging.md | 17 +- tickets/DANDER-103-columnar-staging.md | 9 + tickets/DANDER-104-snowflake-warehouse.md | 16 +- tickets/DANDER-105-redshift-warehouse.md | 31 + 9 files changed, 1406 insertions(+), 42 deletions(-) create mode 100644 docs/cloud-portability-plan.md create mode 100644 tickets/DANDER-105-redshift-warehouse.md diff --git a/HANDOFF.md b/HANDOFF.md index ad549eae..cc536677 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,41 +2,37 @@ ## Finished -- Passed the disposable Redshift Serverless qualification across all five write modes. -- Proved direct and Parquet paths, `SUPER`, transforms, graph execution, replay, and fencing. -- Corrected live system-view and benchmark relation assumptions with regression coverage. -- Destroyed all ten Terraform resources and verified the account cleanup. -- Recorded sanitized acceptance and cost evidence while keeping support experimental. +- Restored the authoritative cloud-portability roadmap onto current protected-main history. +- Re-baselined the Phase 5 exit gate without declaring Phase 5 complete. +- Reconciled completed Snowflake, Redshift, and shared-staging ticket evidence. +- Corrected stale warehouse capability, staging, limitation, and decision documentation. ## Try It -Run `uv run pytest -q tests/providers/test_redshift_warehouse_runtime.py -tests/portability/test_redshift_qualification.py`. The opt-in live command and its existing-profile -boundary remain documented in `docs/redshift.md`. +Review the Phase 5 gate in `docs/cloud-portability-plan.md`, then compare it with +`docs/compatibility-matrix.md` and `docs/known-limitations.md`. ## Checks -- Live qualification passed in 238.579339 seconds. -- AWS reported 4,080 charged RPU-seconds, approximately $0.425 of compute. -- Postflight Terraform state and all named resource lookups returned zero resources. -- Ruff, formatting, strict mypy, and all 1,208 tests passed with PostgreSQL 15. -- Wheel, sdist, source-free installs, runtime-all install, and generated-project validation passed. -- Terraform validation/tests, Helm checks, and non-root/read-only container conformance passed. -- The locked runtime dependency audit found no known vulnerabilities. +- `git diff --check` passed. +- Focused compatibility, distribution-contract, and release-metadata tests passed. +- Protected CI remains required before merge. ## Decisions -- Match Redshift's actual `table_catalog` and `table_schema` system-view coordinates. -- Keep disposable cost enforcement operator-side; do not add provisioning to the harness. -- Preserve experimental status until provisioning and remaining first-class gates pass. +- Earlier phase gates remain binding through current equivalent evidence. +- Provider-specific safe limitations do not imply false parity or support. +- Scale, cost, soak, pairwise-profile, and release qualification remain in Phase 8. ## Remaining -- Push the branch and let protected CI repeat Linux and unavailable local security scans. -- Merge only through the normal protected-main review process. +- Merge this documentation reconciliation through protected review. +- Implement the approved launcher-template request in a separate PR. +- Implement and prove the shared four-warehouse correctness fixture in a separate PR. +- Reassess the Phase 5 exit gate before beginning Phase 6. ## Review First -- `src/dander/providers/redshift/writer.py` -- `scripts/benchmarks/redshift.py` -- `docs/cloud-portability-redshift-qualification.md` +- `docs/cloud-portability-plan.md` +- `docs/compatibility-matrix.md` +- `tickets/DANDER-104-snowflake-warehouse.md` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md new file mode 100644 index 00000000..06a26afe --- /dev/null +++ b/docs/cloud-portability-plan.md @@ -0,0 +1,1299 @@ +# Cloud Portability and First-Class Platform Roadmap + +Status: active roadmap; Phases 0 through 4 completed; Phase 5 implementation merged and exit gate +re-baselined pending shared warehouse conformance + +Initial planning baseline: `origin/main` at `9dc5562` (`0.6.0rc1`) + +Phase 5 implementation checkpoint: protected `main` at +`3a10c121d1d77b7a5074ce7d07a3c9d6b7cb7e1e` after PRs #178 and #179 + +Prepared: 2026-08-06; reconciled: 2026-08-10 + +The baseline descriptions below record the conditions this roadmap was designed to change. They +are retained as architectural rationale rather than claims about the current implementation. The +phase ledger and compatibility documentation are the authoritative current-status record. + +## 1. Outcome + +Dander will ship one documented, immutable OCI runtime that can execute the same logical pipeline +under Cloud Run, ECS/Fargate, Kubernetes, Azure Container Apps Jobs, or Oracle Cloud Container +Instances. A project will select its warehouse, execution-state backend, catalog publisher, secret +source, and launcher independently through validated configuration. + +The complete first-class support target is: + +| Capability | Required providers | +|---|---| +| Warehouse execution | BigQuery, Snowflake, Redshift, PostgreSQL | +| Durable execution state | BigQuery, PostgreSQL | +| Cloud catalog | Dataplex, AWS Glue Data Catalog, none | +| Runtime secret resolution | Environment, GCP Secret Manager, AWS Secrets Manager | +| Launcher-native secret projection | GCP Secret Manager, AWS Secrets Manager, Azure Key Vault, OCI Vault | +| Launching | Cloud Run Jobs, ECS/Fargate, Kubernetes/Helm, Azure Container Apps Jobs, OCI Container Instances | +| Portable artifact | Documented OCI image and invocation/result contract | + +This roadmap preserves the existing GCP/BigQuery behavior while turning it into the first +conforming implementation. Provider support is added through complete vertical slices, not by +shipping unfinished stubs for every provider simultaneously. + +## 2. Definition of “first class” + +A provider is not first class merely because the container starts or a client library connects. +Each provider must satisfy every applicable item below. + +### 2.1 Warehouse + +- Resolve and validate a provider-specific connection without exposing credentials. +- Create or validate Dander-owned raw, staging, transform, and control namespaces. +- Support `replace`, SCD1, SCD2, snapshot, and incremental write modes. +- Support declared schema creation and the common additive-evolution contract. +- Support bounded ingestion without buffering a complete endpoint in memory. +- Use the provider’s bulk path for large loads and a bounded direct path for small loads. +- Materialize tables, views, and incremental models with provider-specific SQL. +- Run not-null, unique, accepted-values, and relationship assertions. +- Preserve idempotency across launcher retries and process interruption. +- Enforce a target-side fencing token at the final publication boundary. +- Emit normalized row, byte, duration, retry, query/job, and cost-attribution metadata. +- Pass unit, adapter-conformance, live correctness, interruption, replay, and scale tests. +- Have setup, permissions, operations, upgrade, rollback, and limitation documentation. + +### 2.2 State backend + +- Persist leases, fencing tokens, watermarks, run history, failure summaries, and metadata snapshots. +- Use backend server time for lease expiry. +- Acquire, heartbeat, and release a lease conditionally and atomically. +- Increment fencing tokens monotonically. +- Compare-and-set watermarks atomically. +- Reconcile runs left active by SIGKILL or platform termination. +- Apply versioned, idempotent schema migrations. +- Retain no source rows, secret values, or unrestricted exception text. +- Pass the same state-conformance suite for BigQuery and PostgreSQL. + +### 2.3 Catalog provider + +- Consume the same canonical metadata manifest used by execution and the local registry. +- Create or update only Dander-owned catalog fields, aspects, tags, or parameters. +- Preserve unrelated provider metadata. +- Read the published object back and normalize it for deterministic comparison. +- Treat `none` as a complete provider that performs no cloud mutation while retaining the local and + durable semantic snapshot. +- Never delete catalog resources by default. + +### 2.4 Launcher + +- Provision or install from reviewed infrastructure as code. +- Run the exact OCI manifest digest tested by the release. +- Support one-off, scheduled, paused, resumed, status, log, cancel, and replay operations. +- Supply workload identity, network placement, secret references, environment, CPU, memory, + ephemeral storage, deadline, retry count, and parallelism. +- Keep scheduling retries distinct from runtime retries and expose both attempt numbers. +- Capture structured runtime output and provider-native execution status. +- Publish logs, metrics, failure alerts, and run correlation identifiers. +- Support immutable upgrades and rollback to the prior digest. +- Pass deployment verification, manual execution, scheduled execution, overlap, SIGTERM, SIGKILL, + credential refresh, and no-change reconciliation tests. +- Document provider limits and unsupported execution-projection fields. + +## 3. Initial baseline and constraints + +The implementation started from a proven GCP slice whose names and types were not yet portable. + +### 3.1 Reusable foundations already present + +- One `PipelineExecutor` owns ingestion, transforms/tests, metadata, and truthful terminal history. +- `WatermarkStore`, `LeaseStore`, `RunHistoryStore`, and `MetadataStore` already have injected + contracts and local/BigQuery implementations. +- `WritePattern` captures the logical write modes. +- The canonical metadata spine and `CatalogPublisher` boundary already exist. +- `SecretStoreProvider` and environment/GCP resolution already exist. +- Hosted pipelines are manifest-driven and individually scheduled, identified, and permissioned. +- BigQuery supports bounded SCD1 ingestion, additive top-level schema evolution, load jobs, + Storage Write staging, run fencing, and replay-safe watermarks. +- The existing Cloud Run Terraform proves immutable images, dedicated identities, scheduling, + failure alerts, and no-change reconciliation. + +### 3.2 Coupling that must be removed deliberately + +- `WriteTarget` contains BigQuery `project/dataset/table` vocabulary. +- raw connector schemas use BigQuery type names and modes. +- transform parsing, quoting, materialization, assertions, and metric calculations are BigQuery SQL. +- graph compilation and graph execution are BigQuery-specific. +- failure classification and operator messages name BigQuery, GCP, and Cloud Run. +- `dander.yaml` contains GCP resource-name validation and one global Cloud Run resource profile. +- the CLI constructs BigQuery writers, state stores, transform runners, and Dataplex directly. +- the packaged dependency set installs `dlt[bigquery]` and GCP SDKs unconditionally. +- the Terraform root, backend, bootstrap identity, registry, scheduler, alerts, and verification are + GCP-specific. +- target DML fencing currently assumes the lease table and destination share BigQuery transaction + semantics. + +### 3.3 Existing work isolation + +The uncommitted `codex/oci-runtime-contract-wip` worktree is not a base branch. Its runtime command, +terminal JSON record, launcher run identifier, OCI annotations, and tests are candidate inputs to +the first foundation ticket only after they are rebased and reviewed. No roadmap phase may merge +that worktree wholesale or overwrite other active work. + +## 4. Target architecture + +### 4.1 Layering + +```text +Project YAML / CLI + | + v +Validated platform profile + | + +--------------------+--------------------+-------------------+ + | | | | + v v v v +Warehouse adapter State backend Catalog publisher Secret provider + | | | | + +--------------------+--------------------+-------------------+ + | + v + Cloud-neutral executor + | + v + Versioned OCI runtime contract + | + +-----------+---------+---------+-----------+-----------+ + | | | | | + v v v v v + Cloud Run Fargate Kubernetes Azure OCI +``` + +The runtime does not call launcher APIs. Launchers project the same execution specification into +their provider and invoke the runtime. Warehouse, state, catalog, and secret adapters are selected +inside the runtime from the validated profile. + +### 4.2 Logical and deployment configuration boundary + +Do not add provider conditionals to the logical pipeline manifest. Configuration version 2 splits +portable pipeline intent from environment/deployment projection so the same pipeline can be run +unchanged in shadow profiles. + +```yaml +# dander.yaml — logical pipeline contract +version: 2 + +pipelines: + salesforce_accounts: + source: salesforce + models: [stg_salesforce__accounts] + +# dander.platforms.yaml — operator/environment contract +version: 1 + +platforms: + production: + warehouse: + provider: bigquery + # provider-specific validated fields + state: + provider: bigquery + catalog: + provider: dataplex + secrets: + provider: gcp_secret_manager + +deployments: + production_cloud_run: + platform: production + launcher: + provider: cloud_run + pipelines: + salesforce_accounts: + schedule: "0 11 * * *" + time_zone: America/New_York + paused: true + secret_bindings: {} + runtime: + # cloud-neutral execution settings +``` + +`dander.yaml` owns sources, endpoints, graphs/models, logical write modes, tests, and metadata. +`dander.platforms.yaml` owns warehouse/state/catalog connections, secret backends, image references, +schedules, identities, secret bindings, resources, networking, logging, and launcher settings. A +runtime invocation selects a platform/deployment explicitly; a pipeline does not bind itself to one +cloud. + +Each provider block is a discriminated union. Unknown fields fail validation. Provider modules own +their names, limits, and defaults; the logical model retains only portable semantics. + +Version 1 manifests must continue to resolve to an implicit GCP/BigQuery profile until the end of +the deprecation window. `dander config migrate` produces the version 2 logical file plus a GCP +deployment file deterministically and supports `--check` before any write. + +### 4.3 Cloud-neutral execution projection + +Every launcher consumes one immutable projection with these fields: + +- OCI manifest digest and command. +- contract version, pipeline identifier, run identifier, launcher attempt, and shard index/count. +- configuration mount or object reference and profile identifier. +- non-secret environment variables and secret-reference bindings. +- workload identity reference. +- CPU, memory, ephemeral storage, deadline, runtime retry count, and launcher retry count. +- task count, maximum parallelism, schedule, time zone, and pause state. +- network placement and provider-specific extension map. +- labels/tags including Dander version, profile, pipeline, and image digest. +- log destination, metric namespace, alert target, and retention settings. + +Unsupported projections fail during plan generation. Launchers must not silently ignore a requested +limit or change a retry count. + +### 4.4 Warehouse adapter composition + +Do not replace BigQuery classes with one large `WarehouseProvider` interface. Define small +capabilities and compose them in a provider factory: + +- `RelationCodec`: validates and renders catalog/namespace/relation identifiers. +- `LogicalTypeMapper`: maps canonical Dander types to provider types and back. +- `SchemaManager`: inspects, creates, and evolves declared relations. +- `RecordWriter`: executes the five logical write modes. +- `BulkLoader`: stages and commits large batches. +- `TransformCompiler`: renders provider SQL from the parsed model representation. +- `TransformRunner`: materializes and tests selected models. +- `TargetFence`: validates the current run token in the same transaction as publication. +- `WarehouseTelemetry`: returns normalized job/query/load statistics. +- `WarehouseCapabilities`: declares supported types, transports, DDL, and limits. + +The executor consumes a `WarehouseRuntime` bundle. It never branches on a provider string. + +### 4.5 Canonical schema and relations + +Replace BigQuery schema vocabulary at shared boundaries with a versioned canonical model covering: + +- boolean, signed integer, decimal precision/scale, floating point, string, binary; +- date, time, timestamp with and without timezone; +- JSON/semi-structured data; +- arrays and records where the provider supports them; +- nullable/required cardinality and provider extension metadata. + +Adapters must reject lossy mappings unless the manifest explicitly chooses a documented fallback. +Existing BigQuery YAML continues to parse through a compatibility mapper during migration. + +Use `RelationRef(catalog, namespace, name)` internally. Provider codecs decide whether the rendered +shape is project/dataset/table, database/schema/table, or database/schema/relation. + +### 4.6 Cross-backend fencing and commit protocol + +The destination warehouse and state backend may differ, so a distributed transaction is not +assumed. + +1. Acquire the pipeline lease and fencing token in the selected state backend. +2. Immediately claim every target-scoped destination fence for the run using the exact state + authority ID, authority epoch, pipeline, target, run ID, and token. +3. Create a run-scoped staging object in the destination warehouse. +4. Load and validate staged data in bounded batches. +5. In one destination transaction, require exact ownership of the already-claimed destination + fence, publish the target, and mark the target commit complete. +6. Atomically compare-and-set the watermark in the state backend. +7. Record terminal history and metadata. + +Claiming occurs before staging so a newer owner invalidates stale work before either run can +publish. If the process dies between steps 5 and 6, the next run re-reads the old watermark and +safely replays against an idempotent target. + +The claim itself is an atomic compare-and-set in the destination: it succeeds only for the +destination's active authority/epoch and a token newer than the recorded owner, or for an exact +idempotent retry of the same run/token. Publication predicates on the full stored tuple, not token +alone. A zero-row conditional update, ownership mismatch, or authority mismatch aborts publication +and is reported as a stale run. + +Each state deployment has an immutable authority ID and monotonically increasing authority epoch. +A state-backend cutover must pause schedules, drain or expire leases, claim an exclusive cutover +lock in every affected destination, advance the target authority epoch, migrate/verify state, and +resume. Tokens from an older authority or epoch can never claim or publish. + +Each warehouse/write/materialization mode must document and test its atomic publication primitive: + +- transactionally fenced DML for keyed writes, snapshots, increments, and stable-table replace; +- object swap/rename only where the provider makes the fence check and swap one atomic operation; +- run-scoped load followed by fenced `DELETE`/`INSERT` for BigQuery stable-table replace; +- provider-specific table/view materialization only after an atomic fence pattern is proven. + +BigQuery permanent DDL cannot be described as transactionally fenced merely because ownership was +checked immediately before it. Hosted table/view materialization must use a proven indirection or +stable-relation DML pattern; otherwise that materialization is marked unsupported for concurrent +hosted runs. Every adapter and mode proves stale-token rejection, including a newer lease acquired +before the newer run publishes. + +### 4.7 Transform and graph portability + +Current model files and graph compilation are BigQuery-specific. Portability requires an explicit +SQL contract rather than assuming arbitrary BigQuery SQL can be translated correctly. + +- Add model metadata `dialect: portable|bigquery|snowflake|redshift|postgres`. +- Define a tested portable SQL AST subset for projections, filters, joins, unions, aggregations, + windows, casts, common scalar functions, and Dander `ref()` relations. +- Parse portable models once with sqlglot, reject unsupported/provider-specific AST nodes, and + render through the selected warehouse dialect. +- Keep existing SQL files as `bigquery` during compatibility migration; do not silently relabel + them portable. +- Allow an explicitly named provider variant only when a model needs semantics outside the + portable subset. Variants share one metadata file, declared output columns, tests, lineage, and + metric definitions. +- Require a model with provider variants to pass normalized result equivalence against the shared + conformance fixture. +- Compile graph operations to the same provider-neutral expression/relational AST used by models; + graph nodes must not construct BigQuery SQL directly. +- Render generic assertions and incremental-materialization control SQL per provider. +- Fail during validation when the selected platform lacks a portable model or an exact provider + variant. + +The portable dialect has canonical semantics, not merely common syntax: + +- Ordering must state `NULLS FIRST` or `NULLS LAST`; window ordering must include a declared unique + tie-breaker whenever peer order can change the result. +- Timestamps cross adapter boundaries as UTC with microsecond precision. Local-time conversion, + daylight-saving behavior, and higher/lower provider precision require an explicit operation. +- Decimals declare precision, scale, and rounding. Overflow and an inexact provider mapping fail + instead of falling back to floating point. +- Portable text comparison is case-sensitive with binary/code-point ordering after Unicode NFC + normalization. Locale collation or case-folding requires a provider variant or explicit portable + normalization operation. +- Only the documented strict cast matrix is portable. Provider permissive/try-casts and implicit + string-to-number/date conversion are rejected unless represented by an explicit portable node. +- JSON is compared and hashed by canonical parsed structure; object-key order and original lexical + formatting are never semantic. Provider-only path, scalar-coercion, or null behavior requires a + provider variant. +- Unquoted logical identifiers are case-stable canonical names; adapters quote rendered physical + identifiers. A model cannot depend on a provider folding unquoted names differently. +- Floating-point result assertions use a declared absolute/relative tolerance; exact types remain + exact. + +The compatibility suite contains positive and rejection fixtures for every rule. Normalized output +comparison starts from the same deterministic source rows, sorts by declared stable keys, applies +the canonical timestamp/decimal/JSON representation, and excludes run IDs, timestamps, provider +job IDs, and other intentionally variable metadata. + +### 4.8 Dependency packaging + +- Python installations expose provider extras: `bigquery`, `snowflake`, `redshift`, `postgres`, + `gcp`, `aws`, `azure`, and `oci`. +- The official full runtime image contains every first-class runtime adapter so the same manifest + can run on every launcher. +- Provider SDK imports remain inside provider packages and are lazy until selected. +- The release builds one multi-platform OCI index for supported Linux architectures, subject to + dependency conformance. Launchers pin the index digest, never a mutable tag. +- The image stays non-root, uses a read-only root filesystem where the launcher supports it, and + writes only to an explicit temporary/work volume. + +### 4.9 Infrastructure organization + +- Keep provider stacks separate so selecting one launcher does not initialize credentials or + provider plugins for every cloud. +- Package versioned roots/modules for `gcp`, `aws`, `azure`, and `oci`; package the Kubernetes + launcher as a Helm chart. +- Project the shared execution specification into provider inputs, but keep registry, IAM, + networking, scheduler, logging, alerting, and remote-state bootstrap in provider modules. +- Use provider-native encrypted remote state: GCS for GCP, S3 for AWS, Azure Storage for Azure, and + OCI Object Storage for OCI. State bootstrap remains an explicit stage-zero operation. +- Preserve existing resource addresses through moves/imports or ship a reviewed migration; never + replace a live GCP job merely to rename the module. +- Extend deployment verification through one provider protocol that returns normalized checks, + while each implementation uses real provider read APIs. + +## 5. OCI runtime contract + +### 5.1 Invocation contract + +The runtime entrypoint is stable and launcher-neutral: + +```text +dander runtime execute \ + --contract io.dander.runtime/v1 \ + --pipeline PIPELINE_ID \ + --platform PLATFORM_PROFILE \ + --config /workspace/dander.yaml +``` + +Launchers may provide the following environment values: + +- `DANDER_RUN_ID` +- `DANDER_LAUNCHER` +- `DANDER_LAUNCHER_EXECUTION_ID` +- `DANDER_ATTEMPT` +- `DANDER_SHARD_INDEX` +- `DANDER_SHARD_COUNT` +- `DANDER_DEADLINE_AT` +- `DANDER_PRINCIPAL` + +The contract specifies validation, maximum lengths, allowed characters, precedence, and how absent +values are generated. Provider identifiers remain opaque strings. + +### 5.2 Output contract + +- Application events are JSON Lines with contract version, event name, timestamp, run ID, + pipeline ID, stage, and non-sensitive dimensions. +- One terminal `runtime.completed` record reports `succeeded`, `skipped`, or `failed`, normalized + aggregates, the stable failure code, and retryability. +- Cursor values, source rows, SQL text containing values, credential material, and unrestricted + exception text never enter the contract. +- Exit codes distinguish success, invalid invocation/configuration, retryable failure, permanent + failure, and graceful cancellation. +- SIGTERM starts bounded graceful cancellation and emits a terminal record if time remains. +- SIGKILL cannot emit a terminal record; lease expiry and the next successful acquisition reconcile + the interrupted run. + +### 5.3 Artifact contract + +- OCI annotations identify source, revision, version, license, documentation, and created time. +- The release records the OCI index digest and per-platform manifest digests. +- The image includes the project runtime, connector/plugin pins, SQL models, graph files, and a + machine-readable capability manifest. +- `dander runtime inspect` reports contract, build, adapter, and plugin metadata without connecting + to providers. +- A generated SBOM and dependency lock accompany the release artifact. +- A local conformance command mounts an example project, runs a credential-free pipeline, and + validates stdout, stderr, exit status, filesystem writes, and signal behavior. +- Build the multi-platform OCI index once in the release pipeline. Promote it to GAR, ECR, ACR, and + OCIR using registry-to-registry OCI copy, never a provider-specific rebuild. +- Resolve and compare the copied index digest and every per-platform manifest digest before a + launcher may deploy it. A registry that rewrites the manifest is a failed promotion, not “the + same image.” + +## 6. Provider implementation plan + +### 6.1 BigQuery warehouse + +Refactor the existing implementation behind the new capability bundle without changing behavior +before adding new functionality. + +Required parity and scale work: + +- Characterize current SQL, DDL, schema, load, fencing, and failure behavior before moves. +- Move provider types and retry policy into `dander.providers.bigquery`. +- Map canonical schema to BigQuery scalar, repeated, and record fields. +- Retain load jobs for general batch loads and Storage Write pending streams where selected. +- Stream all write modes; remove remaining full-logical-batch buffering from SCD2, snapshot, + incremental, and Storage Write orchestration. +- Add GCS/Parquet staging for loads above the measured direct-upload crossover point. +- Preserve run-scoped staging expiration and deterministic cleanup. +- Make partitioning, clustering, schema evolution, and query labels provider configuration. +- Capture bytes processed/billed, slot milliseconds, load bytes, affected rows, and job IDs. +- Support statement maximum-bytes and timeout controls. +- Prove native GCP service-account auth and keyless AWS/Azure workload federation to BigQuery. + +### 6.2 PostgreSQL warehouse + +- Support PostgreSQL 15+ as the first relational reference implementation. +- Use TLS-required connections and a bounded connection pool. +- Use `COPY FROM STDIN` into run-scoped staging for bulk ingestion. +- Use transactional publication with `INSERT ... ON CONFLICT`, `MERGE`, or staged + delete/insert according to write mode and server capability. +- Implement replace by loading a new relation and performing the smallest safe transactional swap + supported by dependencies and constraints. +- Map canonical types, including JSONB and arrays where declared. +- Implement additive columns; reject unsafe type, nullability, and destructive changes. +- Render PostgreSQL identifiers and SQL through the provider dialect. +- Add statement, lock, and idle-transaction timeouts. +- Expose optional partition definitions and required unique indexes for keyed writes. +- Record copy bytes/rows, transaction duration, lock wait, affected rows, pool wait, and retries. +- Test against vanilla PostgreSQL plus one managed representative when credentials are available. + +### 6.3 Snowflake warehouse + +- Use the supported Snowflake Python connector with key-pair or OAuth authentication; secrets are + references, never connection literals in YAML. +- Use compressed Parquet files and `COPY INTO` through a run-scoped internal or external stage for + bulk loads. +- Use bounded direct writes only below a measured crossover threshold. +- Track staged file checksums and load history so retries cannot duplicate a run. +- Publish from transient staging tables with provider `MERGE` and transactional fence checks. +- Implement the five write modes and canonical schema mapping, including `VARIANT` fallbacks only + when explicitly selected. +- Render Snowflake SQL, identifiers, assertions, tables, views, and incremental models. +- Apply query tags containing Dander run and pipeline identifiers. +- Configure statement and queued timeouts. +- Expose warehouse name/size while treating auto-resume, auto-suspend, resource monitors, and + multi-cluster scaling as verified infrastructure settings rather than changing them per run. +- Capture query ID, warehouse, credits/compute time where available, bytes scanned/written, rows, + queue time, spill, and retry metrics. + +### 6.4 Redshift warehouse + +- Support provisioned RA3 and Redshift Serverless through one SQL/data-plane adapter with distinct + infrastructure profiles. +- Stage compressed Parquet files in a run-scoped S3 prefix and load with parallel `COPY` plus a + manifest. +- Use IAM roles for Redshift-to-S3 access; never place AWS access keys in SQL. +- Validate load files before publication and retain normalized load-error summaries. +- Publish through staging tables and Redshift `MERGE`, including target-fence validation. +- Implement the five write modes and canonical schema mapping, including SUPER only when selected. +- Support automatic distribution/sort defaults and explicit validated overrides. +- Render Redshift SQL, identifiers, assertions, tables, views, and incremental models. +- Set query group/run labels for WLM and observability. +- Treat automatic analyze/optimization as the default; request explicit analyze only when measured + table change and workload evidence require it. +- Capture query/load IDs, queue time, execution time, bytes, rows, spill, WLM queue, and retries. + +## 7. State backend plan + +### 7.1 Shared state schema + +Version and migrate these logical tables: + +- `dander_schema_migrations` +- `dander_pipeline_leases` +- `dander_watermarks` +- `dander_runs` +- `dander_metadata_snapshots` +- `dander_target_commits` in each destination warehouse + +Use one lease table keyed by pipeline rather than one physical BigQuery table per pipeline once +parity and query-cost effects have been measured. Migrations must be additive and recoverable; a +release refuses to run against a newer unknown schema version. + +### 7.2 BigQuery state + +- Port existing behavior into the shared schema and conformance suite. +- Preserve server-time lease expiry, monotonically increasing fencing tokens, CAS watermarks, + sanitized history, and interrupted-run reconciliation. +- Parameterize every value and label state jobs with run identifiers. +- Measure query count and minimum billing effects; coalesce safe checkpoints where it reduces cost + without weakening lifecycle truth. + +### 7.3 PostgreSQL state + +- Use short transactions and a dedicated bounded pool separate from warehouse query connections. +- Acquire expired leases using row locking and server time; increment the fence in the same + transaction. +- Heartbeat and release only the exact pipeline/run/token tuple. +- CAS watermarks by expected value and reject stale writers. +- Store JSON metadata snapshots in JSONB with deterministic serialized content. +- Index pipeline/status/start time and source/entity lookups. +- Add retention configuration for terminal history while preserving active/interrupted records. +- Prove behavior during connection loss, database restart, clock differences, lock contention, and + pool exhaustion. + +## 8. Catalog and secret plan + +### 8.1 Canonical catalog changes + +- Remove BigQuery-specific `project/dataset` assumptions from `CatalogAsset`. +- Store canonical relation coordinates, provider, logical types, lineage, tests, metrics, + ownership, and sensitivity. +- Let each warehouse dialect render human calculations while the canonical metric retains a typed + aggregation and field reference. +- Keep the local semantic registry byte-stable across launchers and cloud catalogs. + +### 8.2 Dataplex + +- Preserve current aspect-only updates for first-party BigQuery entries. +- Add custom-entry projection for Snowflake, Redshift, and PostgreSQL assets where a first-party + Dataplex entry is unavailable. +- Normalize generated aspects and custom-entry identifiers for read-back comparison. +- Keep publication explicit and independently retryable after data succeeds. + +### 8.3 AWS Glue Data Catalog + +- Publish canonical databases/tables directly through the Glue API; do not make crawlers the + source of truth for Dander-authored metadata. +- Map columns, relation location/connection, classification, lineage references, ownership, + sensitivity, tests, metrics, and Dander manifest checksum into owned parameters/tags. +- Support Redshift, PostgreSQL, Snowflake, and BigQuery representations without requiring the + warehouse and catalog to share a cloud. +- Read back and normalize the published table. +- Preserve unrelated table parameters and never delete tables by default. + +### 8.4 No catalog + +- Compile and store the canonical snapshot. +- Skip all catalog credentials, IAM, network calls, infrastructure, and alerts. +- Report `catalog.provider=none` and zero published assets without treating the stage as absent or + failed. + +### 8.5 Secret resolution + +- Adopt explicit reference schemes: `env://`, `gcp-sm://`, `aws-sm://`, `azure-kv://`, and + `oci-vault://`. +- Preserve existing environment-name and GCP resource-name forms through compatibility parsing. +- Add AWS Secrets Manager runtime resolution with ambient task/pod identity and audited access. +- Prefer launcher-native projection for Cloud Run, ECS, and Azure Container Apps when the platform + can inject a managed secret without exposing it to Terraform state. +- Add the smallest OCI Vault runtime resolver needed because Container Instances do not provide a + complete arbitrary application-secret projection equivalent. +- Keep direct environment injection for local development and externally managed Kubernetes. +- Audit provider, reference, principal, pipeline, and time; never audit the value. +- Test rotation during retries and token/credential refresh without rebuilding the image. + +### 8.6 Workload identity and credential support matrix + +Identity portability is an explicit allowlist. A combination absent from the released matrix is +unsupported and fails planning. `experimental` combinations require `--allow-experimental` and +print the missing proof; `supported` requires a live bootstrap, token exchange, refresh, rotation, +and revocation test. No supported or experimental hosted profile may use a service-account key, +AWS access-key pair, client secret standing in for workload identity, or another long-lived cloud +credential. + +| Launcher principal | Target services | Credential flow | Release status after proof | +|---|---|---|---| +| Cloud Run service account | BigQuery, Dataplex, GCP Secret Manager | GCP Application Default Credentials from the attached service account | supported | +| ECS task role | Redshift/S3, Glue, AWS Secrets Manager | ECS task-role credentials; the task execution role is separate and limited to image pull/log/declared secret projection | supported | +| ECS task role | BigQuery/Dataplex/GCP Secret Manager | AWS signed workload identity exchanged through Google Workload Identity Federation; Google token refresh occurs inside the running task | supported | +| GKE/EKS/AKS/OKE service account | Same-cloud APIs | The cluster's documented workload-identity binding; each cluster type is a separate tested profile | supported only per proven cluster profile | +| Kubernetes service account | Snowflake/PostgreSQL | TLS plus external-secret/operator-projected OAuth, key-pair, or database credential | supported in the named Kubernetes profile | +| Azure user-assigned managed identity | ACR, Key Vault, Azure logging | Azure managed identity | supported | +| Azure user-assigned managed identity | BigQuery/Dataplex/GCP Secret Manager | Azure workload identity exchanged through Google Workload Identity Federation, including refresh | supported after the Azure portability proof | +| Azure job | Snowflake/PostgreSQL | Key Vault-projected OAuth, key-pair, or database credential over TLS | supported in the named Azure profile | +| OCI Container Instance resource principal | OCIR, Vault, Logging/Monitoring | OCI resource principal and dynamic-group policy | supported | +| OCI job | Snowflake/PostgreSQL | OCI Vault-resolved OAuth, key-pair, or database credential over TLS | supported in the named OCI profile | +| OCI resource principal | BigQuery/Dataplex/GCP Secret Manager | External workload exchange still to be proven with refresh | experimental until Phase 7 proof | +| Any non-AWS principal | Redshift/S3, Glue, AWS Secrets Manager | Short-lived AWS federation is not yet defined or proven | unsupported until a dedicated identity profile passes live proof | + +Environment secrets are supported for local execution and operator-managed Kubernetes only. They +are not a fallback that turns a failed hosted identity flow into a supported profile. Every live +identity test records the non-secret issuer, subject, audience, principal, credential expiry, and +refresh event so the proof distinguishes initial login from sustained execution. + +## 9. Launcher implementation plan + +### 9.1 Cloud Run Jobs + +- Refactor the current `scheduled-job` module into a GCP launcher consuming the shared projection. +- Preserve Artifact Registry, dedicated runtime/scheduler identities, Secret Manager references, + Cloud Scheduler, Cloud Logging, Monitoring alerts, immutable digest validation, and paused-first + deployment. +- Add explicit task count/parallelism, execution attempt/run ID injection, ephemeral storage, + cancellation, and normalized execution status. +- Keep task parallelism at one until a pipeline declares a deterministic shard strategy. +- Verify Cloud Run limits during plan generation. +- Prove behavior parity before deleting or moving existing Terraform addresses. + +### 9.2 ECS/Fargate + +- Provision ECR, ECS cluster, task definitions, execution role, per-pipeline task role, log group, + EventBridge Scheduler schedule/role, a Step Functions execution controller, failure-event rule, + notification/DLQ target, and required VPC inputs. +- Use Fargate platform settings with explicit CPU, memory, ephemeral storage, architecture, + deadline, stop timeout, and read-only filesystem where supported. +- Inject Secrets Manager references through the task definition and use the task role for runtime + AWS access. +- Support manual `RunTask`, scheduled invocation, status, logs, cancellation, and replay through + Dander CLI commands. +- Correlate task ARN, scheduler attempt, container exit code, and terminal runtime event. +- Support Fargate Spot only as an explicit interruption-tolerant profile, never the default. +- First live proof: the same OCI digest as Cloud Run executes the same BigQuery pipeline using + keyless Google federation and normalized output comparison. + +EventBridge delivery and Dander execution attempts are separate counters. The scheduler starts one +idempotently named state-machine execution with the pipeline, scheduled occurrence, and deployment +revision. The controller: + +1. derives the Dander run/idempotency key and returns the existing execution on duplicate delivery; +2. calls `RunTask`, records the task ARN, and polls `DescribeTasks` on a bounded interval; +3. compares provider time to the absolute execution deadline and calls `StopTask` when exceeded; +4. classifies stopped/essential-container state, exit code, and terminal runtime event; +5. starts a new task only for a retryable outcome and while the explicit launcher-attempt limit and + overall deadline both remain; and +6. writes the normalized terminal execution record and sends exhausted controller failures to the + DLQ/alert path. + +Control-plane API retries do not increment the launcher attempt. A runtime self-deadline and ECS +stop timeout assist graceful shutdown but are not the sole hard-deadline mechanism. + +### 9.3 Kubernetes and Helm + +- Publish one versioned Helm chart that targets an existing conforming Kubernetes cluster; cluster + creation is outside the chart. +- Render ConfigMap, ServiceAccount, Job template, CronJob, RBAC, optional secret references, + resources, deadlines, retries, concurrency policy, job-history limits, and cleanup TTL. +- Default CronJob concurrency policy to `Forbid`; Dander leases remain the final overlap defense. +- Use `restartPolicy: Never`, bounded `backoffLimit`, and explicit requests/limits. +- Support indexed jobs only after a pipeline declares stable shards. +- Allow workload-identity annotations and pod labels without embedding one cloud’s identity model in + the chart. +- Integrate with cluster logging/metrics through stdout and standard annotations; do not require a + bundled observability stack. +- Test chart rendering, upgrade, rollback, uninstall retention policy, manual Job creation, CronJob + scheduling, SIGTERM, eviction, duplicate start, and cleanup. + +### 9.4 Azure Container Apps Jobs + +- Provision resource group inputs, ACR, Container Apps environment, scheduled/manual job, + user-assigned managed identity, Key Vault bindings, logging workspace, alerts, and network inputs. +- Map projection fields to replica timeout, retry limit, parallelism, completion count, CPU, memory, + schedule, environment, secrets, and managed identity. +- Use ACR and Key Vault without username/password material in the project manifest. +- Support manual start, schedule pause/resume, execution history, logs, cancellation where the API + permits it, and replay through Dander CLI commands. +- Prove keyless Azure-to-BigQuery access before claiming the BigQuery profile supported. +- Run the same digest and comparison workload used by Cloud Run and Fargate. + +### 9.5 Oracle Cloud Infrastructure + +- Use OCI Container Instances as the native non-Kubernetes runtime, OCIR as the registry, resource + principals/dynamic groups for identity, OCI Vault for application secrets, OCI Logging/Monitoring + for telemetry, and Events for lifecycle reactions. +- Configure `Never` or `OnFailure` restart policy according to the projection, explicit flexible + shape, resource limits, VCN/subnet, graceful shutdown, and immutable image digest. +- Add a narrow OCI Function invoked as a detached call by OCI Resource Scheduler, and manually by + the CLI, to start or create the required Container Instance execution because OCI has no direct + scheduled-container-job primitive equivalent to Cloud Run Jobs or Container Apps Jobs. +- Persist the OCI execution correlation identifier outside the container so status remains + available when the instance becomes inactive. +- Handle create/start, active, inactive, failed, stop, and delete lifecycle states idempotently. +- Support parallel shards by creating distinct run-scoped instances only after the shard contract + is proven. +- Prove OCI ambient identity for OCI services and complete an early feasibility gate for keyless + access to an external warehouse. If OCI-to-Google federation cannot satisfy refresh and workload + identity requirements, the BigQuery-on-OCI profile remains experimental while native PostgreSQL + or Snowflake profiles proceed. +- Keep OKE support under the Kubernetes/Helm launcher; do not duplicate it inside the OCI launcher. + +The Resource Scheduler body contains only pipeline ID, deployment revision, scheduled occurrence, +and schedule ID. Its OCID is placed in a dynamic group with permission to invoke the launch +Function; the Function's resource principal receives only the Container Instance, Vault reference, +tag, log, and network permissions it uses. The Function derives an idempotency key, searches the +execution-record store/tags before creating anything, and persists run ID, attempt, deadline, +Container Instance OCID, image digest, and lifecycle state. + +A separately scheduled reconciliation Function, also triggered early by OCI lifecycle Events, +observes active executions. It stops an instance past its absolute deadline, classifies the runtime +terminal event/container state, starts a fresh run-scoped instance only for an allowed retryable +attempt, and marks exhausted attempts terminal. Duplicate launch, Event, and reconciliation calls +are idempotent. Pause/resume disables/enables the Resource Scheduler schedule; it does not delete +instances or execution history. + +## 10. Efficiency and scale program + +### 10.1 Shared runtime controls + +- Stream source records through bounded queues; memory must be proportional to configured batch + size and schema, not total endpoint size. +- Separate extraction concurrency, transformation concurrency, load concurrency, pipeline + concurrency, and launcher shard parallelism. +- Apply backpressure from the selected writer to extraction. +- Classify retryable errors and use bounded exponential backoff with provider `Retry-After` where + supplied. +- Make batch rows, batch bytes, staged file size, in-flight batches, and worker count independently + configurable within adapter limits. +- Add adaptive batching only after static profiles are benchmarked; preserve deterministic minimum + and maximum bounds. +- Use columnar compressed staging for large warehouse loads. +- Keep run-scoped staging names, expiration, checksums, and cleanup. +- Reuse provider clients and connection pools within a run; never share unsafe sessions across + processes. +- Push transformations into the selected warehouse rather than moving transformed rows through the + runtime. +- Attach run/pipeline labels or query tags to every provider job and statement. +- Expose a normalized `RunPerformance` record with rows, bytes, duration, throughput, peak RSS, + retries, queue time, load time, transform time, catalog time, and provider cost dimensions. + +### 10.2 Provider-specific efficiency controls + +| Provider | Required scale path | Required controls | +|---|---|---| +| BigQuery | GCS/load jobs or Storage Write pending streams | batch bytes/rows, file sizing, partition/clustering, max bytes billed, job labels, slot/load metrics | +| Snowflake | compressed stage files plus `COPY INTO` | file sizing/count, warehouse size, auto-suspend/resume verification, timeout/query tag, merge staging, queue/spill metrics | +| Redshift | parallel S3 Parquet files plus manifest `COPY` | file distribution, WLM/query group, Serverless/RA3 profile, sort/distribution strategy, analyze evidence, queue/spill metrics | +| PostgreSQL | `COPY FROM STDIN` plus transactional staging | pool size, COPY chunk bytes, statement/lock timeout, indexes, partitioning, autovacuum evidence, lock/pool metrics | +| Cloud Run | task count and bounded parallelism | CPU/memory/ephemeral storage, timeout/retries, task index, quota validation, execution metrics | +| Fargate | one task per pipeline or stable shard | task sizing, ephemeral storage, Spot opt-in, VPC placement, scheduler/DLQ retries, task metrics | +| Kubernetes | Job/CronJob, optionally indexed | requests/limits, parallelism/completions, backoff, deadline, concurrency policy, TTL/history, eviction metrics | +| Azure | Container Apps Job replicas | CPU/memory, replica timeout/retries, parallelism/completions, schedule, execution/log metrics | +| OCI | Container Instance per run or shard | flexible shape, restart policy, graceful shutdown, VCN placement, lifecycle work requests/events, instance/container metrics | + +Deadline and retry controls must be mechanically enforced, not merely copied into environment +variables: + +| Launcher | Hard deadline | Retry owner | +|---|---|---| +| Cloud Run | Job task timeout/cancellation plus runtime graceful deadline | Cloud Run task retry count mapped exactly from the projection | +| ECS/Fargate | Step Functions controller observes absolute deadline and calls `StopTask` | Controller starts a new task for each bounded launcher attempt | +| Kubernetes | `activeDeadlineSeconds` and Job deletion/cancellation | Job `backoffLimit`, with pod restarts disabled | +| Azure Container Apps Jobs | Replica timeout and execution stop/cancel reconciliation | Job replica retry limit mapped exactly from the projection | +| OCI Container Instances | Reconciliation Function observes absolute deadline and stops the instance | Reconciler creates a new run-scoped instance for each bounded launcher attempt | + +The runtime owns only in-process transient retries such as an HTTP page retry or warehouse polling +retry. Launcher attempts rerun the whole process and therefore always pass through lease/fence and +idempotency checks. + +### 10.3 Scale test profiles + +Define the SLO document before optimization. Each profile records input rows/bytes, row width, +schema depth, source rate limit, transform complexity, concurrency, allowed latency, peak memory, +and cost ceiling. + +Required benchmark classes: + +- correctness: small deterministic fixture with byte-for-byte normalized output comparison; +- bounded-memory: input materially larger than container memory; +- bulk throughput: wide and narrow tables using each provider bulk path; +- incremental: small deltas against a large existing target; +- concurrent pipelines: independent targets plus controlled contention on one target; +- transform: scan, join, aggregation, incremental merge, and generic tests; +- failure: source throttling, dropped connection, credential expiry, state outage, catalog outage, + process termination, launcher retry, and warehouse cancellation. + +No universal “efficient at scale” claim is made from local mocks. Large paid benchmarks run only +with an explicit budget and approval. Results include provider region, service tier/shape, dataset +shape, configuration, image digest, date, and raw provider job identifiers. + +### 10.4 Optimization gate + +An optimization ships only when it: + +1. improves an agreed SLO or cost metric in a reproducible benchmark; +2. retains adapter conformance and normalized result equality; +3. keeps retries and interruption idempotent; +4. does not make the default unsafe or materially more expensive; and +5. records the before/after evidence and provider configuration. + +### 10.5 Cost and capacity controls + +- Preserve the existing GCP budget preflight as a GCP-only control; remove it from the portable + runtime path unless the selected profile enables it. +- Add provider verification for AWS budgets/alerts, Azure Cost Management budgets, OCI budgets, + and Snowflake resource monitors where the operator chooses those controls. +- Validate service quotas and requested launcher parallelism before enabling a schedule. +- Attribute warehouse queries, staged objects, container executions, and state operations to the + Dander run/pipeline tags supported by each provider. +- Report estimated and observed provider cost dimensions with benchmark evidence; never describe a + budget alert or delayed billing notification as a hard spending cap. +- Require explicit approval for capacity reservations, multi-cluster Snowflake settings, Redshift + concurrency scaling, paid Kubernetes nodes, or any live benchmark expected to incur material + cost. + +## 11. Verification strategy + +### 11.1 Test layers + +| Layer | Scope | Required execution | +|---|---|---| +| Characterization | Existing GCP/BigQuery behavior | Before refactoring each boundary | +| Unit | Canonical models, dialects, factories, validation, retry classification | Every PR | +| Adapter conformance | Warehouse, state, catalog, secret, launcher contracts | Every provider PR | +| Local integration | OCI runtime, PostgreSQL, Helm rendering, failure injection | Every PR where practical | +| Infrastructure validation | Terraform format/validate/tests and Helm lint/template | Every infrastructure PR | +| Live provider smoke | Provision, run, status, logs, rollback, no-change | Provider-gated CI/manual workflow | +| Cross-provider proof | Same digest/config/data with normalized result comparison | Each milestone gate | +| Scale | SLO and cost evidence | Release candidate workflow with approval | + +### 11.2 Avoiding a false Cartesian promise + +Five launchers, four warehouses, two state backends, three catalog modes, and multiple secret +sources create more than one hundred combinations. Dander will not claim every Cartesian +combination without evidence. + +Use three controls: + +- each individual adapter passes its full conformance suite; +- a pairwise integration matrix exercises every provider and every cross-boundary interaction at + least once; +- named production profiles receive end-to-end live proof and an explicit support status. + +The CLI publishes the tested compatibility matrix for the installed version and fails early for a +known-unsupported combination. + +### 11.3 Canonical live profiles + +| Profile | Launcher | Warehouse | State | Catalog | Secret path | +|---|---|---|---|---|---| +| GCP native | Cloud Run | BigQuery | BigQuery | Dataplex | GCP Secret Manager | +| AWS native | ECS/Fargate | Redshift | PostgreSQL | Glue | AWS Secrets Manager | +| Kubernetes portable | Kubernetes/Helm | PostgreSQL | PostgreSQL | none | environment/external secret projection | +| Snowflake multicloud | Azure Container Apps Jobs | Snowflake | PostgreSQL | none | Azure Key Vault projection | +| OCI native runtime | OCI Container Instances | PostgreSQL | PostgreSQL | none | OCI Vault | + +Additional portability proofs run the BigQuery profile from Fargate and Azure using keyless +federation and from Kubernetes using the cluster’s supported workload identity. OCI-to-BigQuery is +gated separately because its external identity path is not equivalent to the documented AWS/Azure +paths. + +The Azure canonical profile uses catalog `none`; Glue is proven in the AWS canonical profile and +may be added to Azure only after a separate short-lived AWS identity proof. + +### 11.4 Required failure assertions + +- Duplicate launcher delivery never duplicates a target business key. +- Overlapping schedules produce one owner and one truthful skipped run. +- A stale fencing token cannot publish after a newer token. +- SIGTERM either commits a complete boundary or leaves replay-safe staging. +- SIGKILL leaves no successful terminal event and is reconciled after lease expiry. +- A crash after target commit but before watermark CAS replays safely. +- Secret rotation is observed without image rebuild or secret logging. +- Catalog failure does not misreport a data write as fully successful. +- Provider throttling exhausts bounded retries with a stable retryability code. +- Schema drift fails before destructive target mutation. +- Upgrade and rollback preserve state-schema compatibility. + +### 11.5 Initial tri-state compatibility matrix + +This is the target release matrix, not a claim about the current code. A row moves to `supported` +only after all referenced adapter, identity, launcher, and live-profile proofs pass in the same +release candidate. + +| Combination | Target status | Required evidence | +|---|---|---| +| Cloud Run + BigQuery state/warehouse + Dataplex + GCP secrets | supported | GCP canonical profile and regression parity | +| Fargate + BigQuery state/warehouse + Dataplex + GCP secrets | supported | identical-digest AWS-to-Google federation and refresh proof | +| Fargate + PostgreSQL state + Redshift + Glue + AWS secrets | supported | AWS canonical profile | +| Kubernetes/Helm + PostgreSQL state/warehouse + no catalog + external secrets | supported | one named cluster profile plus portable chart conformance | +| Azure Container Apps Jobs + PostgreSQL state + Snowflake + no catalog + Key Vault | supported | Azure canonical profile | +| Azure Container Apps Jobs + BigQuery state/warehouse + Dataplex + GCP secrets | supported | Azure-to-Google federation and refresh proof | +| OCI Container Instances + PostgreSQL state/warehouse + no catalog + OCI Vault | supported | OCI canonical profile | +| OCI Container Instances + BigQuery/Dataplex/GCP secrets | experimental | OCI external-identity feasibility, refresh, and revocation proof | +| Non-AWS launcher + Redshift, Glue, or AWS Secrets Manager | unsupported | remains so until an explicit short-lived AWS federation profile is designed and proven | +| Any unlisted combination | unsupported | explicit matrix addition, pairwise coverage, identity proof, and live smoke required | + +Warehouse/catalog combinations that use ordinary TLS credentials may be added without redesigning +the runtime, but they still begin as unsupported. Adapter conformance alone does not promote an +end-to-end combination. + +## 12. Delivery sequence and gates + +### Completed-gate preservation + +Beginning a later phase never waives an earlier exit gate. Each previously completed gate must +remain satisfied on the current protected-main commit or have current equivalent evidence. A +regression in an earlier gate blocks later-phase implementation until the gate is restored. + +| Gate | Current equivalent evidence | +|---|---| +| Phase 0 | `cloud-portability-baseline.md` plus protected-main CI | +| Phase 1 | `cloud-portability-phase1-acceptance.md` and the versioned OCI runtime/projection tests | +| Phase 1B | `cloud-portability-phase1b-acceptance.md` and retained artifact/identity conformance | +| Phase 2 | Version 2 platform profiles, typed lazy factories, canonical schema/relation contracts, and migration parity tests | +| Phase 3 | `cloud-portability-fargate-lifecycle-acceptance.md` plus current GCP compatibility and no-drift evidence | +| Phase 4 | `cloud-portability-postgresql-kubernetes-acceptance.md`, state/warehouse conformance, and the published compatibility matrix | + +The foundation and BigQuery portability proof are serial gates. After Phase 3, bounded provider +lanes may proceed concurrently, but no lane skips its shared-contract dependency or integration +gate. + +```text +Baseline + | +OCI contract + execution projection + | +Artifact-copy + Fargate identity feasibility gate + | +Platform profiles + canonical schema/SQL/state contracts + | +BigQuery + Cloud Run + ECS/Fargate portability proof + | + +----------------------------------+ + | + +--------------+---------------+----------------+ + | | | +PostgreSQL state/warehouse Snowflake/Redshift Kubernetes + | | | + +----------------------+-------+----------------+ + | + Glue catalog proof + | + +---------+---------+ + | | + Azure OCI + | | + +---------+---------+ + | + Pairwise matrix + scale qualification +``` + +### Phase 0 — Baseline and branch safety + +Deliverables: + +- Merge or otherwise resolve all active release and synchronization work before implementation. +- Create the implementation branch/worktree from the then-current `origin/main`. +- Inventory the OCI-runtime WIP and salvage only reviewed commits. +- Record the explicit product-direction change from GCP-native to cloud-selectable, update the + binding steering scope, and retain GCP/BigQuery as the primary compatibility profile. +- Add characterization tests around current BigQuery SQL, state transitions, Cloud Run projection, + CLI behavior, and distribution contents. +- Approve this target matrix and the initial SLO document. + +Exit gate: + +- clean base, no unrelated worktree changes, current full suite green, and no behavior changes. + +### Phase 1 — OCI runtime contract and execution projection + +Tickets: + +1. versioned invocation, event, terminal-result, exit-code, and signal contract; +2. `dander runtime inspect` and local OCI conformance command; +3. OCI annotations, digest recording, SBOM, non-root/read-only filesystem tests; +4. cloud-neutral execution projection and capability/limit validation; +5. Cloud Run consumes the projection with exact behavior parity. + +Exit gate: + +- the same digest runs locally and on Cloud Run; normalized outcomes match; SIGTERM/SIGKILL and + replay behavior are proven; existing Cloud Run deployment reconciles with no unplanned changes. + +### Phase 1B — Artifact-copy and cross-cloud identity feasibility gate + +This deliberately precedes the broad provider refactor. It is a narrow live proof that the central +artifact and identity assumptions work in a real AWS task. + +Tickets: + +1. build the multi-platform runtime image once and publish the release candidate to staging GAR; +2. copy the OCI index from GAR to ECR without rebuilding and verify the index and per-platform + digests at both registries; +3. create an isolated minimal Fargate smoke stack with a task execution role for ECR/logging and a + separate task role for workload access; +4. exchange the task role identity through Google Workload Identity Federation without a Google + service-account key or static AWS access key; +5. run a BigQuery read/query, keep the probe alive long enough to observe the issued Google + credential expire and refresh, then run a second query; and +6. scan configuration, Terraform state, task definition, logs, and image contents for prohibited + long-lived cloud credentials and tear down or leave the paid schedule disabled. + +Exit gate: + +- Cloud Run and Fargate pull byte-identical per-platform content; Fargate reaches BigQuery before + and after credential refresh; task/execution roles are correctly separated; no static cloud key + is present. Failure stops the cross-cloud profile work before factories and adapters are widened. + +### Phase 2 — Platform profiles and factories + +Tickets: + +1. version 2 named platform profiles and discriminated provider configuration; +2. deterministic version 1 migration and compatibility window; +3. provider registry/factories for warehouse, state, catalog, secrets, and launchers; +4. canonical relation and schema model; +5. portable transform/graph AST, explicit dialect metadata, and BigQuery compatibility marking; +6. generic failure codes, logs, and run telemetry; +7. optional provider extras plus full runtime-image dependency assembly. + +Exit gate: + +- the current GCP project migrates with byte-equivalent logical behavior, and provider SDKs are not + imported when their adapter is unselected. + +### Phase 3 — Portable BigQuery vertical slice + +Tickets: + +1. move BigQuery writer/transform/schema/fence/telemetry behind provider capabilities; +2. move BigQuery state behind the shared state schema and migrations; +3. route BigQuery model and graph execution through the dialect/capability boundary with parity; +4. retain Dataplex and GCP secret behavior through provider factories; +5. implement ECS/Fargate launcher, ECR publication, AWS secret path, and AWS-to-Google federation; +6. deterministic local/Cloud Run/Fargate comparison workflow; +7. interruption, overlap, credential refresh, rollback, and no-change proofs. + +Exit gate: + +- one logical BigQuery pipeline and one OCI digest pass locally, Cloud Run, and Fargate with equal + normalized results and keyless cloud identity. This is the first cloud-portable release gate. + +### Phase 4 — PostgreSQL state and warehouse + +Tickets: + +1. PostgreSQL migrations and full state-conformance implementation; +2. destination-side generic target-fence protocol in BigQuery and PostgreSQL; +3. PostgreSQL schema, writer, bulk loader, transform, assertions, and telemetry; +4. Kubernetes/Helm launcher with existing-cluster deployment verification; +5. PostgreSQL native profile and cross-backend BigQuery/PostgreSQL matrix; +6. bounded-memory and concurrent-run benchmarks. + +Exit gate: + +- Kubernetes profile runs PostgreSQL warehouse/state end to end; both state backends pass the same + fault tests; BigQuery behavior remains unchanged. + +### Phase 5 — Snowflake and Redshift + +Tickets: + +1. shared compressed-columnar staging artifact writer; +2. Snowflake stage/COPY, schema, writes, transforms, fence, telemetry, and live profile; +3. Redshift S3/COPY, schema, writes, transforms, fence, telemetry, and live profile; +4. AWS Glue publisher and normalized read-back; +5. warehouse capability matrix and unsupported-type diagnostics; +6. shared deterministic four-warehouse correctness fixture and normalized result comparison. + +Provider-specific throughput, incremental-at-scale, transform, concurrency, crossover, cost, +soak, pairwise-profile, and release qualification are Phase 8 work. Reclassifying those checks does +not weaken them or permit a scale, efficiency, cost, reliability, or support claim before they +pass. + +Exit gate: + +Before Phase 6 implementation begins: + +1. Protected `main` is clean at a recorded commit; required CI is green; all Phase 5 implementation + and evidence PRs are merged; and the roadmap, relevant tickets, compatibility matrix, known + limitations, and provider documentation agree with that commit. +2. Every previously completed phase exit gate remains satisfied or has current equivalent evidence. +3. BigQuery, PostgreSQL, Snowflake, and Redshift are selectable through the same canonical relation, + schema, writer, transform, fencing, telemetry, and provider-factory contracts without provider + branching in the executor. +4. Every capability advertised by each warehouse passes deterministic unit and adapter-conformance + tests, including schema behavior, bounded ingestion, every advertised write mode and transport, + replay/idempotency, stale-fence rejection, supported materializations, assertions, graphs, + cleanup, telemetry availability, and fail-before-mutation diagnostics. +5. One shared deterministic fixture runs against all four warehouses. Results are normalized and + equal for the common canonical semantic intersection. Provider-specific types, fallbacks, + transports, and materializations are tested separately and are not treated as equal merely + because an adapter exists. +6. Known materialization, transport, schema, and fencing limitations fail closed before mutation + and are explicit in the compatibility matrix and known limitations. No second machine-readable + capability-schema revision is required before Phase 6 unless Phase 6 behavior consumes that + granularity. +7. Snowflake and at least one Redshift deployment profile have current live warehouse-correctness + evidence with credentials excluded and exact cleanup verified. + +Provider-specific limitations may remain experimental or unsupported when they are explicit, +safely rejected, and not required by the named Phase 6 profile. They do not become support claims. + +Before a provider or named profile is promoted to supported, every advertised capability must pass +its full conformance suite and the applicable live identity, schedule, duplicate-delivery, overlap, +interruption, replay, stale-fence, secret-rotation, outage, rollback, upgrade, cleanup, and no-drift +proofs. Complete support operations documentation is also required. + +### Phase 6 — Azure first-class launcher + +Tickets: + +1. Azure Terraform bootstrap and remote-state pattern; +2. ACR promotion by OCI copy with digest verification, Container Apps Jobs, managed identity, Key + Vault, logging, alerts, and networking; +3. launcher CLI operations and deployment verification; +4. BigQuery federation feasibility/proof; +5. Snowflake canonical live profile; +6. schedule, retry, parallelism, interruption, rollback, and reconciliation proof. + +Exit gate: + +- the same release digest passes Azure launcher conformance and the named Azure profile. + +### Phase 7 — OCI first-class launcher + +Tickets: + +1. OCI Terraform bootstrap and remote-state pattern; +2. OCIR promotion by OCI copy with digest verification, Container Instances, resource principals, + dynamic groups, Vault, logging, monitoring, and VCN projection; +3. narrow scheduling/launch Function and lifecycle event reconciliation; +4. launcher CLI operations and deployment verification; +5. OCI Vault resolution and credential refresh; +6. PostgreSQL canonical live profile and external-identity feasibility report; +7. schedule, retry, parallelism, interruption, rollback, and reconciliation proof. + +Exit gate: + +- the same release digest passes OCI launcher conformance and the named OCI profile; unsupported + cross-cloud identity combinations are explicitly marked rather than bypassed with static cloud + keys. + +### Phase 8 — Scale qualification and support release + +Tickets: + +1. execute the approved benchmark matrix and publish normalized reports, including + provider-specific throughput, incremental-at-scale, transform, concurrency, bounded-memory, + crossover, and cost evidence; +2. tune only measured bottlenecks under the optimization gate; +3. run the pairwise provider matrix and every canonical live profile; +4. complete install, permission, network, operations, upgrade, rollback, and troubleshooting docs; +5. run security, distribution, image, infrastructure, and dependency audits; +6. perform release-candidate soak with schedules enabled in each approved profile; +7. freeze the tested compatibility matrix and known limitations. + +Exit gate: + +- every first-class definition is evidenced, the complete suite passes, provider live proofs are + current, scale/cost results are attached, and no material adversarial-review blocker remains. + +## 13. Pull-request boundaries + +Keep each PR independently reviewable and releasable: + +- contract/schema changes before provider implementations; +- characterization and compatibility tests in the same PR as boundary moves; +- one state backend, warehouse, catalog, secret provider, or launcher per implementation PR; +- infrastructure module and its deployment verifier together; +- live proof evidence after implementation, not mixed with unrelated features; +- documentation and compatibility-matrix update in every provider PR; +- no PR combines a new warehouse and a new launcher unless it is the explicit vertical-slice gate. + +Every substantial PR must update the decision log, relevant ticket, known limitations, distribution +contents, `HANDOFF.md`, and only the checks actually run. + +## 14. Major risks and controls + +| Risk | Smallest adequate control | +|---|---| +| Generic interfaces hide BigQuery semantics | Characterization tests and small capability protocols before moving code | +| Cross-state/warehouse atomicity is impossible | Destination-side target fence plus replay-safe post-commit watermark CAS | +| A state-backend cutover revives old tokens | Pause/drain, destination cutover lock, authority epoch advance, migration verification, then resume | +| SQL dialect drift corrupts transforms | Parsed model representation, per-dialect compiler, provider conformance fixtures | +| Provider defaults produce silently different rows | Canonical null/time/decimal/text/cast/JSON/window semantics plus rejection fixtures | +| Arbitrary existing BigQuery SQL cannot be portable | Explicit dialect metadata, portable subset, reviewed provider variants | +| Schema mappings are lossy | Canonical types, explicit fallback selection, fail-closed unsupported diagnostics | +| Provider matrix becomes untestable | Adapter conformance + pairwise matrix + named canonical profiles | +| Cross-cloud egress is slow or expensive | Region-aware validation, benchmark cost/latency, explicit warnings and support status | +| OCI scheduling is not a native batch-job primitive | Narrow idempotent launch Function and lifecycle state machine | +| Fargate or OCI tasks outlive the declared deadline | Provider-side execution controller/reconciler stops work and owns bounded whole-task attempts | +| Cross-cloud login works once but not for a long run | Live expiry/refresh/revocation proof for each allowlisted identity path | +| Secrets leak into Terraform state or logs | References only, native injection/runtime fetch, diff/log secret scans | +| Full image becomes too large or slow to start | Lazy imports, measured image/startup budgets, provider extras for library installs | +| Refactor breaks the proven GCP slice | BigQuery/Cloud Run parity gates precede every new provider lane | +| Paid live testing expands silently | Explicit budget, approval, retained provider job IDs, teardown/inventory report | +| Multiple chats overwrite active work | Dedicated worktrees and branches; no edits or merges across active WIP without review | + +## 15. Decisions required before implementation + +These decisions materially affect the product and must be recorded before Phase 2: + +1. Whether one project may use multiple named platform profiles concurrently. Recommended: yes, + because it enables shadow migration and avoids a second configuration format later. +2. Whether the official image contains all provider adapters. Recommended: yes, while Python + installations retain extras, because one portable artifact is the core promise. +3. PostgreSQL minimum version. Recommended: PostgreSQL 15+ for a stable `MERGE` baseline while + retaining `ON CONFLICT` where it is the safer write primitive. +4. Whether Dataplex and Glue must represent non-native warehouses. Recommended: yes through custom + entries/direct table metadata, otherwise catalog selection is coupled to warehouse selection. +5. OCI native launcher shape. Recommended: Container Instances plus a narrow launch Function; + OKE remains the Kubernetes launcher. +6. Initial SLO and paid-test budgets. Required before any scale claim or large live benchmark. + +## 16. Completion checklist + +- [ ] Versioned raw OCI runtime contract is implemented and documented. +- [ ] One immutable OCI index digest passes all launcher conformance suites. +- [ ] BigQuery, Snowflake, Redshift, and PostgreSQL pass warehouse conformance and live proofs. +- [ ] BigQuery and PostgreSQL pass identical state-conformance and fault suites. +- [ ] Dataplex, Glue, and none pass catalog conformance. +- [ ] Environment, GCP, AWS, Azure launcher projection, and OCI Vault secret paths are proven. +- [ ] Cloud Run, Fargate, Kubernetes/Helm, Azure, and OCI launchers meet the first-class definition. +- [ ] Every canonical profile passes manual and scheduled execution, replay, interruption, upgrade, + rollback, and no-change reconciliation. +- [ ] Pairwise compatibility matrix and known unsupported combinations are published. +- [ ] Scale and cost reports satisfy the approved SLOs for every warehouse and launcher. +- [ ] Full tests, lint, typing, packaging, container, Terraform, Helm, live proofs, and release soak + pass without weakening existing tests. + +## 17. Provider references anchoring the plan + +- [Cloud Run Jobs](https://cloud.google.com/run/docs/create-jobs) +- [Amazon ECS scheduled tasks with EventBridge Scheduler](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/tasks-scheduled-eventbridge-scheduler.html) +- [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) +- [Kubernetes CronJobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) +- [Azure Container Apps Jobs](https://learn.microsoft.com/en-us/azure/container-apps/jobs) +- [OCI Container Instances](https://docs.oracle.com/en-us/iaas/Content/container-instances/overview-of-container-instances.htm) +- [OCI Resource Scheduler supported resources](https://docs.oracle.com/en-us/iaas/Content/resource-scheduler/concepts/resourcescheduleroverview-about.htm) +- [OCI scheduled Functions](https://docs.oracle.com/en-us/iaas/Content/Functions/Tasks/functionsscheduling.htm) +- [Step Functions integration with ECS/Fargate](https://docs.aws.amazon.com/step-functions/latest/dg/connect-ecs.html) +- [EventBridge Scheduler to Step Functions](https://docs.aws.amazon.com/step-functions/latest/dg/using-eventbridge-scheduler.html) +- [Artifact Registry image copying](https://cloud.google.com/artifact-registry/docs/docker/copy-images) +- [Snowflake data loading](https://docs.snowflake.com/en/user-guide/data-load-overview) +- [Snowflake warehouse considerations](https://docs.snowflake.com/en/user-guide/warehouses-considerations) +- [Amazon Redshift COPY](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) +- [Amazon Redshift MERGE](https://docs.aws.amazon.com/redshift/latest/dg/r_MERGE.html) +- [PostgreSQL COPY](https://www.postgresql.org/docs/current/sql-copy.html) +- [PostgreSQL MERGE](https://www.postgresql.org/docs/current/sql-merge.html) +- [AWS Glue Data Catalog](https://docs.aws.amazon.com/glue/latest/dg/catalog-and-crawler.html) +- [Google Workload Identity Federation for AWS and Azure](https://docs.cloud.google.com/iam/docs/workload-identity-federation-with-other-clouds) diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index d0a7f241..af08e82b 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -36,15 +36,20 @@ The same command publishes the exact implemented warehouse surface. These declar capabilities, not support promotion: Snowflake, Redshift, and PostgreSQL remain experimental until their named live profiles pass. -| Warehouse | Ingestion modes | Transport | Canonical schema | Models | Graphs | Fence | -|---|---|---|---|---|---|---| -| BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes | yes | yes | -| PostgreSQL | all five | COPY | all v1 types; decimal 1000, time 6 | yes | yes | yes | -| Redshift | all five | direct, COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | yes | yes | -| Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | yes | yes | +| Warehouse | Ingestion modes | Transport | Canonical schema | Table/incremental models | Views | Replace graphs | Destination fence | +|---|---|---|---|---|---|---|---| +| BigQuery | all five | load job, Storage Write | all v1 types; decimal 38, time 6; no nested arrays | yes; permanent DDL is not an atomic cross-backend publication | available; concurrent hosted DDL unsupported | yes | DML and graph publication; not every writer/materialization | +| PostgreSQL | all five | COPY | all v1 types; decimal 1000, time 6 | yes | yes | yes | all advertised modes/materializations | +| Redshift | all five | direct, COPY | scalar types; decimal 38, time 6; explicit JSON-to-SUPER fallback | yes | unsupported | yes | all advertised modes/materializations | +| Snowflake | all five | direct, COPY | scalar types; decimal 38, time 9; explicit JSON-to-VARIANT fallback | yes | unsupported | yes | all advertised modes/materializations | Portable-provider schema validation uses these declarations before source extraction, staging upload, or destination mutation. BigQuery retains its existing provider-native v1 schema path so types without lossless canonical mappings remain compatible. An unsupported portable field reports the provider, field path, type or precision, and supported limit. The packaged JSON and runtime capability constants are checked for drift in the test suite. + +The table deliberately gives more materialization detail than the current machine-readable v1 +report. Phase 6 does not require a second capability-schema revision unless its implementation +actually consumes that granularity. Unsupported materializations and state/warehouse pairs must +continue to fail before provider mutation. diff --git a/docs/decisions.md b/docs/decisions.md index 1d57f1b9..85eef829 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1111,3 +1111,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Exclusions:** The separately approved disposable-account qualification passed and is recorded in `docs/cloud-portability-snowflake-qualification.md`; provider-managed infrastructure, performance claims, and support promotion remain separate gates. + +## 2026-08-10 — Phase 5 separates correctness from support and scale qualification + +- **Gate:** Phase 5 remains open until one deterministic fixture produces equal normalized results + across the common canonical surface of BigQuery, PostgreSQL, Snowflake, and Redshift. Earlier + completed phase gates must remain satisfied or retain current equivalent evidence. +- **Limitations:** Provider-specific materialization, transport, schema, and fencing limits may + remain experimental or unsupported only when documented and rejected before unsafe mutation. A + second machine-readable capability schema is deferred until an approved consumer needs it. +- **Qualification:** Throughput, crossover, incremental-at-scale, concurrency, cost, soak, pairwise + profiles, and release qualification remain mandatory Phase 8 work rather than Phase 5 correctness + evidence. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 9d85d156..bc139dea 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -3,6 +3,11 @@ Dander `0.8.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. +The Phase 5 warehouse implementations and Snowflake/Redshift live correctness proofs are merged, +but the re-baselined Phase 5 exit gate remains open until the shared four-warehouse deterministic +fixture produces equal normalized results and the current evidence ledger is reconciled. Scale, +cost, soak, pairwise-profile, and release qualification remain Phase 8 work. + - AWS ECS/Fargate remains experimental rather than supported. The named Fargate-to-BigQuery/GCP composition has passed source-free manual and scheduled execution, replay, interruption, renewable keyless identity, alert routing, image rollback, cleanup, and no-drift acceptance. diff --git a/docs/warehouse-staging.md b/docs/warehouse-staging.md index 8f8ed9ce..ae5ec51f 100644 --- a/docs/warehouse-staging.md +++ b/docs/warehouse-staging.md @@ -25,8 +25,15 @@ or string representation. Decimal precision above Parquet/Arrow's 76-digit limit file is published. Missing/extra fields, required nulls, malformed JSON, and Arrow conversion errors produce row-indexed messages without echoing record contents. -Redshift's experimental SCD1 adapter consumes this contract through same-region S3 manifest `COPY`, -with IAM-role authentication, deterministic replay history, target fencing, and owned-object -cleanup. Transforms, executable graphs, other write modes, semi-structured `SUPER` mapping, and a -live Redshift profile remain unavailable. Snowflake remains unavailable until its independent -upload, idempotency, fencing, schema, transform, telemetry, cleanup, and live-profile gates pass. +Redshift consumes this contract through same-region S3 manifest `COPY` with IAM-role authentication, +deterministic replay history, target fencing, and owned-object cleanup. Its experimental adapter now +also supports all five write modes, bounded direct staging, explicit JSON-to-`SUPER`, fenced table +and incremental transforms, provider-neutral replace graphs, telemetry, and a live-qualified +Serverless warehouse proof. Views, reusable provider-managed infrastructure, and support promotion +remain unavailable. + +Snowflake consumes the same local artifacts through temporary stage upload and `COPY`. Its +experimental adapter supports all five write modes, bounded direct staging, explicit +JSON-to-`VARIANT`, fenced table and incremental transforms, provider-neutral replace graphs, +telemetry, and a disposable-account live proof. Views, ARRAY/RECORD fallback, measured direct-write +crossover, provider-managed infrastructure, and support promotion remain unavailable. diff --git a/tickets/DANDER-103-columnar-staging.md b/tickets/DANDER-103-columnar-staging.md index a24a3a2a..a7dbebc0 100644 --- a/tickets/DANDER-103-columnar-staging.md +++ b/tickets/DANDER-103-columnar-staging.md @@ -1,3 +1,12 @@ +--- +id: DANDER-103 +title: Add shared bounded columnar staging +status: completed +component: warehouse +epic: cloud-portability +created: 2026-08-08 +--- + # DANDER-103 — Shared bounded columnar staging ## Scope diff --git a/tickets/DANDER-104-snowflake-warehouse.md b/tickets/DANDER-104-snowflake-warehouse.md index a23e6282..37b30de9 100644 --- a/tickets/DANDER-104-snowflake-warehouse.md +++ b/tickets/DANDER-104-snowflake-warehouse.md @@ -1,7 +1,7 @@ --- id: DANDER-104 -title: Add the experimental Snowflake staged-SCD1 warehouse -status: in-review +title: Qualify the experimental Snowflake warehouse +status: completed component: warehouse epic: cloud-portability depends_on: [DANDER-103] @@ -24,8 +24,8 @@ support. - [x] Temporary remote staging and local artifacts clean up after success and handled failure. - [x] Lost fencing ownership blocks target publication. - [x] BigQuery and PostgreSQL compatibility remains unchanged. -- [ ] Full local quality suite and retained GCP no-drift pass. -- [ ] Protected CI passes. +- [x] Full local quality suite and retained GCP no-drift pass. +- [x] Protected CI passes. ## Boundary @@ -41,8 +41,8 @@ the qualification slice also: - [x] selects the target schema explicitly before connector-managed direct binding; - [x] rejects `VARIANT` fields as keys, cursors, or snapshot identity; - [x] provides a disposable-schema live harness with sanitized output and exact cleanup; -- [ ] records a real Snowflake execution under an approved paid-test ceiling; and -- [ ] compares its normalized results with the shared cross-warehouse conformance dataset. +- [x] records a real Snowflake execution under an approved paid-test ceiling. -Until those final two items pass, Snowflake remains experimental and direct thresholds retain their -zero defaults. +The shared four-warehouse result comparison is the separate re-baselined Phase 5 exit gate rather +than an unmerged Snowflake adapter slice. Snowflake remains experimental, views remain unsupported, +and direct thresholds retain their zero defaults until crossover evidence is recorded in Phase 8. diff --git a/tickets/DANDER-105-redshift-warehouse.md b/tickets/DANDER-105-redshift-warehouse.md new file mode 100644 index 00000000..7341914c --- /dev/null +++ b/tickets/DANDER-105-redshift-warehouse.md @@ -0,0 +1,31 @@ +--- +id: DANDER-105 +title: Qualify the experimental Redshift warehouse +status: completed +component: warehouse +epic: cloud-portability +depends_on: [DANDER-103] +created: 2026-08-09 +--- + +## Acceptance Criteria + +- [x] Map native Redshift database/schema coordinates into canonical relations. +- [x] Load bounded compressed Parquet parts through same-region S3 manifest `COPY` with IAM roles. +- [x] Select bounded direct staging only through explicit row and logical-byte thresholds. +- [x] Implement replace, SCD1, SCD2, snapshot, and incremental publication through the exact + destination-fence transaction. +- [x] Reject schema drift and unsupported types before remote staging or destination mutation. +- [x] Support explicit strict JSON-to-`SUPER` fallback without permitting it as row identity. +- [x] Render fenced table/incremental transforms, assertions, and provider-neutral replace graphs. +- [x] Emit bounded, sanitized operation telemetry without inventing unavailable query identifiers. +- [x] Prove replay, monotonic cursors, stale ownership, concurrent claims, readback, and cleanup in a + disposable Redshift Serverless qualification under an approved cost ceiling. +- [x] Pass protected CI after the live-discovered SQL corrections merged. + +## Boundary + +Redshift remains experimental. Views, ARRAY/RECORD fallback, provisioned-RA3 live qualification, +reusable provider-managed infrastructure, support promotion, and Phase 8 scale/cost qualification +remain outside this completed adapter ticket. The shared four-warehouse result comparison is the +separate re-baselined Phase 5 exit gate. From 251dbd976e1976e4e459b9ab27c690ab410b3374 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 15:11:23 -0400 Subject: [PATCH 109/163] refactor launcher template requests --- HANDOFF.md | 39 ++-- docs/decisions.md | 11 ++ docs/execution-projection.md | 7 + src/dander/bootstrap/aws_terraform.py | 32 ++-- src/dander/bootstrap/terraform.py | 38 ++-- src/dander/deployment/__init__.py | 7 +- src/dander/deployment/runtime.py | 54 ++++-- src/dander/providers/cloud_run/runtime.py | 47 ++--- src/dander/providers/fargate/runtime.py | 57 +++--- src/dander/providers/gcp_launcher.py | 50 +++++ src/dander/providers/kubernetes/operations.py | 25 +-- src/dander/providers/kubernetes/runtime.py | 56 ++---- tests/providers/test_kubernetes_runtime.py | 76 ++++---- tests/providers/test_launcher_runtime.py | 179 ++++++++++-------- ...R-106-provider-neutral-template-request.md | 49 +++++ 15 files changed, 448 insertions(+), 279 deletions(-) create mode 100644 src/dander/providers/gcp_launcher.py create mode 100644 tickets/DANDER-106-provider-neutral-template-request.md diff --git a/HANDOFF.md b/HANDOFF.md index cc536677..43770cce 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,40 @@ ## Finished -- Restored the authoritative cloud-portability roadmap onto current protected-main history. -- Re-baselined the Phase 5 exit gate without declaring Phase 5 complete. -- Reconciled completed Snowflake, Redshift, and shared-staging ticket evidence. -- Corrected stale warehouse capability, staging, limitation, and decision documentation. +- Replaced the launcher factory primitive argument list with one immutable resolved request. +- Removed dummy GCP project and guard values from Kubernetes projection callers. +- Captured typed GCP-only values at Cloud Run and Fargate factory construction. +- Preserved the accepted Cloud Run projection and existing launcher fail-closed behavior. ## Try It -Review the Phase 5 gate in `docs/cloud-portability-plan.md`, then compare it with -`docs/compatibility-matrix.md` and `docs/known-limitations.md`. +Run `.venv/bin/pytest -q tests/providers/test_launcher_runtime.py +tests/providers/test_kubernetes_runtime.py`. ## Checks -- `git diff --check` passed. -- Focused compatibility, distribution-contract, and release-metadata tests passed. +- Focused launcher, bootstrap, Kubernetes CLI, and chart tests passed (62 tests). +- Full local pytest suite passed (with expected opt-in skips). +- Ruff lint and format checks passed. +- Mypy passed for every changed source and test file; the full invocation retains one unrelated + Snowflake `unused-ignore` result under this local optional-dependency environment. +- Independent completion review passed with no material findings. - Protected CI remains required before merge. ## Decisions -- Earlier phase gates remain binding through current equivalent evidence. -- Provider-specific safe limitations do not imply false parity or support. -- Scale, cost, soak, pairwise-profile, and release qualification remain in Phase 8. +- The resolved request contains only common launcher intent and no extension bag. +- GCP project and guarded-free-tier values are typed provider-construction context. +- Pipeline containers are defensively copied into read-only equivalents. ## Remaining -- Merge this documentation reconciliation through protected review. -- Implement the approved launcher-template request in a separate PR. -- Implement and prove the shared four-warehouse correctness fixture in a separate PR. -- Reassess the Phase 5 exit gate before beginning Phase 6. +- Complete protected CI for this launcher-contract PR. +- Implement the shared four-warehouse correctness fixture in a separate PR. +- Reassess the Phase 5 gate after both implementation PRs merge. ## Review First -- `docs/cloud-portability-plan.md` -- `docs/compatibility-matrix.md` -- `tickets/DANDER-104-snowflake-warehouse.md` +- `src/dander/deployment/runtime.py` +- `src/dander/providers/gcp_launcher.py` +- `tests/providers/test_launcher_runtime.py` diff --git a/docs/decisions.md b/docs/decisions.md index 85eef829..49d588da 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1123,3 +1123,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Qualification:** Throughput, crossover, incremental-at-scale, concurrency, cost, soak, pairwise profiles, and release qualification remain mandatory Phase 8 work rather than Phase 5 correctness evidence. + +## 2026-08-10 — Launcher factories consume one provider-neutral resolved request + +- **Contract:** `ExecutionTemplateFactory` accepts one frozen `ResolvedTemplateRequest`; its nested + pipeline containers are copied into read-only equivalents at construction. +- **Provider context:** Cloud Run and Fargate capture typed GCP project and guarded-free-tier values + when their selected factories are built. Kubernetes projection no longer receives placeholder + GCP arguments. +- **Boundary:** The existing Cloud Run compatibility projector remains authoritative, and Fargate + and Kubernetes retain their prior fail-closed behavior. No configuration schema, launcher output, + provider capability, warehouse behavior, or cloud implementation changes. diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 736a79bc..6f7feae5 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -22,6 +22,13 @@ existing GCP/BigQuery compatibility profile. A later configuration migration wil pipeline intent from environment-specific deployment settings; it does not change this runtime boundary. +Launcher factories receive one immutable `ResolvedTemplateRequest` containing only resolved, +provider-neutral projection intent. Provider/data-plane construction values are captured when the +selected factory is built: for example, the Cloud Run and Fargate BigQuery profiles receive their +typed GCP project and guarded-free-tier context there. Kubernetes callers therefore do not invent +empty project identifiers or false GCP guard values. The request contains no generic extension bag, +and its pipeline containers are copied into read-only equivalents before projection. + The selected Cloud Run provider supplies these templates to Terraform. Cloud Run consumes their image, command, resources, schedule, environment, secret references, identity, labels, and observability contract. diff --git a/src/dander/bootstrap/aws_terraform.py b/src/dander/bootstrap/aws_terraform.py index 9d5eec38..2bbd91c9 100644 --- a/src/dander/bootstrap/aws_terraform.py +++ b/src/dander/bootstrap/aws_terraform.py @@ -14,8 +14,9 @@ validate_runtime_settings, validate_terraform_pipelines, ) -from dander.deployment import ExecutionProjectionError +from dander.deployment import ExecutionProjectionError, ResolvedTemplateRequest from dander.providers import ProviderFactoryError +from dander.providers.gcp_launcher import GcpLauncherContext if TYPE_CHECKING: from collections.abc import Mapping @@ -122,20 +123,27 @@ def execute( ) try: - launcher = build_launcher_runtime(launcher_config=raw_launcher) + launcher = build_launcher_runtime( + launcher_config=raw_launcher, + gcp_context=GcpLauncherContext( + project=project, + require_guarded_free_tier=require_guarded_free_tier, + ), + ) projections = { pipeline_id: template.as_dict() for pipeline_id, template in launcher.templates.build( - expanded_pipelines, - image=container_image, - project=project, - cpu=runtime_cpu, - memory=runtime_memory, - deadline_seconds=runtime_timeout_seconds, - launcher_retry_count=runtime_max_retries, - batch_rows=runtime_batch_rows, - require_guarded_free_tier=require_guarded_free_tier, - alert_target=None, + ResolvedTemplateRequest( + pipelines=expanded_pipelines, + image=container_image, + profile_id="gcp", + cpu=runtime_cpu, + memory=runtime_memory, + deadline_seconds=runtime_timeout_seconds, + launcher_retry_count=runtime_max_retries, + batch_rows=runtime_batch_rows, + alert_target=None, + ) ).items() } except (ExecutionProjectionError, ProviderFactoryError) as error: diff --git a/src/dander/bootstrap/terraform.py b/src/dander/bootstrap/terraform.py index ba97bd01..12eccf04 100644 --- a/src/dander/bootstrap/terraform.py +++ b/src/dander/bootstrap/terraform.py @@ -8,8 +8,12 @@ from json import dumps from typing import TYPE_CHECKING -from dander.deployment import ExecutionProjectionError, LauncherRuntime +from dander.deployment import ExecutionProjectionError, LauncherRuntime, ResolvedTemplateRequest from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry +from dander.providers.gcp_launcher import ( + GcpLauncherContext, + gcp_launcher_factory_context, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -177,20 +181,25 @@ def execute( launcher = build_launcher_runtime( provider_id=launcher_provider, region=region, + gcp_context=GcpLauncherContext( + project=project, + require_guarded_free_tier=require_guarded_free_tier, + ), ) execution_projections = { pipeline_id: template.as_dict() for pipeline_id, template in launcher.templates.build( - expanded_pipelines, - image=container_image, - project=project, - cpu=runtime_cpu, - memory=runtime_memory, - deadline_seconds=runtime_timeout_seconds, - launcher_retry_count=runtime_max_retries, - batch_rows=runtime_batch_rows, - require_guarded_free_tier=require_guarded_free_tier, - alert_target=failure_alert_email or None, + ResolvedTemplateRequest( + pipelines=expanded_pipelines, + image=container_image, + profile_id="gcp", + cpu=runtime_cpu, + memory=runtime_memory, + deadline_seconds=runtime_timeout_seconds, + launcher_retry_count=runtime_max_retries, + batch_rows=runtime_batch_rows, + alert_target=failure_alert_email or None, + ) ).items() } except (ExecutionProjectionError, ProviderFactoryError) as error: @@ -334,6 +343,7 @@ def build_launcher_runtime( provider_id: str | None = None, region: str | None = None, launcher_config: Mapping[str, object] | None = None, + gcp_context: GcpLauncherContext | None = None, ) -> LauncherRuntime: """Build one launcher through the shared lazy provider registry.""" raw_config = dict(launcher_config or {}) @@ -346,7 +356,11 @@ def build_launcher_runtime( ProviderKind.LAUNCHER, raw_config, ) - runtime = registry.build(ProviderKind.LAUNCHER, config) + runtime = registry.build( + ProviderKind.LAUNCHER, + config, + context=gcp_launcher_factory_context(gcp_context) if gcp_context is not None else None, + ) if not isinstance(runtime, LauncherRuntime): raise ProviderFactoryError("Selected launcher provider returned an invalid runtime") return runtime diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index ba122abb..a674eee1 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -18,7 +18,11 @@ build_gcp_v1_execution_templates, validate_launcher_projection, ) -from dander.deployment.runtime import ExecutionTemplateFactory, LauncherRuntime +from dander.deployment.runtime import ( + ExecutionTemplateFactory, + LauncherRuntime, + ResolvedTemplateRequest, +) __all__ = [ "CLOUD_RUN_CAPABILITIES", @@ -34,6 +38,7 @@ "NetworkPlacement", "ObservabilityProjection", "ResourceProjection", + "ResolvedTemplateRequest", "ScheduleProjection", "SecretReference", "build_gcp_execution_templates", diff --git a/src/dander/deployment/runtime.py b/src/dander/deployment/runtime.py index 9b19f317..a31f2884 100644 --- a/src/dander/deployment/runtime.py +++ b/src/dander/deployment/runtime.py @@ -2,34 +2,54 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Protocol, runtime_checkable if TYPE_CHECKING: - from collections.abc import Mapping - from dander.deployment.projection import ExecutionTemplate, LauncherCapabilities +def _freeze_resolved_value(value: object) -> object: + """Copy container values into read-only equivalents at the contract boundary.""" + if isinstance(value, Mapping): + return MappingProxyType( + {str(key): _freeze_resolved_value(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_resolved_value(item) for item in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze_resolved_value(item) for item in value) + return value + + +@dataclass(frozen=True, slots=True) +class ResolvedTemplateRequest: + """Immutable provider-neutral intent for one launcher template projection.""" + + pipelines: Mapping[str, Mapping[str, object]] + image: str + profile_id: str + cpu: int + memory: str + deadline_seconds: int + launcher_retry_count: int + batch_rows: int + alert_target: str | None + + def __post_init__(self) -> None: + immutable = _freeze_resolved_value(self.pipelines) + if not isinstance(immutable, Mapping): # pragma: no cover - structural invariant + raise TypeError("resolved template pipelines must be a mapping") + object.__setattr__(self, "pipelines", immutable) + + @runtime_checkable class ExecutionTemplateFactory(Protocol): """Build launcher-specific templates from validated deployment inputs.""" - def build( - self, - pipelines: Mapping[str, Mapping[str, object]], - *, - image: str, - project: str, - cpu: int, - memory: str, - deadline_seconds: int, - launcher_retry_count: int, - batch_rows: int, - require_guarded_free_tier: bool, - alert_target: str | None, - profile_id: str = "gcp", - ) -> dict[str, ExecutionTemplate]: + def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: """Return validated execution templates keyed by pipeline ID.""" ... diff --git a/src/dander/providers/cloud_run/runtime.py b/src/dander/providers/cloud_run/runtime.py index 379457f2..02257beb 100644 --- a/src/dander/providers/cloud_run/runtime.py +++ b/src/dander/providers/cloud_run/runtime.py @@ -10,8 +10,9 @@ ExecutionTemplate, build_gcp_execution_templates, ) -from dander.deployment.runtime import LauncherRuntime +from dander.deployment.runtime import LauncherRuntime, ResolvedTemplateRequest from dander.providers.cloud_run.config import CloudRunLauncherConfig +from dander.providers.gcp_launcher import GcpLauncherContext, require_gcp_launcher_context from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind if TYPE_CHECKING: @@ -24,35 +25,23 @@ class CloudRunTemplateFactory: """Preserve the accepted GCP template projection behind a launcher boundary.""" - def build( - self, - pipelines: Mapping[str, Mapping[str, object]], - *, - image: str, - project: str, - cpu: int, - memory: str, - deadline_seconds: int, - launcher_retry_count: int, - batch_rows: int, - require_guarded_free_tier: bool, - alert_target: str | None, - profile_id: str = "gcp", - ) -> dict[str, ExecutionTemplate]: + gcp: GcpLauncherContext + + def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: """Build byte-equivalent Cloud Run templates through the existing projector.""" - if profile_id != "gcp": + if request.profile_id != "gcp": raise ValueError("Cloud Run compatibility projection requires profile_id='gcp'") return build_gcp_execution_templates( - pipelines, - image=image, - project=project, - cpu=cpu, - memory=memory, - deadline_seconds=deadline_seconds, - launcher_retry_count=launcher_retry_count, - batch_rows=batch_rows, - require_guarded_free_tier=require_guarded_free_tier, - alert_target=alert_target, + request.pipelines, + image=request.image, + project=self.gcp.project, + cpu=request.cpu, + memory=request.memory, + deadline_seconds=request.deadline_seconds, + launcher_retry_count=request.launcher_retry_count, + batch_rows=request.batch_rows, + require_guarded_free_tier=self.gcp.require_guarded_free_tier, + alert_target=request.alert_target, ) @@ -61,13 +50,13 @@ def build_cloud_run_launcher( context: Mapping[str, object], ) -> LauncherRuntime: """Build Cloud Run projection behavior only after launcher selection.""" - del context if not isinstance(config, CloudRunLauncherConfig): raise TypeError("Cloud Run launcher factory received the wrong configuration") + gcp = require_gcp_launcher_context(context) return LauncherRuntime( provider_id="cloud_run", region=config.region, - templates=CloudRunTemplateFactory(), + templates=CloudRunTemplateFactory(gcp), capabilities=CLOUD_RUN_CAPABILITIES, ) diff --git a/src/dander/providers/fargate/runtime.py b/src/dander/providers/fargate/runtime.py index 1435b687..893b9077 100644 --- a/src/dander/providers/fargate/runtime.py +++ b/src/dander/providers/fargate/runtime.py @@ -20,8 +20,9 @@ SecretReference, validate_launcher_projection, ) -from dander.deployment.runtime import LauncherRuntime +from dander.deployment.runtime import LauncherRuntime, ResolvedTemplateRequest from dander.providers.fargate.config import FargateLauncherConfig +from dander.providers.gcp_launcher import GcpLauncherContext, require_gcp_launcher_context from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.runtime_contract import RUNTIME_CONTRACT @@ -37,34 +38,21 @@ class FargateTemplateFactory: """Build the GCP data-plane profile for an AWS Fargate launcher.""" config: FargateLauncherConfig + gcp: GcpLauncherContext - def build( - self, - pipelines: Mapping[str, Mapping[str, object]], - *, - image: str, - project: str, - cpu: int, - memory: str, - deadline_seconds: int, - launcher_retry_count: int, - batch_rows: int, - require_guarded_free_tier: bool, - alert_target: str | None, - profile_id: str = "gcp", - ) -> dict[str, ExecutionTemplate]: + def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: """Build fail-closed Fargate templates for the portable BigQuery proof.""" - if profile_id != "gcp": + if request.profile_id != "gcp": raise ExecutionProjectionError("Fargate compatibility projection requires gcp") - if require_guarded_free_tier: + if self.gcp.require_guarded_free_tier: raise ExecutionProjectionError("Fargate cannot run the GCP guarded-free-tier preflight") - if _GCP_PROJECT.fullmatch(project) is None: + if _GCP_PROJECT.fullmatch(self.gcp.project) is None: raise ExecutionProjectionError("invalid GCP project identifier") - memory_mib = _memory_mib(memory) - cpu_millis = cpu * 1_000 + memory_mib = _memory_mib(request.memory) + cpu_millis = request.cpu * 1_000 _validate_fargate_size(cpu_millis=cpu_millis, memory_mib=memory_mib) templates: dict[str, ExecutionTemplate] = {} - for pipeline_id, pipeline in sorted(pipelines.items()): + for pipeline_id, pipeline in sorted(request.pipelines.items()): role_name = str(pipeline["runtime_service_account_id"]) if _ROLE_NAME.fullmatch(role_name) is None: raise ExecutionProjectionError("invalid Fargate task role name") @@ -86,7 +74,7 @@ def build( "--models-dir", "/app/models", "--batch-rows", - str(batch_rows), + str(request.batch_rows), ) if bool(pipeline["build_models"]): command = (*command, "--catalog-output", "/tmp/dander-catalog.json") @@ -96,7 +84,7 @@ def build( pipeline_id=pipeline_id, profile_id="gcp", launcher="fargate", - image=image, + image=request.image, command=command, configuration_reference="/app/dander.yaml", environment=tuple( @@ -106,16 +94,16 @@ def build( "AWS_REGION": self.config.region, "BQ_DATASET_METADATA": "dander_meta", "BQ_DATASET_RAW": "raw", - "DANDER_IMAGE_DIGEST": image.rsplit("@", maxsplit=1)[-1], + "DANDER_IMAGE_DIGEST": request.image.rsplit("@", maxsplit=1)[-1], "DANDER_GCP_SERVICE_ACCOUNT": ( - f"{role_name}@{project}.iam.gserviceaccount.com" + f"{role_name}@{self.gcp.project}.iam.gserviceaccount.com" ), "DANDER_GCP_WIF_AUDIENCE": ( self.config.google_workload_identity_audience ), "DANDER_LAUNCHER": "fargate", "DANDER_PRINCIPAL": identity, - "GCP_PROJECT_ID": project, + "GCP_PROJECT_ID": self.gcp.project, "HOME": "/tmp", "TMPDIR": "/tmp", }.items() @@ -127,7 +115,8 @@ def build( SecretReference( provider="gcp_secret_manager", reference=( - f"gcp-sm://projects/{project}/secrets/{secret_id}/versions/latest" + f"gcp-sm://projects/{self.gcp.project}/secrets/" + f"{secret_id}/versions/latest" ), ), ) @@ -138,9 +127,9 @@ def build( cpu_millis=cpu_millis, memory_mib=memory_mib, ephemeral_storage_mib=self.config.ephemeral_storage_mib, - deadline_seconds=deadline_seconds, + deadline_seconds=request.deadline_seconds, runtime_retry_count=0, - launcher_retry_count=launcher_retry_count, + launcher_retry_count=request.launcher_retry_count, ), schedule=ScheduleProjection( task_count=1, @@ -158,14 +147,14 @@ def build( ), labels=( ("dander_version", __version__), - ("image_digest", image.rsplit("@", maxsplit=1)[-1]), + ("image_digest", request.image.rsplit("@", maxsplit=1)[-1]), ("pipeline", pipeline_id), ("profile", "gcp"), ), observability=ObservabilityProjection( log_destination="cloudwatch_logs", metric_namespace="Dander", - alert_target=alert_target, + alert_target=request.alert_target, retention_days=30, ), extensions=( @@ -187,13 +176,13 @@ def build_fargate_launcher( context: Mapping[str, object], ) -> LauncherRuntime: """Build Fargate projection behavior only after launcher selection.""" - del context if not isinstance(config, FargateLauncherConfig): raise TypeError("Fargate launcher factory received the wrong configuration") + gcp = require_gcp_launcher_context(context) return LauncherRuntime( provider_id="fargate", region=config.region, - templates=FargateTemplateFactory(config), + templates=FargateTemplateFactory(config, gcp), capabilities=FARGATE_CAPABILITIES, ) diff --git a/src/dander/providers/gcp_launcher.py b/src/dander/providers/gcp_launcher.py new file mode 100644 index 00000000..3c56322b --- /dev/null +++ b/src/dander/providers/gcp_launcher.py @@ -0,0 +1,50 @@ +"""Typed GCP data-plane context captured by selected launcher factories.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping + +_CONTEXT_KEY = "gcp_launcher" + + +@dataclass(frozen=True, slots=True) +class GcpLauncherContext: + """GCP-only values that must not leak into provider-neutral template requests.""" + + project: str + require_guarded_free_tier: bool + + +def gcp_launcher_factory_context(context: GcpLauncherContext) -> dict[str, object]: + """Return the provider-registry construction context for one GCP data plane.""" + return {_CONTEXT_KEY: context} + + +def optional_gcp_launcher_context(context: Mapping[str, object]) -> GcpLauncherContext | None: + """Read and validate an optional typed GCP launcher construction context.""" + value = context.get(_CONTEXT_KEY) + if value is None: + return None + if not isinstance(value, GcpLauncherContext): + raise TypeError("launcher factory received an invalid GCP construction context") + return value + + +def require_gcp_launcher_context(context: Mapping[str, object]) -> GcpLauncherContext: + """Require the selected launcher to receive its GCP construction context.""" + value = optional_gcp_launcher_context(context) + if value is None: + raise TypeError("launcher factory requires a GCP construction context") + return value + + +__all__ = [ + "GcpLauncherContext", + "gcp_launcher_factory_context", + "optional_gcp_launcher_context", + "require_gcp_launcher_context", +] diff --git a/src/dander/providers/kubernetes/operations.py b/src/dander/providers/kubernetes/operations.py index b63dcdab..c1b91abd 100644 --- a/src/dander/providers/kubernetes/operations.py +++ b/src/dander/providers/kubernetes/operations.py @@ -14,6 +14,7 @@ import yaml from dander.bootstrap.terraform import build_launcher_runtime +from dander.deployment import ResolvedTemplateRequest from dander.project import ProjectConfigError, load_project_config from dander.providers.kubernetes.chart import build_helm_values from dander.providers.kubernetes.config import KubernetesLauncherConfig @@ -300,19 +301,21 @@ def _resolve_release( ) manifest.validate_references(project_config.parent) launcher_config = KubernetesLauncherConfig.model_validate(manifest.resolved_launcher_config()) + if manifest.platform.safety.require_guarded_free_tier: + raise ProjectConfigError("Kubernetes cannot run the GCP guarded-free-tier preflight") launcher = build_launcher_runtime(launcher_config=manifest.resolved_launcher_config()) templates = launcher.templates.build( - manifest.terraform_pipelines(), - image=image, - project="", - cpu=manifest.platform.runtime.cpu, - memory=manifest.platform.runtime.memory, - deadline_seconds=manifest.platform.runtime.timeout_seconds, - launcher_retry_count=manifest.platform.runtime.max_retries, - batch_rows=manifest.platform.runtime.batch_rows, - require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, - alert_target=None, - profile_id=manifest.deployment_name, + ResolvedTemplateRequest( + pipelines=manifest.terraform_pipelines(), + image=image, + profile_id=manifest.deployment_name, + cpu=manifest.platform.runtime.cpu, + memory=manifest.platform.runtime.memory, + deadline_seconds=manifest.platform.runtime.timeout_seconds, + launcher_retry_count=manifest.platform.runtime.max_retries, + batch_rows=manifest.platform.runtime.batch_rows, + alert_target=None, + ) ) values = build_helm_values( launcher_config, diff --git a/src/dander/providers/kubernetes/runtime.py b/src/dander/providers/kubernetes/runtime.py index ce455eac..a50a2f46 100644 --- a/src/dander/providers/kubernetes/runtime.py +++ b/src/dander/providers/kubernetes/runtime.py @@ -20,7 +20,8 @@ SecretReference, validate_launcher_projection, ) -from dander.deployment.runtime import LauncherRuntime +from dander.deployment.runtime import LauncherRuntime, ResolvedTemplateRequest +from dander.providers.gcp_launcher import optional_gcp_launcher_context from dander.providers.kubernetes.config import KubernetesLauncherConfig from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.runtime_contract import RUNTIME_CONTRACT @@ -37,36 +38,17 @@ class KubernetesTemplateFactory: config: KubernetesLauncherConfig - def build( - self, - pipelines: Mapping[str, Mapping[str, object]], - *, - image: str, - project: str, - cpu: int, - memory: str, - deadline_seconds: int, - launcher_retry_count: int, - batch_rows: int, - require_guarded_free_tier: bool, - alert_target: str | None, - profile_id: str = "gcp", - ) -> dict[str, ExecutionTemplate]: + def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: """Build immutable templates without contacting the selected cluster.""" - del project - if _PROFILE.fullmatch(profile_id) is None: + if _PROFILE.fullmatch(request.profile_id) is None: raise ExecutionProjectionError("invalid Kubernetes deployment selector") - if require_guarded_free_tier: - raise ExecutionProjectionError( - "Kubernetes cannot run the GCP guarded-free-tier preflight" - ) - if alert_target is not None: + if request.alert_target is not None: raise ExecutionProjectionError( "Kubernetes uses cluster observability and cannot provision an alert target" ) - memory_mib = _memory_mib(memory) + memory_mib = _memory_mib(request.memory) templates: dict[str, ExecutionTemplate] = {} - for pipeline_id, pipeline in sorted(pipelines.items()): + for pipeline_id, pipeline in sorted(request.pipelines.items()): secret_env = pipeline["secret_env"] if not isinstance(secret_env, Mapping): raise ExecutionProjectionError("pipeline secret bindings are invalid") @@ -82,13 +64,13 @@ def build( "--pipeline", pipeline_id, "--platform", - profile_id, + request.profile_id, "--config", "/app/dander.yaml", "--models-dir", "/app/models", "--batch-rows", - str(batch_rows), + str(request.batch_rows), ) if bool(pipeline["build_models"]): command = (*command, "--catalog-output", "/tmp/dander-catalog.json") @@ -98,23 +80,23 @@ def build( ) labels = { "dander_version": __version__, - "image_digest": image.rsplit("@", maxsplit=1)[-1], + "image_digest": request.image.rsplit("@", maxsplit=1)[-1], "pipeline": pipeline_id, - "profile": profile_id, + "profile": request.profile_id, } template = ExecutionTemplate( schema=EXECUTION_PROJECTION_SCHEMA, contract=RUNTIME_CONTRACT, pipeline_id=pipeline_id, - profile_id=profile_id, + profile_id=request.profile_id, launcher="kubernetes", - image=image, + image=request.image, command=command, configuration_reference="/app/dander.yaml", environment=tuple( sorted( { - "DANDER_IMAGE_DIGEST": image.rsplit("@", maxsplit=1)[-1], + "DANDER_IMAGE_DIGEST": request.image.rsplit("@", maxsplit=1)[-1], "DANDER_LAUNCHER": "kubernetes", "DANDER_PRINCIPAL": identity, }.items() @@ -129,12 +111,12 @@ def build( ), workload_identity=identity, resources=ResourceProjection( - cpu_millis=cpu * 1_000, + cpu_millis=request.cpu * 1_000, memory_mib=memory_mib, ephemeral_storage_mib=self.config.ephemeral_storage_mib, - deadline_seconds=deadline_seconds, + deadline_seconds=request.deadline_seconds, runtime_retry_count=0, - launcher_retry_count=launcher_retry_count, + launcher_retry_count=request.launcher_retry_count, ), schedule=ScheduleProjection( task_count=1, @@ -162,9 +144,11 @@ def build_kubernetes_launcher( context: Mapping[str, object], ) -> LauncherRuntime: """Build the Kubernetes launcher only after explicit selection.""" - del context if not isinstance(config, KubernetesLauncherConfig): raise TypeError("Kubernetes launcher factory received the wrong configuration") + gcp = optional_gcp_launcher_context(context) + if gcp is not None and gcp.require_guarded_free_tier: + raise ExecutionProjectionError("Kubernetes cannot run the GCP guarded-free-tier preflight") return LauncherRuntime( provider_id="kubernetes", region=config.region, diff --git a/tests/providers/test_kubernetes_runtime.py b/tests/providers/test_kubernetes_runtime.py index 644cafe1..463caa5e 100644 --- a/tests/providers/test_kubernetes_runtime.py +++ b/tests/providers/test_kubernetes_runtime.py @@ -7,8 +7,9 @@ import pytest from pydantic import ValidationError -from dander.deployment import ExecutionProjectionError, LauncherRuntime +from dander.deployment import ExecutionProjectionError, LauncherRuntime, ResolvedTemplateRequest from dander.providers import ProviderKind, default_provider_registry +from dander.providers.gcp_launcher import GcpLauncherContext, gcp_launcher_factory_context from dander.providers.kubernetes import KubernetesLauncherConfig _IMAGE = "ghcr.io/example/dander@sha256:" + "a" * 64 @@ -24,7 +25,25 @@ } -def _runtime(*, secret_name: str | None = "dander-runtime") -> LauncherRuntime: +def _request(*, alert_target: str | None = None) -> ResolvedTemplateRequest: + return ResolvedTemplateRequest( + pipelines=_PIPELINES, + image=_IMAGE, + profile_id="postgres", + cpu=1, + memory="512Mi", + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=100, + alert_target=alert_target, + ) + + +def _runtime( + *, + secret_name: str | None = "dander-runtime", + guarded: bool = False, +) -> LauncherRuntime: registry = default_provider_registry() config = registry.parse( ProviderKind.LAUNCHER, @@ -41,7 +60,17 @@ def _runtime(*, secret_name: str | None = "dander-runtime") -> LauncherRuntime: "pod_labels": {"team": "analytics"}, }, ) - runtime = registry.build(ProviderKind.LAUNCHER, config) + context = ( + gcp_launcher_factory_context( + GcpLauncherContext( + project="unit-project", + require_guarded_free_tier=True, + ) + ) + if guarded + else None + ) + runtime = registry.build(ProviderKind.LAUNCHER, config, context=context) assert isinstance(runtime, LauncherRuntime) return runtime @@ -64,19 +93,7 @@ def test_kubernetes_factory_is_lazy_and_projects_native_profile() -> None: runtime = registry.build(ProviderKind.LAUNCHER, config) assert isinstance(runtime, LauncherRuntime) assert module_name in sys.modules - template = runtime.templates.build( - _PIPELINES, - image=_IMAGE, - project="", - cpu=1, - memory="512Mi", - deadline_seconds=900, - launcher_retry_count=1, - batch_rows=100, - require_guarded_free_tier=False, - alert_target=None, - profile_id="postgres", - )["salesforce_crm"] + template = runtime.templates.build(_request())["salesforce_crm"] assert template.launcher == "kubernetes" assert template.profile_id == "postgres" @@ -98,33 +115,24 @@ def test_kubernetes_factory_is_lazy_and_projects_native_profile() -> None: @pytest.mark.parametrize( - ("secret_name", "guarded", "alert_target", "message"), + ("secret_name", "alert_target", "message"), [ - (None, False, None, "existing_secret_name"), - ("dander-runtime", True, None, "guarded-free-tier"), - ("dander-runtime", False, "operator@example.invalid", "alert target"), + (None, None, "existing_secret_name"), + ("dander-runtime", "operator@example.invalid", "alert target"), ], ) def test_kubernetes_rejects_intent_the_chart_cannot_honor( secret_name: str | None, - guarded: bool, alert_target: str | None, message: str, ) -> None: with pytest.raises(ExecutionProjectionError, match=message): - _runtime(secret_name=secret_name).templates.build( - _PIPELINES, - image=_IMAGE, - project="", - cpu=1, - memory="512Mi", - deadline_seconds=900, - launcher_retry_count=1, - batch_rows=100, - require_guarded_free_tier=guarded, - alert_target=alert_target, - profile_id="postgres", - ) + _runtime(secret_name=secret_name).templates.build(_request(alert_target=alert_target)) + + +def test_kubernetes_rejects_guarded_gcp_context_during_factory_construction() -> None: + with pytest.raises(ExecutionProjectionError, match="guarded-free-tier"): + _runtime(guarded=True) @pytest.mark.parametrize( diff --git a/tests/providers/test_launcher_runtime.py b/tests/providers/test_launcher_runtime.py index 8b36ebec..f8f5d0fa 100644 --- a/tests/providers/test_launcher_runtime.py +++ b/tests/providers/test_launcher_runtime.py @@ -3,15 +3,18 @@ from __future__ import annotations import sys +from dataclasses import FrozenInstanceError import pytest from dander.deployment import ( ExecutionProjectionError, LauncherRuntime, + ResolvedTemplateRequest, build_gcp_execution_templates, ) from dander.providers import ProviderKind, default_provider_registry +from dander.providers.gcp_launcher import GcpLauncherContext, gcp_launcher_factory_context _IMAGE = "us-central1-docker.pkg.dev/unit-project/dander/dander@sha256:" + "a" * 64 _FARGATE_IMAGE = "184463061564.dkr.ecr.us-east-1.amazonaws.com/dander@sha256:" + "b" * 64 @@ -27,20 +30,45 @@ } +def _request( + *, + pipelines: dict[str, dict[str, object]] | None = None, + image: str = _IMAGE, + profile_id: str = "gcp", + cpu: int = 1, + memory: str = "512Mi", + deadline_seconds: int = 300, + launcher_retry_count: int = 1, + batch_rows: int = 10_000, + alert_target: str | None = None, +) -> ResolvedTemplateRequest: + return ResolvedTemplateRequest( + pipelines=_PIPELINES if pipelines is None else pipelines, + image=image, + profile_id=profile_id, + cpu=cpu, + memory=memory, + deadline_seconds=deadline_seconds, + launcher_retry_count=launcher_retry_count, + batch_rows=batch_rows, + alert_target=alert_target, + ) + + +def _gcp_context(*, guarded: bool = False) -> dict[str, object]: + return gcp_launcher_factory_context( + GcpLauncherContext( + project="unit-project", + require_guarded_free_tier=guarded, + ) + ) + + def _templates(runtime: LauncherRuntime) -> dict[str, object]: return { pipeline_id: template.as_dict() for pipeline_id, template in runtime.templates.build( - _PIPELINES, - image=_IMAGE, - project="unit-project", - cpu=1, - memory="512Mi", - deadline_seconds=300, - launcher_retry_count=1, - batch_rows=10_000, - require_guarded_free_tier=False, - alert_target="operator@example.invalid", + _request(alert_target="operator@example.invalid") ).items() } @@ -55,7 +83,7 @@ def test_cloud_run_factory_is_lazy_and_matches_the_accepted_projection() -> None ) assert module_name not in sys.modules - runtime = registry.build(ProviderKind.LAUNCHER, config) + runtime = registry.build(ProviderKind.LAUNCHER, config, context=_gcp_context()) assert isinstance(runtime, LauncherRuntime) assert runtime.provider_id == "cloud_run" @@ -80,7 +108,7 @@ def test_cloud_run_factory_is_lazy_and_matches_the_accepted_projection() -> None assert _templates(runtime) == expected -def _fargate_runtime() -> LauncherRuntime: +def _fargate_runtime(*, guarded: bool = False) -> LauncherRuntime: registry = default_provider_registry() config = registry.parse( ProviderKind.LAUNCHER, @@ -98,7 +126,11 @@ def _fargate_runtime() -> LauncherRuntime: "assign_public_ip": True, }, ) - runtime = registry.build(ProviderKind.LAUNCHER, config) + runtime = registry.build( + ProviderKind.LAUNCHER, + config, + context=_gcp_context(guarded=guarded), + ) assert isinstance(runtime, LauncherRuntime) return runtime @@ -123,7 +155,7 @@ def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> ) assert module_name not in sys.modules - runtime = registry.build(ProviderKind.LAUNCHER, config) + runtime = registry.build(ProviderKind.LAUNCHER, config, context=_gcp_context()) assert isinstance(runtime, LauncherRuntime) assert module_name in sys.modules pipelines = { @@ -134,16 +166,14 @@ def test_fargate_factory_is_lazy_and_projects_bigquery_without_credentials() -> } } template = runtime.templates.build( - pipelines, - image=_FARGATE_IMAGE, - project="unit-project", - cpu=1, - memory="2Gi", - deadline_seconds=900, - launcher_retry_count=1, - batch_rows=1_000, - require_guarded_free_tier=False, - alert_target="arn:aws:sns:us-east-1:184463061564:dander-failures", + _request( + pipelines=pipelines, + image=_FARGATE_IMAGE, + memory="2Gi", + deadline_seconds=900, + batch_rows=1_000, + alert_target="arn:aws:sns:us-east-1:184463061564:dander-failures", + ) )["salesforce_crm"] assert template.launcher == "fargate" @@ -187,32 +217,24 @@ def test_fargate_rejects_unhonored_runtime_intent( message: str, ) -> None: with pytest.raises(ExecutionProjectionError, match=message): - _fargate_runtime().templates.build( - _PIPELINES, - image=_FARGATE_IMAGE, - project="unit-project", - cpu=1, - memory=memory, - deadline_seconds=900, - launcher_retry_count=1, - batch_rows=1_000, - require_guarded_free_tier=guarded, - alert_target=None, + _fargate_runtime(guarded=guarded).templates.build( + _request( + image=_FARGATE_IMAGE, + memory=memory, + deadline_seconds=900, + batch_rows=1_000, + ) ) def test_fargate_accepts_a_run_longer_than_one_task_role_session() -> None: template = _fargate_runtime().templates.build( - _PIPELINES, - image=_FARGATE_IMAGE, - project="unit-project", - cpu=1, - memory="2Gi", - deadline_seconds=3_601, - launcher_retry_count=1, - batch_rows=1_000, - require_guarded_free_tier=False, - alert_target=None, + _request( + image=_FARGATE_IMAGE, + memory="2Gi", + deadline_seconds=3_601, + batch_rows=1_000, + ) )["greenhouse_jobs"] assert template.resources.deadline_seconds == 3_601 @@ -221,16 +243,12 @@ def test_fargate_accepts_a_run_longer_than_one_task_role_session() -> None: def test_fargate_rejects_a_run_longer_than_one_day() -> None: with pytest.raises(ExecutionProjectionError, match="deadline"): _fargate_runtime().templates.build( - _PIPELINES, - image=_FARGATE_IMAGE, - project="unit-project", - cpu=1, - memory="2Gi", - deadline_seconds=86_401, - launcher_retry_count=1, - batch_rows=1_000, - require_guarded_free_tier=False, - alert_target=None, + _request( + image=_FARGATE_IMAGE, + memory="2Gi", + deadline_seconds=86_401, + batch_rows=1_000, + ) ) @@ -243,16 +261,13 @@ def test_fargate_rejects_cron_semantics_eventbridge_cannot_preserve() -> None: } with pytest.raises(ExecutionProjectionError, match="both day fields"): _fargate_runtime().templates.build( - pipelines, - image=_FARGATE_IMAGE, - project="unit-project", - cpu=1, - memory="2Gi", - deadline_seconds=900, - launcher_retry_count=1, - batch_rows=1_000, - require_guarded_free_tier=False, - alert_target=None, + _request( + pipelines=pipelines, + image=_FARGATE_IMAGE, + memory="2Gi", + deadline_seconds=900, + batch_rows=1_000, + ) ) @@ -265,14 +280,28 @@ def test_fargate_rejects_provider_specific_cron_syntax() -> None: } with pytest.raises(ExecutionProjectionError, match="valid five-field"): _fargate_runtime().templates.build( - pipelines, - image=_FARGATE_IMAGE, - project="unit-project", - cpu=1, - memory="2Gi", - deadline_seconds=900, - launcher_retry_count=1, - batch_rows=1_000, - require_guarded_free_tier=False, - alert_target=None, + _request( + pipelines=pipelines, + image=_FARGATE_IMAGE, + memory="2Gi", + deadline_seconds=900, + batch_rows=1_000, + ) ) + + +def test_resolved_template_request_is_provider_neutral_and_immutable() -> None: + request = _request() + + assert "project" not in request.__dataclass_fields__ + assert "require_guarded_free_tier" not in request.__dataclass_fields__ + with pytest.raises(TypeError): + request.pipelines["other"] = {} # type: ignore[index] + with pytest.raises(TypeError): + request.pipelines["greenhouse_jobs"]["paused"] = False # type: ignore[index] + secret_env = request.pipelines["greenhouse_jobs"]["secret_env"] + assert isinstance(secret_env, dict) is False + with pytest.raises(TypeError): + secret_env["TOKEN"] = "secret-id" # type: ignore[index] + with pytest.raises(FrozenInstanceError): + request.profile_id = "postgres" # type: ignore[misc] diff --git a/tickets/DANDER-106-provider-neutral-template-request.md b/tickets/DANDER-106-provider-neutral-template-request.md new file mode 100644 index 00000000..b0c7f54c --- /dev/null +++ b/tickets/DANDER-106-provider-neutral-template-request.md @@ -0,0 +1,49 @@ +--- +id: DANDER-106 +title: Resolve launcher templates through one provider-neutral request +status: in-review +component: deployment +epic: cloud-portability +depends_on: [DANDER-89, DANDER-90] +created: 2026-08-10 +--- + +## Context + +The Phase 5.5 architecture checkpoint found one narrow pre-Phase-6 correction: the launcher +factory protocol exposed GCP project and guarded-free-tier primitives to every launcher. Kubernetes +callers consequently supplied placeholder GCP values even though its projection did not consume +them. + +## Acceptance Criteria + +- [x] `ExecutionTemplateFactory` accepts one immutable provider-neutral resolved request. +- [x] Cloud Run, Fargate, and Kubernetes projections consume that request. +- [x] GCP-only construction values remain typed and outside the provider-neutral request. +- [x] Non-GCP callers pass no dummy GCP project or guarded-free-tier values. +- [x] Existing Cloud Run output remains exactly equal to the accepted compatibility projector. +- [x] Existing Fargate and Kubernetes projection and fail-closed behavior remains covered. +- [x] Focused launcher, bootstrap, Kubernetes CLI, and chart tests pass. +- [ ] Full protected CI passes. +- [x] Independent completion review passes. + +## Design + +Introduce a frozen `ResolvedTemplateRequest` at the existing deployment boundary and recursively +copy its resolved pipeline containers into read-only equivalents. Capture the typed GCP data-plane +context when Cloud Run or Fargate is selected through the existing provider-registry construction +context. Keep direct compatibility projectors and all manifest/configuration schemas unchanged. + +## Implementation Notes + +Cloud Run delegates the request to the unchanged GCP projector. Fargate reads GCP-only values from +its frozen factory context. Kubernetes receives only the selected profile and common execution +intent; its existing guarded-free-tier rejection remains at manifest resolution. No extension bag, +Azure/OCI code, warehouse work, SQL, or user-visible projection change is included. + +## Review Log + +### 2026-08-10 — PASS + +Independent read-only completion review found no material defect or unnecessary scope. Protected +CI remains required before merge. From 527a7c4101ce76fd0e7576dfdbe1524e13fcd183 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 15:14:59 -0400 Subject: [PATCH 110/163] record launcher completion evidence --- HANDOFF.md | 4 ++-- tickets/DANDER-106-provider-neutral-template-request.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 43770cce..8fc491ef 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -20,7 +20,7 @@ tests/providers/test_kubernetes_runtime.py`. - Mypy passed for every changed source and test file; the full invocation retains one unrelated Snowflake `unused-ignore` result under this local optional-dependency environment. - Independent completion review passed with no material findings. -- Protected CI remains required before merge. +- Protected CI passed all five required jobs on PR #181. ## Decisions @@ -30,7 +30,7 @@ tests/providers/test_kubernetes_runtime.py`. ## Remaining -- Complete protected CI for this launcher-contract PR. +- Merge the protected launcher-contract PR. - Implement the shared four-warehouse correctness fixture in a separate PR. - Reassess the Phase 5 gate after both implementation PRs merge. diff --git a/tickets/DANDER-106-provider-neutral-template-request.md b/tickets/DANDER-106-provider-neutral-template-request.md index b0c7f54c..e7340865 100644 --- a/tickets/DANDER-106-provider-neutral-template-request.md +++ b/tickets/DANDER-106-provider-neutral-template-request.md @@ -1,7 +1,7 @@ --- id: DANDER-106 title: Resolve launcher templates through one provider-neutral request -status: in-review +status: completed component: deployment epic: cloud-portability depends_on: [DANDER-89, DANDER-90] @@ -24,7 +24,7 @@ them. - [x] Existing Cloud Run output remains exactly equal to the accepted compatibility projector. - [x] Existing Fargate and Kubernetes projection and fail-closed behavior remains covered. - [x] Focused launcher, bootstrap, Kubernetes CLI, and chart tests pass. -- [ ] Full protected CI passes. +- [x] Full protected CI passes. - [x] Independent completion review passes. ## Design @@ -46,4 +46,4 @@ Azure/OCI code, warehouse work, SQL, or user-visible projection change is includ ### 2026-08-10 — PASS Independent read-only completion review found no material defect or unnecessary scope. Protected -CI remains required before merge. +CI passed all five required jobs on PR #181. From f1033f6652fd5deaef3778436468d2ea39b31e5c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 15:34:04 -0400 Subject: [PATCH 111/163] add four-warehouse correctness gate --- HANDOFF.md | 41 +- docs/decisions.md | 12 + docs/known-limitations.md | 7 +- docs/warehouse-correctness-conformance.md | 86 ++ pyproject.toml | 2 + scripts/benchmarks/warehouse_correctness.py | 1076 +++++++++++++++++ .../portability/test_warehouse_correctness.py | 291 +++++ .../DANDER-107-four-warehouse-correctness.md | 51 + 8 files changed, 1541 insertions(+), 25 deletions(-) create mode 100644 docs/warehouse-correctness-conformance.md create mode 100644 scripts/benchmarks/warehouse_correctness.py create mode 100644 tests/portability/test_warehouse_correctness.py create mode 100644 tickets/DANDER-107-four-warehouse-correctness.md diff --git a/HANDOFF.md b/HANDOFF.md index 8fc491ef..bd4bf6ca 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,37 @@ ## Finished -- Replaced the launcher factory primitive argument list with one immutable resolved request. -- Removed dummy GCP project and guard values from Kubernetes projection callers. -- Captured typed GCP-only values at Cloud Run and Fargate factory construction. -- Preserved the accepted Cloud Run projection and existing launcher fail-closed behavior. +- Added one deterministic SCD1 fixture over the four warehouses' common scalar intersection. +- Added canonical readback normalization and exact expected/replay equality checks. +- Added sanitized per-provider evidence and exact four-provider comparison gates. +- Added owned cleanup verification and mandatory reviewed ceiling metadata. ## Try It -Run `.venv/bin/pytest -q tests/providers/test_launcher_runtime.py -tests/providers/test_kubernetes_runtime.py`. +Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. ## Checks -- Focused launcher, bootstrap, Kubernetes CLI, and chart tests passed (62 tests). -- Full local pytest suite passed (with expected opt-in skips). -- Ruff lint and format checks passed. -- Mypy passed for every changed source and test file; the full invocation retains one unrelated - Snowflake `unused-ignore` result under this local optional-dependency environment. -- Independent completion review passed with no material findings. -- Protected CI passed all five required jobs on PR #181. +- Credential-free conformance tests pass (6 tests). +- Full local pytest suite passes with expected opt-in skips. +- Ruff lint/format and focused strict mypy checks pass. +- Release metadata, wheel/sdist build, distribution inspection, and artifact inclusion pass. +- Protected CI remains required. ## Decisions -- The resolved request contains only common launcher intent and no extension bag. -- GCP project and guarded-free-tier values are typed provider-construction context. -- Pipeline containers are defensively copied into read-only equivalents. +- Rows and credentials never enter evidence; only hashes, counts, and reviewed metadata do. +- Physical transport equality is excluded; provider-specific conformance remains separate. +- Live mutation requires a separately reviewed ceiling and stable approval reference. ## Remaining -- Merge the protected launcher-contract PR. -- Implement the shared four-warehouse correctness fixture in a separate PR. -- Reassess the Phase 5 gate after both implementation PRs merge. +- Obtain reviewed ceilings and run all four providers on one protected-main commit. +- Verify exact cleanup and retained GCP no-drift after those live runs. +- Merge sanitized equal-result evidence, then reassess the Phase 5 gate. ## Review First -- `src/dander/deployment/runtime.py` -- `src/dander/providers/gcp_launcher.py` -- `tests/providers/test_launcher_runtime.py` +- `scripts/benchmarks/warehouse_correctness.py` +- `tests/portability/test_warehouse_correctness.py` +- `docs/warehouse-correctness-conformance.md` diff --git a/docs/decisions.md b/docs/decisions.md index 49d588da..3441e88c 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1134,3 +1134,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** The existing Cloud Run compatibility projector remains authoritative, and Fargate and Kubernetes retain their prior fail-closed behavior. No configuration schema, launcher output, provider capability, warehouse behavior, or cloud implementation changes. + +## 2026-08-10 — Four-warehouse equality compares hashes from one bounded scalar fixture + +- **Fixture:** BigQuery, PostgreSQL, Snowflake, and Redshift receive the same SCD1 waves and replay + over their common canonical scalar intersection. Canonical normalization must match the expected + rows before provider evidence can be emitted. +- **Evidence:** Per-provider records retain schema, fixture, and normalized-result hashes, row + count, candidate commit, cleanup, transport, and reviewed ceiling metadata; row values and + credentials are excluded. The final gate accepts exactly one equal record per provider. +- **Boundary:** Physical transports and provider-specific types, fallbacks, write modes, + materializations, and fencing remain separate conformance concerns. Scale, cost, crossover, soak, + pairwise-profile, and release qualification remain in Phase 8. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index bc139dea..76f27057 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -4,9 +4,10 @@ Dander `0.8.x` is beta and proves a focused GCP-native vertical slice. It remain evaluate these limits before using it for an unattended system containing business-critical data. The Phase 5 warehouse implementations and Snowflake/Redshift live correctness proofs are merged, -but the re-baselined Phase 5 exit gate remains open until the shared four-warehouse deterministic -fixture produces equal normalized results and the current evidence ledger is reconciled. Scale, -cost, soak, pairwise-profile, and release qualification remain Phase 8 work. +but the re-baselined Phase 5 exit gate remains open until the +[shared four-warehouse deterministic fixture](warehouse-correctness-conformance.md) produces equal +normalized results on one protected-main commit and the current evidence ledger is reconciled. +Scale, cost, soak, pairwise-profile, and release qualification remain Phase 8 work. - AWS ECS/Fargate remains experimental rather than supported. The named Fargate-to-BigQuery/GCP composition has passed source-free manual and scheduled execution, replay, interruption, diff --git a/docs/warehouse-correctness-conformance.md b/docs/warehouse-correctness-conformance.md new file mode 100644 index 00000000..bcedeebe --- /dev/null +++ b/docs/warehouse-correctness-conformance.md @@ -0,0 +1,86 @@ +# Four-warehouse correctness conformance + +Phase 5 uses one bounded fixture to compare the common canonical scalar semantics of BigQuery, +PostgreSQL, Snowflake, and Redshift. The harness is +`scripts/benchmarks/warehouse_correctness.py`. It is a correctness gate, not a throughput, cost, +scale, soak, pairwise-profile, or support qualification. + +## What is equal + +Every provider receives the same two SCD1 input waves and one exact replay. The schema covers only +the shared scalar intersection: boolean, signed 64-bit integer, exact `DECIMAL(38,9)`, 64-bit +float, string, binary, date, microsecond time, microsecond UTC timestamp, and microsecond local +timestamp. Rows include duplicate-key last-write-wins, an update, an insert, a nullable value, +Unicode normalization, binary values, and exact temporal values. + +Readback is normalized by the canonical schema: decimals use fixed scale, binary is base64, +strings use NFC, zoned timestamps use UTC, local timestamps remain timezone-free, and rows sort by +the canonical key. A provider run fails unless its normalized rows equal the fixture's expected +rows before and after replay. + +Physical transports are deliberately not compared. BigQuery uses its load-job SCD1 adapter, +PostgreSQL uses COPY, and the bounded Snowflake and Redshift profiles force their direct paths. +Provider-specific types, JSON fallbacks, staged transports, other write modes, materializations, +and fencing behavior retain their separate adapter/qualification tests. + +## Live execution + +Create one uncommitted JSON profile per provider. Profiles contain only non-secret coordinates and +credential references already accepted by the provider registry: + +```json +{"provider":"bigquery","project":"PROJECT","location":"US","dataset":"raw"} +``` + +```json +{"provider":"postgresql","database":"DATABASE","dsn_env":"DANDER_POSTGRES_DSN"} +``` + +Snowflake uses its normal `account`, `user`, `database`, `warehouse`, optional `role`, and `auth` +reference block. Redshift uses its normal deployment, endpoint, database, region, IAM role, and S3 +staging coordinates. Never put a DSN, OAuth token, private key, AWS credential, or row in a profile +or evidence file. + +Obtain a separately reviewed cost ceiling for each renewed live invocation. The CLI has no default +and refuses to mutate a provider unless both a ceiling and stable approval reference are supplied. +The recorded ceiling is evidence, not a hard provider spending cap. + +Run each profile against the same full protected-main commit: + +```bash +uv run python scripts/benchmarks/warehouse_correctness.py run \ + --profile-json /secure/bigquery-profile.json \ + --candidate-commit COMMIT_SHA \ + --approved-cost-ceiling-usd 1.00 \ + --cost-approval-reference REVIEW_REFERENCE \ + --output /secure/bigquery-evidence.json +``` + +Repeat for PostgreSQL, Snowflake, and Redshift, then compare exactly four records: + +```bash +uv run python scripts/benchmarks/warehouse_correctness.py compare \ + --evidence /secure/bigquery-evidence.json \ + --evidence /secure/postgresql-evidence.json \ + --evidence /secure/snowflake-evidence.json \ + --evidence /secure/redshift-evidence.json \ + --output /secure/four-warehouse-comparison.json +``` + +Provider evidence contains only fixture/schema/result SHA-256 hashes, normalized row count, +transport name, replay and cleanup booleans, candidate commit, Dander version, timestamps, and cost +approval metadata. It never contains normalized or source rows. The comparison fails on a missing +or duplicate provider, different candidate, unequal hash/count, failed replay, or unverified +cleanup. + +Each run owns a random target scope and cleans it in `finally`: one BigQuery table and its exact +staging-name prefix, one PostgreSQL schema, one Snowflake schema, or one Redshift schema plus one S3 +prefix. After all four pass, verify the retained GCP Terraform plan is no-drift and record only the +sanitized evidence and reviewed ceilings. + +## Current evidence status + +The deterministic harness and credential-free contract tests are implemented. No renewed paid +provider execution is claimed by this document. Phase 5 remains open until all four same-commit +live records compare equal, exact cleanup is verified, retained GCP no-drift passes afterward, and +the resulting sanitized evidence is merged through protected CI. diff --git a/pyproject.toml b/pyproject.toml index 0982e401..55047595 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,6 +244,7 @@ include = [ "docs/snowflake.md", "docs/upgrading.md", "docs/warehouse-staging.md", + "docs/warehouse-correctness-conformance.md", "examples", "infra", "graphs", @@ -252,6 +253,7 @@ include = [ "scripts/__init__.py", "scripts/benchmarks/__init__.py", "scripts/benchmarks/postgresql.py", + "scripts/benchmarks/warehouse_correctness.py", "src", "tests", "uv.lock", diff --git a/scripts/benchmarks/warehouse_correctness.py b/scripts/benchmarks/warehouse_correctness.py new file mode 100644 index 00000000..7a888663 --- /dev/null +++ b/scripts/benchmarks/warehouse_correctness.py @@ -0,0 +1,1076 @@ +#!/usr/bin/env python3 +"""Run and compare the bounded four-warehouse correctness fixture.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import math +import re +import unicodedata +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass +from datetime import UTC, date, datetime, time +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from dander import __version__ +from dander.concurrency import FencingToken +from dander.providers import ProviderKind, default_provider_registry +from dander.warehouse import ( + CanonicalField, + CanonicalType, + FieldCardinality, + LogicalTypeKind, + RelationRef, + RelationSchema, + WarehouseRuntime, + normalize_staging_record, +) +from dander.writer import SchemaEvolution, WriteField, WriteMode, WriteTarget, WriteTransport + +if TYPE_CHECKING: + from typing import Any, Protocol + + from dander.providers.postgresql.fence import PostgreSQLTargetFence + from dander.providers.redshift.fence import RedshiftTargetFence + from dander.providers.redshift.runtime import RedshiftWriterFactory + from dander.providers.snowflake.fence import SnowflakeTargetFence + from dander.telemetry import OperationTelemetry + + class _S3Inspector(Protocol): + def list_objects_v2(self, **kwargs: object) -> Mapping[str, object]: ... + + def delete_objects(self, **kwargs: object) -> Mapping[str, object]: ... + + +_EVIDENCE_SCHEMA = "io.dander.conformance.warehouse-correctness/v1" +_COMPARISON_SCHEMA = "io.dander.conformance.warehouse-correctness-comparison/v1" +_FIXTURE_VERSION = "warehouse-common-scalar-v1" +_PROVIDERS = frozenset({"bigquery", "postgresql", "snowflake", "redshift"}) +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_COMMIT = re.compile(r"^[0-9a-f]{40}$") +_APPROVAL_REFERENCE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/#-]{2,255}$") +_TARGET_PREFIX = "dander_conform_" + + +class WarehouseCorrectnessError(RuntimeError): + """Raised with a sanitized four-warehouse conformance failure.""" + + +COMMON_SCHEMA = RelationSchema( + fields=( + CanonicalField( + name="id", + data_type=CanonicalType(kind=LogicalTypeKind.STRING), + cardinality=FieldCardinality.REQUIRED, + ), + CanonicalField( + name="active", + data_type=CanonicalType(kind=LogicalTypeKind.BOOLEAN), + cardinality=FieldCardinality.REQUIRED, + ), + CanonicalField( + name="units", + data_type=CanonicalType(kind=LogicalTypeKind.INTEGER, bit_width=64), + ), + CanonicalField( + name="amount", + data_type=CanonicalType(kind=LogicalTypeKind.DECIMAL, precision=38, scale=9), + ), + CanonicalField( + name="ratio", + data_type=CanonicalType(kind=LogicalTypeKind.FLOAT, bit_width=64), + ), + CanonicalField(name="label", data_type=CanonicalType(kind=LogicalTypeKind.STRING)), + CanonicalField(name="payload", data_type=CanonicalType(kind=LogicalTypeKind.BINARY)), + CanonicalField(name="event_date", data_type=CanonicalType(kind=LogicalTypeKind.DATE)), + CanonicalField( + name="event_time", + data_type=CanonicalType( + kind=LogicalTypeKind.TIME, + fractional_second_precision=6, + ), + ), + CanonicalField( + name="observed_at", + data_type=CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=True, + fractional_second_precision=6, + ), + ), + CanonicalField( + name="local_at", + data_type=CanonicalType( + kind=LogicalTypeKind.TIMESTAMP, + with_timezone=False, + fractional_second_precision=6, + ), + ), + ) +) + +LEGACY_BIGQUERY_FIELDS = ( + WriteField(name="id", data_type="STRING", mode="REQUIRED"), + WriteField(name="active", data_type="BOOLEAN", mode="REQUIRED"), + WriteField(name="units", data_type="INT64"), + WriteField(name="amount", data_type="NUMERIC"), + WriteField(name="ratio", data_type="FLOAT64"), + WriteField(name="label", data_type="STRING"), + WriteField(name="payload", data_type="BYTES"), + WriteField(name="event_date", data_type="DATE"), + WriteField(name="event_time", data_type="TIME"), + WriteField(name="observed_at", data_type="TIMESTAMP"), + WriteField(name="local_at", data_type="DATETIME"), +) + +_INITIAL_ROWS: tuple[dict[str, object], ...] = ( + { + "id": "alpha", + "active": True, + "units": 1, + "amount": Decimal("10.250000000"), + "ratio": 0.5, + "label": "older", + "payload": b"\x00\x7f", + "event_date": date(2026, 1, 2), + "event_time": time(3, 4, 5, 123456), + "observed_at": datetime(2026, 1, 2, 3, 4, 5, 123456, tzinfo=UTC), + "local_at": datetime(2026, 1, 2, 3, 4, 5, 123456), + }, + { + "id": "beta", + "active": False, + "units": -2, + "amount": Decimal("0.000000001"), + "ratio": -2.0, + "label": None, + "payload": b"dander", + "event_date": date(2026, 2, 3), + "event_time": time(6, 7, 8), + "observed_at": datetime(2026, 2, 3, 6, 7, 8, tzinfo=UTC), + "local_at": datetime(2026, 2, 3, 6, 7, 8), + }, + { + "id": "alpha", + "active": True, + "units": 2, + "amount": Decimal("11.250000000"), + "ratio": 0.25, + "label": "newer", + "payload": b"\x00\x7f", + "event_date": date(2026, 1, 2), + "event_time": time(3, 4, 5, 123456), + "observed_at": datetime(2026, 1, 2, 3, 4, 5, 123456, tzinfo=UTC), + "local_at": datetime(2026, 1, 2, 3, 4, 5, 123456), + }, +) + +_UPDATE_ROWS: tuple[dict[str, object], ...] = ( + { + "id": "alpha", + "active": False, + "units": 3, + "amount": Decimal("12.500000000"), + "ratio": 0.5, + "label": "caf\N{LATIN SMALL LETTER E WITH ACUTE}", + "payload": b"updated", + "event_date": date(2026, 3, 4), + "event_time": time(9, 10, 11, 654321), + "observed_at": datetime(2026, 3, 4, 9, 10, 11, 654321, tzinfo=UTC), + "local_at": datetime(2026, 3, 4, 9, 10, 11, 654321), + }, + { + "id": "gamma", + "active": True, + "units": 0, + "amount": Decimal("999.000000000"), + "ratio": 0.0, + "label": "third", + "payload": b"\xff", + "event_date": date(2026, 4, 5), + "event_time": time(12, 13, 14), + "observed_at": datetime(2026, 4, 5, 12, 13, 14, tzinfo=UTC), + "local_at": datetime(2026, 4, 5, 12, 13, 14), + }, +) + +_EXPECTED_ROWS = (_UPDATE_ROWS[0], _INITIAL_ROWS[1], _UPDATE_ROWS[1]) + + +@dataclass(frozen=True, slots=True) +class ApprovedCostCeiling: + """Human-reviewed maximum for one bounded live provider invocation.""" + + usd: str + approval_reference: str + + def __post_init__(self) -> None: + try: + amount = Decimal(self.usd) + except InvalidOperation as error: + raise ValueError("approved cost ceiling must be a decimal USD amount") from error + if not amount.is_finite() or amount < 0: + raise ValueError("approved cost ceiling must be finite and nonnegative") + if not _APPROVAL_REFERENCE.fullmatch(self.approval_reference): + raise ValueError("approval reference must be a stable non-secret identifier") + object.__setattr__(self, "usd", format(amount, "f")) + + +@dataclass(frozen=True, slots=True) +class WarehouseCorrectnessEvidence: + """Sanitized proof for one provider; normalized rows are represented only by hashes.""" + + schema: str + fixture_version: str + provider: str + candidate_commit: str + dander_version: str + started_at_utc: str + ended_at_utc: str + fixture_hash: str + canonical_schema_hash: str + normalized_rows_hash: str + normalized_row_count: int + write_mode: str + transport: str + replay_equal: bool + cleanup_verified: bool + approved_cost_ceiling_usd: str + cost_approval_reference: str + status: str = "passed" + + def __post_init__(self) -> None: + if self.schema != _EVIDENCE_SCHEMA or self.fixture_version != _FIXTURE_VERSION: + raise ValueError("warehouse correctness evidence uses an unknown schema") + if self.provider not in _PROVIDERS: + raise ValueError("warehouse correctness evidence names an unknown provider") + if not _COMMIT.fullmatch(self.candidate_commit): + raise ValueError("warehouse correctness evidence requires a full commit SHA") + for value in ( + self.fixture_hash, + self.canonical_schema_hash, + self.normalized_rows_hash, + ): + if not _SHA256.fullmatch(value): + raise ValueError("warehouse correctness evidence contains an invalid hash") + if self.normalized_row_count < 0: + raise ValueError("warehouse correctness row count must be nonnegative") + if self.write_mode != WriteMode.SCD1.value: + raise ValueError("warehouse correctness evidence must use SCD1") + if not self.replay_equal or not self.cleanup_verified or self.status != "passed": + raise ValueError("warehouse correctness evidence is not a passing proof") + ApprovedCostCeiling( + self.approved_cost_ceiling_usd, + self.cost_approval_reference, + ) + + def to_json(self) -> str: + return json.dumps(asdict(self), separators=(",", ":"), sort_keys=True) + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> WarehouseCorrectnessEvidence: + expected = {field.name for field in cls.__dataclass_fields__.values()} + if set(value) != expected: + raise ValueError("warehouse correctness evidence has an invalid shape") + return cls(**cast("Any", dict(value))) + + +@dataclass(frozen=True, slots=True) +class WarehouseCorrectnessComparison: + """Sanitized equality gate across exactly four provider evidence records.""" + + schema: str + fixture_version: str + candidate_commit: str + providers: tuple[str, ...] + fixture_hash: str + canonical_schema_hash: str + normalized_rows_hash: str + normalized_row_count: int + all_rows_equal: bool + all_cleanup_verified: bool + status: str = "passed" + + def to_json(self) -> str: + return json.dumps(asdict(self), separators=(",", ":"), sort_keys=True) + + +@dataclass(slots=True) +class _WarehouseSession: + provider: str + runtime: WarehouseRuntime + relation: RelationRef + schema_inputs: Sequence[object] + transport: WriteTransport + read_rows: Callable[[], Sequence[Mapping[str, object]]] + cleanup: Callable[[], None] + cleanup_verified: Callable[[], bool] + close: Callable[[], None] + + +def normalized_rows( + rows: Sequence[Mapping[str, object]], + schema: RelationSchema = COMMON_SCHEMA, +) -> tuple[dict[str, object], ...]: + """Normalize provider readback by canonical field semantics and stable key order.""" + normalized = [ + _normalize_row(normalize_staging_record(row, schema, row_index=index), schema) + for index, row in enumerate(rows) + ] + normalized.sort(key=lambda row: cast("str", row["id"])) + return tuple(normalized) + + +def compare_evidence( + evidence: Sequence[WarehouseCorrectnessEvidence], +) -> WarehouseCorrectnessComparison: + """Require one passing, same-candidate proof per warehouse and exact hash equality.""" + providers = [item.provider for item in evidence] + if len(evidence) != len(_PROVIDERS) or set(providers) != _PROVIDERS: + raise WarehouseCorrectnessError( + "comparison requires one unique BigQuery, PostgreSQL, Snowflake, and Redshift proof" + ) + attributes = ( + "candidate_commit", + "fixture_hash", + "canonical_schema_hash", + "normalized_rows_hash", + "normalized_row_count", + ) + for attribute in attributes: + if len({getattr(item, attribute) for item in evidence}) != 1: + raise WarehouseCorrectnessError( + f"warehouse correctness comparison found unequal {attribute}" + ) + first = evidence[0] + return WarehouseCorrectnessComparison( + schema=_COMPARISON_SCHEMA, + fixture_version=_FIXTURE_VERSION, + candidate_commit=first.candidate_commit, + providers=tuple(sorted(providers)), + fixture_hash=first.fixture_hash, + canonical_schema_hash=first.canonical_schema_hash, + normalized_rows_hash=first.normalized_rows_hash, + normalized_row_count=first.normalized_row_count, + all_rows_equal=True, + all_cleanup_verified=all(item.cleanup_verified for item in evidence), + ) + + +def run_provider( + profile: Mapping[str, object], + *, + candidate_commit: str, + cost_ceiling: ApprovedCostCeiling, +) -> WarehouseCorrectnessEvidence: + """Execute the common fixture through one selected live warehouse runtime.""" + if not _COMMIT.fullmatch(candidate_commit): + raise ValueError("candidate_commit must be a full lowercase commit SHA") + provider = profile.get("provider") + if provider not in _PROVIDERS: + raise ValueError("profile must select one of the four conformance warehouses") + session = _open_session(profile) + started = _now() + try: + evidence = _run_session( + session, + candidate_commit=candidate_commit, + cost_ceiling=cost_ceiling, + started_at_utc=started, + ) + finally: + session.close() + return evidence + + +def _run_session( + session: _WarehouseSession, + *, + candidate_commit: str, + cost_ceiling: ApprovedCostCeiling, + started_at_utc: str, +) -> WarehouseCorrectnessEvidence: + runtime = session.runtime + runtime.capabilities.schema_support.require(COMMON_SCHEMA) + mapped = runtime.schema_mapper.canonical_schema(session.schema_inputs) + if _without_extensions(mapped) != COMMON_SCHEMA: + raise WarehouseCorrectnessError( + f"{session.provider} schema mapper changed the common canonical fixture" + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=len(_INITIAL_ROWS), + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.SCD1, + ) + publication = None + if writer.requires_publication_fence: + publication = runtime.target_fence.claim( + session.relation, + FencingToken( + lease_table=None, + pipeline_id="warehouse_correctness", + run_id="fixture-v1", + token=1, + authority_id=f"{session.provider}:warehouse-correctness", + ), + ) + target = WriteTarget( + relation=session.relation, + business_key=("id",), + schema=LEGACY_BIGQUERY_FIELDS, + declared_schema=COMMON_SCHEMA, + publication_fence=publication, + ) + operations: list[OperationTelemetry] = [] + cleanup_error: Exception | None = None + primary_error: Exception | None = None + before_replay: tuple[dict[str, object], ...] = () + after_replay: tuple[dict[str, object], ...] = () + try: + writer.write(_INITIAL_ROWS, target) + operations.extend(writer.drain_telemetry()) + writer.write(_UPDATE_ROWS, target) + operations.extend(writer.drain_telemetry()) + before_replay = normalized_rows(session.read_rows()) + writer.write(_UPDATE_ROWS, target) + operations.extend(writer.drain_telemetry()) + after_replay = normalized_rows(session.read_rows()) + expected = normalized_rows(_EXPECTED_ROWS) + if before_replay != expected or after_replay != expected: + raise WarehouseCorrectnessError( + f"{session.provider} normalized rows differ from the common fixture" + ) + _require_transport(session, operations) + except Exception as error: + primary_error = error + finally: + try: + session.cleanup() + except Exception as error: + cleanup_error = error + + cleaned = False + if cleanup_error is None: + try: + cleaned = session.cleanup_verified() + except Exception as error: + cleanup_error = error + if primary_error is not None or cleanup_error is not None: + errors = [error for error in (primary_error, cleanup_error) if error is not None] + raise WarehouseCorrectnessError( + f"{session.provider} correctness run failed; owned cleanup was attempted" + ) from ExceptionGroup("warehouse correctness failures", errors) + if not cleaned: + raise WarehouseCorrectnessError( + f"{session.provider} correctness cleanup could not be verified" + ) + + fixture_hash = _fixture_hash() + schema_hash = _hash_json(COMMON_SCHEMA.model_dump(mode="json", by_alias=True)) + result_hash = _hash_json(after_replay) + return WarehouseCorrectnessEvidence( + schema=_EVIDENCE_SCHEMA, + fixture_version=_FIXTURE_VERSION, + provider=session.provider, + candidate_commit=candidate_commit, + dander_version=__version__, + started_at_utc=started_at_utc, + ended_at_utc=_now(), + fixture_hash=fixture_hash, + canonical_schema_hash=schema_hash, + normalized_rows_hash=result_hash, + normalized_row_count=len(after_replay), + write_mode=WriteMode.SCD1.value, + transport=session.transport.value, + replay_equal=before_replay == after_replay, + cleanup_verified=cleaned, + approved_cost_ceiling_usd=cost_ceiling.usd, + cost_approval_reference=cost_ceiling.approval_reference, + ) + + +def _open_session(profile: Mapping[str, object]) -> _WarehouseSession: + provider = profile["provider"] + if provider == "bigquery": + return _open_bigquery(profile) + if provider == "postgresql": + return _open_postgresql(profile) + if provider == "snowflake": + return _open_snowflake(profile) + if provider == "redshift": + return _open_redshift(profile) + raise AssertionError("provider validation is exhaustive") + + +def _open_bigquery(profile: Mapping[str, object]) -> _WarehouseSession: + from google.api_core.exceptions import NotFound + from google.cloud import bigquery + + values = dict(profile) + project = values.pop("project", None) + if not isinstance(project, str) or not project: + raise ValueError("BigQuery conformance profile requires project") + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, values) + dataset = getattr(config, "dataset", None) + if not isinstance(dataset, str) or not dataset: + raise ValueError("BigQuery conformance profile requires an existing dataset") + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={"catalog": project}, + ) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("BigQuery provider returned an invalid warehouse runtime") + client = bigquery.Client(project=project) + relation = RelationRef( + catalog=project, + namespace=dataset, + name=f"{_TARGET_PREFIX}{uuid.uuid4().hex[:12]}", + ) + table_id = ".".join(relation.coordinates) + stage_prefix = f"_dander_stage_{relation.name}_" + + def read_rows() -> Sequence[Mapping[str, object]]: + fields = ", ".join(f"`{field.name}`" for field in COMMON_SCHEMA.fields) + query = f"SELECT {fields} FROM `{table_id}` ORDER BY `id`" + job_config = bigquery.QueryJobConfig( + maximum_bytes_billed=10_000_000, + labels={"dander-proof": "warehouse-correctness"}, + ) + return tuple(dict(row) for row in client.query(query, job_config=job_config).result()) + + def owned_tables() -> tuple[str, ...]: + query = ( + f"SELECT table_name FROM `{project}.{dataset}.INFORMATION_SCHEMA.TABLES` " + "WHERE table_name = @target OR STARTS_WITH(table_name, @stage_prefix)" + ) + config = bigquery.QueryJobConfig( + maximum_bytes_billed=10_000_000, + query_parameters=[ + bigquery.ScalarQueryParameter("target", "STRING", relation.name), + bigquery.ScalarQueryParameter("stage_prefix", "STRING", stage_prefix), + ], + labels={"dander-proof": "warehouse-correctness"}, + ) + return tuple( + str(row["table_name"]) for row in client.query(query, job_config=config).result() + ) + + def cleanup() -> None: + for table_name in owned_tables(): + if table_name != relation.name and not table_name.startswith(stage_prefix): + raise WarehouseCorrectnessError("BigQuery cleanup escaped its owned table prefix") + client.delete_table(f"{project}.{dataset}.{table_name}", not_found_ok=True) + + def cleanup_verified() -> bool: + if owned_tables(): + return False + try: + client.get_table(table_id) + except NotFound: + return True + return False + + return _WarehouseSession( + provider="bigquery", + runtime=runtime, + relation=relation, + schema_inputs=LEGACY_BIGQUERY_FIELDS, + transport=WriteTransport.LOAD_JOB, + read_rows=read_rows, + cleanup=cleanup, + cleanup_verified=cleanup_verified, + close=cast("Callable[[], None]", client.close), + ) + + +def _open_postgresql(profile: Mapping[str, object]) -> _WarehouseSession: + from psycopg import sql + + values = dict(profile) + schema_name = f"{_TARGET_PREFIX}{uuid.uuid4().hex[:12]}" + values["schema"] = schema_name + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, values) + runtime = registry.build(ProviderKind.WAREHOUSE, config) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("PostgreSQL provider returned an invalid warehouse runtime") + fence = cast("PostgreSQLTargetFence", runtime.target_fence) + relation = RelationRef( + catalog=fence.catalog, + namespace=schema_name, + name="records", + ) + + def read_rows() -> Sequence[Mapping[str, object]]: + statement = sql.SQL("SELECT {} FROM {} ORDER BY {}").format( + sql.SQL(", ").join(sql.Identifier(field.name) for field in COMMON_SCHEMA.fields), + sql.Identifier(schema_name, relation.name), + sql.Identifier("id"), + ) + with fence.pool.connection() as connection: + return tuple(cast("Mapping[str, object]", row) for row in connection.execute(statement)) + + def cleanup() -> None: + with fence.pool.connection() as connection: + connection.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_name)) + ) + + def cleanup_verified() -> bool: + with fence.pool.connection() as connection: + row = connection.execute( + "SELECT COUNT(*) AS count FROM pg_namespace WHERE nspname = %s", + (schema_name,), + ).fetchone() + return row is not None and _integer(row["count"]) == 0 + + return _WarehouseSession( + provider="postgresql", + runtime=runtime, + relation=relation, + schema_inputs=COMMON_SCHEMA.fields, + transport=WriteTransport.COPY, + read_rows=read_rows, + cleanup=cleanup, + cleanup_verified=cleanup_verified, + close=fence.pool.close, + ) + + +def _open_snowflake(profile: Mapping[str, object]) -> _WarehouseSession: + from dander.providers.snowflake.session import execute, open_connection + + values = dict(profile) + schema_name = f"DANDER_CONFORM_{uuid.uuid4().hex[:12].upper()}" + values.update( + { + "schema": schema_name, + "direct_max_rows": 10, + "direct_max_logical_bytes": 1_048_576, + } + ) + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, values) + database = getattr(config, "database", None) + if not isinstance(database, str): + raise TypeError("Snowflake conformance profile requires a database") + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={"catalog": database}, + ) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("Snowflake provider returned an invalid warehouse runtime") + fence = cast("SnowflakeTargetFence", runtime.target_fence) + relation = RelationRef(catalog=database, namespace=schema_name, name="records") + + def read_rows() -> Sequence[Mapping[str, object]]: + columns = ", ".join(_quote(field.name) for field in COMMON_SCHEMA.fields) + with open_connection(fence.connection_factory) as connection: + rows = execute( + connection, + f"SELECT {columns} FROM {runtime.relation_codec.render(relation)} " + f"ORDER BY {_quote('id')}", + fetch="all", + ).rows + return _rows_from_sequences(rows) + + def cleanup() -> None: + with open_connection(fence.connection_factory) as connection: + execute( + connection, + f"DROP SCHEMA IF EXISTS {_quote(database)}.{_quote(schema_name)} CASCADE", + ) + connection.commit() + + def cleanup_verified() -> bool: + with open_connection(fence.connection_factory) as connection: + row = execute( + connection, + f"SELECT COUNT(*) FROM {_quote(database)}.INFORMATION_SCHEMA.SCHEMATA " + "WHERE SCHEMA_NAME = ?", + (schema_name,), + fetch="one", + ).row + return _count(row) == 0 + + return _WarehouseSession( + provider="snowflake", + runtime=runtime, + relation=relation, + schema_inputs=COMMON_SCHEMA.fields, + transport=WriteTransport.DIRECT, + read_rows=read_rows, + cleanup=cleanup, + cleanup_verified=cleanup_verified, + close=lambda: None, + ) + + +def _open_redshift(profile: Mapping[str, object]) -> _WarehouseSession: + from dander.providers.redshift.session import execute, open_connection + + values = dict(profile) + suffix = uuid.uuid4().hex[:12] + schema_name = f"{_TARGET_PREFIX}{suffix}" + prefix = values.get("staging_prefix", "dander/staging") + if not isinstance(prefix, str): + raise ValueError("Redshift staging_prefix must be a string") + owned_prefix = f"{prefix.strip('/')}/conformance/{suffix}" + values.update( + { + "schema": schema_name, + "staging_prefix": owned_prefix, + "direct_max_rows": 10, + "direct_max_logical_bytes": 1_048_576, + } + ) + registry = default_provider_registry() + config = registry.parse(ProviderKind.WAREHOUSE, values) + database = getattr(config, "database", None) + if not isinstance(database, str): + raise TypeError("Redshift conformance profile requires a database") + runtime = registry.build( + ProviderKind.WAREHOUSE, + config, + context={"catalog": database}, + ) + if not isinstance(runtime, WarehouseRuntime): + raise TypeError("Redshift provider returned an invalid warehouse runtime") + fence = cast("RedshiftTargetFence", runtime.target_fence) + writers = cast("RedshiftWriterFactory", runtime.writers) + s3 = cast("_S3Inspector", writers.s3_client) + relation = RelationRef(catalog=database, namespace=schema_name, name="records") + + def read_rows() -> Sequence[Mapping[str, object]]: + columns = ", ".join(_quote(field.name) for field in COMMON_SCHEMA.fields) + with open_connection(fence.connection_factory) as connection: + rows = execute( + connection, + f"SELECT {columns} FROM {runtime.relation_codec.render(relation)} " + f"ORDER BY {_quote('id')}", + fetch="all", + ).rows + return _rows_from_sequences(rows) + + def owned_keys() -> tuple[str, ...]: + keys: list[str] = [] + token: str | None = None + while True: + arguments: dict[str, object] = { + "Bucket": writers.staging.bucket, + "Prefix": f"{owned_prefix}/", + } + if token is not None: + arguments["ContinuationToken"] = token + response = s3.list_objects_v2(**arguments) + contents = response.get("Contents", []) + if not isinstance(contents, list): + raise WarehouseCorrectnessError("Redshift cleanup returned malformed contents") + for item in contents: + if not isinstance(item, Mapping) or not isinstance(item.get("Key"), str): + raise WarehouseCorrectnessError("Redshift cleanup returned a malformed key") + key = cast("str", item["Key"]) + if not key.startswith(f"{owned_prefix}/"): + raise WarehouseCorrectnessError("Redshift cleanup escaped its owned prefix") + keys.append(key) + if response.get("IsTruncated") is not True: + break + next_token = response.get("NextContinuationToken") + if not isinstance(next_token, str) or not next_token: + raise WarehouseCorrectnessError("Redshift cleanup pagination was malformed") + token = next_token + return tuple(keys) + + def cleanup() -> None: + with open_connection(fence.connection_factory) as connection: + execute(connection, f"DROP SCHEMA IF EXISTS {_quote(schema_name)} CASCADE") + connection.commit() + keys = owned_keys() + for index in range(0, len(keys), 1_000): + response = s3.delete_objects( + Bucket=writers.staging.bucket, + Delete={"Objects": [{"Key": key} for key in keys[index : index + 1_000]]}, + ) + if response.get("Errors"): + raise WarehouseCorrectnessError("Redshift cleanup reported undeleted objects") + + def cleanup_verified() -> bool: + with open_connection(fence.connection_factory) as connection: + row = execute( + connection, + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = %s", + (schema_name,), + fetch="one", + ).row + return _count(row) == 0 and not owned_keys() + + return _WarehouseSession( + provider="redshift", + runtime=runtime, + relation=relation, + schema_inputs=COMMON_SCHEMA.fields, + transport=WriteTransport.DIRECT, + read_rows=read_rows, + cleanup=cleanup, + cleanup_verified=cleanup_verified, + close=lambda: None, + ) + + +def _normalize_row(row: Mapping[str, object], schema: RelationSchema) -> dict[str, object]: + return { + field.name: _normalize_scalar(row[field.name], field.data_type) for field in schema.fields + } + + +def _normalize_scalar(value: object, data_type: CanonicalType) -> object: + if value is None: + return None + match data_type.kind: + case LogicalTypeKind.BOOLEAN: + if not isinstance(value, bool): + raise WarehouseCorrectnessError("canonical boolean readback has the wrong type") + return value + case LogicalTypeKind.INTEGER: + if isinstance(value, bool) or not isinstance(value, int): + raise WarehouseCorrectnessError("canonical integer readback has the wrong type") + return value + case LogicalTypeKind.DECIMAL: + try: + decimal = value if isinstance(value, Decimal) else Decimal(str(value)) + assert data_type.scale is not None + quantum = Decimal(1).scaleb(-data_type.scale) + normalized = decimal.quantize(quantum) + except (InvalidOperation, ValueError) as error: + raise WarehouseCorrectnessError("canonical decimal readback is invalid") from error + if normalized != decimal: + raise WarehouseCorrectnessError("canonical decimal readback lost scale precision") + return format(normalized, "f") + case LogicalTypeKind.FLOAT: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise WarehouseCorrectnessError("canonical float readback has the wrong type") + result = float(value) + if not math.isfinite(result): + raise WarehouseCorrectnessError("canonical float readback must be finite") + return 0.0 if result == 0 else result + case LogicalTypeKind.STRING: + if not isinstance(value, str): + raise WarehouseCorrectnessError("canonical string readback has the wrong type") + return unicodedata.normalize("NFC", value) + case LogicalTypeKind.BINARY: + if isinstance(value, memoryview): + value = value.tobytes() + if isinstance(value, bytearray): + value = bytes(value) + if not isinstance(value, bytes): + raise WarehouseCorrectnessError("canonical binary readback has the wrong type") + return base64.b64encode(value).decode("ascii") + case LogicalTypeKind.DATE: + if isinstance(value, datetime) or not isinstance(value, date): + raise WarehouseCorrectnessError("canonical date readback has the wrong type") + return value.isoformat() + case LogicalTypeKind.TIME: + if not isinstance(value, time) or value.utcoffset() is not None: + raise WarehouseCorrectnessError("canonical time readback has the wrong type") + return value.isoformat(timespec="microseconds") + case LogicalTypeKind.TIMESTAMP: + if not isinstance(value, datetime): + raise WarehouseCorrectnessError("canonical timestamp readback has the wrong type") + if data_type.with_timezone: + if value.utcoffset() is None: + raise WarehouseCorrectnessError( + "canonical zoned timestamp readback lost its timezone" + ) + return ( + value.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") + ) + if value.utcoffset() is not None: + raise WarehouseCorrectnessError( + "canonical local timestamp readback gained a timezone" + ) + return value.isoformat(timespec="microseconds") + case _: + raise WarehouseCorrectnessError( + "fixture contains a type outside the common scalar intersection" + ) + + +def _without_extensions(schema: RelationSchema) -> RelationSchema: + return RelationSchema( + fields=tuple(field.model_copy(update={"extensions": ()}) for field in schema.fields) + ) + + +def _rows_from_sequences(rows: object) -> tuple[dict[str, object], ...]: + if not isinstance(rows, Sequence): + raise WarehouseCorrectnessError("provider readback did not return a row sequence") + names = tuple(field.name for field in COMMON_SCHEMA.fields) + result: list[dict[str, object]] = [] + for row in rows: + if not isinstance(row, Sequence) or isinstance(row, (str, bytes, bytearray)): + raise WarehouseCorrectnessError("provider readback returned a malformed row") + if len(row) != len(names): + raise WarehouseCorrectnessError("provider readback returned the wrong column count") + result.append(dict(zip(names, row, strict=True))) + return tuple(result) + + +def _require_transport( + session: _WarehouseSession, + operations: Sequence[OperationTelemetry], +) -> None: + observed = tuple( + operation.transport for operation in operations if operation.transport is not None + ) + if observed and any(item is not session.transport for item in observed): + raise WarehouseCorrectnessError( + f"{session.provider} conformance used an unexpected transport" + ) + if session.provider in {"snowflake", "redshift"} and not observed: + raise WarehouseCorrectnessError( + f"{session.provider} conformance emitted no transport evidence" + ) + + +def _fixture_hash() -> str: + payload = { + "version": _FIXTURE_VERSION, + "initial": normalized_rows(_INITIAL_ROWS), + "update": normalized_rows(_UPDATE_ROWS), + "expected": normalized_rows(_EXPECTED_ROWS), + } + return _hash_json(payload) + + +def _hash_json(value: object) -> str: + encoded = json.dumps(value, separators=(",", ":"), sort_keys=True).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _count(row: object) -> int: + if not isinstance(row, Sequence) or isinstance(row, (str, bytes, bytearray)) or not row: + raise WarehouseCorrectnessError("provider cleanup count was malformed") + return _integer(row[0]) + + +def _integer(value: object) -> int: + if isinstance(value, bool): + raise WarehouseCorrectnessError("provider cleanup count was not an integer") + if isinstance(value, int): + return value + if not isinstance(value, (str, bytes, bytearray)): + raise WarehouseCorrectnessError("provider cleanup count was not an integer") + try: + return int(value) + except (TypeError, ValueError) as error: + raise WarehouseCorrectnessError("provider cleanup count was not an integer") from error + + +def _quote(value: str) -> str: + return f'"{value.replace(chr(34), chr(34) * 2)}"' + + +def _now() -> str: + return datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _load_object(path: Path) -> Mapping[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, Mapping): + raise ValueError("JSON input must contain one object") + return cast("Mapping[str, object]", value) + + +def _write_json(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{content}\n", encoding="utf-8") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + run = subparsers.add_parser("run", help="run one provider and write sanitized evidence") + run.add_argument("--profile-json", type=Path, required=True) + run.add_argument("--candidate-commit", required=True) + run.add_argument("--approved-cost-ceiling-usd", required=True) + run.add_argument("--cost-approval-reference", required=True) + run.add_argument("--output", type=Path, required=True) + compare = subparsers.add_parser("compare", help="require equal evidence from all providers") + compare.add_argument("--evidence", type=Path, action="append", required=True) + compare.add_argument("--output", type=Path, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + if arguments.command == "compare": + try: + records = tuple( + WarehouseCorrectnessEvidence.from_mapping(_load_object(path)) + for path in arguments.evidence + ) + comparison = compare_evidence(records) + _write_json(arguments.output, comparison.to_json()) + except Exception: + print( + json.dumps( + { + "schema": _COMPARISON_SCHEMA, + "status": "failed", + "summary": "Four-warehouse correctness evidence did not compare equal.", + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 1 + print(comparison.to_json()) + return 0 + + profile: Mapping[str, object] = {} + try: + profile = _load_object(arguments.profile_json) + ceiling = ApprovedCostCeiling( + arguments.approved_cost_ceiling_usd, + arguments.cost_approval_reference, + ) + provider_evidence = run_provider( + profile, + candidate_commit=arguments.candidate_commit, + cost_ceiling=ceiling, + ) + _write_json(arguments.output, provider_evidence.to_json()) + except Exception: + provider = profile.get("provider") + print( + json.dumps( + { + "schema": _EVIDENCE_SCHEMA, + "provider": provider if provider in _PROVIDERS else "unknown", + "status": "failed", + "summary": ( + "Warehouse correctness run failed; inspect provider logs and verify " + "owned cleanup before retrying." + ), + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 1 + print(provider_evidence.to_json()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/portability/test_warehouse_correctness.py b/tests/portability/test_warehouse_correctness.py new file mode 100644 index 00000000..e8c4a591 --- /dev/null +++ b/tests/portability/test_warehouse_correctness.py @@ -0,0 +1,291 @@ +"""Credential-free contract checks for the four-warehouse correctness gate.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from typing import TYPE_CHECKING, cast + +import pytest +from scripts.benchmarks import warehouse_correctness as correctness + +from dander.concurrency import TargetFence +from dander.providers.bigquery.runtime import BIGQUERY_SCHEMA_SUPPORT, BigQuerySchemaMapper +from dander.providers.postgresql.runtime import ( + POSTGRESQL_SCHEMA_SUPPORT, + PostgreSQLSchemaMapper, +) +from dander.providers.redshift.runtime import REDSHIFT_SCHEMA_SUPPORT, RedshiftSchemaMapper +from dander.providers.snowflake.runtime import SNOWFLAKE_SCHEMA_SUPPORT, SnowflakeSchemaMapper +from dander.warehouse import LogicalTypeKind, RelationRef +from dander.warehouse.runtime import WarehouseCapabilities, WarehouseRuntime +from dander.writer import WriteMode, WritePattern, WriteTransport + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping, Sequence + from pathlib import Path + from typing import Any + + from dander.concurrency import FencingToken + from dander.warehouse import RelationSchema + from dander.warehouse.runtime import PreparedWarehouseStatement + from dander.writer import SchemaEvolution, WriteTarget + + +class _SchemaMapper: + def canonical_schema(self, fields: Sequence[object]) -> RelationSchema: + del fields + return correctness.COMMON_SCHEMA + + +class _Codec: + provider_id = "bigquery" + + def render(self, relation: RelationRef) -> str: + return ".".join(relation.coordinates) + + +class _Fence: + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + return TargetFence( + fence_table=f"{target.namespace}.dander_target_commits", + target_id=".".join(target.coordinates), + authority_id=fence.resolved_authority_id, + authority_epoch=fence.authority_epoch, + pipeline_id=fence.pipeline_id, + run_id=fence.run_id, + token=fence.token, + ) + + def prepare_dml(self, statement: str, fence: TargetFence) -> PreparedWarehouseStatement: + del statement, fence + raise AssertionError("fake writer does not prepare SQL") + + +class _Writer(WritePattern): + mode = WriteMode.SCD1 + requires_publication_fence = True + + def __init__(self, rows: dict[str, dict[str, object]]) -> None: + self.rows = rows + + def write( + self, + records: Iterable[Mapping[str, object]], + target: WriteTarget, + ) -> int: + assert target.publication_fence is not None + count = 0 + for record in records: + row = dict(record) + self.rows[cast("str", row["id"])] = row + count += 1 + return count + + +class _Writers: + def __init__(self, rows: dict[str, dict[str, object]]) -> None: + self.rows = rows + + def build_ingestion_writer( + self, + *, + sandbox: bool, + batch_rows: int, + schema_evolution: SchemaEvolution, + mode: WriteMode = WriteMode.SCD1, + cursor_field: str | None = None, + snapshot_field: str | None = None, + ) -> WritePattern: + del batch_rows, schema_evolution, cursor_field, snapshot_field + assert sandbox is False + assert mode is WriteMode.SCD1 + return _Writer(self.rows) + + +def _evidence(provider: str) -> correctness.WarehouseCorrectnessEvidence: + return correctness.WarehouseCorrectnessEvidence( + schema="io.dander.conformance.warehouse-correctness/v1", + fixture_version="warehouse-common-scalar-v1", + provider=provider, + candidate_commit="a" * 40, + dander_version="test", + started_at_utc="2026-08-10T00:00:00Z", + ended_at_utc="2026-08-10T00:00:01Z", + fixture_hash="b" * 64, + canonical_schema_hash="c" * 64, + normalized_rows_hash="d" * 64, + normalized_row_count=3, + write_mode="scd1", + transport="direct" if provider in {"snowflake", "redshift"} else "copy", + replay_equal=True, + cleanup_verified=True, + approved_cost_ceiling_usd="1.00", + cost_approval_reference="review/phase5-conformance", + ) + + +def test_fixture_uses_only_the_declared_common_scalar_intersection() -> None: + supports = ( + BIGQUERY_SCHEMA_SUPPORT, + POSTGRESQL_SCHEMA_SUPPORT, + SNOWFLAKE_SCHEMA_SUPPORT, + REDSHIFT_SCHEMA_SUPPORT, + ) + + assert {field.data_type.kind for field in correctness.COMMON_SCHEMA.fields} == { + LogicalTypeKind.BOOLEAN, + LogicalTypeKind.INTEGER, + LogicalTypeKind.DECIMAL, + LogicalTypeKind.FLOAT, + LogicalTypeKind.STRING, + LogicalTypeKind.BINARY, + LogicalTypeKind.DATE, + LogicalTypeKind.TIME, + LogicalTypeKind.TIMESTAMP, + } + for support in supports: + assert support.require(correctness.COMMON_SCHEMA) is correctness.COMMON_SCHEMA + + assert ( + correctness._without_extensions( + BigQuerySchemaMapper().canonical_schema(correctness.LEGACY_BIGQUERY_FIELDS) + ) + == correctness.COMMON_SCHEMA + ) + for mapper in (PostgreSQLSchemaMapper(), SnowflakeSchemaMapper(), RedshiftSchemaMapper()): + assert ( + mapper.canonical_schema(correctness.COMMON_SCHEMA.fields) == correctness.COMMON_SCHEMA + ) + + +def test_normalization_preserves_canonical_time_binary_decimal_and_unicode_semantics() -> None: + rows = [dict(correctness._UPDATE_ROWS[0])] + rows[0]["payload"] = memoryview(b"updated") + rows[0]["label"] = "cafe\N{COMBINING ACUTE ACCENT}" + rows[0]["observed_at"] = datetime( + 2026, + 3, + 4, + 4, + 10, + 11, + 654321, + tzinfo=timezone(-timedelta(hours=5)), + ) + + normalized = correctness.normalized_rows(rows)[0] + + assert normalized["amount"] == "12.500000000" + assert normalized["payload"] == "dXBkYXRlZA==" + assert normalized["label"] == "caf\N{LATIN SMALL LETTER E WITH ACUTE}" + assert normalized["observed_at"] == "2026-03-04T09:10:11.654321Z" + assert normalized["local_at"] == "2026-03-04T09:10:11.654321" + + +def test_shared_runner_uses_runtime_contract_replays_and_cleans_without_row_evidence() -> None: + rows: dict[str, dict[str, object]] = {} + cleaned = False + schema_support = BIGQUERY_SCHEMA_SUPPORT + capabilities = WarehouseCapabilities( + provider_id="bigquery", + schema_contract_version=1, + write_modes=frozenset({WriteMode.SCD1}), + transports=frozenset({WriteTransport.LOAD_JOB}), + supports_transforms=False, + supports_graphs=False, + supports_target_fencing=True, + schema_support=schema_support, + ) + runtime = WarehouseRuntime( + provider_id="bigquery", + relation_codec=_Codec(), + schema_mapper=_SchemaMapper(), + writers=_Writers(rows), + transforms=cast("Any", SimpleNamespace()), + target_fence=_Fence(), + telemetry=cast("Any", SimpleNamespace()), + capabilities=capabilities, + ) + + def cleanup() -> None: + nonlocal cleaned + rows.clear() + cleaned = True + + session = correctness._WarehouseSession( + provider="bigquery", + runtime=runtime, + relation=RelationRef(catalog="project", namespace="raw", name="records"), + schema_inputs=correctness.COMMON_SCHEMA.fields, + transport=WriteTransport.LOAD_JOB, + read_rows=lambda: tuple(rows.values()), + cleanup=cleanup, + cleanup_verified=lambda: cleaned and not rows, + close=lambda: None, + ) + + evidence = correctness._run_session( + session, + candidate_commit="a" * 40, + cost_ceiling=correctness.ApprovedCostCeiling("1.00", "review/phase5-conformance"), + started_at_utc="2026-08-10T00:00:00Z", + ) + payload = evidence.to_json() + + assert evidence.normalized_row_count == 3 + assert evidence.replay_equal is True + assert evidence.cleanup_verified is True + assert "alpha" not in payload + assert "café" not in payload + assert "payload" not in payload + + +def test_comparison_requires_exact_four_provider_hash_and_candidate_equality() -> None: + evidence = tuple(_evidence(provider) for provider in sorted(correctness._PROVIDERS)) + + comparison = correctness.compare_evidence(evidence) + + assert comparison.providers == ("bigquery", "postgresql", "redshift", "snowflake") + assert comparison.all_rows_equal is True + assert comparison.all_cleanup_verified is True + with pytest.raises(correctness.WarehouseCorrectnessError, match="normalized_rows_hash"): + correctness.compare_evidence( + (*evidence[:-1], replace(evidence[-1], normalized_rows_hash="e" * 64)) + ) + with pytest.raises(correctness.WarehouseCorrectnessError, match="one unique"): + correctness.compare_evidence(evidence[:-1]) + + +def test_cost_ceiling_and_evidence_shape_fail_closed() -> None: + with pytest.raises(ValueError, match="nonnegative"): + correctness.ApprovedCostCeiling("-0.01", "review/phase5-conformance") + with pytest.raises(ValueError, match="approval reference"): + correctness.ApprovedCostCeiling("1.00", "not a stable reference") + + payload = json.loads(_evidence("bigquery").to_json()) + payload["rows"] = [{"id": "must-not-be-accepted"}] + with pytest.raises(ValueError, match="invalid shape"): + correctness.WarehouseCorrectnessEvidence.from_mapping(payload) + + +def test_compare_cli_writes_only_sanitized_equal_result_evidence(tmp_path: Path) -> None: + paths: list[Path] = [] + for provider in sorted(correctness._PROVIDERS): + path = tmp_path / f"{provider}.json" + path.write_text(f"{_evidence(provider).to_json()}\n", encoding="utf-8") + paths.append(path) + output = tmp_path / "comparison.json" + arguments = ["compare"] + for path in paths: + arguments.extend(("--evidence", str(path))) + arguments.extend(("--output", str(output))) + + assert correctness.main(arguments) == 0 + payload = output.read_text(encoding="utf-8") + assert '"all_rows_equal":true' in payload + assert '"providers":["bigquery","postgresql","redshift","snowflake"]' in payload + assert "alpha" not in payload + assert "café" not in payload diff --git a/tickets/DANDER-107-four-warehouse-correctness.md b/tickets/DANDER-107-four-warehouse-correctness.md new file mode 100644 index 00000000..b8f6593b --- /dev/null +++ b/tickets/DANDER-107-four-warehouse-correctness.md @@ -0,0 +1,51 @@ +--- +id: DANDER-107 +title: Gate Phase 5 on equal four-warehouse canonical rows +status: in-review +component: warehouse +epic: cloud-portability +depends_on: [DANDER-104, DANDER-105] +created: 2026-08-10 +--- + +## Context + +The re-baselined Phase 5 exit gate requires one deterministic fixture to produce equal normalized +rows across BigQuery, PostgreSQL, Snowflake, and Redshift. Existing provider qualifications prove +their individual capabilities but do not compare one shared fixture. + +## Acceptance Criteria + +- [x] One versioned fixture uses only the common canonical scalar intersection. +- [x] All provider runs use the existing registry, runtime, schema, writer, and qualification + boundaries. +- [x] Canonical normalization requires equal expected rows before and after exact replay. +- [x] Comparison requires one same-commit passing record from each of the four warehouses. +- [x] Provider-specific transports, types, fallbacks, and materializations remain separate. +- [x] Evidence excludes credentials and row data and records reviewed cost ceilings. +- [x] Every provider run attempts exact owned cleanup and requires cleanup readback. +- [x] Credential-free fixture, normalization, lifecycle, evidence, and comparison tests pass. +- [ ] Same protected-main live evidence from all four providers compares equal under reviewed + ceilings. +- [ ] Retained GCP no-drift passes after the live executions. +- [ ] Protected CI passes. + +## Design + +Keep the gate in the benchmark/qualification layer. One core fixture validates the canonical +schema, builds the selected provider's SCD1 writer, writes two waves, reads normalized results, +replays, and cleans. Thin provider sessions own readback and cleanup only. One provider evidence +record contains hashes and booleans; a separate comparison accepts exactly four records and fails +on any candidate, schema, fixture, result, count, replay, or cleanup mismatch. + +## Implementation Notes + +BigQuery uses load jobs, PostgreSQL uses COPY, and Snowflake/Redshift use their bounded direct +paths. Transport equality is intentionally not asserted. Live execution is not claimed because +this instruction supplied no renewed paid-provider ceilings; the CLI requires explicit ceiling and +approval arguments before any mutation. + +## Review Log + +Implementation review and protected CI remain required. The live comparison and post-run GCP +no-drift evidence remain an explicit Phase 5 blocker. From e676444c5764f2f2dfd52d2f2e91548b18578155 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 15:38:37 -0400 Subject: [PATCH 112/163] record four-warehouse gate CI evidence --- HANDOFF.md | 2 +- tickets/DANDER-107-four-warehouse-correctness.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index bd4bf6ca..185a3ef9 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -17,7 +17,7 @@ Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. - Full local pytest suite passes with expected opt-in skips. - Ruff lint/format and focused strict mypy checks pass. - Release metadata, wheel/sdist build, distribution inspection, and artifact inclusion pass. -- Protected CI remains required. +- Protected CI passed on PR #182 at implementation commit `f1033f66`. ## Decisions diff --git a/tickets/DANDER-107-four-warehouse-correctness.md b/tickets/DANDER-107-four-warehouse-correctness.md index b8f6593b..f876bda1 100644 --- a/tickets/DANDER-107-four-warehouse-correctness.md +++ b/tickets/DANDER-107-four-warehouse-correctness.md @@ -28,7 +28,7 @@ their individual capabilities but do not compare one shared fixture. - [ ] Same protected-main live evidence from all four providers compares equal under reviewed ceilings. - [ ] Retained GCP no-drift passes after the live executions. -- [ ] Protected CI passes. +- [x] Protected CI passes. ## Design @@ -47,5 +47,6 @@ approval arguments before any mutation. ## Review Log -Implementation review and protected CI remain required. The live comparison and post-run GCP -no-drift evidence remain an explicit Phase 5 blocker. +Implementation review and protected CI passed on PR #182 at commit +`f1033f6652fd5deaef3778436468d2ea39b31e5c`. The live comparison and post-run GCP no-drift +evidence remain an explicit Phase 5 blocker. From 5f049cc13bd3faf0d0a7f96f7806cfad2575a1d6 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 22:33:51 -0400 Subject: [PATCH 113/163] fix: harden warehouse correctness failure handling --- HANDOFF.md | 31 +- docs/ci.md | 2 +- docs/warehouse-correctness-conformance.md | 26 +- scripts/benchmarks/warehouse_correctness.py | 291 +++++++++++++++--- .../portability/test_warehouse_correctness.py | 105 +++++++ .../DANDER-107-four-warehouse-correctness.md | 17 +- 6 files changed, 404 insertions(+), 68 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 185a3ef9..767f1a9a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,10 +2,11 @@ ## Finished -- Added one deterministic SCD1 fixture over the four warehouses' common scalar intersection. -- Added canonical readback normalization and exact expected/replay equality checks. -- Added sanitized per-provider evidence and exact four-provider comparison gates. -- Added owned cleanup verification and mandatory reviewed ceiling metadata. +- Ran the bounded live fixture once per provider under the approved ceilings; PostgreSQL passed. +- Cleaned every owned provider object after BigQuery, Snowflake, and Redshift failed. +- Verified retained GCP stage-zero and platform Terraform both report exact `No changes.` +- Fixed BigQuery's metadata-query floor and expanded cleanup around fence acquisition. +- Added sanitized failed-run stage/type evidence without provider messages, SQL, credentials, or rows. ## Try It @@ -13,23 +14,23 @@ Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. ## Checks -- Credential-free conformance tests pass (6 tests). -- Full local pytest suite passes with expected opt-in skips. -- Ruff lint/format and focused strict mypy checks pass. -- Release metadata, wheel/sdist build, distribution inspection, and artifact inclusion pass. -- Protected CI passed on PR #182 at implementation commit `f1033f66`. +- Focused conformance tests pass (9 tests). +- Full pytest passes (1,200 passed, 28 expected opt-in skips). +- Repository-wide Ruff lint/format and strict mypy pass with the protected-CI dependency set. +- Protected CI for this correction is pending. ## Decisions -- Rows and credentials never enter evidence; only hashes, counts, and reviewed metadata do. -- Physical transport equality is excluded; provider-specific conformance remains separate. -- Live mutation requires a separately reviewed ceiling and stable approval reference. +- Failed evidence records only safe stage/type metadata and cannot satisfy comparison. +- The BigQuery query cap is exactly its 10 MiB minimum, still far below the approved $1 ceiling. +- No paid provider receives an automatic rerun after a failed attempt. ## Remaining -- Obtain reviewed ceilings and run all four providers on one protected-main commit. -- Verify exact cleanup and retained GCP no-drift after those live runs. -- Merge sanitized equal-result evidence, then reassess the Phase 5 gate. +- Merge the focused harness correction through protected CI. +- Obtain new explicit approvals before any BigQuery, Snowflake, or Redshift rerun. +- Run all four providers on one protected-main correction commit. +- Merge equal passing evidence and reassess the Phase 5 exit gate. ## Review First diff --git a/docs/ci.md b/docs/ci.md index e1180445..728a3ead 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -6,7 +6,7 @@ work. It runs on pull requests, pushes to `main`, and manual dispatch without an The local equivalent is: ```bash -uv sync --frozen --extra dev +uv sync --frozen --extra dev --extra postgres uv run ruff check . uv run ruff format --check . uv run mypy src tests diff --git a/docs/warehouse-correctness-conformance.md b/docs/warehouse-correctness-conformance.md index bcedeebe..03ca534f 100644 --- a/docs/warehouse-correctness-conformance.md +++ b/docs/warehouse-correctness-conformance.md @@ -73,6 +73,11 @@ approval metadata. It never contains normalized or source rows. The comparison f or duplicate provider, different candidate, unequal hash/count, failed replay, or unverified cleanup. +A failed run writes a separate sanitized failure record. It identifies only the bounded execution +stage, exception type names, cleanup attempt/result, candidate, timestamps, and reviewed ceiling. +Provider messages, SQL, coordinates, credentials, and rows remain excluded. Failed evidence is not +accepted by the four-provider comparison. + Each run owns a random target scope and cleans it in `finally`: one BigQuery table and its exact staging-name prefix, one PostgreSQL schema, one Snowflake schema, or one Redshift schema plus one S3 prefix. After all four pass, verify the retained GCP Terraform plan is no-drift and record only the @@ -80,7 +85,20 @@ sanitized evidence and reviewed ceilings. ## Current evidence status -The deterministic harness and credential-free contract tests are implemented. No renewed paid -provider execution is claimed by this document. Phase 5 remains open until all four same-commit -live records compare equal, exact cleanup is verified, retained GCP no-drift passes afterward, and -the resulting sanitized evidence is merged through protected CI. +An authorized one-attempt run was made on 2026-08-10 against protected-main commit +`4927d5f66c787c6d5da700baa06edcef8b4e4c6b`, with ceilings of BigQuery $1, PostgreSQL $0, +Snowflake $2, and Redshift $3 and no automatic paid reruns. PostgreSQL passed the fixture, replay, +and cleanup. The other three did not produce passing records, so no comparison is claimed. + +The attempt exposed a harness defect: BigQuery metadata and cleanup queries used a 10,000,000-byte +limit, below BigQuery's 10 MiB minimum. The one owned table left by that cleanup failure was deleted +and metadata confirmed zero matching tables. Snowflake's temporary user, database, warehouse, +role, and resource monitor each returned zero matches after cleanup. Redshift's disposable +workgroup, namespace, bucket, role, VPC, and Terraform state each returned zero after destroy. +Post-run retained GCP stage-zero and platform plans both reported exact `No changes.` + +This correction raises the BigQuery limit to exactly 10 MiB, puts fence acquisition inside the +owned cleanup boundary, and persists sanitized failure classification. It does not authorize a +provider rerun. Phase 5 remains open until all four same-commit live records compare equal under +newly reviewed paid-provider approvals and the resulting passing evidence is merged through +protected CI. diff --git a/scripts/benchmarks/warehouse_correctness.py b/scripts/benchmarks/warehouse_correctness.py index 7a888663..b99d998e 100644 --- a/scripts/benchmarks/warehouse_correctness.py +++ b/scripts/benchmarks/warehouse_correctness.py @@ -12,7 +12,7 @@ import unicodedata import uuid from collections.abc import Callable, Mapping, Sequence -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from datetime import UTC, date, datetime, time from decimal import Decimal, InvalidOperation from pathlib import Path @@ -49,19 +49,46 @@ def delete_objects(self, **kwargs: object) -> Mapping[str, object]: ... _EVIDENCE_SCHEMA = "io.dander.conformance.warehouse-correctness/v1" +_FAILURE_EVIDENCE_SCHEMA = "io.dander.conformance.warehouse-correctness-failure/v1" _COMPARISON_SCHEMA = "io.dander.conformance.warehouse-correctness-comparison/v1" _FIXTURE_VERSION = "warehouse-common-scalar-v1" _PROVIDERS = frozenset({"bigquery", "postgresql", "snowflake", "redshift"}) _SHA256 = re.compile(r"^[0-9a-f]{64}$") _COMMIT = re.compile(r"^[0-9a-f]{40}$") _APPROVAL_REFERENCE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/#-]{2,255}$") +_ERROR_TYPE = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,127}$") _TARGET_PREFIX = "dander_conform_" +_BIGQUERY_MAXIMUM_BYTES_BILLED = 10 * 1_024 * 1_024 +_FAILURE_STAGES = frozenset( + { + "open_session", + "schema_contract", + "fence", + "initial_write", + "update_write", + "readback", + "replay_write", + "replay_readback", + "transport", + "cleanup", + "cleanup_verification", + "close_session", + } +) class WarehouseCorrectnessError(RuntimeError): """Raised with a sanitized four-warehouse conformance failure.""" +class _WarehouseCorrectnessRunError(WarehouseCorrectnessError): + """Carry one sanitized failed-run record without provider messages or payloads.""" + + def __init__(self, evidence: WarehouseCorrectnessFailureEvidence) -> None: + super().__init__(f"{evidence.provider} warehouse correctness run failed") + self.evidence = evidence + + COMMON_SCHEMA = RelationSchema( fields=( CanonicalField( @@ -281,6 +308,51 @@ def from_mapping(cls, value: Mapping[str, object]) -> WarehouseCorrectnessEviden return cls(**cast("Any", dict(value))) +@dataclass(frozen=True, slots=True) +class WarehouseCorrectnessFailureEvidence: + """Sanitized failed-run record containing no provider messages, SQL, credentials, or rows.""" + + schema: str + fixture_version: str + provider: str + candidate_commit: str + dander_version: str + started_at_utc: str + ended_at_utc: str + stage: str + primary_error_types: tuple[str, ...] + cleanup_error_types: tuple[str, ...] + cleanup_attempted: bool + cleanup_verified: bool + approved_cost_ceiling_usd: str + cost_approval_reference: str + status: str = "failed" + + def __post_init__(self) -> None: + if self.schema != _FAILURE_EVIDENCE_SCHEMA or self.fixture_version != _FIXTURE_VERSION: + raise ValueError("warehouse correctness failure evidence uses an unknown schema") + if self.provider not in _PROVIDERS: + raise ValueError("warehouse correctness failure evidence names an unknown provider") + if not _COMMIT.fullmatch(self.candidate_commit): + raise ValueError("warehouse correctness failure evidence requires a full commit SHA") + if self.stage not in _FAILURE_STAGES: + raise ValueError("warehouse correctness failure evidence has an unknown stage") + error_types = (*self.primary_error_types, *self.cleanup_error_types) + if not error_types or any(not _ERROR_TYPE.fullmatch(value) for value in error_types): + raise ValueError("warehouse correctness failure evidence has invalid error types") + if self.cleanup_verified and not self.cleanup_attempted: + raise ValueError("warehouse correctness cleanup cannot pass when it was not attempted") + if self.status != "failed": + raise ValueError("warehouse correctness failure evidence must be failed") + ApprovedCostCeiling( + self.approved_cost_ceiling_usd, + self.cost_approval_reference, + ) + + def to_json(self) -> str: + return json.dumps(asdict(self), separators=(",", ":"), sort_keys=True) + + @dataclass(frozen=True, slots=True) class WarehouseCorrectnessComparison: """Sanitized equality gate across exactly four provider evidence records.""" @@ -375,8 +447,25 @@ def run_provider( provider = profile.get("provider") if provider not in _PROVIDERS: raise ValueError("profile must select one of the four conformance warehouses") - session = _open_session(profile) started = _now() + try: + session = _open_session(profile) + except Exception as error: + failure = _failure_evidence( + provider=provider, + candidate_commit=candidate_commit, + cost_ceiling=cost_ceiling, + started_at_utc=started, + stage="open_session", + primary_errors=(error,), + cleanup_errors=(), + cleanup_attempted=False, + cleanup_verified=False, + ) + raise _WarehouseCorrectnessRunError(failure) from error + + evidence: WarehouseCorrectnessEvidence | None = None + run_failure: _WarehouseCorrectnessRunError | None = None try: evidence = _run_session( session, @@ -384,8 +473,36 @@ def run_provider( cost_ceiling=cost_ceiling, started_at_utc=started, ) - finally: + except _WarehouseCorrectnessRunError as error: + run_failure = error + try: session.close() + except Exception as error: + if run_failure is None: + failure = _failure_evidence( + provider=provider, + candidate_commit=candidate_commit, + cost_ceiling=cost_ceiling, + started_at_utc=started, + stage="close_session", + primary_errors=(error,), + cleanup_errors=(), + cleanup_attempted=evidence is not None, + cleanup_verified=evidence is not None and evidence.cleanup_verified, + ) + else: + failure = replace( + run_failure.evidence, + cleanup_error_types=_merge_error_types( + run_failure.evidence.cleanup_error_types, + _exception_types((error,)), + ), + ended_at_utc=_now(), + ) + raise _WarehouseCorrectnessRunError(failure) from error + if run_failure is not None: + raise run_failure + assert evidence is not None return evidence @@ -397,56 +514,64 @@ def _run_session( started_at_utc: str, ) -> WarehouseCorrectnessEvidence: runtime = session.runtime - runtime.capabilities.schema_support.require(COMMON_SCHEMA) - mapped = runtime.schema_mapper.canonical_schema(session.schema_inputs) - if _without_extensions(mapped) != COMMON_SCHEMA: - raise WarehouseCorrectnessError( - f"{session.provider} schema mapper changed the common canonical fixture" - ) - writer = runtime.writers.build_ingestion_writer( - sandbox=False, - batch_rows=len(_INITIAL_ROWS), - schema_evolution=SchemaEvolution.STRICT, - mode=WriteMode.SCD1, - ) - publication = None - if writer.requires_publication_fence: - publication = runtime.target_fence.claim( - session.relation, - FencingToken( - lease_table=None, - pipeline_id="warehouse_correctness", - run_id="fixture-v1", - token=1, - authority_id=f"{session.provider}:warehouse-correctness", - ), - ) - target = WriteTarget( - relation=session.relation, - business_key=("id",), - schema=LEGACY_BIGQUERY_FIELDS, - declared_schema=COMMON_SCHEMA, - publication_fence=publication, - ) operations: list[OperationTelemetry] = [] cleanup_error: Exception | None = None primary_error: Exception | None = None + primary_stage = "schema_contract" before_replay: tuple[dict[str, object], ...] = () after_replay: tuple[dict[str, object], ...] = () try: + runtime.capabilities.schema_support.require(COMMON_SCHEMA) + mapped = runtime.schema_mapper.canonical_schema(session.schema_inputs) + if _without_extensions(mapped) != COMMON_SCHEMA: + raise WarehouseCorrectnessError( + f"{session.provider} schema mapper changed the common canonical fixture" + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=len(_INITIAL_ROWS), + schema_evolution=SchemaEvolution.STRICT, + mode=WriteMode.SCD1, + ) + primary_stage = "fence" + publication = None + if writer.requires_publication_fence: + publication = runtime.target_fence.claim( + session.relation, + FencingToken( + lease_table=None, + pipeline_id="warehouse_correctness", + run_id="fixture-v1", + token=1, + authority_id=f"{session.provider}:warehouse-correctness", + ), + ) + target = WriteTarget( + relation=session.relation, + business_key=("id",), + schema=LEGACY_BIGQUERY_FIELDS, + declared_schema=COMMON_SCHEMA, + publication_fence=publication, + ) + primary_stage = "initial_write" writer.write(_INITIAL_ROWS, target) operations.extend(writer.drain_telemetry()) + primary_stage = "update_write" writer.write(_UPDATE_ROWS, target) operations.extend(writer.drain_telemetry()) + primary_stage = "readback" before_replay = normalized_rows(session.read_rows()) + primary_stage = "replay_write" writer.write(_UPDATE_ROWS, target) operations.extend(writer.drain_telemetry()) + primary_stage = "replay_readback" after_replay = normalized_rows(session.read_rows()) expected = normalized_rows(_EXPECTED_ROWS) if before_replay != expected or after_replay != expected: raise WarehouseCorrectnessError( f"{session.provider} normalized rows differ from the common fixture" ) + primary_stage = "transport" _require_transport(session, operations) except Exception as error: primary_error = error @@ -463,14 +588,37 @@ def _run_session( except Exception as error: cleanup_error = error if primary_error is not None or cleanup_error is not None: - errors = [error for error in (primary_error, cleanup_error) if error is not None] - raise WarehouseCorrectnessError( - f"{session.provider} correctness run failed; owned cleanup was attempted" - ) from ExceptionGroup("warehouse correctness failures", errors) + primary_errors = (primary_error,) if primary_error is not None else () + cleanup_errors = (cleanup_error,) if cleanup_error is not None else () + stage = primary_stage if primary_error is not None else "cleanup" + failure = _failure_evidence( + provider=session.provider, + candidate_commit=candidate_commit, + cost_ceiling=cost_ceiling, + started_at_utc=started_at_utc, + stage=stage, + primary_errors=primary_errors, + cleanup_errors=cleanup_errors, + cleanup_attempted=True, + cleanup_verified=cleaned, + ) + errors = [*primary_errors, *cleanup_errors] + raise _WarehouseCorrectnessRunError(failure) from ExceptionGroup( + "warehouse correctness failures", errors + ) if not cleaned: - raise WarehouseCorrectnessError( - f"{session.provider} correctness cleanup could not be verified" + failure = _failure_evidence( + provider=session.provider, + candidate_commit=candidate_commit, + cost_ceiling=cost_ceiling, + started_at_utc=started_at_utc, + stage="cleanup_verification", + primary_errors=(WarehouseCorrectnessError("cleanup verification returned false"),), + cleanup_errors=(), + cleanup_attempted=True, + cleanup_verified=False, ) + raise _WarehouseCorrectnessRunError(failure) fixture_hash = _fixture_hash() schema_hash = _hash_json(COMMON_SCHEMA.model_dump(mode="json", by_alias=True)) @@ -496,6 +644,61 @@ def _run_session( ) +def _failure_evidence( + *, + provider: str, + candidate_commit: str, + cost_ceiling: ApprovedCostCeiling, + started_at_utc: str, + stage: str, + primary_errors: Sequence[BaseException], + cleanup_errors: Sequence[BaseException], + cleanup_attempted: bool, + cleanup_verified: bool, +) -> WarehouseCorrectnessFailureEvidence: + return WarehouseCorrectnessFailureEvidence( + schema=_FAILURE_EVIDENCE_SCHEMA, + fixture_version=_FIXTURE_VERSION, + provider=provider, + candidate_commit=candidate_commit, + dander_version=__version__, + started_at_utc=started_at_utc, + ended_at_utc=_now(), + stage=stage, + primary_error_types=_exception_types(primary_errors), + cleanup_error_types=_exception_types(cleanup_errors), + cleanup_attempted=cleanup_attempted, + cleanup_verified=cleanup_verified, + approved_cost_ceiling_usd=cost_ceiling.usd, + cost_approval_reference=cost_ceiling.approval_reference, + ) + + +def _exception_types(errors: Sequence[BaseException]) -> tuple[str, ...]: + names: set[str] = set() + pending = list(errors) + seen: set[int] = set() + while pending: + error = pending.pop() + if id(error) in seen: + continue + seen.add(id(error)) + name = type(error).__name__ + if name not in {"ExceptionGroup", "_WarehouseCorrectnessRunError"}: + names.add(name if _ERROR_TYPE.fullmatch(name) else "Exception") + if isinstance(error, BaseExceptionGroup): + pending.extend(error.exceptions) + if error.__cause__ is not None: + pending.append(error.__cause__) + elif error.__context__ is not None: + pending.append(error.__context__) + return tuple(sorted(names)) + + +def _merge_error_types(*groups: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({value for group in groups for value in group})) + + def _open_session(profile: Mapping[str, object]) -> _WarehouseSession: provider = profile["provider"] if provider == "bigquery": @@ -542,7 +745,7 @@ def read_rows() -> Sequence[Mapping[str, object]]: fields = ", ".join(f"`{field.name}`" for field in COMMON_SCHEMA.fields) query = f"SELECT {fields} FROM `{table_id}` ORDER BY `id`" job_config = bigquery.QueryJobConfig( - maximum_bytes_billed=10_000_000, + maximum_bytes_billed=_BIGQUERY_MAXIMUM_BYTES_BILLED, labels={"dander-proof": "warehouse-correctness"}, ) return tuple(dict(row) for row in client.query(query, job_config=job_config).result()) @@ -553,7 +756,7 @@ def owned_tables() -> tuple[str, ...]: "WHERE table_name = @target OR STARTS_WITH(table_name, @stage_prefix)" ) config = bigquery.QueryJobConfig( - maximum_bytes_billed=10_000_000, + maximum_bytes_billed=_BIGQUERY_MAXIMUM_BYTES_BILLED, query_parameters=[ bigquery.ScalarQueryParameter("target", "STRING", relation.name), bigquery.ScalarQueryParameter("stage_prefix", "STRING", stage_prefix), @@ -1050,6 +1253,10 @@ def main(argv: Sequence[str] | None = None) -> int: cost_ceiling=ceiling, ) _write_json(arguments.output, provider_evidence.to_json()) + except _WarehouseCorrectnessRunError as error: + _write_json(arguments.output, error.evidence.to_json()) + print(error.evidence.to_json()) + return 1 except Exception: provider = profile.get("provider") print( diff --git a/tests/portability/test_warehouse_correctness.py b/tests/portability/test_warehouse_correctness.py index e8c4a591..dc588c4a 100644 --- a/tests/portability/test_warehouse_correctness.py +++ b/tests/portability/test_warehouse_correctness.py @@ -243,6 +243,71 @@ def cleanup() -> None: assert "payload" not in payload +def test_shared_runner_cleans_when_publication_fence_fails() -> None: + rows: dict[str, dict[str, object]] = {} + cleaned = False + + class _FailingFence(_Fence): + def claim(self, target: RelationRef, fence: FencingToken) -> TargetFence: + del target, fence + raise RuntimeError("provider detail must not enter failure evidence") + + runtime = WarehouseRuntime( + provider_id="bigquery", + relation_codec=_Codec(), + schema_mapper=_SchemaMapper(), + writers=_Writers(rows), + transforms=cast("Any", SimpleNamespace()), + target_fence=_FailingFence(), + telemetry=cast("Any", SimpleNamespace()), + capabilities=WarehouseCapabilities( + provider_id="bigquery", + schema_contract_version=1, + write_modes=frozenset({WriteMode.SCD1}), + transports=frozenset({WriteTransport.LOAD_JOB}), + supports_transforms=False, + supports_graphs=False, + supports_target_fencing=True, + schema_support=BIGQUERY_SCHEMA_SUPPORT, + ), + ) + + def cleanup() -> None: + nonlocal cleaned + cleaned = True + + session = correctness._WarehouseSession( + provider="bigquery", + runtime=runtime, + relation=RelationRef(catalog="project", namespace="raw", name="records"), + schema_inputs=correctness.COMMON_SCHEMA.fields, + transport=WriteTransport.LOAD_JOB, + read_rows=lambda: (), + cleanup=cleanup, + cleanup_verified=lambda: cleaned, + close=lambda: None, + ) + + with pytest.raises(correctness._WarehouseCorrectnessRunError) as captured: + correctness._run_session( + session, + candidate_commit="a" * 40, + cost_ceiling=correctness.ApprovedCostCeiling("1.00", "review/phase5-conformance"), + started_at_utc="2026-08-10T00:00:00Z", + ) + + evidence = captured.value.evidence + assert evidence.stage == "fence" + assert evidence.primary_error_types == ("RuntimeError",) + assert evidence.cleanup_attempted is True + assert evidence.cleanup_verified is True + assert "provider detail" not in evidence.to_json() + + +def test_bigquery_metadata_queries_allow_the_minimum_billable_partition() -> None: + assert correctness._BIGQUERY_MAXIMUM_BYTES_BILLED == 10 * 1_024 * 1_024 + + def test_comparison_requires_exact_four_provider_hash_and_candidate_equality() -> None: evidence = tuple(_evidence(provider) for provider in sorted(correctness._PROVIDERS)) @@ -289,3 +354,43 @@ def test_compare_cli_writes_only_sanitized_equal_result_evidence(tmp_path: Path) assert '"providers":["bigquery","postgresql","redshift","snowflake"]' in payload assert "alpha" not in payload assert "café" not in payload + + +def test_run_cli_writes_sanitized_failure_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + profile = tmp_path / "profile.json" + profile.write_text('{"provider":"bigquery"}\n', encoding="utf-8") + output = tmp_path / "failure.json" + + def fail_open(_profile: Mapping[str, object]) -> correctness._WarehouseSession: + raise RuntimeError("credential-and-provider-detail-must-not-escape") + + monkeypatch.setattr(correctness, "_open_session", fail_open) + + assert ( + correctness.main( + [ + "run", + "--profile-json", + str(profile), + "--candidate-commit", + "a" * 40, + "--approved-cost-ceiling-usd", + "1.00", + "--cost-approval-reference", + "review/phase5-conformance", + "--output", + str(output), + ] + ) + == 1 + ) + payload = output.read_text(encoding="utf-8") + record = json.loads(payload) + assert record["schema"] == "io.dander.conformance.warehouse-correctness-failure/v1" + assert record["stage"] == "open_session" + assert record["primary_error_types"] == ["RuntimeError"] + assert record["cleanup_attempted"] is False + assert "credential" not in payload diff --git a/tickets/DANDER-107-four-warehouse-correctness.md b/tickets/DANDER-107-four-warehouse-correctness.md index f876bda1..8f06dddc 100644 --- a/tickets/DANDER-107-four-warehouse-correctness.md +++ b/tickets/DANDER-107-four-warehouse-correctness.md @@ -27,7 +27,7 @@ their individual capabilities but do not compare one shared fixture. - [x] Credential-free fixture, normalization, lifecycle, evidence, and comparison tests pass. - [ ] Same protected-main live evidence from all four providers compares equal under reviewed ceilings. -- [ ] Retained GCP no-drift passes after the live executions. +- [x] Retained GCP no-drift passes after the live executions. - [x] Protected CI passes. ## Design @@ -41,12 +41,17 @@ on any candidate, schema, fixture, result, count, replay, or cleanup mismatch. ## Implementation Notes BigQuery uses load jobs, PostgreSQL uses COPY, and Snowflake/Redshift use their bounded direct -paths. Transport equality is intentionally not asserted. Live execution is not claimed because -this instruction supplied no renewed paid-provider ceilings; the CLI requires explicit ceiling and -approval arguments before any mutation. +paths. Transport equality is intentionally not asserted. The CLI requires explicit ceiling and +approval arguments before any mutation. Failed runs persist only sanitized stage, exception-type, +cleanup, candidate, timestamp, and ceiling metadata; they cannot satisfy comparison. ## Review Log Implementation review and protected CI passed on PR #182 at commit -`f1033f6652fd5deaef3778436468d2ea39b31e5c`. The live comparison and post-run GCP no-drift -evidence remain an explicit Phase 5 blocker. +`f1033f6652fd5deaef3778436468d2ea39b31e5c`. An authorized one-attempt run on protected-main +commit `4927d5f66c787c6d5da700baa06edcef8b4e4c6b` produced one passing PostgreSQL record and three +failed records, so no four-provider equality is claimed and paid providers were not rerun. +Provider-owned cleanup was completed and verified, and retained GCP stage-zero and platform plans +both reported exact `No changes.` The correction prompted by that attempt fixes BigQuery's 10 MiB +metadata-query minimum, expands cleanup around fence acquisition, and adds sanitized failure +evidence. Equal same-commit live evidence remains the explicit Phase 5 blocker. From 53d28ec79851122d9539c4b514aa726b2a5d5699 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Mon, 10 Aug 2026 22:37:24 -0400 Subject: [PATCH 114/163] docs: record correction CI evidence --- HANDOFF.md | 2 +- tickets/DANDER-107-four-warehouse-correctness.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 767f1a9a..3d16f06d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -17,7 +17,7 @@ Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. - Focused conformance tests pass (9 tests). - Full pytest passes (1,200 passed, 28 expected opt-in skips). - Repository-wide Ruff lint/format and strict mypy pass with the protected-CI dependency set. -- Protected CI for this correction is pending. +- Protected CI passed all five checks on PR #183 for the implementation commit. ## Decisions diff --git a/tickets/DANDER-107-four-warehouse-correctness.md b/tickets/DANDER-107-four-warehouse-correctness.md index 8f06dddc..8d79d44f 100644 --- a/tickets/DANDER-107-four-warehouse-correctness.md +++ b/tickets/DANDER-107-four-warehouse-correctness.md @@ -54,4 +54,5 @@ failed records, so no four-provider equality is claimed and paid providers were Provider-owned cleanup was completed and verified, and retained GCP stage-zero and platform plans both reported exact `No changes.` The correction prompted by that attempt fixes BigQuery's 10 MiB metadata-query minimum, expands cleanup around fence acquisition, and adds sanitized failure -evidence. Equal same-commit live evidence remains the explicit Phase 5 blocker. +evidence. Its implementation commit passed all five protected checks on PR #183. Equal +same-commit live evidence remains the explicit Phase 5 blocker. From abfd11974ec0d959cbff94bcbb439ce3fdfd651c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 05:49:16 -0400 Subject: [PATCH 115/163] Encode BigQuery bytes for JSON loads --- HANDOFF.md | 39 ++++++++++++++-------------- src/dander/writer/bigquery.py | 7 +++++ tests/writer/test_bigquery_writer.py | 3 +++ 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 3d16f06d..ba1f1179 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,37 @@ ## Finished -- Ran the bounded live fixture once per provider under the approved ceilings; PostgreSQL passed. -- Cleaned every owned provider object after BigQuery, Snowflake, and Redshift failed. -- Verified retained GCP stage-zero and platform Terraform both report exact `No changes.` -- Fixed BigQuery's metadata-query floor and expanded cleanup around fence acquisition. -- Added sanitized failed-run stage/type evidence without provider messages, SQL, credentials, or rows. +- Re-ran the four-warehouse gate from protected-main commit `4270643` under renewed ceilings. +- Confirmed PostgreSQL passes the full fixture, replay, canonical hash, and owned cleanup. +- Proved BigQuery cleanup succeeds after its first live write fails on raw Python `bytes`. +- Added BigQuery JSON-load encoding for canonical binary values. +- Added focused regression coverage for the `BYTES` projection. ## Try It -Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. +Run `.venv/bin/pytest -q tests/writer/test_bigquery_writer.py`. ## Checks -- Focused conformance tests pass (9 tests). -- Full pytest passes (1,200 passed, 28 expected opt-in skips). -- Repository-wide Ruff lint/format and strict mypy pass with the protected-CI dependency set. -- Protected CI passed all five checks on PR #183 for the implementation commit. +- BigQuery writer plus warehouse-correctness tests pass (44 tests). +- Repository-wide Ruff lint/format, strict mypy, and full pytest pass in the protected-CI dependency set. +- Protected CI pending. ## Decisions -- Failed evidence records only safe stage/type metadata and cannot satisfy comparison. -- The BigQuery query cap is exactly its 10 MiB minimum, still far below the approved $1 ceiling. -- No paid provider receives an automatic rerun after a failed attempt. +- BigQuery `BYTES` values use the base64 string representation required by its JSON load API. +- Passing evidence must move to the corrected protected-main commit; relabeling modified code as + commit `4270643` would be invalid evidence. +- Existing Snowflake and Redshift cost guards remain active while protected CI runs. ## Remaining -- Merge the focused harness correction through protected CI. -- Obtain new explicit approvals before any BigQuery, Snowflake, or Redshift rerun. -- Run all four providers on one protected-main correction commit. -- Merge equal passing evidence and reassess the Phase 5 exit gate. +- Merge this focused BigQuery correction through protected CI. +- Restart all four provider records on the resulting protected-main commit. +- Compare equal evidence, verify cleanup and GCP no-drift, then merge sanitized proof. +- Reassess the revised Phase 5 exit gate without beginning Phase 6. ## Review First -- `scripts/benchmarks/warehouse_correctness.py` -- `tests/portability/test_warehouse_correctness.py` -- `docs/warehouse-correctness-conformance.md` +- `src/dander/writer/bigquery.py` +- `tests/writer/test_bigquery_writer.py` diff --git a/src/dander/writer/bigquery.py b/src/dander/writer/bigquery.py index 9d0d354e..8bc06ba0 100644 --- a/src/dander/writer/bigquery.py +++ b/src/dander/writer/bigquery.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import re from collections.abc import Mapping from datetime import UTC, date, datetime, time, timedelta @@ -842,6 +843,12 @@ def _json_load_value(value: object) -> object: """Encode validated Python scalars for BigQuery's JSON load client.""" if isinstance(value, Decimal): return format(value, "f") + if isinstance(value, memoryview): + value = value.tobytes() + if isinstance(value, bytearray): + value = bytes(value) + if isinstance(value, bytes): + return base64.b64encode(value).decode("ascii") if isinstance(value, (datetime, date, time)): return value.isoformat() if isinstance(value, Mapping): diff --git a/tests/writer/test_bigquery_writer.py b/tests/writer/test_bigquery_writer.py index 8ea3ffcd..0f932c3e 100644 --- a/tests/writer/test_bigquery_writer.py +++ b/tests/writer/test_bigquery_writer.py @@ -236,6 +236,7 @@ def test_scd1_encodes_typed_scalars_for_bigquery_json_load() -> None: schema=( WriteField(name="id", data_type="STRING", mode="REQUIRED"), WriteField(name="amount", data_type="NUMERIC"), + WriteField(name="payload", data_type="BYTES"), WriteField(name="observed_at", data_type="TIMESTAMP"), WriteField( name="details", @@ -254,6 +255,7 @@ def test_scd1_encodes_typed_scalars_for_bigquery_json_load() -> None: { "id": "one", "amount": Decimal("125.50"), + "payload": b"\x00\xff", "observed_at": datetime(2026, 8, 3, 2, 30, tzinfo=UTC), "details": { "amount": Decimal("0.000000001"), @@ -269,6 +271,7 @@ def test_scd1_encodes_typed_scalars_for_bigquery_json_load() -> None: { "id": "one", "amount": "125.50", + "payload": "AP8=", "observed_at": "2026-08-03T02:30:00+00:00", "details": { "amount": "0.000000001", From bdc83cf45c4e065a93e6ba6184ef1c8aba30868d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 06:05:35 -0400 Subject: [PATCH 116/163] Fix Redshift binary conformance --- HANDOFF.md | 42 ++++++++------- docs/warehouse-correctness-conformance.md | 8 ++- scripts/benchmarks/warehouse_correctness.py | 39 +++++++++++++- src/dander/providers/redshift/writer.py | 5 +- .../portability/test_warehouse_correctness.py | 25 +++++++++ .../test_redshift_warehouse_runtime.py | 52 ++++++++++++++++++- 6 files changed, 146 insertions(+), 25 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index ba1f1179..7aa8340e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,41 @@ ## Finished -- Re-ran the four-warehouse gate from protected-main commit `4270643` under renewed ceilings. -- Confirmed PostgreSQL passes the full fixture, replay, canonical hash, and owned cleanup. -- Proved BigQuery cleanup succeeds after its first live write fails on raw Python `bytes`. -- Added BigQuery JSON-load encoding for canonical binary values. -- Added focused regression coverage for the `BYTES` projection. +- Corrected Redshift direct-load binary serialization at the driver boundary. +- Projected Redshift `VARBYTE` as base64 text and strictly decoded it to bytes before normalization. +- Preserved the shared canonical schema, expected hashes, transport selection, and fencing behavior. +- Added credential-free direct-load, projection, decode, malformed-value, and wrong-type coverage. +- Corrected the documented CLI invocation so Snowflake's package cannot be shadowed by the script path. ## Try It -Run `.venv/bin/pytest -q tests/writer/test_bigquery_writer.py`. +Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. ## Checks -- BigQuery writer plus warehouse-correctness tests pass (44 tests). -- Repository-wide Ruff lint/format, strict mypy, and full pytest pass in the protected-CI dependency set. -- Protected CI pending. +- Warehouse-correctness contract tests pass (10 tests). +- Redshift runtime plus warehouse-correctness tests pass (64 tests). +- Focused Ruff lint/format and mypy pass. +- Protected-CI dependency profile passes Ruff, mypy, and full pytest (1,202 passed, 28 skipped). +- Bounded live Redshift validation passes the expected hash, replay, and owned cleanup. ## Decisions -- BigQuery `BYTES` values use the base64 string representation required by its JSON load API. -- Passing evidence must move to the corrected protected-main commit; relabeling modified code as - commit `4270643` would be invalid evidence. -- Existing Snowflake and Redshift cost guards remain active while protected CI runs. +- The Redshift driver sends byte parameters as hex text and decodes raw `VARBYTE` as UTF-8, so both + conversions remain at its direct-load or live-qualification boundaries. +- Base64 is validated strictly and converted back to bytes so canonical normalization remains + provider-neutral. +- No schema, transport selection, fencing, or non-binary behavior changes are included. ## Remaining -- Merge this focused BigQuery correction through protected CI. -- Restart all four provider records on the resulting protected-main commit. -- Compare equal evidence, verify cleanup and GCP no-drift, then merge sanitized proof. -- Reassess the revised Phase 5 exit gate without beginning Phase 6. +- Complete protected CI and merge the focused correction. +- Re-run all four providers on the resulting protected-main commit and compare equal evidence. +- Verify provider cleanup and retained GCP no-drift, then merge sanitized evidence. +- Reassess the revised Phase 5 gate without beginning Phase 6. ## Review First -- `src/dander/writer/bigquery.py` -- `tests/writer/test_bigquery_writer.py` +- `scripts/benchmarks/warehouse_correctness.py` +- `tests/portability/test_warehouse_correctness.py` +- `docs/warehouse-correctness-conformance.md` diff --git a/docs/warehouse-correctness-conformance.md b/docs/warehouse-correctness-conformance.md index 03ca534f..28e9a0af 100644 --- a/docs/warehouse-correctness-conformance.md +++ b/docs/warehouse-correctness-conformance.md @@ -20,6 +20,10 @@ rows before and after replay. Physical transports are deliberately not compared. BigQuery uses its load-job SCD1 adapter, PostgreSQL uses COPY, and the bounded Snowflake and Redshift profiles force their direct paths. +The current Redshift Python driver sends byte parameters as hex text, so the direct path explicitly +decodes binary placeholders with `TO_VARBYTE`; for readback, the harness projects `VARBYTE` as +base64 text and strictly decodes it back to bytes before canonical normalization because the same +driver otherwise treats raw binary as UTF-8 text. Provider-specific types, JSON fallbacks, staged transports, other write modes, materializations, and fencing behavior retain their separate adapter/qualification tests. @@ -48,7 +52,7 @@ The recorded ceiling is evidence, not a hard provider spending cap. Run each profile against the same full protected-main commit: ```bash -uv run python scripts/benchmarks/warehouse_correctness.py run \ +uv run python -m scripts.benchmarks.warehouse_correctness run \ --profile-json /secure/bigquery-profile.json \ --candidate-commit COMMIT_SHA \ --approved-cost-ceiling-usd 1.00 \ @@ -59,7 +63,7 @@ uv run python scripts/benchmarks/warehouse_correctness.py run \ Repeat for PostgreSQL, Snowflake, and Redshift, then compare exactly four records: ```bash -uv run python scripts/benchmarks/warehouse_correctness.py compare \ +uv run python -m scripts.benchmarks.warehouse_correctness compare \ --evidence /secure/bigquery-evidence.json \ --evidence /secure/postgresql-evidence.json \ --evidence /secure/snowflake-evidence.json \ diff --git a/scripts/benchmarks/warehouse_correctness.py b/scripts/benchmarks/warehouse_correctness.py index b99d998e..cb8d43ec 100644 --- a/scripts/benchmarks/warehouse_correctness.py +++ b/scripts/benchmarks/warehouse_correctness.py @@ -5,6 +5,7 @@ import argparse import base64 +import binascii import hashlib import json import math @@ -955,7 +956,7 @@ def _open_redshift(profile: Mapping[str, object]) -> _WarehouseSession: relation = RelationRef(catalog=database, namespace=schema_name, name="records") def read_rows() -> Sequence[Mapping[str, object]]: - columns = ", ".join(_quote(field.name) for field in COMMON_SCHEMA.fields) + columns = _redshift_read_projection(COMMON_SCHEMA) with open_connection(fence.connection_factory) as connection: rows = execute( connection, @@ -963,7 +964,7 @@ def read_rows() -> Sequence[Mapping[str, object]]: f"ORDER BY {_quote('id')}", fetch="all", ).rows - return _rows_from_sequences(rows) + return _decode_redshift_read_rows(rows) def owned_keys() -> tuple[str, ...]: keys: list[str] = [] @@ -1030,6 +1031,40 @@ def cleanup_verified() -> bool: ) +def _redshift_read_projection(schema: RelationSchema) -> str: + return ", ".join( + ( + f"FROM_VARBYTE({_quote(field.name)}, 'base64') AS {_quote(field.name)}" + if field.data_type.kind is LogicalTypeKind.BINARY + else _quote(field.name) + ) + for field in schema.fields + ) + + +def _decode_redshift_read_rows(rows: object) -> tuple[dict[str, object], ...]: + decoded_rows = _rows_from_sequences(rows) + binary_names = tuple( + field.name + for field in COMMON_SCHEMA.fields + if field.data_type.kind is LogicalTypeKind.BINARY + ) + for row in decoded_rows: + for name in binary_names: + value = row[name] + if value is None: + continue + if not isinstance(value, str): + raise WarehouseCorrectnessError("Redshift binary readback has the wrong type") + try: + row[name] = base64.b64decode(value, validate=True) + except (binascii.Error, ValueError) as error: + raise WarehouseCorrectnessError( + "Redshift binary readback is not valid base64" + ) from error + return decoded_rows + + def _normalize_row(row: Mapping[str, object], schema: RelationSchema) -> dict[str, object]: return { field.name: _normalize_scalar(row[field.name], field.data_type) for field in schema.fields diff --git a/src/dander/providers/redshift/writer.py b/src/dander/providers/redshift/writer.py index 5b53ad6d..f45e6d79 100644 --- a/src/dander/providers/redshift/writer.py +++ b/src/dander/providers/redshift/writer.py @@ -557,7 +557,10 @@ def _direct_insert( rows: tuple[dict[str, object], ...], ) -> tuple[str, tuple[tuple[object, ...], ...]]: columns = ", ".join(_quote(field.name) for field in schema.fields) - placeholders = ", ".join("%s" for _field in schema.fields) + placeholders = ", ".join( + "TO_VARBYTE(%s, 'hex')" if field.data_type.kind is LogicalTypeKind.BINARY else "%s" + for field in schema.fields + ) statement = f"INSERT INTO {_quote(temporary)} ({columns}) VALUES ({placeholders})" parameters = tuple(tuple(row[field.name] for field in schema.fields) for row in rows) return statement, parameters diff --git a/tests/portability/test_warehouse_correctness.py b/tests/portability/test_warehouse_correctness.py index dc588c4a..921f24bd 100644 --- a/tests/portability/test_warehouse_correctness.py +++ b/tests/portability/test_warehouse_correctness.py @@ -308,6 +308,31 @@ def test_bigquery_metadata_queries_allow_the_minimum_billable_partition() -> Non assert correctness._BIGQUERY_MAXIMUM_BYTES_BILLED == 10 * 1_024 * 1_024 +def test_redshift_readback_projects_binary_through_strict_base64() -> None: + projection = correctness._redshift_read_projection(correctness.COMMON_SCHEMA) + values: list[object] = [None] * len(correctness.COMMON_SCHEMA.fields) + payload_index = next( + index + for index, field in enumerate(correctness.COMMON_SCHEMA.fields) + if field.name == "payload" + ) + values[payload_index] = "AP8=" + + rows = correctness._decode_redshift_read_rows((tuple(values),)) + + assert 'FROM_VARBYTE("payload", \'base64\') AS "payload"' in projection + assert projection.count("FROM_VARBYTE(") == 1 + assert rows[0]["payload"] == b"\x00\xff" + + values[payload_index] = "not-base64!" + with pytest.raises(correctness.WarehouseCorrectnessError, match="not valid base64"): + correctness._decode_redshift_read_rows((tuple(values),)) + + values[payload_index] = b"already-bytes" + with pytest.raises(correctness.WarehouseCorrectnessError, match="wrong type"): + correctness._decode_redshift_read_rows((tuple(values),)) + + def test_comparison_requires_exact_four_provider_hash_and_candidate_equality() -> None: evidence = tuple(_evidence(provider) for provider in sorted(correctness._PROVIDERS)) diff --git a/tests/providers/test_redshift_warehouse_runtime.py b/tests/providers/test_redshift_warehouse_runtime.py index f28ca67a..d5110801 100644 --- a/tests/providers/test_redshift_warehouse_runtime.py +++ b/tests/providers/test_redshift_warehouse_runtime.py @@ -1225,7 +1225,7 @@ def test_redshift_direct_load_parses_explicit_super_without_s3_or_query_attribut for _, statement, _ in backend.statements if statement.startswith("CREATE TEMP TABLE") ) - assert direct_statement.endswith("VALUES (%s, %s, %s)") + assert direct_statement.endswith("VALUES (%s, TO_VARBYTE(%s, 'hex'), %s)") assert direct_parameters == (("one", b'{"ready":true}', 0),) assert 'JSON_PARSE(staged."payload") AS "payload"' in next( statement for _, statement, _ in backend.statements if statement.startswith("MERGE INTO") @@ -1250,6 +1250,56 @@ def test_redshift_direct_load_parses_explicit_super_without_s3_or_query_attribut assert not tuple(tmp_path.iterdir()) +def test_redshift_direct_load_decodes_driver_hex_for_canonical_binary(tmp_path: Path) -> None: + runtime, backend, s3 = _direct_redshift_runtime(tmp_path) + backend.schema_rows = [ + ("id", "character varying", 65_535, None, None, "NO"), + ("payload", "binary varying", None, None, None, "YES"), + ] + relation = RelationRef(catalog="analytics", namespace="raw", name="binary_records") + publication = runtime.target_fence.claim( + relation, + FencingToken( + lease_table=None, + pipeline_id="redshift_binary_records", + run_id="run-binary", + token=8, + authority_id="postgresql:test-state", + ), + ) + writer = runtime.writers.build_ingestion_writer( + sandbox=False, + batch_rows=1, + schema_evolution=SchemaEvolution.ADDITIVE, + ) + target = WriteTarget( + relation=relation, + business_key=("id",), + schema=( + WriteField(name="id", data_type="STRING"), + WriteField(name="payload", data_type="BYTES"), + ), + publication_fence=publication, + ) + + assert writer.write(({"id": "one", "payload": b"\x00\xff"},), target) == 1 + + direct_statement, direct_parameters = next( + (statement, parameters) + for _, statement, parameters in backend.statements + if statement.startswith('INSERT INTO "dander_stage_') + and "dander_stage_loads" not in statement + ) + assert direct_statement.endswith("VALUES (%s, TO_VARBYTE(%s, 'hex'), %s)") + assert direct_parameters == (("one", b"\x00\xff", 0),) + assert s3.region_checks == 0 + assert not s3.uploads and not s3.puts and not s3.deleted + assert all( + operation.transport is WriteTransport.DIRECT for operation in writer.drain_telemetry() + ) + assert not tuple(tmp_path.iterdir()) + + def test_redshift_direct_load_falls_back_to_copy_above_the_varbyte_limit( tmp_path: Path, ) -> None: From e347564b5431094e41daec5a142ed0bc13dad550 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 06:42:59 -0400 Subject: [PATCH 117/163] Record four-warehouse live correctness evidence --- HANDOFF.md | 42 +++++++++---------- docs/cloud-portability-plan.md | 9 ++-- docs/compatibility-matrix.md | 7 ++++ docs/decisions.md | 11 +++++ .../2026-08-11/bigquery.json | 1 + .../2026-08-11/comparison.json | 1 + .../2026-08-11/postgresql.json | 1 + .../2026-08-11/redshift.json | 1 + .../2026-08-11/snowflake.json | 1 + docs/known-limitations.md | 11 ++--- docs/release-audit.md | 7 +++- docs/session-resume.md | 12 ++++-- docs/warehouse-correctness-conformance.md | 38 +++++++++-------- tickets/DANDER-104-snowflake-warehouse.md | 7 ++-- tickets/DANDER-105-redshift-warehouse.md | 4 +- .../DANDER-107-four-warehouse-correctness.md | 22 ++++++++-- 16 files changed, 115 insertions(+), 60 deletions(-) create mode 100644 docs/evidence/warehouse-correctness/2026-08-11/bigquery.json create mode 100644 docs/evidence/warehouse-correctness/2026-08-11/comparison.json create mode 100644 docs/evidence/warehouse-correctness/2026-08-11/postgresql.json create mode 100644 docs/evidence/warehouse-correctness/2026-08-11/redshift.json create mode 100644 docs/evidence/warehouse-correctness/2026-08-11/snowflake.json diff --git a/HANDOFF.md b/HANDOFF.md index 7aa8340e..b86bb749 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,41 +2,39 @@ ## Finished -- Corrected Redshift direct-load binary serialization at the driver boundary. -- Projected Redshift `VARBYTE` as base64 text and strictly decoded it to bytes before normalization. -- Preserved the shared canonical schema, expected hashes, transport selection, and fencing behavior. -- Added credential-free direct-load, projection, decode, malformed-value, and wrong-type coverage. -- Corrected the documented CLI invocation so Snowflake's package cannot be shadowed by the script path. +- Merged the BigQuery and Redshift binary correctness fixes through protected PRs #184 and #185. +- Ran BigQuery, PostgreSQL, Snowflake, and Redshift from protected-main commit `c0f3e2c`. +- Produced one equal normalized three-row hash after exact replay on all four warehouses. +- Verified provider-owned cleanup and removed all disposable PostgreSQL, Snowflake, and AWS assets. +- Recorded sanitized evidence and reconciled the Phase 5 roadmap, tickets, matrix, and limitations. ## Try It -Run `.venv/bin/pytest -q tests/portability/test_warehouse_correctness.py`. +Run `.venv/bin/python -m scripts.benchmarks.warehouse_correctness compare` with the four committed +provider records and a temporary output path. ## Checks -- Warehouse-correctness contract tests pass (10 tests). -- Redshift runtime plus warehouse-correctness tests pass (64 tests). -- Focused Ruff lint/format and mypy pass. -- Protected-CI dependency profile passes Ruff, mypy, and full pytest (1,202 passed, 28 skipped). -- Bounded live Redshift validation passes the expected hash, replay, and owned cleanup. +- Four live provider records pass; the comparison reports equal rows and verified cleanup. +- Retained GCP stage-zero and current-equivalent platform plans report exact `No changes.` +- PR #185 protected CI and post-merge protected-main CI passed all five jobs. +- Evidence JSON shape and forbidden-field checks pass. ## Decisions -- The Redshift driver sends byte parameters as hex text and decodes raw `VARBYTE` as UTF-8, so both - conversions remain at its direct-load or live-qualification boundaries. -- Base64 is validated strictly and converted back to bytes so canonical normalization remains - provider-neutral. -- No schema, transport selection, fencing, or non-binary behavior changes are included. +- The live proof covers only the common canonical scalar intersection. +- Snowflake, Redshift, and PostgreSQL/Kubernetes remain experimental despite Phase 5 correctness. +- Scale, throughput, crossover, cost, soak, pairwise profiles, and release qualification stay in + Phase 8 or provider-promotion gates. ## Remaining -- Complete protected CI and merge the focused correction. -- Re-run all four providers on the resulting protected-main commit and compare equal evidence. -- Verify provider cleanup and retained GCP no-drift, then merge sanitized evidence. -- Reassess the revised Phase 5 gate without beginning Phase 6. +- Complete protected CI and merge the sanitized evidence PR. +- Reassess the revised Phase 5 exit gate and report the binary Phase 6 recommendation. +- Do not begin Phase 6 or Azure implementation in this task. ## Review First -- `scripts/benchmarks/warehouse_correctness.py` -- `tests/portability/test_warehouse_correctness.py` +- `docs/evidence/warehouse-correctness/2026-08-11/comparison.json` +- `tickets/DANDER-107-four-warehouse-correctness.md` - `docs/warehouse-correctness-conformance.md` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 06a26afe..83769d3b 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1,14 +1,17 @@ # Cloud Portability and First-Class Platform Roadmap -Status: active roadmap; Phases 0 through 4 completed; Phase 5 implementation merged and exit gate -re-baselined pending shared warehouse conformance +Status: active roadmap; Phases 0 through 5 exit gates satisfied; Phase 6 has not begun Initial planning baseline: `origin/main` at `9dc5562` (`0.6.0rc1`) Phase 5 implementation checkpoint: protected `main` at `3a10c121d1d77b7a5074ce7d07a3c9d6b7cb7e1e` after PRs #178 and #179 -Prepared: 2026-08-06; reconciled: 2026-08-10 +Phase 5 exit-evidence candidate: protected `main` at +`c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad` after PRs #184 and #185. The shared live records are +stored under `docs/evidence/warehouse-correctness/2026-08-11/`. + +Prepared: 2026-08-06; reconciled: 2026-08-11 The baseline descriptions below record the conditions this roadmap was designed to change. They are retained as architectural rationale rather than claims about the current implementation. The diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index af08e82b..3e5b273c 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -53,3 +53,10 @@ The table deliberately gives more materialization detail than the current machin report. Phase 6 does not require a second capability-schema revision unless its implementation actually consumes that granularity. Unsupported materializations and state/warehouse pairs must continue to fail before provider mutation. + +The bounded live fixture on protected-main commit +`c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad` produced equal normalized rows across BigQuery, +PostgreSQL, Snowflake, and Redshift over the common canonical scalar intersection. The sanitized +[comparison record](evidence/warehouse-correctness/2026-08-11/comparison.json) proves shared +correctness only; provider-specific materializations, transports, types, profiles, and support +promotion retain their separate gates. diff --git a/docs/decisions.md b/docs/decisions.md index 3441e88c..4fc66efe 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1146,3 +1146,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Physical transports and provider-specific types, fallbacks, write modes, materializations, and fencing remain separate conformance concerns. Scale, cost, crossover, soak, pairwise-profile, and release qualification remain in Phase 8. + +## 2026-08-11 — Shared warehouse correctness closes the Phase 5 gate + +- **Result:** BigQuery, PostgreSQL, Snowflake, and Redshift produced the same normalized three-row + hash from one protected-main commit after exact replay, with provider-owned cleanup verified. +- **Evidence:** Only fixture, schema, and result hashes, counts, booleans, timestamps, transports, + commit identity, and reviewed ceilings are retained under + `docs/evidence/warehouse-correctness/2026-08-11/`; credentials, coordinates, and rows are absent. +- **Boundary:** This satisfies Phase 5 common-intersection correctness. Provider/profile support, + scale, throughput, crossover, cost, soak, and pairwise qualification remain separate Phase 8 or + promotion requirements. diff --git a/docs/evidence/warehouse-correctness/2026-08-11/bigquery.json b/docs/evidence/warehouse-correctness/2026-08-11/bigquery.json new file mode 100644 index 00000000..04630f89 --- /dev/null +++ b/docs/evidence/warehouse-correctness/2026-08-11/bigquery.json @@ -0,0 +1 @@ +{"approved_cost_ceiling_usd":"1.00","candidate_commit":"c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad","canonical_schema_hash":"e7789f4068ac9564ddf83e22884bafec64b822d1e4f77d04aef4bd1feb6c4ccd","cleanup_verified":true,"cost_approval_reference":"user-approved/2026-08-11/warehouse-correctness-reruns","dander_version":"0.8.0rc8","ended_at_utc":"2026-08-11T10:19:11Z","fixture_hash":"23767566255eaf86140f56e3418f7b55a62d3e27859f7dc953b670d3a011d83a","fixture_version":"warehouse-common-scalar-v1","normalized_row_count":3,"normalized_rows_hash":"86d374f2d75aeac324b72a0b7870a876fba6301c9499a7b1961fe3c25274523c","provider":"bigquery","replay_equal":true,"schema":"io.dander.conformance.warehouse-correctness/v1","started_at_utc":"2026-08-11T10:18:41Z","status":"passed","transport":"load_job","write_mode":"scd1"} diff --git a/docs/evidence/warehouse-correctness/2026-08-11/comparison.json b/docs/evidence/warehouse-correctness/2026-08-11/comparison.json new file mode 100644 index 00000000..5caa959d --- /dev/null +++ b/docs/evidence/warehouse-correctness/2026-08-11/comparison.json @@ -0,0 +1 @@ +{"all_cleanup_verified":true,"all_rows_equal":true,"candidate_commit":"c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad","canonical_schema_hash":"e7789f4068ac9564ddf83e22884bafec64b822d1e4f77d04aef4bd1feb6c4ccd","fixture_hash":"23767566255eaf86140f56e3418f7b55a62d3e27859f7dc953b670d3a011d83a","fixture_version":"warehouse-common-scalar-v1","normalized_row_count":3,"normalized_rows_hash":"86d374f2d75aeac324b72a0b7870a876fba6301c9499a7b1961fe3c25274523c","providers":["bigquery","postgresql","redshift","snowflake"],"schema":"io.dander.conformance.warehouse-correctness-comparison/v1","status":"passed"} diff --git a/docs/evidence/warehouse-correctness/2026-08-11/postgresql.json b/docs/evidence/warehouse-correctness/2026-08-11/postgresql.json new file mode 100644 index 00000000..93529052 --- /dev/null +++ b/docs/evidence/warehouse-correctness/2026-08-11/postgresql.json @@ -0,0 +1 @@ +{"approved_cost_ceiling_usd":"0.00","candidate_commit":"c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad","canonical_schema_hash":"e7789f4068ac9564ddf83e22884bafec64b822d1e4f77d04aef4bd1feb6c4ccd","cleanup_verified":true,"cost_approval_reference":"user-approved/2026-08-11/warehouse-correctness-reruns","dander_version":"0.8.0rc8","ended_at_utc":"2026-08-11T10:18:41Z","fixture_hash":"23767566255eaf86140f56e3418f7b55a62d3e27859f7dc953b670d3a011d83a","fixture_version":"warehouse-common-scalar-v1","normalized_row_count":3,"normalized_rows_hash":"86d374f2d75aeac324b72a0b7870a876fba6301c9499a7b1961fe3c25274523c","provider":"postgresql","replay_equal":true,"schema":"io.dander.conformance.warehouse-correctness/v1","started_at_utc":"2026-08-11T10:18:41Z","status":"passed","transport":"copy","write_mode":"scd1"} diff --git a/docs/evidence/warehouse-correctness/2026-08-11/redshift.json b/docs/evidence/warehouse-correctness/2026-08-11/redshift.json new file mode 100644 index 00000000..b0b68213 --- /dev/null +++ b/docs/evidence/warehouse-correctness/2026-08-11/redshift.json @@ -0,0 +1 @@ +{"approved_cost_ceiling_usd":"3.00","candidate_commit":"c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad","canonical_schema_hash":"e7789f4068ac9564ddf83e22884bafec64b822d1e4f77d04aef4bd1feb6c4ccd","cleanup_verified":true,"cost_approval_reference":"user-approved/2026-08-11/warehouse-correctness-reruns","dander_version":"0.8.0rc8","ended_at_utc":"2026-08-11T10:20:55Z","fixture_hash":"23767566255eaf86140f56e3418f7b55a62d3e27859f7dc953b670d3a011d83a","fixture_version":"warehouse-common-scalar-v1","normalized_row_count":3,"normalized_rows_hash":"86d374f2d75aeac324b72a0b7870a876fba6301c9499a7b1961fe3c25274523c","provider":"redshift","replay_equal":true,"schema":"io.dander.conformance.warehouse-correctness/v1","started_at_utc":"2026-08-11T10:18:41Z","status":"passed","transport":"direct","write_mode":"scd1"} diff --git a/docs/evidence/warehouse-correctness/2026-08-11/snowflake.json b/docs/evidence/warehouse-correctness/2026-08-11/snowflake.json new file mode 100644 index 00000000..df6f34c3 --- /dev/null +++ b/docs/evidence/warehouse-correctness/2026-08-11/snowflake.json @@ -0,0 +1 @@ +{"approved_cost_ceiling_usd":"2.00","candidate_commit":"c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad","canonical_schema_hash":"e7789f4068ac9564ddf83e22884bafec64b822d1e4f77d04aef4bd1feb6c4ccd","cleanup_verified":true,"cost_approval_reference":"user-approved/2026-08-11/warehouse-correctness-reruns","dander_version":"0.8.0rc8","ended_at_utc":"2026-08-11T10:19:02Z","fixture_hash":"23767566255eaf86140f56e3418f7b55a62d3e27859f7dc953b670d3a011d83a","fixture_version":"warehouse-common-scalar-v1","normalized_row_count":3,"normalized_rows_hash":"86d374f2d75aeac324b72a0b7870a876fba6301c9499a7b1961fe3c25274523c","provider":"snowflake","replay_equal":true,"schema":"io.dander.conformance.warehouse-correctness/v1","started_at_utc":"2026-08-11T10:18:41Z","status":"passed","transport":"direct","write_mode":"scd1"} diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 76f27057..b2f1118c 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -3,11 +3,12 @@ Dander `0.8.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. -The Phase 5 warehouse implementations and Snowflake/Redshift live correctness proofs are merged, -but the re-baselined Phase 5 exit gate remains open until the -[shared four-warehouse deterministic fixture](warehouse-correctness-conformance.md) produces equal -normalized results on one protected-main commit and the current evidence ledger is reconciled. -Scale, cost, soak, pairwise-profile, and release qualification remain Phase 8 work. +The Phase 5 warehouse implementations and the +[shared four-warehouse deterministic fixture](warehouse-correctness-conformance.md) pass on one +protected-main commit with equal normalized rows and exact cleanup. This closes the Phase 5 +correctness gate; it does not promote Snowflake, Redshift, PostgreSQL/Kubernetes, or any new +state/warehouse pair to supported. Scale, cost, soak, pairwise-profile, and release qualification +remain Phase 8 work. - AWS ECS/Fargate remains experimental rather than supported. The named Fargate-to-BigQuery/GCP composition has passed source-free manual and scheduled execution, replay, interruption, diff --git a/docs/release-audit.md b/docs/release-audit.md index bd4c8500..91aea944 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -1,6 +1,6 @@ # Dander Platform Release Audit -Audited on 2026-08-10 against the product promise in `steering/00-project-overview.md`. +Audited on 2026-08-11 against the product promise in `steering/00-project-overview.md`. “Live-proven” means the behavior was observed in a disposable provider account or retained GCP project. “Implemented” means automated tests cover the contract while an optional provider or cloud path remains outside the live proof. @@ -19,7 +19,7 @@ cloud path remains outside the live proof. | Durable state and optional read capabilities | Live-proven | Run history, leases, watermarks, count, connection-check, and targeted-read paths were exercised for the relevant retained plugins without provider write-back. | | Single metadata spine | Live-proven | The run ledger and catalog store pipeline lifecycle, source/model schema, lineage, tests, and governed metrics in BigQuery; Dataplex publication remains optional. | | Canonical visual authoring | Live-proven for the bounded graph slice | Druff opens and saves canonical `PipelineGraph`, discovers presentation-safe connector and operation descriptors, starts an already-deployed run, and previews a non-applyable full-manifest plan through Dander's loopback service. | -| Infrastructure reconciliation safety | Live-proven | The retained platform plan after Druff deployment reported `0` add, `0` change, and `0` destroy across `111` no-op resources. | +| Infrastructure reconciliation safety | Live-proven | Fresh retained stage-zero and platform plans after the four-warehouse run reported exact `No changes.`; the current-equivalent platform plan contained `113` no-op resources. | ## Current release and deployment record @@ -41,6 +41,9 @@ cloud path remains outside the live proof. alert routing, image rollback, cleanup, and four final no-drift plans. Fargate remains experimental pending scale/profile qualification. See `docs/cloud-portability-fargate-lifecycle-acceptance.md`. +- Protected-main Dander `0.8.0rc8` passed the bounded Phase 5 common-scalar fixture on BigQuery, + PostgreSQL, Snowflake, and Redshift with one equal normalized hash, exact replay, and verified + cleanup. This is warehouse correctness evidence, not provider or profile support promotion. - Retained source-free Dander image: `sha256:68e112c43b365018b735be7934446e15dfe6169fc64062b62b8bb97ea4f93b96`, built with Dander `0.7.1`, Salesforce `0.3.1`, and ServiceNow `0.2.2`. - Retained Druff image: `sha256:a5e255d6…871c`; public static URL: diff --git a/docs/session-resume.md b/docs/session-resume.md index de1fe25b..0c48c095 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -1,4 +1,4 @@ -# Session Resume — 2026-08-10 +# Session Resume — 2026-08-11 Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `docs/release-audit.md` before changing code or cloud resources. @@ -33,6 +33,12 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Latest operating evidence +- On 2026-08-11, the Phase 5 common-scalar fixture passed BigQuery, PostgreSQL, Snowflake, and + Redshift on protected-main commit `c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad`. All four records + produced one equal normalized hash after replay and verified owned cleanup. Provider account + objects were removed, and fresh retained GCP stage-zero and current-equivalent platform plans + each reported exact `No changes.` + - On 2026-08-10, public Dander `0.8.0rc8` passed the complete lifecycle gate for the named Fargate-to-BigQuery/GCP composition. The source-free, byte-identical GAR/ECR image completed manual, scheduled, replay, interruption, alert-routing, rollback, cleanup, and no-drift checks. @@ -55,8 +61,8 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `dander-servicenow-incidents-6g72x` all completed successfully. - The latest manual executable-graph run, `dander-greenhouse-graph-7gn9z`, completed successfully on 2026-08-04; its schedule remains intentionally paused. -- The latest retained stage-zero and platform plans each reported exactly `No changes.` after the - isolated Fargate proof. No retained-project apply occurred. +- The latest retained stage-zero and current-equivalent platform plans each reported exactly + `No changes.` after the four-warehouse proof. No retained-project apply occurred. - Continue the 30-day operating record in GitHub issue #26. The next normal scheduled runs are the remaining observation point for the reconciled stable image. diff --git a/docs/warehouse-correctness-conformance.md b/docs/warehouse-correctness-conformance.md index 28e9a0af..ddb534bd 100644 --- a/docs/warehouse-correctness-conformance.md +++ b/docs/warehouse-correctness-conformance.md @@ -89,20 +89,24 @@ sanitized evidence and reviewed ceilings. ## Current evidence status -An authorized one-attempt run was made on 2026-08-10 against protected-main commit -`4927d5f66c787c6d5da700baa06edcef8b4e4c6b`, with ceilings of BigQuery $1, PostgreSQL $0, -Snowflake $2, and Redshift $3 and no automatic paid reruns. PostgreSQL passed the fixture, replay, -and cleanup. The other three did not produce passing records, so no comparison is claimed. - -The attempt exposed a harness defect: BigQuery metadata and cleanup queries used a 10,000,000-byte -limit, below BigQuery's 10 MiB minimum. The one owned table left by that cleanup failure was deleted -and metadata confirmed zero matching tables. Snowflake's temporary user, database, warehouse, -role, and resource monitor each returned zero matches after cleanup. Redshift's disposable -workgroup, namespace, bucket, role, VPC, and Terraform state each returned zero after destroy. -Post-run retained GCP stage-zero and platform plans both reported exact `No changes.` - -This correction raises the BigQuery limit to exactly 10 MiB, puts fence acquisition inside the -owned cleanup boundary, and persists sanitized failure classification. It does not authorize a -provider rerun. Phase 5 remains open until all four same-commit live records compare equal under -newly reviewed paid-provider approvals and the resulting passing evidence is merged through -protected CI. +An authorized run on 2026-08-11 used protected-main commit +`c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad`, after the BigQuery binary-load correction in PR #184 +and the Redshift binary direct-load/readback correction in PR #185. The reviewed per-attempt +ceilings were BigQuery $1, PostgreSQL $0, Snowflake $2, and Redshift $3, with no unapproved paid +rerun. + +All four providers passed with the same fixture, canonical-schema, and normalized-row hashes. The +three normalized rows were equal before and after exact replay, every provider verified owned +cleanup, and the [comparison record](evidence/warehouse-correctness/2026-08-11/comparison.json) +reports `all_rows_equal=true` and `all_cleanup_verified=true`. The directory also contains the four +sanitized provider records and no coordinates, credentials, DSNs, private keys, or row values. + +Post-run verification found zero BigQuery owned tables, zero PostgreSQL test containers, zero +Snowflake test users/databases/warehouses/roles/resource monitors, and zero Redshift Terraform or +named AWS proof resources. Fresh retained GCP stage-zero and current-equivalent platform plans both +reported exact `No changes.`; the platform plan contained 113 no-op resources. No retained GCP +apply occurred. + +This closes the Phase 5 shared correctness requirement. Provider-specific types, fallbacks, +transports, materializations, scale, cost, soak, pairwise profiles, and support promotion remain +outside this proof and retain their documented conformance or Phase 8 gates. diff --git a/tickets/DANDER-104-snowflake-warehouse.md b/tickets/DANDER-104-snowflake-warehouse.md index 37b30de9..950d4648 100644 --- a/tickets/DANDER-104-snowflake-warehouse.md +++ b/tickets/DANDER-104-snowflake-warehouse.md @@ -43,6 +43,7 @@ the qualification slice also: - [x] provides a disposable-schema live harness with sanitized output and exact cleanup; - [x] records a real Snowflake execution under an approved paid-test ceiling. -The shared four-warehouse result comparison is the separate re-baselined Phase 5 exit gate rather -than an unmerged Snowflake adapter slice. Snowflake remains experimental, views remain unsupported, -and direct thresholds retain their zero defaults until crossover evidence is recorded in Phase 8. +The separate shared four-warehouse result comparison passed on protected-main commit +`c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad`. Snowflake remains experimental, views remain +unsupported, and direct thresholds retain their zero defaults until crossover evidence is recorded +in Phase 8. diff --git a/tickets/DANDER-105-redshift-warehouse.md b/tickets/DANDER-105-redshift-warehouse.md index 7341914c..3db9bf02 100644 --- a/tickets/DANDER-105-redshift-warehouse.md +++ b/tickets/DANDER-105-redshift-warehouse.md @@ -27,5 +27,5 @@ created: 2026-08-09 Redshift remains experimental. Views, ARRAY/RECORD fallback, provisioned-RA3 live qualification, reusable provider-managed infrastructure, support promotion, and Phase 8 scale/cost qualification -remain outside this completed adapter ticket. The shared four-warehouse result comparison is the -separate re-baselined Phase 5 exit gate. +remain outside this completed adapter ticket. The separate shared four-warehouse result comparison +passed on protected-main commit `c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad`. diff --git a/tickets/DANDER-107-four-warehouse-correctness.md b/tickets/DANDER-107-four-warehouse-correctness.md index 8d79d44f..e66429fa 100644 --- a/tickets/DANDER-107-four-warehouse-correctness.md +++ b/tickets/DANDER-107-four-warehouse-correctness.md @@ -1,7 +1,7 @@ --- id: DANDER-107 title: Gate Phase 5 on equal four-warehouse canonical rows -status: in-review +status: completed component: warehouse epic: cloud-portability depends_on: [DANDER-104, DANDER-105] @@ -25,7 +25,7 @@ their individual capabilities but do not compare one shared fixture. - [x] Evidence excludes credentials and row data and records reviewed cost ceilings. - [x] Every provider run attempts exact owned cleanup and requires cleanup readback. - [x] Credential-free fixture, normalization, lifecycle, evidence, and comparison tests pass. -- [ ] Same protected-main live evidence from all four providers compares equal under reviewed +- [x] Same protected-main live evidence from all four providers compares equal under reviewed ceilings. - [x] Retained GCP no-drift passes after the live executions. - [x] Protected CI passes. @@ -55,4 +55,20 @@ Provider-owned cleanup was completed and verified, and retained GCP stage-zero a both reported exact `No changes.` The correction prompted by that attempt fixes BigQuery's 10 MiB metadata-query minimum, expands cleanup around fence acquisition, and adds sanitized failure evidence. Its implementation commit passed all five protected checks on PR #183. Equal -same-commit live evidence remains the explicit Phase 5 blocker. +same-commit live evidence was the remaining explicit Phase 5 blocker before the run below. + +The authorized 2026-08-11 run used protected-main commit +`c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad` and the reviewed ceilings BigQuery $1, +PostgreSQL $0, Snowflake $2, and Redshift $3. All four records contain fixture hash +`23767566255eaf86140f56e3418f7b55a62d3e27859f7dc953b670d3a011d83a`, canonical-schema hash +`e7789f4068ac9564ddf83e22884bafec64b822d1e4f77d04aef4bd1feb6c4ccd`, and normalized-row hash +`86d374f2d75aeac324b72a0b7870a876fba6301c9499a7b1961fe3c25274523c` for three rows. Exact +replay and owned cleanup passed for every provider, and the +[comparison record](../docs/evidence/warehouse-correctness/2026-08-11/comparison.json) reports +`all_rows_equal=true` and `all_cleanup_verified=true`. + +BigQuery reported zero owned tables; PostgreSQL's disposable container was removed; Snowflake +reported zero matching users, databases, warehouses, roles, and resource monitors; and the +Redshift Terraform state, workgroup, namespace, bucket, role, and VPC inventory returned zero. +Fresh retained GCP stage-zero and current-equivalent platform plans each reported exact +`No changes.` No provider coordinate, credential, private key, DSN, or row value is committed. From c0477ed6fdf810ab8c816439338d8fc6e79567b3 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 06:43:44 -0400 Subject: [PATCH 118/163] Clarify retained no-drift baseline --- docs/cloud-portability-plan.md | 3 ++- docs/session-resume.md | 4 +++- docs/warehouse-correctness-conformance.md | 5 ++++- tickets/DANDER-107-four-warehouse-correctness.md | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 83769d3b..55df8fa2 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -9,7 +9,8 @@ Phase 5 implementation checkpoint: protected `main` at Phase 5 exit-evidence candidate: protected `main` at `c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad` after PRs #184 and #185. The shared live records are -stored under `docs/evidence/warehouse-correctness/2026-08-11/`. +stored under `docs/evidence/warehouse-correctness/2026-08-11/` and reconciled through protected +evidence PR #186. Prepared: 2026-08-06; reconciled: 2026-08-11 diff --git a/docs/session-resume.md b/docs/session-resume.md index 0c48c095..62aee421 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -37,7 +37,9 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and Redshift on protected-main commit `c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad`. All four records produced one equal normalized hash after replay and verified owned cleanup. Provider account objects were removed, and fresh retained GCP stage-zero and current-equivalent platform plans - each reported exact `No changes.` + each reported exact `No changes.` The platform proof reused the last accepted source-free + deployment inputs; a current-default diagnostic that proposed only timeout/version-label changes + was not applied. - On 2026-08-10, public Dander `0.8.0rc8` passed the complete lifecycle gate for the named Fargate-to-BigQuery/GCP composition. The source-free, byte-identical GAR/ECR image completed diff --git a/docs/warehouse-correctness-conformance.md b/docs/warehouse-correctness-conformance.md index ddb534bd..a792e347 100644 --- a/docs/warehouse-correctness-conformance.md +++ b/docs/warehouse-correctness-conformance.md @@ -105,7 +105,10 @@ Post-run verification found zero BigQuery owned tables, zero PostgreSQL test con Snowflake test users/databases/warehouses/roles/resource monitors, and zero Redshift Terraform or named AWS proof resources. Fresh retained GCP stage-zero and current-equivalent platform plans both reported exact `No changes.`; the platform plan contained 113 no-op resources. No retained GCP -apply occurred. +apply occurred. The current-equivalent platform plan replayed the latest accepted source-free +deployment inputs, including its 600-second job timeout and deployed `0.7.0` projection-label +metadata. A diagnostic plan using current repository defaults proposed only five job +timeout/version-label updates; it was rejected as the wrong retained baseline and was not applied. This closes the Phase 5 shared correctness requirement. Provider-specific types, fallbacks, transports, materializations, scale, cost, soak, pairwise profiles, and support promotion remain diff --git a/tickets/DANDER-107-four-warehouse-correctness.md b/tickets/DANDER-107-four-warehouse-correctness.md index e66429fa..e9211e1d 100644 --- a/tickets/DANDER-107-four-warehouse-correctness.md +++ b/tickets/DANDER-107-four-warehouse-correctness.md @@ -72,3 +72,4 @@ reported zero matching users, databases, warehouses, roles, and resource monitor Redshift Terraform state, workgroup, namespace, bucket, role, and VPC inventory returned zero. Fresh retained GCP stage-zero and current-equivalent platform plans each reported exact `No changes.` No provider coordinate, credential, private key, DSN, or row value is committed. +The sanitized evidence ledger is reconciled through protected PR #186. From 1f91a1bb669e792e10f306edd16795d05652f695 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 06:51:56 -0400 Subject: [PATCH 119/163] Finalize Phase 5 handoff --- HANDOFF.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index b86bb749..bc211dd1 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -10,14 +10,14 @@ ## Try It -Run `.venv/bin/python -m scripts.benchmarks.warehouse_correctness compare` with the four committed -provider records and a temporary output path. +Run `uv run python -m scripts.benchmarks.warehouse_correctness compare`, passing each committed +provider record with its own `--evidence` argument and a temporary `--output` path. ## Checks - Four live provider records pass; the comparison reports equal rows and verified cleanup. - Retained GCP stage-zero and current-equivalent platform plans report exact `No changes.` -- PR #185 protected CI and post-merge protected-main CI passed all five jobs. +- PRs #184, #185, and #186 merged through protection; final protected-main CI passed all five jobs. - Evidence JSON shape and forbidden-field checks pass. ## Decisions @@ -29,9 +29,10 @@ provider records and a temporary output path. ## Remaining -- Complete protected CI and merge the sanitized evidence PR. -- Reassess the revised Phase 5 exit gate and report the binary Phase 6 recommendation. -- Do not begin Phase 6 or Azure implementation in this task. +- Phase 6 may begin under the revised Phase 5 exit gate, but this task deliberately starts neither + Phase 6 nor Azure implementation. +- Complete provider-promotion conformance and Phase 8 qualification before making broader support, + scale, cost, soak, or release claims. ## Review First From 9623e66a1382d1f23da077688840af91c894a382 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 11:34:22 -0400 Subject: [PATCH 120/163] Add Azure launcher contract --- HANDOFF.md | 42 ++-- docs/cloud-portability-plan.md | 2 +- docs/execution-projection.md | 9 + docs/known-limitations.md | 4 + src/dander/deployment/__init__.py | 2 + src/dander/deployment/projection.py | 23 ++ src/dander/project/config.py | 6 +- src/dander/project/portable_config.py | 25 +- .../azure_container_apps/__init__.py | 5 + .../providers/azure_container_apps/config.py | 68 ++++++ .../providers/azure_container_apps/runtime.py | 219 ++++++++++++++++++ .../providers/azure_key_vault/__init__.py | 5 + .../providers/azure_key_vault/config.py | 13 ++ .../providers/azure_key_vault/runtime.py | 45 ++++ src/dander/providers/defaults.py | 18 ++ src/dander/security/azure_key_vault.py | 69 ++++++ tests/project/test_portable_config.py | 76 ++++++ .../test_azure_container_apps_runtime.py | 164 +++++++++++++ .../providers/test_azure_key_vault_runtime.py | 97 ++++++++ tickets/DANDER-108-azure-launcher-contract.md | 29 +++ 20 files changed, 893 insertions(+), 28 deletions(-) create mode 100644 src/dander/providers/azure_container_apps/__init__.py create mode 100644 src/dander/providers/azure_container_apps/config.py create mode 100644 src/dander/providers/azure_container_apps/runtime.py create mode 100644 src/dander/providers/azure_key_vault/__init__.py create mode 100644 src/dander/providers/azure_key_vault/config.py create mode 100644 src/dander/providers/azure_key_vault/runtime.py create mode 100644 src/dander/security/azure_key_vault.py create mode 100644 tests/providers/test_azure_container_apps_runtime.py create mode 100644 tests/providers/test_azure_key_vault_runtime.py create mode 100644 tickets/DANDER-108-azure-launcher-contract.md diff --git a/HANDOFF.md b/HANDOFF.md index bc211dd1..c9c0c816 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,38 @@ ## Finished -- Merged the BigQuery and Redshift binary correctness fixes through protected PRs #184 and #185. -- Ran BigQuery, PostgreSQL, Snowflake, and Redshift from protected-main commit `c0f3e2c`. -- Produced one equal normalized three-row hash after exact replay on all four warehouses. -- Verified provider-owned cleanup and removed all disposable PostgreSQL, Snowflake, and AWS assets. -- Recorded sanitized evidence and reconciled the Phase 5 roadmap, tickets, matrix, and limitations. +- Re-verified the Phase 5 exit gate on protected `main` and began Phase 6 locally. +- Added typed Azure Container Apps Jobs and Azure Key Vault provider contracts. +- Added deterministic ACR digest, managed identity, UTC schedule, resource, and secret projections. +- Kept all Azure provider writes behind a separate explicit cost/approval gate. ## Try It -Run `uv run python -m scripts.benchmarks.warehouse_correctness compare`, passing each committed -provider record with its own `--evidence` argument and a temporary `--output` path. +Run `uv run --extra dev pytest -q tests/providers/test_azure_container_apps_runtime.py +tests/providers/test_azure_key_vault_runtime.py tests/project/test_portable_config.py`. ## Checks -- Four live provider records pass; the comparison reports equal rows and verified cleanup. -- Retained GCP stage-zero and current-equivalent platform plans report exact `No changes.` -- PRs #184, #185, and #186 merged through protection; final protected-main CI passed all five jobs. -- Evidence JSON shape and forbidden-field checks pass. +- Focused Azure, launcher, secret-provider, dependency, and portable-config tests pass. +- Ruff passes on the Phase 6 contract scope. +- Azure subscription preflight is enabled; required resource providers remain unregistered. ## Decisions -- The live proof covers only the common canonical scalar intersection. -- Snowflake, Redshift, and PostgreSQL/Kubernetes remain experimental despite Phase 5 correctness. -- Scale, throughput, crossover, cost, soak, pairwise profiles, and release qualification stay in - Phase 8 or provider-promotion gates. +- Azure cron is UTC-only and fails closed for other time zones. +- The first canonical Azure shape is Snowflake + PostgreSQL state + no catalog + Key Vault. +- Live proof must use a newly approved candidate built after the Azure implementation merges. ## Remaining -- Phase 6 may begin under the revised Phase 5 exit gate, but this task deliberately starts neither - Phase 6 nor Azure implementation. -- Complete provider-promotion conformance and Phase 8 qualification before making broader support, - scale, cost, soak, or release claims. +- Merge this contract through a focused protected PR. +- Add plan-first Azure state/ACR/Container Apps Terraform plus deployment verification. +- Add ACR copy and launcher operations without performing provider writes. +- Obtain publication and provider-cost approvals before candidate or live proof mutations. +- Run approved identity, lifecycle, cleanup, rollback, and no-drift acceptance. ## Review First -- `docs/evidence/warehouse-correctness/2026-08-11/comparison.json` -- `tickets/DANDER-107-four-warehouse-correctness.md` -- `docs/warehouse-correctness-conformance.md` +- `src/dander/providers/azure_container_apps/runtime.py` +- `src/dander/security/azure_key_vault.py` +- `tests/providers/test_azure_container_apps_runtime.py` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 55df8fa2..0c7382f4 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1,6 +1,6 @@ # Cloud Portability and First-Class Platform Roadmap -Status: active roadmap; Phases 0 through 5 exit gates satisfied; Phase 6 has not begun +Status: active roadmap; Phases 0 through 5 exit gates satisfied; Phase 6 in progress Initial planning baseline: `origin/main` at `9dc5562` (`0.6.0rc1`) diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 6f7feae5..046652ee 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -55,3 +55,12 @@ references, optional cloud-neutral ServiceAccount annotations, explicit pod reso deadlines, and bounded retries. The chart adds `Forbid` schedule concurrency and completed-Job TTL cleanup while Dander leases remain the final overlap defense. Rendering and read-only verification do not qualify a hosted Kubernetes profile; a real cluster and PostgreSQL acceptance run remain. + +The Azure Container Apps Jobs provider projects a selected Azure profile into one UTC-scheduled or +manual job contract. It requires an immutable digest in the selected ACR, one pre-authorized +user-assigned managed identity, exact Container Apps environment placement, supported 1 CPU/2 GiB +or 2 CPU/4 GiB sizing, launcher-owned retries and deadline, Log Analytics destinations, and +versionless Key Vault references for declared secrets. Azure evaluates scheduled-job cron in UTC, +so a non-UTC projection fails before provider access rather than silently changing wall-clock +behavior. This contract is locally conformance-tested only; Terraform and live-profile evidence +remain separate Phase 6 gates. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index b2f1118c..311f94fe 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -46,6 +46,10 @@ remain Phase 8 work. schema limits, transforms, graphs, and fencing through `dander runtime compatibility`, while the packaged capability manifest remains the support boundary. Local PostgreSQL benchmark results are regression evidence, not a paid or controlled-memory scale qualification. +- Azure Container Apps Jobs and Azure Key Vault have a typed, deterministic local projection only. + Azure schedules are UTC-only and currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. + Resource-provider registration, Terraform, ACR image copy, identity federation, operations, and + live Snowflake/PostgreSQL acceptance have not yet run, so Azure remains unsupported. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index a674eee1..1d1ef477 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -1,6 +1,7 @@ """Cloud-neutral deployment and execution projections.""" from dander.deployment.projection import ( + AZURE_CONTAINER_APPS_CAPABILITIES, CLOUD_RUN_CAPABILITIES, EXECUTION_PROJECTION_SCHEMA, FARGATE_CAPABILITIES, @@ -25,6 +26,7 @@ ) __all__ = [ + "AZURE_CONTAINER_APPS_CAPABILITIES", "CLOUD_RUN_CAPABILITIES", "EXECUTION_PROJECTION_SCHEMA", "FARGATE_CAPABILITIES", diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index 25835290..ef4c86f2 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -380,6 +380,29 @@ class LauncherCapabilities: ) +AZURE_CONTAINER_APPS_CAPABILITIES = LauncherCapabilities( + launcher="azure_container_apps", + cpu_millis=frozenset({1_000, 2_000}), + minimum_memory_mib=2_048, + maximum_memory_mib=4_096, + maximum_deadline_seconds=86_400, + maximum_launcher_retries=10, + maximum_task_count=1, + maximum_parallelism=1, + supports_ephemeral_storage=False, + supports_schedules=True, + supports_time_zones=True, + supports_network_placement=True, + extension_names=frozenset( + { + "azure_acr_login_server", + "azure_key_vault_uri", + "azure_managed_identity_client_id", + } + ), +) + + def validate_launcher_projection( template: ExecutionTemplate, capabilities: LauncherCapabilities, diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 1b35a5ab..f0e2f6fb 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -240,12 +240,12 @@ class DanderProject(BaseModel): default_factory=_default_catalog_config, exclude=True, ) - secret_provider: Literal["gcp_secret_manager", "environment"] = Field( + secret_provider: Literal["gcp_secret_manager", "environment", "azure_key_vault"] = Field( default="gcp_secret_manager", exclude=True, ) - launcher_provider: Literal["cloud_run", "fargate", "kubernetes"] = Field( - default="cloud_run", exclude=True + launcher_provider: Literal["cloud_run", "fargate", "kubernetes", "azure_container_apps"] = ( + Field(default="cloud_run", exclude=True) ) launcher_config: dict[str, object] = Field(default_factory=dict, exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 00d65161..324bce63 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -23,6 +23,10 @@ ProjectConfigError, _load_yaml_mapping, ) +from dander.providers.azure_container_apps import ( # noqa: TC001 + AzureContainerAppsLauncherConfig, +) +from dander.providers.azure_key_vault import AzureKeyVaultConfig # noqa: TC001 from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig # noqa: TC001 from dander.providers.cloud_run import CloudRunLauncherConfig # noqa: TC001 from dander.providers.dataplex import DataplexCatalogConfig # noqa: TC001 @@ -134,13 +138,16 @@ def validate_pipeline_ids( SecretProviderSpec = Annotated[ - GcpSecretManagerConfig | EnvironmentSecretConfig, + GcpSecretManagerConfig | EnvironmentSecretConfig | AzureKeyVaultConfig, Field(discriminator="provider"), ] LauncherSpec = Annotated[ - CloudRunLauncherConfig | FargateLauncherConfig | KubernetesLauncherConfig, + CloudRunLauncherConfig + | FargateLauncherConfig + | KubernetesLauncherConfig + | AzureContainerAppsLauncherConfig, Field(discriminator="provider"), ] @@ -336,6 +343,13 @@ def _resolve( selected_name = _select_deployment(platforms, deployment) selected = platforms.deployments[selected_name] profile = platforms.platforms[selected.platform] + if ( + profile.secrets.provider == "azure_key_vault" + and selected.launcher.provider != "azure_container_apps" + ): + raise ProjectConfigError( + "Azure Key Vault projection currently requires launcher.provider='azure_container_apps'" + ) if ( selected.launcher.provider == "cloud_run" and profile.secrets.provider != "gcp_secret_manager" @@ -344,6 +358,13 @@ def _resolve( "Cloud Run requires secrets.provider='gcp_secret_manager'; " "environment secrets are local or operator-managed Kubernetes only" ) + if ( + selected.launcher.provider == "azure_container_apps" + and profile.secrets.provider != "azure_key_vault" + ): + raise ProjectConfigError( + "Azure Container Apps currently requires secrets.provider='azure_key_vault'" + ) unknown_pipelines = sorted(set(selected.pipelines) - set(logical.pipelines)) if unknown_pipelines: raise ProjectConfigError( diff --git a/src/dander/providers/azure_container_apps/__init__.py b/src/dander/providers/azure_container_apps/__init__.py new file mode 100644 index 00000000..b84a20f3 --- /dev/null +++ b/src/dander/providers/azure_container_apps/__init__.py @@ -0,0 +1,5 @@ +"""Azure Container Apps Jobs launcher configuration.""" + +from dander.providers.azure_container_apps.config import AzureContainerAppsLauncherConfig + +__all__ = ["AzureContainerAppsLauncherConfig"] diff --git a/src/dander/providers/azure_container_apps/config.py b/src/dander/providers/azure_container_apps/config.py new file mode 100644 index 00000000..9266f335 --- /dev/null +++ b/src/dander/providers/azure_container_apps/config.py @@ -0,0 +1,68 @@ +"""Validated Azure Container Apps Jobs launcher configuration without SDK imports.""" + +from __future__ import annotations + +import re +from typing import Literal +from uuid import UUID # noqa: TC003 - Pydantic resolves this annotation at runtime. + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +_RESOURCE_NAME = re.compile(r"^[a-z][a-z0-9-]{1,62}[a-z0-9]$") +_ACR_NAME = re.compile(r"^[a-z][a-z0-9]{4,49}$") +_KEY_VAULT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,22}[a-z0-9]$") +_LOCATION = re.compile(r"^[a-z][a-z0-9]{1,31}$") + + +class AzureContainerAppsLauncherConfig(BaseModel): + """Select one pre-authorized Azure identity and Container Apps environment.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["azure_container_apps"] + region: str = Field(pattern=_LOCATION.pattern) + subscription_id: UUID + resource_group_name: str = Field(pattern=_RESOURCE_NAME.pattern) + container_app_environment_name: str = Field(pattern=_RESOURCE_NAME.pattern) + acr_name: str = Field(pattern=_ACR_NAME.pattern) + key_vault_name: str = Field(pattern=_KEY_VAULT_NAME.pattern) + managed_identity_name: str = Field(pattern=_RESOURCE_NAME.pattern) + managed_identity_client_id: UUID + + @field_validator("subscription_id", "managed_identity_client_id") + @classmethod + def require_rfc4122_uuid(cls, value: UUID) -> UUID: + if value.variant != "specified in RFC 4122": + raise ValueError("Azure identifiers must be RFC 4122 UUIDs") + return value + + @field_validator("key_vault_name") + @classmethod + def reject_ambiguous_key_vault_name(cls, value: str) -> str: + if "--" in value: + raise ValueError("Azure Key Vault names must not contain consecutive hyphens") + return value + + @property + def acr_login_server(self) -> str: + return f"{self.acr_name}.azurecr.io" + + @property + def key_vault_uri(self) -> str: + return f"https://{self.key_vault_name}.vault.azure.net" + + @property + def managed_identity_resource_id(self) -> str: + return ( + f"/subscriptions/{self.subscription_id}/resourceGroups/{self.resource_group_name}" + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + f"{self.managed_identity_name}" + ) + + @property + def container_app_environment_resource_id(self) -> str: + return ( + f"/subscriptions/{self.subscription_id}/resourceGroups/{self.resource_group_name}" + "/providers/Microsoft.App/managedEnvironments/" + f"{self.container_app_environment_name}" + ) diff --git a/src/dander/providers/azure_container_apps/runtime.py b/src/dander/providers/azure_container_apps/runtime.py new file mode 100644 index 00000000..9b5a4ec3 --- /dev/null +++ b/src/dander/providers/azure_container_apps/runtime.py @@ -0,0 +1,219 @@ +"""Azure Container Apps Jobs execution projection.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander import __version__ +from dander.deployment.projection import ( + AZURE_CONTAINER_APPS_CAPABILITIES, + EXECUTION_PROJECTION_SCHEMA, + ExecutionProjectionError, + ExecutionTemplate, + NetworkPlacement, + ObservabilityProjection, + ResourceProjection, + ScheduleProjection, + SecretReference, + validate_launcher_projection, +) +from dander.deployment.runtime import LauncherRuntime, ResolvedTemplateRequest +from dander.providers.azure_container_apps.config import AzureContainerAppsLauncherConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.runtime_contract import RUNTIME_CONTRACT + +if TYPE_CHECKING: + from pydantic import BaseModel + +_IMAGE_PATH = re.compile(r"^[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$") +_CRON_PART = re.compile(r"^[A-Za-z0-9*/,-]+$") +_KEY_VAULT_SECRET = re.compile(r"^[A-Za-z0-9-]{1,127}$") + + +@dataclass(frozen=True, slots=True) +class AzureContainerAppsTemplateFactory: + """Project one selected profile into Azure Container Apps Jobs.""" + + config: AzureContainerAppsLauncherConfig + + def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: + """Build immutable Azure templates without contacting Azure.""" + prefix = f"{self.config.acr_login_server}/" + image_path = request.image.removeprefix(prefix) + if not request.image.startswith(prefix) or _IMAGE_PATH.fullmatch(image_path) is None: + raise ExecutionProjectionError( + "Azure Container Apps requires an immutable image in the selected ACR" + ) + memory_mib = _memory_mib(request.memory) + cpu_millis = request.cpu * 1_000 + _validate_azure_size(cpu_millis=cpu_millis, memory_mib=memory_mib) + templates: dict[str, ExecutionTemplate] = {} + for pipeline_id, pipeline in sorted(request.pipelines.items()): + secret_env = pipeline["secret_env"] + if not isinstance(secret_env, Mapping): + raise ExecutionProjectionError("pipeline secret bindings are invalid") + if any( + not isinstance(secret_id, str) or _KEY_VAULT_SECRET.fullmatch(secret_id) is None + for secret_id in secret_env.values() + ): + raise ExecutionProjectionError( + "Azure Key Vault secret ids must use letters, numbers, or hyphens" + ) + schedule = _azure_schedule( + expression=str(pipeline["schedule"]), + time_zone=str(pipeline["time_zone"]), + ) + command: tuple[str, ...] = ( + "runtime", + "execute", + "--contract", + RUNTIME_CONTRACT, + "--pipeline", + pipeline_id, + "--platform", + request.profile_id, + "--config", + "/app/dander.yaml", + "--models-dir", + "/app/models", + "--batch-rows", + str(request.batch_rows), + ) + if bool(pipeline["build_models"]): + command = (*command, "--catalog-output", "/tmp/dander-catalog.json") + digest = request.image.rsplit("@", maxsplit=1)[-1] + labels = { + "dander_version": __version__, + "image_digest": digest, + "pipeline": pipeline_id, + "profile": request.profile_id, + } + template = ExecutionTemplate( + schema=EXECUTION_PROJECTION_SCHEMA, + contract=RUNTIME_CONTRACT, + pipeline_id=pipeline_id, + profile_id=request.profile_id, + launcher="azure_container_apps", + image=request.image, + command=command, + configuration_reference="/app/dander.yaml", + environment=tuple( + sorted( + { + "AZURE_CLIENT_ID": str(self.config.managed_identity_client_id), + "AZURE_RESOURCE_GROUP": self.config.resource_group_name, + "AZURE_SUBSCRIPTION_ID": str(self.config.subscription_id), + "DANDER_IMAGE_DIGEST": digest, + "DANDER_LAUNCHER": "azure_container_apps", + "DANDER_PRINCIPAL": self.config.managed_identity_resource_id, + "HOME": "/tmp", + "TMPDIR": "/tmp", + }.items() + ) + ), + secret_bindings=tuple( + ( + str(environment_name), + SecretReference( + provider="azure_key_vault", + reference=( + f"azure-kv://{self.config.key_vault_uri}/secrets/{secret_id}" + ), + ), + ) + for environment_name, secret_id in sorted(secret_env.items()) + ), + workload_identity=self.config.managed_identity_resource_id, + resources=ResourceProjection( + cpu_millis=cpu_millis, + memory_mib=memory_mib, + ephemeral_storage_mib=None, + deadline_seconds=request.deadline_seconds, + runtime_retry_count=0, + launcher_retry_count=request.launcher_retry_count, + ), + schedule=ScheduleProjection( + task_count=1, + maximum_parallelism=1, + expression=schedule, + time_zone="UTC", + paused=bool(pipeline["paused"]), + ), + network=NetworkPlacement( + placement=self.config.container_app_environment_resource_id, + ), + labels=tuple(sorted(labels.items())), + observability=ObservabilityProjection( + log_destination="log_analytics", + metric_namespace="Microsoft.App/jobs", + alert_target=request.alert_target, + retention_days=30, + ), + extensions=( + ("azure_acr_login_server", self.config.acr_login_server), + ("azure_key_vault_uri", self.config.key_vault_uri), + ( + "azure_managed_identity_client_id", + str(self.config.managed_identity_client_id), + ), + ), + ) + validate_launcher_projection(template, AZURE_CONTAINER_APPS_CAPABILITIES) + templates[pipeline_id] = template + return templates + + +def build_azure_container_apps_launcher( + config: BaseModel, + context: Mapping[str, object], +) -> LauncherRuntime: + """Build Azure projection behavior only after explicit launcher selection.""" + del context + if not isinstance(config, AzureContainerAppsLauncherConfig): + raise TypeError("Azure Container Apps factory received the wrong configuration") + return LauncherRuntime( + provider_id="azure_container_apps", + region=config.region, + templates=AzureContainerAppsTemplateFactory(config), + capabilities=AZURE_CONTAINER_APPS_CAPABILITIES, + ) + + +def _memory_mib(value: str) -> int: + match = re.fullmatch(r"([1-9][0-9]*)(Mi|Gi)", value) + if match is None: + raise ExecutionProjectionError("runtime memory must use Mi or Gi") + quantity = int(match.group(1)) + return quantity if match.group(2) == "Mi" else quantity * 1_024 + + +def _validate_azure_size(*, cpu_millis: int, memory_mib: int) -> None: + supported = {1_000: 2_048, 2_000: 4_096} + if supported.get(cpu_millis) != memory_mib: + raise ExecutionProjectionError( + "Azure Container Apps does not support the requested CPU/memory pair" + ) + + +def _azure_schedule(*, expression: str, time_zone: str) -> str: + if time_zone != "UTC": + raise ExecutionProjectionError("Azure Container Apps scheduled jobs require UTC") + parts = expression.split() + if len(parts) != 5 or any(_CRON_PART.fullmatch(part) is None for part in parts): + raise ExecutionProjectionError( + "Azure Container Apps requires a valid five-field cron schedule" + ) + return " ".join(parts) + + +AZURE_CONTAINER_APPS_LAUNCHER_FACTORY: ProviderFactory[LauncherRuntime] = ProviderFactory( + kind=ProviderKind.LAUNCHER, + provider_id="azure_container_apps", + api_version=PROVIDER_API_VERSION, + build=build_azure_container_apps_launcher, +) + +__all__ = ["AZURE_CONTAINER_APPS_LAUNCHER_FACTORY"] diff --git a/src/dander/providers/azure_key_vault/__init__.py b/src/dander/providers/azure_key_vault/__init__.py new file mode 100644 index 00000000..c5a558c9 --- /dev/null +++ b/src/dander/providers/azure_key_vault/__init__.py @@ -0,0 +1,5 @@ +"""Azure Key Vault provider configuration.""" + +from dander.providers.azure_key_vault.config import AzureKeyVaultConfig + +__all__ = ["AzureKeyVaultConfig"] diff --git a/src/dander/providers/azure_key_vault/config.py b/src/dander/providers/azure_key_vault/config.py new file mode 100644 index 00000000..32342a60 --- /dev/null +++ b/src/dander/providers/azure_key_vault/config.py @@ -0,0 +1,13 @@ +"""Dependency-light Azure Key Vault configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class AzureKeyVaultConfig(BaseModel): + """Select launcher-native Azure Key Vault secret projection.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["azure_key_vault"] diff --git a/src/dander/providers/azure_key_vault/runtime.py b/src/dander/providers/azure_key_vault/runtime.py new file mode 100644 index 00000000..f3cc02e6 --- /dev/null +++ b/src/dander/providers/azure_key_vault/runtime.py @@ -0,0 +1,45 @@ +"""Azure Key Vault runtime selected through the provider registry.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from dander.providers.azure_key_vault.config import AzureKeyVaultConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.security.azure_key_vault import AzureKeyVaultSecretStore +from dander.security.runtime import SecretCapabilities, SecretRuntime + +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + + from pydantic import BaseModel + + +def build_azure_key_vault( + config: BaseModel, + context: Mapping[str, object], +) -> SecretRuntime: + """Build a lazy Key Vault resolver without constructing Azure SDK clients eagerly.""" + if not isinstance(config, AzureKeyVaultConfig): + raise TypeError("Azure Key Vault factory received the wrong configuration") + return SecretRuntime( + provider_id="azure_key_vault", + store=AzureKeyVaultSecretStore(client=cast("Any", context.get("client"))), + capabilities=SecretCapabilities( + provider_id="azure_key_vault", + reference_forms=frozenset({"azure_key_vault_secret_uri"}), + environment_indirection=False, + audited_access=True, + ), + ) + + +AZURE_KEY_VAULT_FACTORY: ProviderFactory[SecretRuntime] = ProviderFactory( + kind=ProviderKind.SECRETS, + provider_id="azure_key_vault", + api_version=PROVIDER_API_VERSION, + build=build_azure_key_vault, +) + +__all__ = ["AZURE_KEY_VAULT_FACTORY"] diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 84f236a1..3371903b 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -1,6 +1,8 @@ """Built-in provider registrations with lazy implementation loading.""" from dander.providers.aws_secrets_manager import AwsSecretsManagerConfig +from dander.providers.azure_container_apps import AzureContainerAppsLauncherConfig +from dander.providers.azure_key_vault import AzureKeyVaultConfig from dander.providers.bigquery import BigQueryStateConfig, BigQueryWarehouseConfig from dander.providers.cloud_run import CloudRunLauncherConfig from dander.providers.dataplex import DataplexCatalogConfig @@ -113,6 +115,14 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.aws_secrets_manager.runtime:AWS_SECRET_MANAGER_FACTORY" ), ) + registry.register( + kind=ProviderKind.SECRETS, + provider_id="azure_key_vault", + config_model=AzureKeyVaultConfig, + load_factory=lazy_provider_factory( + "dander.providers.azure_key_vault.runtime:AZURE_KEY_VAULT_FACTORY" + ), + ) registry.register( kind=ProviderKind.LAUNCHER, provider_id="cloud_run", @@ -137,4 +147,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.kubernetes.runtime:KUBERNETES_LAUNCHER_FACTORY" ), ) + registry.register( + kind=ProviderKind.LAUNCHER, + provider_id="azure_container_apps", + config_model=AzureContainerAppsLauncherConfig, + load_factory=lazy_provider_factory( + "dander.providers.azure_container_apps.runtime:AZURE_CONTAINER_APPS_LAUNCHER_FACTORY" + ), + ) return registry diff --git a/src/dander/security/azure_key_vault.py b/src/dander/security/azure_key_vault.py new file mode 100644 index 00000000..afc02794 --- /dev/null +++ b/src/dander/security/azure_key_vault.py @@ -0,0 +1,69 @@ +"""Secret resolution from versionless or versioned Azure Key Vault references.""" + +from __future__ import annotations + +import re +from importlib import import_module +from typing import TYPE_CHECKING, Protocol, cast + +from dander.security.secret_manager import SecretResolutionError, audit_secret_access + +if TYPE_CHECKING: + from typing import Any + +_REFERENCE = re.compile( + r"^azure-kv://(?Phttps://[a-z][a-z0-9-]{1,22}[a-z0-9]\.vault\.azure\.net)" + r"/secrets/(?P[A-Za-z0-9-]{1,127})(?:/(?P[A-Za-z0-9]+))?$" +) + + +class _Secret(Protocol): + value: str | None + + +class _SecretClient(Protocol): + def get_secret(self, name: str, version: str | None = None) -> object: + """Read one Azure Key Vault secret.""" + + +class AzureKeyVaultSecretStore: + """Resolve full Key Vault references without accepting literal secret values.""" + + def __init__(self, client: _SecretClient | None = None) -> None: + self._client = client + self._vault_uri: str | None = None + + def get_secret(self, reference: str) -> str: + """Return one UTF-8 text secret from an exact Key Vault URI.""" + match = _REFERENCE.fullmatch(reference) + if match is None or "--" in match.group("vault"): + raise SecretResolutionError( + "Azure Key Vault references must be full azure-kv://https://.../secrets/... URIs" + ) + vault_uri = match.group("vault") + client = self._client_for(vault_uri) + secret = cast("_Secret", client.get_secret(match.group("name"), match.group("version"))) + if not isinstance(secret.value, str) or not secret.value: + raise SecretResolutionError("Azure Key Vault secret is empty or not text") + audit_secret_access(reference, "azure_key_vault") + return secret.value + + def _client_for(self, vault_uri: str) -> _SecretClient: + if self._client is not None: + if self._vault_uri is None: + self._vault_uri = vault_uri + elif self._vault_uri != vault_uri: + raise SecretResolutionError( + "One Azure Key Vault resolver cannot cross vault boundaries" + ) + return self._client + identity_module = import_module("azure.identity") + secrets_module = import_module("azure.keyvault.secrets") + credential = cast("Any", identity_module).DefaultAzureCredential() + client = cast("Any", secrets_module).SecretClient( + vault_url=vault_uri, + credential=credential, + ) + self._client = cast("_SecretClient", client) + self._vault_uri = vault_uri + return self._client diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index e3d54ea4..a3d473dd 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -358,6 +358,82 @@ def test_version_two_resolves_complete_fargate_launcher(tmp_path: Path) -> None: } +def test_version_two_resolves_complete_azure_snowflake_profile(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + profile = platforms["platforms"].pop("gcp") + profile["warehouse"] = { + "provider": "snowflake", + "account": "unit-account", + "user": "DANDER_RUNTIME", + "database": "DANDER", + "schema": "RAW", + "warehouse": "DANDER_XS", + "auth": { + "method": "oauth", + "token_env": "DANDER_SNOWFLAKE_OAUTH_TOKEN", + }, + } + profile["state"] = { + "provider": "postgresql", + "authority_id": "postgresql:azure-phase6", + "dsn_env": "DANDER_POSTGRES_DSN", + } + profile["catalog"] = {"provider": "none"} + profile["secrets"] = {"provider": "azure_key_vault"} + platforms["platforms"]["azure_snowflake"] = profile + deployment = platforms["deployments"].pop("gcp_cloud_run") + deployment["platform"] = "azure_snowflake" + deployment["launcher"] = { + "provider": "azure_container_apps", + "region": "eastus", + "subscription_id": "11111111-1111-4111-8111-111111111111", + "resource_group_name": "dander-phase6", + "container_app_environment_name": "dander-phase6-env", + "acr_name": "danderphase6", + "key_vault_name": "dander-phase6-kv", + "managed_identity_name": "dander-phase6-runtime", + "managed_identity_client_id": "22222222-2222-4222-8222-222222222222", + } + deployment["runtime"]["memory"] = "2Gi" + deployment["pipelines"]["example_records"]["time_zone"] = "UTC" + deployment["pipelines"]["example_records"]["secret_bindings"] = { + "DANDER_POSTGRES_DSN": "postgres-dsn", + "DANDER_SNOWFLAKE_OAUTH_TOKEN": "snowflake-oauth-token", + } + platforms["deployments"]["azure_snowflake"] = deployment + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path, deployment="azure_snowflake") + + assert resolved.platform_name == "azure_snowflake" + assert resolved.warehouse_provider == "snowflake" + assert resolved.state_provider == "postgresql" + assert resolved.catalog_provider == "none" + assert resolved.secret_provider == "azure_key_vault" + assert resolved.launcher_provider == "azure_container_apps" + assert resolved.resolved_launcher_config()["acr_name"] == "danderphase6" + assert resolved.platform.runtime.memory == "2Gi" + + +def test_azure_key_vault_profile_fails_closed_on_another_launcher(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + platforms["platforms"]["gcp"]["secrets"] = {"provider": "azure_key_vault"} + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + with pytest.raises(ProjectConfigError, match="Azure Key Vault projection"): + load_project_config(project_path) + + def test_config_migrate_check_is_read_only_then_write_is_atomic(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_azure_container_apps_runtime.py b/tests/providers/test_azure_container_apps_runtime.py new file mode 100644 index 00000000..d748e0ac --- /dev/null +++ b/tests/providers/test_azure_container_apps_runtime.py @@ -0,0 +1,164 @@ +"""Azure Container Apps Jobs launcher projection and limit coverage.""" + +from __future__ import annotations + +import sys + +import pytest + +from dander.deployment import ExecutionProjectionError, LauncherRuntime, ResolvedTemplateRequest +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry + +_SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" +_CLIENT_ID = "22222222-2222-4222-8222-222222222222" +_IMAGE = "danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64 +_LAUNCHER = { + "provider": "azure_container_apps", + "region": "eastus", + "subscription_id": _SUBSCRIPTION_ID, + "resource_group_name": "dander-phase6", + "container_app_environment_name": "dander-phase6-env", + "acr_name": "danderphase6", + "key_vault_name": "dander-phase6-kv", + "managed_identity_name": "dander-phase6-runtime", + "managed_identity_client_id": _CLIENT_ID, +} +_PIPELINES: dict[str, dict[str, object]] = { + "warehouse_fixture": { + "runtime_service_account_id": "dander-runtime", + "build_models": True, + "schedule": "15 4 * * *", + "time_zone": "UTC", + "paused": True, + "secret_env": { + "DANDER_POSTGRES_DSN": "postgres-dsn", + "DANDER_SNOWFLAKE_OAUTH_TOKEN": "snowflake-oauth-token", + }, + } +} + + +def _runtime() -> LauncherRuntime: + registry = default_provider_registry() + config = registry.parse(ProviderKind.LAUNCHER, _LAUNCHER) + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + return runtime + + +def _request( + *, + pipelines: dict[str, dict[str, object]] | None = None, + image: str = _IMAGE, + memory: str = "2Gi", +) -> ResolvedTemplateRequest: + return ResolvedTemplateRequest( + pipelines=_PIPELINES if pipelines is None else pipelines, + image=image, + profile_id="azure_snowflake", + cpu=1, + memory=memory, + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=1_000, + alert_target=( + "/subscriptions/11111111-1111-4111-8111-111111111111/" + "resourceGroups/dander-phase6/providers/microsoft.insights/" + "actionGroups/dander-phase6" + ), + ) + + +def test_azure_factory_is_lazy_and_projects_only_non_secret_identity() -> None: + module_name = "dander.providers.azure_container_apps.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + config = registry.parse(ProviderKind.LAUNCHER, _LAUNCHER) + + assert module_name not in sys.modules + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + assert module_name in sys.modules + + template = runtime.templates.build(_request())["warehouse_fixture"] + environment = dict(template.environment) + secrets = dict(template.secret_bindings) + + assert runtime.provider_id == "azure_container_apps" + assert runtime.region == "eastus" + assert template.image == _IMAGE + assert template.profile_id == "azure_snowflake" + assert template.resources.cpu_millis == 1_000 + assert template.resources.memory_mib == 2_048 + assert template.resources.ephemeral_storage_mib is None + assert template.schedule.expression == "15 4 * * *" + assert template.schedule.time_zone == "UTC" + assert template.schedule.paused is True + assert template.network.placement is not None + assert template.network.placement.endswith( + "/providers/Microsoft.App/managedEnvironments/dander-phase6-env" + ) + assert template.workload_identity.endswith( + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + ) + assert environment["AZURE_CLIENT_ID"] == _CLIENT_ID + assert environment["HOME"] == "/tmp" + assert "DANDER_POSTGRES_DSN" not in environment + assert secrets["DANDER_POSTGRES_DSN"].reference == ( + "azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn" + ) + assert secrets["DANDER_SNOWFLAKE_OAUTH_TOKEN"].provider == "azure_key_vault" + assert dict(template.extensions) == { + "azure_acr_login_server": "danderphase6.azurecr.io", + "azure_key_vault_uri": "https://dander-phase6-kv.vault.azure.net", + "azure_managed_identity_client_id": _CLIENT_ID, + } + serialized = repr(template.as_dict()) + assert "oauth-token-value" not in serialized + assert "postgresql://" not in serialized + + +@pytest.mark.parametrize( + ("update", "message"), + [ + ({"time_zone": "America/New_York"}, "require UTC"), + ({"schedule": "0 9 * * ?"}, "five-field"), + ({"secret_env": {"TOKEN": "not_a_key_vault_name"}}, "secret ids"), + ], +) +def test_azure_projection_rejects_unhonored_schedule_or_secret_intent( + update: dict[str, object], + message: str, +) -> None: + pipeline = {**_PIPELINES["warehouse_fixture"], **update} + with pytest.raises(ExecutionProjectionError, match=message): + _runtime().templates.build(_request(pipelines={"warehouse_fixture": pipeline})) + + +@pytest.mark.parametrize( + ("image", "memory", "message"), + [ + ( + "other.azurecr.io/dander/runtime@sha256:" + "a" * 64, + "2Gi", + "selected ACR", + ), + (_IMAGE, "512Mi", "CPU/memory"), + ], +) +def test_azure_projection_rejects_unowned_images_and_invalid_resource_pairs( + image: str, + memory: str, + message: str, +) -> None: + with pytest.raises(ExecutionProjectionError, match=message): + _runtime().templates.build(_request(image=image, memory=memory)) + + +def test_azure_launcher_configuration_rejects_ambiguous_names() -> None: + registry = default_provider_registry() + with pytest.raises(ProviderFactoryError, match="key_vault_name"): + registry.parse( + ProviderKind.LAUNCHER, + {**_LAUNCHER, "key_vault_name": "dander--phase6"}, + ) diff --git a/tests/providers/test_azure_key_vault_runtime.py b/tests/providers/test_azure_key_vault_runtime.py new file mode 100644 index 00000000..dddaa857 --- /dev/null +++ b/tests/providers/test_azure_key_vault_runtime.py @@ -0,0 +1,97 @@ +"""Azure Key Vault provider selection and safe-reference coverage.""" + +from __future__ import annotations + +import logging +import sys +from dataclasses import dataclass + +import pytest + +from dander.providers import ProviderKind, default_provider_registry +from dander.security import SecretResolutionError, SecretRuntime + + +@dataclass +class _Secret: + value: str | None + + +class _Client: + def __init__(self, value: str | None = "resolved-value") -> None: + self.value = value + self.requests: list[tuple[str, str | None]] = [] + + def get_secret(self, name: str, version: str | None = None) -> _Secret: + self.requests.append((name, version)) + return _Secret(self.value) + + +def _runtime(client: _Client) -> SecretRuntime: + registry = default_provider_registry() + config = registry.parse(ProviderKind.SECRETS, {"provider": "azure_key_vault"}) + runtime = registry.build(ProviderKind.SECRETS, config, context={"client": client}) + assert isinstance(runtime, SecretRuntime) + return runtime + + +def test_azure_key_vault_provider_is_lazy_and_audits_without_secret_values( + caplog: pytest.LogCaptureFixture, +) -> None: + runtime_module = "dander.providers.azure_key_vault.runtime" + sys.modules.pop(runtime_module, None) + sys.modules.pop("azure.identity", None) + sys.modules.pop("azure.keyvault.secrets", None) + client = _Client() + registry = default_provider_registry() + config = registry.parse(ProviderKind.SECRETS, {"provider": "azure_key_vault"}) + + assert runtime_module not in sys.modules + runtime = registry.build(ProviderKind.SECRETS, config, context={"client": client}) + assert isinstance(runtime, SecretRuntime) + assert runtime_module in sys.modules + assert "azure.identity" not in sys.modules + assert "azure.keyvault.secrets" not in sys.modules + reference = ( + "azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/snowflake-token/version1" + ) + with caplog.at_level(logging.INFO): + assert runtime.store.get_secret(reference) == "resolved-value" + + assert client.requests == [("snowflake-token", "version1")] + assert caplog.records[-1].__dict__["secret_backend"] == "azure_key_vault" + assert "resolved-value" not in caplog.text + assert runtime.capabilities.reference_forms == frozenset({"azure_key_vault_secret_uri"}) + + +@pytest.mark.parametrize( + ("reference", "client", "message"), + [ + ("azure-kv://short-name", _Client(), "full"), + ( + "azure-kv://https://dander--vault.vault.azure.net/secrets/token", + _Client(), + "full", + ), + ( + "azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/token", + _Client(value=None), + "empty", + ), + ], +) +def test_azure_key_vault_rejects_ambiguous_or_non_text_secrets( + reference: str, + client: _Client, + message: str, +) -> None: + with pytest.raises(SecretResolutionError, match=message): + _runtime(client).store.get_secret(reference) + + +def test_azure_key_vault_resolver_cannot_cross_vaults() -> None: + runtime = _runtime(_Client()) + runtime.store.get_secret("azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/token") + + with pytest.raises(SecretResolutionError, match="cross vault"): + runtime.store.get_secret("azure-kv://https://other-phase6-kv.vault.azure.net/secrets/token") diff --git a/tickets/DANDER-108-azure-launcher-contract.md b/tickets/DANDER-108-azure-launcher-contract.md new file mode 100644 index 00000000..26758961 --- /dev/null +++ b/tickets/DANDER-108-azure-launcher-contract.md @@ -0,0 +1,29 @@ +# DANDER-108 — Azure launcher and Key Vault contract + +Status: implemented locally; live qualification pending + +## Requirement + +Add Azure Container Apps Jobs and Azure Key Vault to the existing typed provider boundaries before +Terraform or live-provider work. The contract must consume the immutable provider-neutral +`ResolvedTemplateRequest`, preserve the OCI runtime command, require an ACR digest, keep secret +values out of projections, and reject Azure semantics that cannot be represented exactly. + +## Acceptance + +- [x] `azure_container_apps` is a lazy launcher provider with explicit resource and extension limits. +- [x] The launcher projects an immutable ACR digest, user-assigned managed identity, Container Apps + environment, bounded retries/deadline, logs, and Key Vault references. +- [x] Scheduled jobs accept only five-field UTC cron because Azure evaluates job schedules in UTC. +- [x] Unsupported CPU/memory pairs, foreign registries, malformed secret ids, and incompatible + launcher/secret-provider combinations fail before provider access. +- [x] `azure_key_vault` resolves only full vault secret URIs, remains SDK-lazy, audits references, + excludes values, and cannot cross vault boundaries through one resolver. +- [x] Version 2 configuration resolves the named Azure/Snowflake/PostgreSQL/no-catalog shape. +- [ ] Terraform, ACR promotion, Azure operations, federation, and live profile proofs remain separate + Phase 6 tickets and require explicit approval before provider writes. + +## Evidence boundary + +This ticket is deterministic local contract evidence only. It creates no Azure resources, registers +no resource providers, copies no image, and makes no support claim. From 808e30c58d13387c3ea4740bb7626b685771b43b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 12:11:44 -0400 Subject: [PATCH 121/163] Add plan-first Azure infrastructure --- .github/workflows/ci.yml | 10 + HANDOFF.md | 38 +- docs/decisions.md | 14 + docs/execution-projection.md | 5 +- docs/known-limitations.md | 7 +- infra/README.md | 4 + infra/azure/.terraform.lock.hcl | 22 + infra/azure/README.md | 74 +++ .../azure/bootstrap-admin/.terraform.lock.hcl | 22 + infra/azure/bootstrap-admin/README.md | 15 + infra/azure/bootstrap-admin/main.tf | 94 ++++ infra/azure/bootstrap-admin/outputs.tf | 39 ++ .../tests/bootstrap.tftest.hcl | 57 +++ infra/azure/bootstrap-admin/variables.tf | 91 ++++ infra/azure/bootstrap-admin/versions.tf | 18 + infra/azure/main.tf | 43 ++ .../container-apps-jobs/.terraform.lock.hcl | 22 + .../modules/container-apps-jobs/README.md | 15 + .../azure/modules/container-apps-jobs/main.tf | 209 +++++++++ .../modules/container-apps-jobs/outputs.tf | 24 + .../container-apps-jobs/tests/jobs.tftest.hcl | 88 ++++ .../modules/container-apps-jobs/variables.tf | 82 ++++ .../modules/container-apps-jobs/versions.tf | 10 + infra/azure/outputs.tf | 19 + infra/azure/terraform.tfvars.example | 13 + infra/azure/tests/azure.tftest.hcl | 83 ++++ infra/azure/variables.tf | 95 ++++ infra/azure/versions.tf | 20 + pyproject.toml | 22 + scripts/check_distribution.py | 5 + src/dander/bootstrap/__init__.py | 9 + src/dander/bootstrap/azure_admin.py | 408 ++++++++++++++++ src/dander/bootstrap/azure_terraform.py | 325 +++++++++++++ src/dander/cli/azure_command.py | 306 ++++++++++++ src/dander/cli/main.py | 2 + .../azure_container_apps/__init__.py | 14 +- .../azure_container_apps/verification.py | 439 ++++++++++++++++++ tests/bootstrap/test_azure_admin.py | 209 +++++++++ tests/bootstrap/test_azure_terraform.py | 174 +++++++ .../test_azure_deployment_verification.py | 253 ++++++++++ ...R-109-azure-terraform-plan-verification.md | 35 ++ 41 files changed, 3410 insertions(+), 24 deletions(-) create mode 100644 infra/azure/.terraform.lock.hcl create mode 100644 infra/azure/README.md create mode 100644 infra/azure/bootstrap-admin/.terraform.lock.hcl create mode 100644 infra/azure/bootstrap-admin/README.md create mode 100644 infra/azure/bootstrap-admin/main.tf create mode 100644 infra/azure/bootstrap-admin/outputs.tf create mode 100644 infra/azure/bootstrap-admin/tests/bootstrap.tftest.hcl create mode 100644 infra/azure/bootstrap-admin/variables.tf create mode 100644 infra/azure/bootstrap-admin/versions.tf create mode 100644 infra/azure/main.tf create mode 100644 infra/azure/modules/container-apps-jobs/.terraform.lock.hcl create mode 100644 infra/azure/modules/container-apps-jobs/README.md create mode 100644 infra/azure/modules/container-apps-jobs/main.tf create mode 100644 infra/azure/modules/container-apps-jobs/outputs.tf create mode 100644 infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl create mode 100644 infra/azure/modules/container-apps-jobs/variables.tf create mode 100644 infra/azure/modules/container-apps-jobs/versions.tf create mode 100644 infra/azure/outputs.tf create mode 100644 infra/azure/terraform.tfvars.example create mode 100644 infra/azure/tests/azure.tftest.hcl create mode 100644 infra/azure/variables.tf create mode 100644 infra/azure/versions.tf create mode 100644 src/dander/bootstrap/azure_admin.py create mode 100644 src/dander/bootstrap/azure_terraform.py create mode 100644 src/dander/cli/azure_command.py create mode 100644 src/dander/providers/azure_container_apps/verification.py create mode 100644 tests/bootstrap/test_azure_admin.py create mode 100644 tests/bootstrap/test_azure_terraform.py create mode 100644 tests/providers/test_azure_deployment_verification.py create mode 100644 tickets/DANDER-109-azure-terraform-plan-verification.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 535cea3d..586b9ef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,16 @@ jobs: terraform -chdir=aws validate terraform -chdir=aws/modules/fargate init -backend=false -input=false terraform -chdir=aws/modules/fargate test + - name: Validate Azure Terraform stacks + run: | + terraform -chdir=azure/bootstrap-admin init -backend=false -input=false + terraform -chdir=azure/bootstrap-admin validate + terraform -chdir=azure/bootstrap-admin test + terraform -chdir=azure init -backend=false -input=false + terraform -chdir=azure validate + terraform -chdir=azure test + terraform -chdir=azure/modules/container-apps-jobs init -backend=false -input=false + terraform -chdir=azure/modules/container-apps-jobs test - name: Validate Kubernetes Helm chart run: | helm lint kubernetes/chart/dander --values ../tests/fixtures/kubernetes-values.yaml diff --git a/HANDOFF.md b/HANDOFF.md index c9c0c816..0d9639ef 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,40 @@ ## Finished -- Re-verified the Phase 5 exit gate on protected `main` and began Phase 6 locally. -- Added typed Azure Container Apps Jobs and Azure Key Vault provider contracts. -- Added deterministic ACR digest, managed identity, UTC schedule, resource, and secret projections. -- Kept all Azure provider writes behind a separate explicit cost/approval gate. +- Merged the Azure launcher/Key Vault contract through protected PR #188 with green main CI. +- Added plan-first Azure stage zero for firewall-restricted Storage state, Basic ACR, and one user-assigned runtime identity. +- Added Container Apps Jobs Terraform for exact projections, Key Vault references, logs, alerts, and optional internal networking. +- Added separate plan/apply CLI paths and a read-only, secret-free Azure deployment verifier. +- Kept provider registration, candidate publication, resource creation, image copy, and job execution behind later approvals. ## Try It -Run `uv run --extra dev pytest -q tests/providers/test_azure_container_apps_runtime.py -tests/providers/test_azure_key_vault_runtime.py tests/project/test_portable_config.py`. +Run `terraform -chdir=infra/azure test` and `uv run pytest -q +tests/bootstrap/test_azure_admin.py tests/bootstrap/test_azure_terraform.py +tests/providers/test_azure_deployment_verification.py`. ## Checks -- Focused Azure, launcher, secret-provider, dependency, and portable-config tests pass. -- Ruff passes on the Phase 6 contract scope. -- Azure subscription preflight is enabled; required resource providers remain unregistered. +- Full `pytest -q`, Ruff, strict mypy, Terraform validate/provider-mocked tests, wheel/sdist inspection, and Trivy HIGH/CRITICAL config scan pass. +- Protected main CI is green at merged PR #188; protected CI for this infrastructure slice is pending. +- Azure CLI is signed in; required resource providers remain unregistered and no Azure resource exists from this work. ## Decisions -- Azure cron is UTC-only and fails closed for other time zones. -- The first canonical Azure shape is Snowflake + PostgreSQL state + no catalog + Key Vault. -- Live proof must use a newly approved candidate built after the Azure implementation merges. +- Terraform disables automatic provider registration and uses saved-plan-only applies with Entra-authenticated state. +- State and Key Vault networks default to deny and admit one reviewed exact operator IP; Key Vault also keeps Azure's trusted-service path. +- Paused projections are manual jobs; active schedules retain the already validated UTC cron. ## Remaining -- Merge this contract through a focused protected PR. -- Add plan-first Azure state/ACR/Container Apps Terraform plus deployment verification. -- Add ACR copy and launcher operations without performing provider writes. +- Merge this infrastructure/verifier slice through a focused protected PR. +- Add digest-preserving ACR copy and launcher lifecycle operations without provider writes. +- Prepare Azure-to-Google federation and named Snowflake proof tooling locally. - Obtain publication and provider-cost approvals before candidate or live proof mutations. - Run approved identity, lifecycle, cleanup, rollback, and no-drift acceptance. ## Review First -- `src/dander/providers/azure_container_apps/runtime.py` -- `src/dander/security/azure_key_vault.py` -- `tests/providers/test_azure_container_apps_runtime.py` +- `infra/azure/modules/container-apps-jobs/main.tf` +- `src/dander/bootstrap/azure_admin.py` +- `src/dander/providers/azure_container_apps/verification.py` diff --git a/docs/decisions.md b/docs/decisions.md index 4fc66efe..176f4da9 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1157,3 +1157,17 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** This satisfies Phase 5 common-intersection correctness. Provider/profile support, scale, throughput, crossover, cost, soak, and pairwise qualification remain separate Phase 8 or promotion requirements. + +## 2026-08-11 — Azure bootstrap uses reviewed plans and Entra-authenticated state + +- **Stage zero:** One saved plan creates the resource group, default-deny operator-`/32` versioned + private Storage container, Basic ACR with administrator credentials disabled, and user-assigned + runtime identity. Automatic resource-provider registration is disabled and remains a separately + approved live mutation. +- **Projection:** Paused pipelines become manual Container Apps Jobs; active pipelines use the + validated UTC cron. Deadline, retry, CPU/memory, identity, ACR digest, Key Vault references, + default-deny vault networking with one operator IP, logging, optional internal subnet, and + existing Action Group routing remain exact inputs. +- **Boundary:** Terraform contains secret references but never values. Local/provider-mocked tests + do not qualify Azure; image copy, operations, federation, paid proof, cleanup, and support + promotion remain separate Phase 6 gates. diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 046652ee..76eec49b 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -62,5 +62,6 @@ user-assigned managed identity, exact Container Apps environment placement, supp or 2 CPU/4 GiB sizing, launcher-owned retries and deadline, Log Analytics destinations, and versionless Key Vault references for declared secrets. Azure evaluates scheduled-job cron in UTC, so a non-UTC projection fails before provider access rather than silently changing wall-clock -behavior. This contract is locally conformance-tested only; Terraform and live-profile evidence -remain separate Phase 6 gates. +behavior. Plan-first Terraform now consumes this projection exactly and has provider-mocked +contract evidence; no Azure resource has been created, and live-profile evidence remains a +separate Phase 6 gate. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 311f94fe..30ed6354 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -46,9 +46,10 @@ remain Phase 8 work. schema limits, transforms, graphs, and fencing through `dander runtime compatibility`, while the packaged capability manifest remains the support boundary. Local PostgreSQL benchmark results are regression evidence, not a paid or controlled-memory scale qualification. -- Azure Container Apps Jobs and Azure Key Vault have a typed, deterministic local projection only. - Azure schedules are UTC-only and currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. - Resource-provider registration, Terraform, ACR image copy, identity federation, operations, and +- Azure Container Apps Jobs and Azure Key Vault have a typed projection plus locally validated, + plan-first Terraform and read-only deployment verification. Azure schedules are UTC-only and + currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. Resource-provider registration, + credentialed Azure planning/apply, ACR image copy, identity federation, lifecycle operations, and live Snowflake/PostgreSQL acceptance have not yet run, so Azure remains unsupported. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or diff --git a/infra/README.md b/infra/README.md index 65c09d4a..ee2498dd 100644 --- a/infra/README.md +++ b/infra/README.md @@ -14,6 +14,8 @@ call sites (mirrors the `SecretStoreProvider` / `ComputeProvider` abstractions i | `modules/github-wif` | Repository/ref-scoped GitHub OIDC and a keyless deployment identity. **Implemented.** | | `modules/cost-guard` | Project budget, Pub/Sub, and simulation-first Gen 2 billing kill switch. **Implemented.** | | `kubernetes/chart/dander` | Versioned Helm chart for an existing conforming cluster. **Locally validated; not live-qualified.** | +| `azure/bootstrap-admin` | Azure Storage state, ACR, and user-assigned runtime identity. **Locally validated; not applied.** | +| `azure/modules/container-apps-jobs` | Container Apps Jobs, Key Vault references, Log Analytics, alerts, and optional network placement. **Locally validated; not live-qualified.** | The main root always calls `modules/bigquery` and can opt into the remaining workload modules. The one-time `infra/bootstrap-admin` root creates the remote-state bucket, the Artifact Registry @@ -102,6 +104,8 @@ Each scheduler identity can invoke only its pipeline's named Cloud Run Job. ## Rules (see `steering/01-security.md` and `steering/languages/terraform.md`) - **Remote GCS backend** for state — never local state committed to the repo. +- Azure uses an Entra-authenticated Azure Storage backend; its temporary bootstrap state and saved + plans remain outside the repository. - **No secret values** in `.tf`/`.tfvars`; reference Secret Manager. Commit only `*.tfvars.example`. - Project id / region are always parameterized, never hard-coded. - Stage-zero applies run through the CLI using the approved administrator. Platform applies use diff --git a/infra/azure/.terraform.lock.hcl b/infra/azure/.terraform.lock.hcl new file mode 100644 index 00000000..c455d6da --- /dev/null +++ b/infra/azure/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.81.0" + constraints = "~> 4.72" + hashes = [ + "h1:XhToZua4gtih1Kv8RdStcfND83G4Tmb6GZFT4jEUhDU=", + "zh:0732e7b74264ddfa2b90ba69d01c283d3cbae9f72ed3e506c6ac92529fed7fd3", + "zh:12afb524e232fe4e3d6161927724af5dfa4831d71edd9c174917ca9b7377bfae", + "zh:169d619ae202c4145e02fb706fb7c3679445ab3e3ff722edbf89597517a8c92e", + "zh:6beb95a3ef2f2d9c76abaa48e5450e90686a3fb6a47f1cb0ff7c5e94b6960151", + "zh:705e075fb5ffc4bf66fd7cbabf1a65007a41621e80030a2c158a4c83b6046216", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79a8d17fefe647040fcb9ee8821a4f09f395427c4fd49493489b9a93a9a1038e", + "zh:8cc3f900b3774c0ae37ae42365c4579a199cf9e5edf88e476fdf5ab1048f84ea", + "zh:dec373b9390fa95e257291acd018ed65a7d512b428645d35e22cdbe8b245a08b", + "zh:e60f1e9fb45df6defade2855ed6e68547409ea75d30655c556adb0c08579749b", + "zh:f901d12ec82f3f8b5880a27b5cbcd7bd0d97e60c9367a2d7ed82fdd1157b39ff", + "zh:facf68ea5bf0f2b8ba720e7fba5f86492e1d4c591100460bb91c3f79f391f4b6", + ] +} diff --git a/infra/azure/README.md b/infra/azure/README.md new file mode 100644 index 00000000..9256485d --- /dev/null +++ b/infra/azure/README.md @@ -0,0 +1,74 @@ +# Azure Container Apps Jobs Terraform + +This root turns validated `io.dander.execution/v1` Azure projections into Container Apps Jobs. It +uses Azure Storage remote state, one existing ACR and user-assigned managed identity from the +stage-zero root, a Log Analytics-backed Container Apps environment, an RBAC-enabled Key Vault, and +optional failed-execution alerts to an existing Action Group. Terraform creates secret references, +never secret values. + +Planning and applying are deliberately separate. Both use the signed-in Azure CLI identity and +Entra authentication for Storage; neither Terraform provider automatically registers Azure +resource providers. + +## Stage zero + +Choose globally unique lowercase Storage and ACR names. Keep the operator artifact directory +outside the checkout because it contains the saved plan and temporary local state: + +```bash +dander init-azure-admin-plan \ + --subscription-id 11111111-1111-4111-8111-111111111111 \ + --location eastus \ + --resource-group dander-phase6 \ + --state-storage-account danderphase6state \ + --state-allowed-ip 203.0.113.10 \ + --acr-name danderphase6 \ + --managed-identity-name dander-phase6-runtime \ + --operator-artifact-dir "$HOME/Library/Application Support/Dander/terraform/azure-admin" +``` + +Replace the documentation-only IP with the operator's current public IPv4 address. Azure treats +this exact-IP firewall entry with `/32` semantics. +After separate approval for provider registration and resource costs, review the saved plan and +run the matching `init-azure-admin-apply` command. The apply migrates its initial local state into +the firewall-restricted, private, versioned Azure Storage container. It grants the authenticated +operator only the Blob data role needed to use that Entra-authenticated backend. Record +`runtime_identity_client_id` from the printed Terraform output command and place that non-secret +identifier in the named launcher configuration. + +## Platform plan + +After an accepted OCI release is copied into ACR by digest, plan the selected deployment: + +```bash +dander init-azure-plan \ + --deployment azure_snowflake \ + --state-resource-group dander-phase6 \ + --state-storage-account danderphase6state \ + --key-vault-allowed-ip 203.0.113.10 \ + --container-image danderphase6.azurecr.io/dander/runtime@sha256:DIGEST +``` + +This creates a saved plan only. An optional existing delegated subnet can be selected with +`--infrastructure-subnet-id`; selecting it makes the environment internal. An existing reviewed +Action Group can be selected with `--alert-action-group-id`. Paused pipelines remain manual jobs; +active schedules use Azure's UTC-only five-field cron. + +Replace the documentation-only Key Vault IP with the operator's current public IPv4 address. Key +Vault defaults to deny, admits that exact operator IP for secret administration, and retains the +Azure trusted-service path used by the managed-identity secret reference. + +Run `dander init-azure-apply` only after reviewing the exact saved plan and receiving explicit +approval for the live Azure changes. Then use `dander azure verify` for read-only checks of the +subscription, environment, logs, job trigger/resources/image, managed identity, ACR, and Key Vault. + +## Boundaries + +- ACR administrator credentials and Storage shared keys remain disabled; state network access + defaults to deny and admits only the reviewed operator `/32`. +- The runtime identity receives only `AcrPull` and `Key Vault Secrets User` in this root. +- Key Vault network access defaults to deny and admits only Azure's trusted-service path plus the + reviewed operator IP. +- Secret values are an operator action outside Terraform and must never enter a plan or state. +- Image copy, job lifecycle operations, external identity federation, live profile proof, and + support promotion are separate Phase 6 gates. diff --git a/infra/azure/bootstrap-admin/.terraform.lock.hcl b/infra/azure/bootstrap-admin/.terraform.lock.hcl new file mode 100644 index 00000000..c455d6da --- /dev/null +++ b/infra/azure/bootstrap-admin/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.81.0" + constraints = "~> 4.72" + hashes = [ + "h1:XhToZua4gtih1Kv8RdStcfND83G4Tmb6GZFT4jEUhDU=", + "zh:0732e7b74264ddfa2b90ba69d01c283d3cbae9f72ed3e506c6ac92529fed7fd3", + "zh:12afb524e232fe4e3d6161927724af5dfa4831d71edd9c174917ca9b7377bfae", + "zh:169d619ae202c4145e02fb706fb7c3679445ab3e3ff722edbf89597517a8c92e", + "zh:6beb95a3ef2f2d9c76abaa48e5450e90686a3fb6a47f1cb0ff7c5e94b6960151", + "zh:705e075fb5ffc4bf66fd7cbabf1a65007a41621e80030a2c158a4c83b6046216", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79a8d17fefe647040fcb9ee8821a4f09f395427c4fd49493489b9a93a9a1038e", + "zh:8cc3f900b3774c0ae37ae42365c4579a199cf9e5edf88e476fdf5ab1048f84ea", + "zh:dec373b9390fa95e257291acd018ed65a7d512b428645d35e22cdbe8b245a08b", + "zh:e60f1e9fb45df6defade2855ed6e68547409ea75d30655c556adb0c08579749b", + "zh:f901d12ec82f3f8b5880a27b5cbcd7bd0d97e60c9367a2d7ed82fdd1157b39ff", + "zh:facf68ea5bf0f2b8ba720e7fba5f86492e1d4c591100460bb91c3f79f391f4b6", + ] +} diff --git a/infra/azure/bootstrap-admin/README.md b/infra/azure/bootstrap-admin/README.md new file mode 100644 index 00000000..671e898a --- /dev/null +++ b/infra/azure/bootstrap-admin/README.md @@ -0,0 +1,15 @@ +# Azure stage-zero bootstrap + +This one-time root creates the resource group, firewall-restricted private/versioned Azure Storage state container, +Basic ACR with administrator credentials disabled, and one user-assigned runtime identity. It uses +the signed-in Azure CLI principal and deliberately disables automatic resource-provider +registration. + +The Dander CLI first plans this root against secured local state in an operator directory outside +the repository. After an explicitly approved apply, it migrates that state into the new Azure +Storage backend using Entra authentication and records only the non-secret backend coordinates. +Subsequent plans must match that record exactly. + +Do not commit the saved plan, local state, backend metadata, or `.terraform/` directory. The +generated identity client id and reviewed exact operator IP are non-secret reproducibility inputs; +secret values and credentials are not outputs of this root. diff --git a/infra/azure/bootstrap-admin/main.tf b/infra/azure/bootstrap-admin/main.tf new file mode 100644 index 00000000..a41376ea --- /dev/null +++ b/infra/azure/bootstrap-admin/main.tf @@ -0,0 +1,94 @@ +locals { + tags = merge(var.tags, { + component = "azure-stage-zero" + managed-by = "dander" + }) +} + +data "azurerm_client_config" "current" {} + +check "authenticated_subscription_matches_target" { + assert { + condition = data.azurerm_client_config.current.subscription_id == var.subscription_id + error_message = "Authenticated Azure subscription does not match subscription_id." + } +} + +resource "azurerm_resource_group" "dander" { + name = var.resource_group_name + location = var.location + tags = local.tags +} + +resource "azurerm_storage_account" "terraform_state" { + name = var.storage_account_name + resource_group_name = azurerm_resource_group.dander.name + location = azurerm_resource_group.dander.location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" + access_tier = "Hot" + min_tls_version = "TLS1_2" + https_traffic_only_enabled = true + allow_nested_items_to_be_public = false + public_network_access_enabled = true + shared_access_key_enabled = false + tags = local.tags + + blob_properties { + versioning_enabled = true + + container_delete_retention_policy { + days = 14 + } + + delete_retention_policy { + days = 14 + } + } + + network_rules { + default_action = "Deny" + bypass = ["AzureServices"] + ip_rules = [var.state_allowed_ip_rule] + } + + lifecycle { + prevent_destroy = true + } +} + +resource "azurerm_storage_container" "terraform_state" { + name = var.state_container_name + storage_account_id = azurerm_storage_account.terraform_state.id + container_access_type = "private" + + lifecycle { + prevent_destroy = true + } + + depends_on = [azurerm_role_assignment.terraform_state_operator] +} + +resource "azurerm_role_assignment" "terraform_state_operator" { + scope = azurerm_storage_account.terraform_state.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_container_registry" "runtime" { + name = var.acr_name + resource_group_name = azurerm_resource_group.dander.name + location = azurerm_resource_group.dander.location + sku = "Basic" + admin_enabled = false + public_network_access_enabled = true + tags = local.tags +} + +resource "azurerm_user_assigned_identity" "runtime" { + name = var.managed_identity_name + resource_group_name = azurerm_resource_group.dander.name + location = azurerm_resource_group.dander.location + tags = local.tags +} diff --git a/infra/azure/bootstrap-admin/outputs.tf b/infra/azure/bootstrap-admin/outputs.tf new file mode 100644 index 00000000..b0b4cf95 --- /dev/null +++ b/infra/azure/bootstrap-admin/outputs.tf @@ -0,0 +1,39 @@ +output "resource_group_name" { + description = "Resource group containing the Azure deployment." + value = azurerm_resource_group.dander.name +} + +output "state_storage_account_name" { + description = "Storage account containing remote Terraform state." + value = azurerm_storage_account.terraform_state.name +} + +output "state_container_name" { + description = "Blob container containing remote Terraform state." + value = azurerm_storage_container.terraform_state.name +} + +output "runtime_registry_id" { + description = "Azure Container Registry resource id." + value = azurerm_container_registry.runtime.id +} + +output "runtime_registry_login_server" { + description = "Azure Container Registry login server." + value = azurerm_container_registry.runtime.login_server +} + +output "runtime_identity_id" { + description = "User-assigned managed identity resource id." + value = azurerm_user_assigned_identity.runtime.id +} + +output "runtime_identity_client_id" { + description = "User-assigned managed identity client id required by the launcher config." + value = azurerm_user_assigned_identity.runtime.client_id +} + +output "runtime_identity_principal_id" { + description = "User-assigned managed identity principal id used for role assignments." + value = azurerm_user_assigned_identity.runtime.principal_id +} diff --git a/infra/azure/bootstrap-admin/tests/bootstrap.tftest.hcl b/infra/azure/bootstrap-admin/tests/bootstrap.tftest.hcl new file mode 100644 index 00000000..566b4ad0 --- /dev/null +++ b/infra/azure/bootstrap-admin/tests/bootstrap.tftest.hcl @@ -0,0 +1,57 @@ +mock_provider "azurerm" { + mock_data "azurerm_client_config" { + defaults = { + subscription_id = "11111111-1111-4111-8111-111111111111" + tenant_id = "22222222-2222-4222-8222-222222222222" + client_id = "33333333-3333-4333-8333-333333333333" + object_id = "44444444-4444-4444-8444-444444444444" + } + } +} + +variables { + subscription_id = "11111111-1111-4111-8111-111111111111" + location = "eastus" + resource_group_name = "dander-phase6" + storage_account_name = "danderphase6state" + state_container_name = "tfstate" + state_allowed_ip_rule = "203.0.113.10" + acr_name = "danderphase6" + managed_identity_name = "dander-phase6-runtime" +} + +run "hardened_stage_zero" { + command = plan + + assert { + condition = ( + azurerm_storage_account.terraform_state.min_tls_version == "TLS1_2" && + azurerm_storage_account.terraform_state.https_traffic_only_enabled && + !azurerm_storage_account.terraform_state.allow_nested_items_to_be_public && + !azurerm_storage_account.terraform_state.shared_access_key_enabled && + azurerm_storage_account.terraform_state.network_rules[0].default_action == "Deny" + ) + error_message = "Terraform state must use Entra authentication, TLS, and private blobs." + } + + assert { + condition = ( + azurerm_storage_account.terraform_state.blob_properties[0].versioning_enabled && + azurerm_storage_container.terraform_state.container_access_type == "private" + ) + error_message = "Terraform state must be versioned in a private container." + } + + assert { + condition = ( + !azurerm_container_registry.runtime.admin_enabled && + azurerm_container_registry.runtime.sku == "Basic" + ) + error_message = "The runtime registry must disable static administrator credentials." + } + + assert { + condition = azurerm_role_assignment.terraform_state_operator.role_definition_name == "Storage Blob Data Contributor" + error_message = "The authenticated operator must be able to use the Entra-authenticated state backend." + } +} diff --git a/infra/azure/bootstrap-admin/variables.tf b/infra/azure/bootstrap-admin/variables.tf new file mode 100644 index 00000000..904f0689 --- /dev/null +++ b/infra/azure/bootstrap-admin/variables.tf @@ -0,0 +1,91 @@ +variable "subscription_id" { + description = "Azure subscription receiving Dander stage-zero resources." + type = string + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$", var.subscription_id)) + error_message = "subscription_id must be an RFC 4122 UUID." + } +} + +variable "location" { + description = "Azure location for the first Dander deployment." + type = string + default = "eastus" + + validation { + condition = can(regex("^[a-z][a-z0-9]{1,31}$", var.location)) + error_message = "location must be a normalized Azure location name." + } +} + +variable "resource_group_name" { + description = "Resource group containing Azure stage-zero and runtime resources." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,62}[a-z0-9]$", var.resource_group_name)) + error_message = "resource_group_name must contain 3-64 lowercase letters, numbers, or hyphens." + } +} + +variable "storage_account_name" { + description = "Globally unique Storage account used for Terraform state." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9]{2,23}$", var.storage_account_name)) + error_message = "storage_account_name must contain 3-24 lowercase letters and numbers." + } +} + +variable "state_container_name" { + description = "Private blob container used for Terraform state." + type = string + default = "tfstate" + + validation { + condition = can(regex("^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])?$", var.state_container_name)) + error_message = "state_container_name must be a valid Azure blob container name." + } +} + +variable "state_allowed_ip_rule" { + description = "One reviewed operator public IPv4 address for exact state access." + type = string + + validation { + condition = ( + !strcontains(var.state_allowed_ip_rule, "/") && + length(split(".", var.state_allowed_ip_rule)) == 4 && + can(cidrnetmask("${var.state_allowed_ip_rule}/32")) + ) + error_message = "state_allowed_ip_rule must be one exact IPv4 address." + } +} + +variable "acr_name" { + description = "Globally unique Azure Container Registry name for Dander images." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9]{4,49}$", var.acr_name)) + error_message = "acr_name must contain 5-50 lowercase letters and numbers." + } +} + +variable "managed_identity_name" { + description = "User-assigned identity used by Container Apps Jobs." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,62}[a-z0-9]$", var.managed_identity_name)) + error_message = "managed_identity_name must contain 3-64 lowercase letters, numbers, or hyphens." + } +} + +variable "tags" { + description = "Additional non-secret tags applied to stage-zero resources." + type = map(string) + default = {} +} diff --git a/infra/azure/bootstrap-admin/versions.tf b/infra/azure/bootstrap-admin/versions.tf new file mode 100644 index 00000000..817e2b94 --- /dev/null +++ b/infra/azure/bootstrap-admin/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.9, < 2.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.72" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + resource_provider_registrations = "none" + storage_use_azuread = true + + features {} +} diff --git a/infra/azure/main.tf b/infra/azure/main.tf new file mode 100644 index 00000000..ce4191df --- /dev/null +++ b/infra/azure/main.tf @@ -0,0 +1,43 @@ +data "azurerm_client_config" "current" {} + +check "authenticated_subscription_matches_target" { + assert { + condition = data.azurerm_client_config.current.subscription_id == var.subscription_id + error_message = "Authenticated Azure subscription does not match subscription_id." + } +} + +data "azurerm_resource_group" "dander" { + name = var.resource_group_name +} + +data "azurerm_container_registry" "runtime" { + name = var.acr_name + resource_group_name = data.azurerm_resource_group.dander.name +} + +data "azurerm_user_assigned_identity" "runtime" { + name = var.managed_identity_name + resource_group_name = data.azurerm_resource_group.dander.name +} + +module "container_apps_jobs" { + source = "./modules/container-apps-jobs" + + name = var.name + subscription_id = var.subscription_id + tenant_id = data.azurerm_client_config.current.tenant_id + location = var.location + resource_group_name = data.azurerm_resource_group.dander.name + container_app_environment_name = var.container_app_environment_name + acr_id = data.azurerm_container_registry.runtime.id + acr_login_server = data.azurerm_container_registry.runtime.login_server + key_vault_name = var.key_vault_name + key_vault_allowed_ip_rule = var.key_vault_allowed_ip_rule + managed_identity_id = data.azurerm_user_assigned_identity.runtime.id + managed_identity_client_id = data.azurerm_user_assigned_identity.runtime.client_id + managed_identity_principal_id = data.azurerm_user_assigned_identity.runtime.principal_id + execution_projections = var.execution_projections + infrastructure_subnet_id = var.infrastructure_subnet_id + tags = var.tags +} diff --git a/infra/azure/modules/container-apps-jobs/.terraform.lock.hcl b/infra/azure/modules/container-apps-jobs/.terraform.lock.hcl new file mode 100644 index 00000000..c455d6da --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.81.0" + constraints = "~> 4.72" + hashes = [ + "h1:XhToZua4gtih1Kv8RdStcfND83G4Tmb6GZFT4jEUhDU=", + "zh:0732e7b74264ddfa2b90ba69d01c283d3cbae9f72ed3e506c6ac92529fed7fd3", + "zh:12afb524e232fe4e3d6161927724af5dfa4831d71edd9c174917ca9b7377bfae", + "zh:169d619ae202c4145e02fb706fb7c3679445ab3e3ff722edbf89597517a8c92e", + "zh:6beb95a3ef2f2d9c76abaa48e5450e90686a3fb6a47f1cb0ff7c5e94b6960151", + "zh:705e075fb5ffc4bf66fd7cbabf1a65007a41621e80030a2c158a4c83b6046216", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79a8d17fefe647040fcb9ee8821a4f09f395427c4fd49493489b9a93a9a1038e", + "zh:8cc3f900b3774c0ae37ae42365c4579a199cf9e5edf88e476fdf5ab1048f84ea", + "zh:dec373b9390fa95e257291acd018ed65a7d512b428645d35e22cdbe8b245a08b", + "zh:e60f1e9fb45df6defade2855ed6e68547409ea75d30655c556adb0c08579749b", + "zh:f901d12ec82f3f8b5880a27b5cbcd7bd0d97e60c9367a2d7ed82fdd1157b39ff", + "zh:facf68ea5bf0f2b8ba720e7fba5f86492e1d4c591100460bb91c3f79f391f4b6", + ] +} diff --git a/infra/azure/modules/container-apps-jobs/README.md b/infra/azure/modules/container-apps-jobs/README.md new file mode 100644 index 00000000..cc1e429a --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/README.md @@ -0,0 +1,15 @@ +# Container Apps Jobs module + +This module accepts only validated Azure execution projections. It creates one Container Apps Job +per pipeline with the exact immutable ACR digest, user-assigned identity, CPU/memory pair, deadline, +launcher retry count, environment, Key Vault references, trigger, labels, and observability target. + +Paused projections use a manual trigger. Active projections use the already validated UTC cron. +The selected managed identity pulls from ACR and reads only the selected Key Vault; Terraform never +receives a secret value. The vault defaults to deny and allows the Azure service path plus one +reviewed operator IP for secret administration. A projection that names a different registry, +identity, environment, client id, vault, or secret provider fails its Terraform check. + +An optional existing infrastructure subnet creates an internal Container Apps environment. Failed +execution alerts use the Azure `Microsoft.App/jobs` `Executions` metric filtered to `Failed` and +send to the projection's existing Action Group id. diff --git a/infra/azure/modules/container-apps-jobs/main.tf b/infra/azure/modules/container-apps-jobs/main.tf new file mode 100644 index 00000000..8ede22e6 --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/main.tf @@ -0,0 +1,209 @@ +locals { + container_app_environment_id = "/subscriptions/${var.subscription_id}/resourceGroups/${var.resource_group_name}/providers/Microsoft.App/managedEnvironments/${var.container_app_environment_name}" + key_vault_uri = "https://${var.key_vault_name}.vault.azure.net" + resource_names = { + for id in keys(var.execution_projections) : + id => "${substr(var.name, 0, 12)}-${substr(sha1(id), 0, 12)}" + } + tags = merge(var.tags, { + component = "azure-container-apps-jobs" + managed-by = "dander" + }) +} + +check "projections_match_selected_azure_resources" { + assert { + condition = alltrue([ + for projection in values(var.execution_projections) : ( + projection.launcher == "azure_container_apps" && + projection.workload_identity == var.managed_identity_id && + projection.network.placement == local.container_app_environment_id && + projection.extensions.azure_acr_login_server == var.acr_login_server && + projection.extensions.azure_managed_identity_client_id == var.managed_identity_client_id && + projection.extensions.azure_key_vault_uri == local.key_vault_uri + ) + ]) + error_message = "Azure projections must match the selected registry, identity, environment, and Key Vault." + } +} + +check "secret_references_match_selected_key_vault" { + assert { + condition = alltrue(flatten([ + for projection in values(var.execution_projections) : [ + for binding in values(projection.secret_bindings) : ( + binding.provider == "azure_key_vault" && + startswith(binding.reference, "azure-kv://${local.key_vault_uri}/secrets/") + ) + ] + ])) + error_message = "Azure secret references must use the selected Key Vault." + } +} + +resource "azurerm_log_analytics_workspace" "runtime" { + name = "${var.name}-logs" + location = var.location + resource_group_name = var.resource_group_name + sku = "PerGB2018" + retention_in_days = 30 + tags = local.tags +} + +resource "azurerm_container_app_environment" "runtime" { + name = var.container_app_environment_name + location = var.location + resource_group_name = var.resource_group_name + log_analytics_workspace_id = azurerm_log_analytics_workspace.runtime.id + infrastructure_subnet_id = var.infrastructure_subnet_id + internal_load_balancer_enabled = var.infrastructure_subnet_id == null ? null : true + zone_redundancy_enabled = var.infrastructure_subnet_id == null ? null : false + tags = local.tags +} + +resource "azurerm_key_vault" "runtime" { + name = var.key_vault_name + location = var.location + resource_group_name = var.resource_group_name + tenant_id = var.tenant_id + sku_name = "standard" + rbac_authorization_enabled = true + purge_protection_enabled = true + public_network_access_enabled = true + soft_delete_retention_days = 90 + tags = local.tags + + network_acls { + bypass = "AzureServices" + default_action = "Deny" + ip_rules = [var.key_vault_allowed_ip_rule] + } +} + +resource "azurerm_role_assignment" "acr_pull" { + scope = var.acr_id + role_definition_name = "AcrPull" + principal_id = var.managed_identity_principal_id + principal_type = "ServicePrincipal" +} + +resource "azurerm_role_assignment" "key_vault_secrets" { + scope = azurerm_key_vault.runtime.id + role_definition_name = "Key Vault Secrets User" + principal_id = var.managed_identity_principal_id + principal_type = "ServicePrincipal" +} + +resource "azurerm_container_app_job" "pipeline" { + for_each = var.execution_projections + + name = local.resource_names[each.key] + location = var.location + resource_group_name = var.resource_group_name + container_app_environment_id = azurerm_container_app_environment.runtime.id + replica_timeout_in_seconds = each.value.resources.deadline_seconds + replica_retry_limit = each.value.resources.launcher_retry_count + tags = merge(local.tags, each.value.labels, { pipeline = each.key }) + + identity { + type = "UserAssigned" + identity_ids = [var.managed_identity_id] + } + + registry { + server = var.acr_login_server + identity = var.managed_identity_id + } + + dynamic "secret" { + for_each = each.value.secret_bindings + content { + name = "secret-${substr(sha1(secret.key), 0, 16)}" + identity = var.managed_identity_id + key_vault_secret_id = trimprefix(secret.value.reference, "azure-kv://") + } + } + + template { + container { + name = "runtime" + image = each.value.image + cpu = each.value.resources.cpu_millis / 1000 + memory = "${each.value.resources.memory_mib / 1024}Gi" + args = each.value.command + + dynamic "env" { + for_each = each.value.environment + content { + name = env.key + value = env.value + } + } + + dynamic "env" { + for_each = each.value.secret_bindings + content { + name = env.key + secret_name = "secret-${substr(sha1(env.key), 0, 16)}" + } + } + } + } + + dynamic "manual_trigger_config" { + for_each = each.value.schedule.paused ? [true] : [] + content { + parallelism = each.value.schedule.maximum_parallelism + replica_completion_count = each.value.schedule.task_count + } + } + + dynamic "schedule_trigger_config" { + for_each = each.value.schedule.paused ? [] : [true] + content { + cron_expression = each.value.schedule.expression + parallelism = each.value.schedule.maximum_parallelism + replica_completion_count = each.value.schedule.task_count + } + } + + depends_on = [ + azurerm_role_assignment.acr_pull, + azurerm_role_assignment.key_vault_secrets, + ] +} + +resource "azurerm_monitor_metric_alert" "failed_execution" { + for_each = { + for id, projection in var.execution_projections : + id => projection if projection.observability.alert_target != null + } + + name = "${local.resource_names[each.key]}-failed" + resource_group_name = var.resource_group_name + scopes = [azurerm_container_app_job.pipeline[each.key].id] + description = "Dander Container Apps Job reported a failed execution." + severity = 1 + frequency = "PT1M" + window_size = "PT5M" + auto_mitigate = true + tags = local.tags + + criteria { + metric_namespace = "Microsoft.App/jobs" + metric_name = "Executions" + aggregation = "Total" + operator = "GreaterThan" + threshold = 0 + + dimension { + name = "state" + operator = "Include" + values = ["Failed"] + } + } + + action { + action_group_id = each.value.observability.alert_target + } +} diff --git a/infra/azure/modules/container-apps-jobs/outputs.tf b/infra/azure/modules/container-apps-jobs/outputs.tf new file mode 100644 index 00000000..a0b4f8eb --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/outputs.tf @@ -0,0 +1,24 @@ +output "container_app_environment_id" { + description = "Container Apps environment containing Dander jobs." + value = azurerm_container_app_environment.runtime.id +} + +output "key_vault_uri" { + description = "Key Vault URI used by versionless job secret references." + value = trimsuffix(azurerm_key_vault.runtime.vault_uri, "/") +} + +output "jobs" { + description = "Container Apps Job names and ids keyed by pipeline." + value = { + for id, job in azurerm_container_app_job.pipeline : id => { + id = job.id + name = job.name + } + } +} + +output "log_analytics_workspace_id" { + description = "Log Analytics workspace receiving environment logs." + value = azurerm_log_analytics_workspace.runtime.id +} diff --git a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl new file mode 100644 index 00000000..131e3a6d --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl @@ -0,0 +1,88 @@ +mock_provider "azurerm" { + mock_resource "azurerm_container_app_environment" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.App/managedEnvironments/dander-phase6-env" + } + } + + mock_resource "azurerm_key_vault" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.KeyVault/vaults/dander-phase6-kv" + vault_uri = "https://dander-phase6-kv.vault.azure.net/" + } + } +} + +variables { + name = "dander" + subscription_id = "11111111-1111-4111-8111-111111111111" + tenant_id = "22222222-2222-4222-8222-222222222222" + location = "eastus" + resource_group_name = "dander-phase6" + container_app_environment_name = "dander-phase6-env" + acr_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ContainerRegistry/registries/danderphase6" + acr_login_server = "danderphase6.azurecr.io" + key_vault_name = "dander-phase6-kv" + key_vault_allowed_ip_rule = "203.0.113.10" + managed_identity_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + managed_identity_client_id = "33333333-3333-4333-8333-333333333333" + managed_identity_principal_id = "44444444-4444-4444-8444-444444444444" + execution_projections = { + warehouse_fixture = { + launcher = "azure_container_apps" + image = "danderphase6.azurecr.io/dander/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command = ["runtime", "execute", "--contract", "io.dander.runtime/v1", "--pipeline", "warehouse_fixture", "--platform", "azure_snowflake"] + workload_identity = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + environment = { AZURE_CLIENT_ID = "33333333-3333-4333-8333-333333333333", HOME = "/tmp" } + labels = { pipeline = "warehouse_fixture", profile = "azure_snowflake" } + secret_bindings = { DANDER_POSTGRES_DSN = { provider = "azure_key_vault", reference = "azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn" } } + resources = { cpu_millis = 1000, memory_mib = 2048, deadline_seconds = 900, launcher_retry_count = 1 } + schedule = { paused = true, expression = "15 4 * * *", maximum_parallelism = 1, task_count = 1 } + network = { placement = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.App/managedEnvironments/dander-phase6-env" } + observability = { alert_target = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/microsoft.insights/actionGroups/dander-phase6" } + extensions = { azure_acr_login_server = "danderphase6.azurecr.io", azure_key_vault_uri = "https://dander-phase6-kv.vault.azure.net", azure_managed_identity_client_id = "33333333-3333-4333-8333-333333333333" } + } + } +} + +run "projects_exact_job_contract" { + command = plan + + assert { + condition = ( + azurerm_container_app_job.pipeline["warehouse_fixture"].replica_timeout_in_seconds == 900 && + azurerm_container_app_job.pipeline["warehouse_fixture"].replica_retry_limit == 1 + ) + error_message = "Container Apps must preserve the projected deadline and launcher retry count." + } + + assert { + condition = ( + azurerm_container_app_job.pipeline["warehouse_fixture"].template[0].container[0].args[0] == "runtime" && + azurerm_container_app_job.pipeline["warehouse_fixture"].template[0].container[0].image == "danderphase6.azurecr.io/dander/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ) + error_message = "Container Apps must preserve the OCI digest and runtime arguments." + } + + assert { + condition = ( + length(azurerm_container_app_job.pipeline["warehouse_fixture"].manual_trigger_config) == 1 && + length(azurerm_container_app_job.pipeline["warehouse_fixture"].schedule_trigger_config) == 0 + ) + error_message = "Paused pipelines must remain manual-only." + } + + assert { + condition = azurerm_monitor_metric_alert.failed_execution["warehouse_fixture"].criteria[0].metric_name == "Executions" + error_message = "A projected alert target must receive failed-execution monitoring." + } + + + assert { + condition = ( + azurerm_key_vault.runtime.network_acls[0].default_action == "Deny" && + azurerm_key_vault.runtime.network_acls[0].bypass == "AzureServices" + ) + error_message = "Key Vault must default-deny network access while allowing the Azure service path." + } +} diff --git a/infra/azure/modules/container-apps-jobs/variables.tf b/infra/azure/modules/container-apps-jobs/variables.tf new file mode 100644 index 00000000..8141fe7d --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/variables.tf @@ -0,0 +1,82 @@ +variable "name" { + type = string + description = "Stable Dander deployment prefix." +} + +variable "subscription_id" { + type = string + description = "Azure subscription owning the deployment." +} + +variable "tenant_id" { + type = string + description = "Entra tenant used by Key Vault RBAC." +} + +variable "location" { + type = string + description = "Azure location for runtime resources." +} + +variable "resource_group_name" { + type = string + description = "Existing resource group for runtime resources." +} + +variable "container_app_environment_name" { + type = string + description = "Container Apps managed environment name." +} + +variable "acr_id" { + type = string + description = "Existing Azure Container Registry resource id." +} + +variable "acr_login_server" { + type = string + description = "Existing Azure Container Registry login server." +} + +variable "key_vault_name" { + type = string + description = "Key Vault name for declared secret containers." +} + +variable "key_vault_allowed_ip_rule" { + type = string + description = "One reviewed operator public IPv4 address for Key Vault data access." +} + +variable "managed_identity_id" { + type = string + description = "User-assigned managed identity resource id." +} + +variable "managed_identity_client_id" { + type = string + description = "User-assigned managed identity client id." +} + +variable "managed_identity_principal_id" { + type = string + description = "User-assigned managed identity principal id." +} + +variable "execution_projections" { + type = any + description = "Validated Azure execution templates keyed by pipeline." +} + +variable "infrastructure_subnet_id" { + type = string + description = "Optional existing delegated subnet for the managed environment." + default = null + nullable = true +} + +variable "tags" { + type = map(string) + description = "Additional non-secret Azure tags." + default = {} +} diff --git a/infra/azure/modules/container-apps-jobs/versions.tf b/infra/azure/modules/container-apps-jobs/versions.tf new file mode 100644 index 00000000..660c8895 --- /dev/null +++ b/infra/azure/modules/container-apps-jobs/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.9, < 2.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.72" + } + } +} diff --git a/infra/azure/outputs.tf b/infra/azure/outputs.tf new file mode 100644 index 00000000..65ad66f1 --- /dev/null +++ b/infra/azure/outputs.tf @@ -0,0 +1,19 @@ +output "container_app_environment_id" { + description = "Container Apps environment containing Dander jobs." + value = module.container_apps_jobs.container_app_environment_id +} + +output "key_vault_uri" { + description = "Key Vault URI referenced by Dander jobs." + value = module.container_apps_jobs.key_vault_uri +} + +output "jobs" { + description = "Container Apps Job names and ids keyed by pipeline." + value = module.container_apps_jobs.jobs +} + +output "log_analytics_workspace_id" { + description = "Log Analytics workspace receiving Container Apps environment logs." + value = module.container_apps_jobs.log_analytics_workspace_id +} diff --git a/infra/azure/terraform.tfvars.example b/infra/azure/terraform.tfvars.example new file mode 100644 index 00000000..1e5a4070 --- /dev/null +++ b/infra/azure/terraform.tfvars.example @@ -0,0 +1,13 @@ +subscription_id = "11111111-1111-4111-8111-111111111111" +location = "eastus" +name = "dander" +resource_group_name = "dander-phase6" +container_app_environment_name = "dander-phase6-env" +acr_name = "danderphase6" +key_vault_name = "dander-phase6-kv" +key_vault_allowed_ip_rule = "203.0.113.10" +managed_identity_name = "dander-phase6-runtime" +infrastructure_subnet_id = null + +# Supply execution_projections only through `dander init-azure-plan`; do not hand-author them. +execution_projections = {} diff --git a/infra/azure/tests/azure.tftest.hcl b/infra/azure/tests/azure.tftest.hcl new file mode 100644 index 00000000..62a15bca --- /dev/null +++ b/infra/azure/tests/azure.tftest.hcl @@ -0,0 +1,83 @@ +mock_provider "azurerm" { + mock_data "azurerm_client_config" { + defaults = { + subscription_id = "11111111-1111-4111-8111-111111111111" + tenant_id = "22222222-2222-4222-8222-222222222222" + client_id = "33333333-3333-4333-8333-333333333333" + object_id = "44444444-4444-4444-8444-444444444444" + } + } + + mock_data "azurerm_resource_group" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6" + name = "dander-phase6" + location = "eastus" + } + } + + mock_data "azurerm_container_registry" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ContainerRegistry/registries/danderphase6" + login_server = "danderphase6.azurecr.io" + } + } + + mock_data "azurerm_user_assigned_identity" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + client_id = "33333333-3333-4333-8333-333333333333" + principal_id = "44444444-4444-4444-8444-444444444444" + } + } + + mock_resource "azurerm_container_app_environment" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.App/managedEnvironments/dander-phase6-env" + } + } + + mock_resource "azurerm_key_vault" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.KeyVault/vaults/dander-phase6-kv" + vault_uri = "https://dander-phase6-kv.vault.azure.net/" + } + } +} + +variables { + subscription_id = "11111111-1111-4111-8111-111111111111" + location = "eastus" + name = "dander" + resource_group_name = "dander-phase6" + container_app_environment_name = "dander-phase6-env" + acr_name = "danderphase6" + key_vault_name = "dander-phase6-kv" + key_vault_allowed_ip_rule = "203.0.113.10" + managed_identity_name = "dander-phase6-runtime" + execution_projections = { + warehouse_fixture = { + launcher = "azure_container_apps" + image = "danderphase6.azurecr.io/dander/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command = ["runtime", "execute", "--contract", "io.dander.runtime/v1", "--pipeline", "warehouse_fixture", "--platform", "azure_snowflake"] + workload_identity = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + environment = { AZURE_CLIENT_ID = "33333333-3333-4333-8333-333333333333", HOME = "/tmp" } + labels = { pipeline = "warehouse_fixture", profile = "azure_snowflake" } + secret_bindings = { DANDER_POSTGRES_DSN = { provider = "azure_key_vault", reference = "azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn" } } + resources = { cpu_millis = 1000, memory_mib = 2048, deadline_seconds = 900, launcher_retry_count = 1 } + schedule = { paused = true, expression = "15 4 * * *", maximum_parallelism = 1, task_count = 1 } + network = { placement = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.App/managedEnvironments/dander-phase6-env" } + observability = { alert_target = null } + extensions = { azure_acr_login_server = "danderphase6.azurecr.io", azure_key_vault_uri = "https://dander-phase6-kv.vault.azure.net", azure_managed_identity_client_id = "33333333-3333-4333-8333-333333333333" } + } + } +} + +run "wires_stage_zero_resources_into_jobs" { + command = plan + + assert { + condition = module.container_apps_jobs.jobs["warehouse_fixture"].name == "dander-00626d3b5f01" + error_message = "The Azure root must preserve deterministic manifest-to-job naming." + } +} diff --git a/infra/azure/variables.tf b/infra/azure/variables.tf new file mode 100644 index 00000000..82915b0f --- /dev/null +++ b/infra/azure/variables.tf @@ -0,0 +1,95 @@ +variable "subscription_id" { + description = "Azure subscription that owns the selected deployment." + type = string + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$", var.subscription_id)) + error_message = "subscription_id must be an RFC 4122 UUID." + } +} + +variable "location" { + description = "Azure location for Container Apps resources." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9]{1,31}$", var.location)) + error_message = "location must be a normalized Azure location name." + } +} + +variable "name" { + description = "Stable name prefix for this Dander deployment." + type = string + default = "dander" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,22}[a-z0-9]$", var.name)) + error_message = "name must contain 3-24 lowercase letters, numbers, or hyphens." + } +} + +variable "resource_group_name" { + description = "Existing stage-zero resource group." + type = string +} + +variable "container_app_environment_name" { + description = "Container Apps managed environment to create." + type = string +} + +variable "acr_name" { + description = "Existing stage-zero Azure Container Registry name." + type = string +} + +variable "key_vault_name" { + description = "Key Vault to create for declared secret containers." + type = string +} + +variable "key_vault_allowed_ip_rule" { + description = "One reviewed operator public IPv4 address for exact Key Vault data access." + type = string + + validation { + condition = ( + !strcontains(var.key_vault_allowed_ip_rule, "/") && + length(split(".", var.key_vault_allowed_ip_rule)) == 4 && + can(cidrnetmask("${var.key_vault_allowed_ip_rule}/32")) + ) + error_message = "key_vault_allowed_ip_rule must be one exact IPv4 address." + } +} + +variable "managed_identity_name" { + description = "Existing stage-zero user-assigned managed identity name." + type = string +} + +variable "execution_projections" { + description = "Validated io.dander.execution/v1 Azure templates keyed by pipeline id." + type = any +} + +variable "infrastructure_subnet_id" { + description = "Optional existing delegated subnet for the Container Apps environment." + type = string + default = null + nullable = true + + validation { + condition = ( + var.infrastructure_subnet_id == null || + can(regex("^/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[^/]+/providers/[Mm]icrosoft\\.[Nn]etwork/virtualNetworks/[^/]+/subnets/[^/]+$", var.infrastructure_subnet_id)) + ) + error_message = "infrastructure_subnet_id must be an Azure virtual-network subnet resource id." + } +} + +variable "tags" { + description = "Additional non-secret Azure tags." + type = map(string) + default = {} +} diff --git a/infra/azure/versions.tf b/infra/azure/versions.tf new file mode 100644 index 00000000..8001f593 --- /dev/null +++ b/infra/azure/versions.tf @@ -0,0 +1,20 @@ +terraform { + required_version = ">= 1.9, < 2.0" + + backend "azurerm" {} + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.72" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + resource_provider_registrations = "none" + storage_use_azuread = true + + features {} +} diff --git a/pyproject.toml b/pyproject.toml index 55047595..af16f8a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,6 +153,28 @@ packages = ["src/dander"] "infra/aws/terraform.tfvars.example" = "dander/templates/project/infra/aws/terraform.tfvars.example" "infra/aws/variables.tf" = "dander/templates/project/infra/aws/variables.tf" "infra/aws/versions.tf" = "dander/templates/project/infra/aws/versions.tf" +"infra/azure/.terraform.lock.hcl" = "dander/templates/project/infra/azure/.terraform.lock.hcl" +"infra/azure/README.md" = "dander/templates/project/infra/azure/README.md" +"infra/azure/bootstrap-admin/.terraform.lock.hcl" = "dander/templates/project/infra/azure/bootstrap-admin/.terraform.lock.hcl" +"infra/azure/bootstrap-admin/README.md" = "dander/templates/project/infra/azure/bootstrap-admin/README.md" +"infra/azure/bootstrap-admin/main.tf" = "dander/templates/project/infra/azure/bootstrap-admin/main.tf" +"infra/azure/bootstrap-admin/outputs.tf" = "dander/templates/project/infra/azure/bootstrap-admin/outputs.tf" +"infra/azure/bootstrap-admin/tests/bootstrap.tftest.hcl" = "dander/templates/project/infra/azure/bootstrap-admin/tests/bootstrap.tftest.hcl" +"infra/azure/bootstrap-admin/variables.tf" = "dander/templates/project/infra/azure/bootstrap-admin/variables.tf" +"infra/azure/bootstrap-admin/versions.tf" = "dander/templates/project/infra/azure/bootstrap-admin/versions.tf" +"infra/azure/main.tf" = "dander/templates/project/infra/azure/main.tf" +"infra/azure/modules/container-apps-jobs/.terraform.lock.hcl" = "dander/templates/project/infra/azure/modules/container-apps-jobs/.terraform.lock.hcl" +"infra/azure/modules/container-apps-jobs/README.md" = "dander/templates/project/infra/azure/modules/container-apps-jobs/README.md" +"infra/azure/modules/container-apps-jobs/main.tf" = "dander/templates/project/infra/azure/modules/container-apps-jobs/main.tf" +"infra/azure/modules/container-apps-jobs/outputs.tf" = "dander/templates/project/infra/azure/modules/container-apps-jobs/outputs.tf" +"infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl" = "dander/templates/project/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl" +"infra/azure/modules/container-apps-jobs/variables.tf" = "dander/templates/project/infra/azure/modules/container-apps-jobs/variables.tf" +"infra/azure/modules/container-apps-jobs/versions.tf" = "dander/templates/project/infra/azure/modules/container-apps-jobs/versions.tf" +"infra/azure/outputs.tf" = "dander/templates/project/infra/azure/outputs.tf" +"infra/azure/terraform.tfvars.example" = "dander/templates/project/infra/azure/terraform.tfvars.example" +"infra/azure/tests/azure.tftest.hcl" = "dander/templates/project/infra/azure/tests/azure.tftest.hcl" +"infra/azure/variables.tf" = "dander/templates/project/infra/azure/variables.tf" +"infra/azure/versions.tf" = "dander/templates/project/infra/azure/versions.tf" "infra/functions/__init__.py" = "dander/templates/project/infra/functions/__init__.py" "infra/functions/stop_billing/__init__.py" = "dander/templates/project/infra/functions/stop_billing/__init__.py" "infra/functions/stop_billing/handler.py" = "dander/templates/project/infra/functions/stop_billing/handler.py" diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 0962bad4..3fb3efda 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -22,6 +22,9 @@ "dander/templates/project/infra/kubernetes/chart/dander/Chart.yaml", "dander/templates/project/infra/kubernetes/chart/dander/templates/cronjobs.yaml", "dander/templates/project/infra/kubernetes/chart/dander/values.schema.json", + "dander/templates/project/infra/azure/bootstrap-admin/main.tf", + "dander/templates/project/infra/azure/main.tf", + "dander/templates/project/infra/azure/modules/container-apps-jobs/main.tf", "dander/templates/project/infra/main.tf", "dander/templates/project/models/staging/stg_greenhouse__jobs.sql", "dander/templates/project/models/staging/stg_greenhouse__jobs.yml", @@ -47,6 +50,8 @@ "examples/salesforce/dander.yaml", "infra/main.tf", "infra/kubernetes/chart/dander/Chart.yaml", + "infra/azure/bootstrap-admin/main.tf", + "infra/azure/modules/container-apps-jobs/main.tf", "pyproject.toml", "scripts/benchmarks/postgresql.py", "src/dander/project/scaffold.py", diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index e646c4ce..e191cf3b 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -6,6 +6,11 @@ AwsAdministrativeBootstrapError, ) from dander.bootstrap.aws_terraform import AwsTerraformBootstrap, AwsTerraformBootstrapError +from dander.bootstrap.azure_admin import ( + AzureAdministrativeBootstrap, + AzureAdministrativeBootstrapError, +) +from dander.bootstrap.azure_terraform import AzureTerraformBootstrap, AzureTerraformBootstrapError from dander.bootstrap.permissions import require_stage_zero_permissions from dander.bootstrap.project import ( ProjectBootstrapError, @@ -32,6 +37,10 @@ "AwsAdministrativeBootstrapError", "AwsTerraformBootstrap", "AwsTerraformBootstrapError", + "AzureAdministrativeBootstrap", + "AzureAdministrativeBootstrapError", + "AzureTerraformBootstrap", + "AzureTerraformBootstrapError", "DeploymentSummary", "DeploymentVerificationError", "DeploymentVerifier", diff --git a/src/dander/bootstrap/azure_admin.py b/src/dander/bootstrap/azure_admin.py new file mode 100644 index 00000000..b46099e4 --- /dev/null +++ b/src/dander/bootstrap/azure_admin.py @@ -0,0 +1,408 @@ +"""Plan-first Azure stage-zero bootstrap with post-apply remote-state migration.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from ipaddress import IPv4Address, ip_address +from typing import TYPE_CHECKING +from uuid import UUID + +if TYPE_CHECKING: + from pathlib import Path + +_LOCATION = re.compile(r"^[a-z][a-z0-9]{1,31}$") +_RESOURCE_NAME = re.compile(r"^[a-z][a-z0-9-]{1,62}[a-z0-9]$") +_STORAGE_ACCOUNT = re.compile(r"^[a-z][a-z0-9]{2,23}$") +_CONTAINER = re.compile(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])?$") +_ACR_NAME = re.compile(r"^[a-z][a-z0-9]{4,49}$") +_STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") +_BACKEND_SCHEMA = "io.dander.azure-bootstrap-backend/v1" + + +class AzureAdministrativeBootstrapError(RuntimeError): + """Raised when Azure stage zero cannot be planned or applied safely.""" + + +class AzureAdministrativeBootstrap: + """Create Azure state, registry, and workload-identity prerequisites through a saved plan.""" + + def __init__(self, infra_dir: Path, operator_artifact_dir: Path) -> None: + self._infra_dir = infra_dir.resolve() + self._repository_dir = self._infra_dir.parent.parent.parent + self._operator_artifact_dir = operator_artifact_dir.expanduser().resolve() + if self._is_within(self._operator_artifact_dir, self._repository_dir): + raise AzureAdministrativeBootstrapError( + "Azure operator artifact directory must be outside the repository checkout" + ) + self._workspace = self._operator_artifact_dir / "terraform-workspace" + self._tf_data_dir = self._operator_artifact_dir / "terraform-data" + self._plan_path = self._operator_artifact_dir / "dander-azure-admin-bootstrap.tfplan" + self._backend_path = self._workspace / "backend.tf.json" + self._backend_record_path = self._operator_artifact_dir / "backend.json" + self._local_state_path = self._operator_artifact_dir / "terraform.tfstate" + + def execute( + self, + *, + subscription_id: str, + location: str, + resource_group_name: str, + storage_account_name: str, + state_container_name: str, + state_allowed_ip_rule: str, + state_key: str, + acr_name: str, + managed_identity_name: str, + ) -> Path: + """Save a stage-zero plan without applying or creating Azure resources.""" + self._validate( + subscription_id=subscription_id, + location=location, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_allowed_ip_rule=state_allowed_ip_rule, + state_key=state_key, + acr_name=acr_name, + managed_identity_name=managed_identity_name, + ) + self._prepare_operator_directories() + backend = self._load_backend_record() + if backend is None: + self._write_backend("local", {"path": str(self._local_state_path)}) + else: + self._require_matching_backend( + backend, + subscription_id=subscription_id, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + self._write_azurerm_backend( + subscription_id=subscription_id, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + self._run("terraform", "init", "-reconfigure", "-input=false") + self._run( + "terraform", + "plan", + "-input=false", + f"-var=subscription_id={subscription_id}", + f"-var=location={location}", + f"-var=resource_group_name={resource_group_name}", + f"-var=storage_account_name={storage_account_name}", + f"-var=state_container_name={state_container_name}", + f"-var=state_allowed_ip_rule={state_allowed_ip_rule}", + f"-var=acr_name={acr_name}", + f"-var=managed_identity_name={managed_identity_name}", + f"-out={self._plan_path}", + ) + self._secure_plan() + return self._plan_path + + def apply_saved_plan( + self, + *, + subscription_id: str, + location: str, + resource_group_name: str, + storage_account_name: str, + state_container_name: str, + state_allowed_ip_rule: str, + state_key: str, + acr_name: str, + managed_identity_name: str, + ) -> Path: + """Apply only the reviewed plan, then migrate initial local state into Azure Storage.""" + self._validate( + subscription_id=subscription_id, + location=location, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_allowed_ip_rule=state_allowed_ip_rule, + state_key=state_key, + acr_name=acr_name, + managed_identity_name=managed_identity_name, + ) + self._prepare_operator_directories(preserve_plan=True) + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AzureAdministrativeBootstrapError( + f"No reviewed Azure stage-zero plan exists at {self._plan_path}" + ) + backend = self._load_backend_record() + if backend is None: + self._write_backend("local", {"path": str(self._local_state_path)}) + else: + self._require_matching_backend( + backend, + subscription_id=subscription_id, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + self._write_azurerm_backend( + subscription_id=subscription_id, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + self._run("terraform", "init", "-reconfigure", "-input=false") + self._run("terraform", "apply", "-input=false", str(self._plan_path)) + if backend is None: + self._migrate_state( + subscription_id=subscription_id, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + return self._plan_path + + def _migrate_state( + self, + *, + subscription_id: str, + resource_group_name: str, + storage_account_name: str, + state_container_name: str, + state_key: str, + ) -> None: + self._write_azurerm_backend( + subscription_id=subscription_id, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + try: + self._run("terraform", "init", "-migrate-state", "-force-copy", "-input=false") + except AzureAdministrativeBootstrapError: + self._write_backend("local", {"path": str(self._local_state_path)}) + raise + self._write_json( + self._backend_record_path, + { + "schema": _BACKEND_SCHEMA, + "subscription_id": subscription_id, + "resource_group_name": resource_group_name, + "storage_account_name": storage_account_name, + "container_name": state_container_name, + "key": state_key, + }, + ) + + def _prepare_operator_directories(self, *, preserve_plan: bool = False) -> None: + try: + for path, label in ( + (self._operator_artifact_dir, "operator artifact directory"), + (self._workspace, "Terraform workspace"), + (self._tf_data_dir, "terraform-data"), + ): + if path.is_symlink(): + raise AzureAdministrativeBootstrapError(f"{label} must not be a symlink") + if path.exists() and not path.is_dir(): + raise AzureAdministrativeBootstrapError(f"{label} must be a directory") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + path.chmod(0o700) + if self._plan_path.is_symlink(): + raise AzureAdministrativeBootstrapError("Saved Azure plan must not be a symlink") + if self._plan_path.exists() and not self._plan_path.is_file(): + raise AzureAdministrativeBootstrapError("Saved Azure plan must be a regular file") + if self._plan_path.exists() and not preserve_plan: + self._plan_path.unlink() + for existing in self._workspace.iterdir(): + is_source = existing.suffix == ".tf" or ( + existing.name.endswith(".tf.json") and existing != self._backend_path + ) + if not is_source and existing.name != ".terraform.lock.hcl": + continue + if existing.is_symlink() or not existing.is_file(): + raise AzureAdministrativeBootstrapError( + "Terraform workspace files must be regular files, not symlinks" + ) + existing.unlink() + for source in self._infra_dir.iterdir(): + if source.is_file() and ( + source.suffix == ".tf" or source.name == ".terraform.lock.hcl" + ): + shutil.copy2(source, self._workspace / source.name) + except OSError as error: + raise AzureAdministrativeBootstrapError( + "Could not prepare the Azure operator artifact directory" + ) from error + + def _secure_plan(self) -> None: + try: + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AzureAdministrativeBootstrapError( + f"Terraform did not create a regular saved plan at {self._plan_path}" + ) + self._plan_path.chmod(0o600) + except OSError as error: + raise AzureAdministrativeBootstrapError( + "Could not secure the saved Azure Terraform plan" + ) from error + + def _load_backend_record(self) -> dict[str, object] | None: + if not self._backend_record_path.exists(): + return None + if not self._backend_record_path.is_file() or self._backend_record_path.is_symlink(): + raise AzureAdministrativeBootstrapError("Azure backend record must be a regular file") + try: + document = json.loads(self._backend_record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AzureAdministrativeBootstrapError("Azure backend record is unreadable") from error + if not isinstance(document, dict) or document.get("schema") != _BACKEND_SCHEMA: + raise AzureAdministrativeBootstrapError( + "Azure backend record has an unsupported schema" + ) + return document + + @staticmethod + def _require_matching_backend( + backend: dict[str, object], + *, + subscription_id: str, + resource_group_name: str, + storage_account_name: str, + state_container_name: str, + state_key: str, + ) -> None: + expected = { + "subscription_id": subscription_id, + "resource_group_name": resource_group_name, + "storage_account_name": storage_account_name, + "container_name": state_container_name, + "key": state_key, + } + if any(backend.get(key) != value for key, value in expected.items()): + raise AzureAdministrativeBootstrapError( + "Azure backend inputs do not match the previously migrated stage-zero state" + ) + + def _write_azurerm_backend( + self, + *, + subscription_id: str, + resource_group_name: str, + storage_account_name: str, + state_container_name: str, + state_key: str, + ) -> None: + self._write_backend( + "azurerm", + { + "subscription_id": subscription_id, + "resource_group_name": resource_group_name, + "storage_account_name": storage_account_name, + "container_name": state_container_name, + "key": state_key, + "use_azuread_auth": True, + }, + ) + + def _write_backend(self, backend_type: str, config: dict[str, object]) -> None: + self._write_json( + self._backend_path, + {"terraform": {"backend": {backend_type: config}}}, + ) + + @staticmethod + def _write_json(path: Path, document: dict[str, object]) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + try: + temporary.write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.chmod(0o600) + os.replace(temporary, path) + path.chmod(0o600) + except OSError as error: + raise AzureAdministrativeBootstrapError( + "Could not write Azure backend metadata" + ) from error + finally: + temporary.unlink(missing_ok=True) + + @staticmethod + def _validate( + *, + subscription_id: str, + location: str, + resource_group_name: str, + storage_account_name: str, + state_container_name: str, + state_allowed_ip_rule: str, + state_key: str, + acr_name: str, + managed_identity_name: str, + ) -> None: + try: + parsed = UUID(subscription_id) + except ValueError as error: + raise AzureAdministrativeBootstrapError("Invalid Azure subscription id") from error + if parsed.variant != "specified in RFC 4122": + raise AzureAdministrativeBootstrapError("Invalid Azure subscription id") + try: + address = ip_address(state_allowed_ip_rule) + except ValueError as error: + raise AzureAdministrativeBootstrapError( + "Invalid Azure state allowed IP rule" + ) from error + if not isinstance(address, IPv4Address): + raise AzureAdministrativeBootstrapError("Invalid Azure state allowed IP rule") + checks = ( + (_LOCATION, location, "location"), + (_RESOURCE_NAME, resource_group_name, "resource group name"), + (_STORAGE_ACCOUNT, storage_account_name, "storage account name"), + (_CONTAINER, state_container_name, "state container name"), + (_STATE_KEY, state_key, "state key"), + (_ACR_NAME, acr_name, "ACR name"), + (_RESOURCE_NAME, managed_identity_name, "managed identity name"), + ) + for pattern, value, label in checks: + if pattern.fullmatch(value) is None: + raise AzureAdministrativeBootstrapError(f"Invalid Azure {label}: {value!r}") + + def _run(self, *args: str) -> None: + environment = os.environ.copy() + environment["TF_DATA_DIR"] = str(self._tf_data_dir) + try: + subprocess.run( + args, + cwd=self._workspace, + check=True, + env=environment, + umask=0o077, + ) + except FileNotFoundError as error: + raise AzureAdministrativeBootstrapError( + "Terraform is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + command = " ".join(args[:2]) + raise AzureAdministrativeBootstrapError( + f"{command} failed with exit code {error.returncode}" + ) from error + + @staticmethod + def _is_within(candidate: Path, parent: Path) -> bool: + try: + candidate.relative_to(parent) + except ValueError: + return False + return True + + +__all__ = ["AzureAdministrativeBootstrap", "AzureAdministrativeBootstrapError"] diff --git a/src/dander/bootstrap/azure_terraform.py b/src/dander/bootstrap/azure_terraform.py new file mode 100644 index 00000000..ba868d3d --- /dev/null +++ b/src/dander/bootstrap/azure_terraform.py @@ -0,0 +1,325 @@ +"""Saved-plan Terraform lifecycle for one manifest-defined Azure deployment.""" + +from __future__ import annotations + +import os +import re +import subprocess +from ipaddress import IPv4Address, ip_address +from json import dumps +from typing import TYPE_CHECKING +from uuid import UUID + +from dander.bootstrap.terraform import ( + TerraformBootstrapError, + build_launcher_runtime, + validate_runtime_settings, + validate_terraform_pipelines, +) +from dander.deployment import ExecutionProjectionError, ResolvedTemplateRequest +from dander.providers import ProviderFactoryError + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + +_LOCATION = re.compile(r"^[a-z][a-z0-9]{1,31}$") +_DEPLOYMENT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,22}[a-z0-9]$") +_RESOURCE_NAME = re.compile(r"^[a-z][a-z0-9-]{1,62}[a-z0-9]$") +_STORAGE_ACCOUNT = re.compile(r"^[a-z][a-z0-9]{2,23}$") +_CONTAINER = re.compile(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])?$") +_STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") +_ACR_NAME = re.compile(r"^[a-z][a-z0-9]{4,49}$") +_KEY_VAULT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,22}[a-z0-9]$") +_ACTION_GROUP_ID = re.compile( + r"^/subscriptions/(?P[0-9a-fA-F-]{36})/resourceGroups/[^/]+/" + r"providers/[Mm]icrosoft\.[Ii]nsights/actionGroups/[^/]+$" +) +_SUBNET_ID = re.compile( + r"^/subscriptions/(?P[0-9a-fA-F-]{36})/resourceGroups/[^/]+/" + r"providers/[Mm]icrosoft\.[Nn]etwork/virtualNetworks/[^/]+/subnets/[^/]+$" +) +_ACR_IMAGE = re.compile( + r"^(?P[a-z][a-z0-9]{4,49})\.azurecr\.io/" + r"(?P[A-Za-z0-9._/-]+)@sha256:(?P[0-9a-f]{64})$" +) + + +class AzureTerraformBootstrapError(RuntimeError): + """Raised when the Azure saved-plan lifecycle cannot complete safely.""" + + +class AzureTerraformBootstrap: + """Plan or apply the packaged Container Apps Jobs root using Azure Storage state.""" + + def __init__(self, infra_dir: Path) -> None: + self._infra_dir = infra_dir.resolve() + self._plan_path = self._infra_dir / "dander-azure.tfplan" + + def execute( + self, + *, + deployment_name: str, + state_resource_group_name: str, + state_storage_account_name: str, + state_container_name: str, + state_key: str, + container_image: str, + launcher_config: Mapping[str, object], + key_vault_allowed_ip_rule: str, + runtime_cpu: int, + runtime_memory: str, + runtime_timeout_seconds: int, + runtime_max_retries: int, + runtime_batch_rows: int, + require_guarded_free_tier: bool, + pipelines: Mapping[str, Mapping[str, object]], + apply: bool, + alert_target: str | None = None, + infrastructure_subnet_id: str | None = None, + name: str = "dander", + ) -> Path: + """Produce one immutable Azure plan and optionally apply that exact plan.""" + raw_launcher = dict(launcher_config) + subscription_id = self._subscription_id(raw_launcher.get("subscription_id")) + self._validate_backend( + state_resource_group_name=state_resource_group_name, + state_storage_account_name=state_storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + if not _DEPLOYMENT_NAME.fullmatch(name): + raise AzureTerraformBootstrapError( + "Azure deployment name must contain 3-24 lowercase letters, numbers, or hyphens" + ) + if not deployment_name.strip(): + raise AzureTerraformBootstrapError("Manifest deployment name must not be blank") + try: + key_vault_address = ip_address(key_vault_allowed_ip_rule) + except ValueError as error: + raise AzureTerraformBootstrapError("Invalid Azure Key Vault allowed IP rule") from error + if not isinstance(key_vault_address, IPv4Address): + raise AzureTerraformBootstrapError("Invalid Azure Key Vault allowed IP rule") + expanded_pipelines = dict(pipelines) + if not expanded_pipelines: + raise AzureTerraformBootstrapError("Azure planning requires at least one pipeline") + try: + validate_runtime_settings( + cpu=runtime_cpu, + memory=runtime_memory, + timeout_seconds=runtime_timeout_seconds, + max_retries=runtime_max_retries, + batch_rows=runtime_batch_rows, + require_guarded_free_tier=require_guarded_free_tier, + ) + validate_terraform_pipelines(expanded_pipelines) + except TerraformBootstrapError as error: + raise AzureTerraformBootstrapError(str(error)) from error + if raw_launcher.get("provider") != "azure_container_apps": + raise AzureTerraformBootstrapError( + "Azure planning requires launcher.provider='azure_container_apps'" + ) + location = self._required(raw_launcher, "region", _LOCATION) + resource_group_name = self._required(raw_launcher, "resource_group_name", _RESOURCE_NAME) + environment_name = self._required( + raw_launcher, "container_app_environment_name", _RESOURCE_NAME + ) + acr_name = self._required(raw_launcher, "acr_name", _ACR_NAME) + key_vault_name = self._required(raw_launcher, "key_vault_name", _KEY_VAULT_NAME) + identity_name = self._required(raw_launcher, "managed_identity_name", _RESOURCE_NAME) + image_match = _ACR_IMAGE.fullmatch(container_image) + if image_match is None or image_match.group("registry") != acr_name: + raise AzureTerraformBootstrapError( + "Azure requires an immutable image in the selected ACR" + ) + alert_match = _ACTION_GROUP_ID.fullmatch(alert_target or "") + if alert_target is not None and ( + alert_match is None + or alert_match.group("subscription").lower() != subscription_id.lower() + ): + raise AzureTerraformBootstrapError( + "Azure alert target must be an Action Group in the selected subscription" + ) + subnet_match = _SUBNET_ID.fullmatch(infrastructure_subnet_id or "") + if infrastructure_subnet_id is not None and ( + subnet_match is None + or subnet_match.group("subscription").lower() != subscription_id.lower() + ): + raise AzureTerraformBootstrapError( + "Azure infrastructure subnet must be a subnet in the selected subscription" + ) + try: + launcher = build_launcher_runtime(launcher_config=raw_launcher) + projections = { + pipeline_id: template.as_dict() + for pipeline_id, template in launcher.templates.build( + ResolvedTemplateRequest( + pipelines=expanded_pipelines, + image=container_image, + profile_id="azure_snowflake", + cpu=runtime_cpu, + memory=runtime_memory, + deadline_seconds=runtime_timeout_seconds, + launcher_retry_count=runtime_max_retries, + batch_rows=runtime_batch_rows, + alert_target=alert_target, + ) + ).items() + } + except (ExecutionProjectionError, ProviderFactoryError) as error: + raise AzureTerraformBootstrapError(str(error)) from error + + self._run( + "terraform", + "init", + "-reconfigure", + "-input=false", + f"-backend-config=subscription_id={subscription_id}", + f"-backend-config=resource_group_name={state_resource_group_name}", + f"-backend-config=storage_account_name={state_storage_account_name}", + f"-backend-config=container_name={state_container_name}", + f"-backend-config=key={state_key}", + "-backend-config=use_azuread_auth=true", + ) + plan_args = [ + "terraform", + "plan", + "-input=false", + f"-var=subscription_id={subscription_id}", + f"-var=location={location}", + f"-var=name={name}", + f"-var=resource_group_name={resource_group_name}", + f"-var=container_app_environment_name={environment_name}", + f"-var=acr_name={acr_name}", + f"-var=key_vault_name={key_vault_name}", + f"-var=key_vault_allowed_ip_rule={key_vault_allowed_ip_rule}", + f"-var=managed_identity_name={identity_name}", + "-var=execution_projections=" + + dumps(projections, sort_keys=True, separators=(",", ":")), + "-var=tags=" + + dumps( + { + "dander-deployment": deployment_name, + "dander-profile": "azure_snowflake", + }, + sort_keys=True, + separators=(",", ":"), + ), + f"-out={self._plan_path.name}", + ] + if infrastructure_subnet_id is not None: + plan_args.insert(-1, f"-var=infrastructure_subnet_id={infrastructure_subnet_id}") + self._run(*plan_args) + self._secure_plan() + if apply: + self._run("terraform", "apply", "-input=false", self._plan_path.name) + return self._plan_path + + def apply_saved_plan( + self, + *, + subscription_id: str, + state_resource_group_name: str, + state_storage_account_name: str, + state_container_name: str, + state_key: str, + ) -> Path: + """Apply only the saved plan produced by ``execute``.""" + subscription_id = self._subscription_id(subscription_id) + self._validate_backend( + state_resource_group_name=state_resource_group_name, + state_storage_account_name=state_storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AzureTerraformBootstrapError(f"Saved Azure plan is missing: {self._plan_path}") + self._run( + "terraform", + "init", + "-reconfigure", + "-input=false", + f"-backend-config=subscription_id={subscription_id}", + f"-backend-config=resource_group_name={state_resource_group_name}", + f"-backend-config=storage_account_name={state_storage_account_name}", + f"-backend-config=container_name={state_container_name}", + f"-backend-config=key={state_key}", + "-backend-config=use_azuread_auth=true", + ) + self._run("terraform", "apply", "-input=false", self._plan_path.name) + return self._plan_path + + @staticmethod + def _subscription_id(value: object) -> str: + if not isinstance(value, str): + raise AzureTerraformBootstrapError("Azure launcher requires a valid subscription id") + try: + parsed = UUID(value) + except ValueError as error: + raise AzureTerraformBootstrapError( + "Azure launcher requires a valid subscription id" + ) from error + if parsed.variant != "specified in RFC 4122": + raise AzureTerraformBootstrapError("Azure launcher requires a valid subscription id") + return value + + @staticmethod + def _required(values: Mapping[str, object], key: str, pattern: re.Pattern[str]) -> str: + value = values.get(key) + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise AzureTerraformBootstrapError(f"Azure launcher has invalid {key}") + return value + + @staticmethod + def _validate_backend( + *, + state_resource_group_name: str, + state_storage_account_name: str, + state_container_name: str, + state_key: str, + ) -> None: + checks = ( + (_RESOURCE_NAME, state_resource_group_name, "state resource group"), + (_STORAGE_ACCOUNT, state_storage_account_name, "state storage account"), + (_CONTAINER, state_container_name, "state container"), + (_STATE_KEY, state_key, "state key"), + ) + for pattern, value, label in checks: + if pattern.fullmatch(value) is None: + raise AzureTerraformBootstrapError(f"Invalid Azure {label}: {value!r}") + + def _secure_plan(self) -> None: + try: + if not self._plan_path.is_file() or self._plan_path.is_symlink(): + raise AzureTerraformBootstrapError( + f"Terraform did not create a regular saved plan at {self._plan_path}" + ) + self._plan_path.chmod(0o600) + except OSError as error: + raise AzureTerraformBootstrapError( + "Could not secure the saved Azure Terraform plan" + ) from error + + def _run(self, *args: str) -> None: + environment = os.environ.copy() + environment["ARM_USE_AZUREAD"] = "true" + try: + subprocess.run( + args, + cwd=self._infra_dir, + check=True, + env=environment, + umask=0o077, + ) + except FileNotFoundError as error: + raise AzureTerraformBootstrapError( + "Terraform is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + command = " ".join(args[:2]) + raise AzureTerraformBootstrapError( + f"{command} failed with exit code {error.returncode}" + ) from error + + +__all__ = ["AzureTerraformBootstrap", "AzureTerraformBootstrapError"] diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py new file mode 100644 index 00000000..27988e9c --- /dev/null +++ b/src/dander/cli/azure_command.py @@ -0,0 +1,306 @@ +"""Azure stage-zero, Terraform, and read-only verification CLI commands.""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +import typer +from click import ClickException +from rich.console import Console + +from dander.bootstrap import ( + AzureAdministrativeBootstrap, + AzureAdministrativeBootstrapError, + AzureTerraformBootstrap, + AzureTerraformBootstrapError, +) +from dander.project import ProjectConfigError, load_project_config +from dander.providers.azure_container_apps import ( + AzureDeploymentBinding, + AzureDeploymentVerificationError, + AzureDeploymentVerifier, +) + +_DEFAULT_AZURE_INFRA_DIR = Path("infra/azure") +_DEFAULT_AZURE_BOOTSTRAP_ADMIN_DIR = Path("infra/azure/bootstrap-admin") +_DEFAULT_PROJECT_CONFIG = Path("dander.yaml") +console = Console() +azure_app = typer.Typer(help="Operate and verify manifest-bound Azure Container Apps Jobs.") + + +@azure_app.command("verify") +def azure_verify( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + expected_image: str = typer.Option(..., "--expected-image"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Verify one deployed Azure pipeline through read-only provider checks.""" + try: + binding = AzureDeploymentBinding.from_project( + config=config, + deployment=deployment, + pipeline_id=pipeline, + name=name, + ) + verification = AzureDeploymentVerifier(binding).verify(expected_image=expected_image) + except AzureDeploymentVerificationError as error: + raise ClickException(str(error)) from error + console.print_json(data=verification.as_dict()) + + +def init_azure_admin_plan( + subscription_id: str = typer.Option(..., "--subscription-id"), + location: str = typer.Option("eastus", "--location"), + resource_group_name: str = typer.Option(..., "--resource-group"), + storage_account_name: str = typer.Option(..., "--state-storage-account"), + state_container_name: str = typer.Option("tfstate", "--state-container"), + state_allowed_ip_rule: str = typer.Option(..., "--state-allowed-ip"), + state_key: str = typer.Option("dander/azure/bootstrap-admin/terraform.tfstate", "--state-key"), + acr_name: str = typer.Option(..., "--acr-name"), + managed_identity_name: str = typer.Option(..., "--managed-identity-name"), + operator_artifact_dir: Path = typer.Option( # noqa: B008 + ..., + "--operator-artifact-dir", + help="Secured directory outside the repository for plans, state, and Terraform metadata.", + ), + infra_dir: Path = typer.Option( # noqa: B008 + _DEFAULT_AZURE_BOOTSTRAP_ADMIN_DIR, hidden=True + ), +) -> None: + """Plan Azure state, registry, and managed-identity prerequisites without applying.""" + arguments = { + "subscription_id": subscription_id, + "location": location, + "resource_group_name": resource_group_name, + "storage_account_name": storage_account_name, + "state_container_name": state_container_name, + "state_allowed_ip_rule": state_allowed_ip_rule, + "state_key": state_key, + "acr_name": acr_name, + "managed_identity_name": managed_identity_name, + } + try: + plan_path = AzureAdministrativeBootstrap(infra_dir, operator_artifact_dir).execute( + **arguments + ) + except AzureAdministrativeBootstrapError as error: + raise ClickException(str(error)) from error + workspace = operator_artifact_dir.expanduser().resolve() / "terraform-workspace" + console.print(f"[green]Azure administrative bootstrap planned.[/green] Saved plan: {plan_path}") + console.print( + "Review: " + + shlex.join(("terraform", f"-chdir={workspace}", "show", "-no-color", str(plan_path))), + soft_wrap=True, + ) + console.print( + "No Azure resources were created. Provider registration and apply require separate " + "explicit cost approval." + ) + console.print( + "Next after review and approval: " + + shlex.join( + ( + "dander", + "init-azure-admin-apply", + "--subscription-id", + subscription_id, + "--location", + location, + "--resource-group", + resource_group_name, + "--state-storage-account", + storage_account_name, + "--state-container", + state_container_name, + "--state-allowed-ip", + state_allowed_ip_rule, + "--state-key", + state_key, + "--acr-name", + acr_name, + "--managed-identity-name", + managed_identity_name, + "--operator-artifact-dir", + str(operator_artifact_dir), + ) + ), + soft_wrap=True, + ) + + +def init_azure_admin_apply( + subscription_id: str = typer.Option(..., "--subscription-id"), + location: str = typer.Option("eastus", "--location"), + resource_group_name: str = typer.Option(..., "--resource-group"), + storage_account_name: str = typer.Option(..., "--state-storage-account"), + state_container_name: str = typer.Option("tfstate", "--state-container"), + state_allowed_ip_rule: str = typer.Option(..., "--state-allowed-ip"), + state_key: str = typer.Option("dander/azure/bootstrap-admin/terraform.tfstate", "--state-key"), + acr_name: str = typer.Option(..., "--acr-name"), + managed_identity_name: str = typer.Option(..., "--managed-identity-name"), + operator_artifact_dir: Path = typer.Option(..., "--operator-artifact-dir"), # noqa: B008 + infra_dir: Path = typer.Option( # noqa: B008 + _DEFAULT_AZURE_BOOTSTRAP_ADMIN_DIR, hidden=True + ), +) -> None: + """Apply the reviewed Azure stage-zero plan and migrate state into Azure Storage.""" + if not typer.confirm( + f"Apply the reviewed Azure stage-zero plan to subscription {subscription_id!r}?", + default=False, + ): + raise typer.Abort() + try: + plan_path = AzureAdministrativeBootstrap(infra_dir, operator_artifact_dir).apply_saved_plan( + subscription_id=subscription_id, + location=location, + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + state_container_name=state_container_name, + state_allowed_ip_rule=state_allowed_ip_rule, + state_key=state_key, + acr_name=acr_name, + managed_identity_name=managed_identity_name, + ) + except AzureAdministrativeBootstrapError as error: + raise ClickException(str(error)) from error + workspace = operator_artifact_dir.expanduser().resolve() / "terraform-workspace" + console.print(f"[green]Azure administrative bootstrap applied.[/green] Saved plan: {plan_path}") + console.print( + "Record the generated launcher client id with: " + + shlex.join( + ( + "terraform", + f"-chdir={workspace}", + "output", + "-raw", + "runtime_identity_client_id", + ) + ), + soft_wrap=True, + ) + + +def init_azure_plan( + state_resource_group_name: str = typer.Option(..., "--state-resource-group"), + state_storage_account_name: str = typer.Option(..., "--state-storage-account"), + state_container_name: str = typer.Option("tfstate", "--state-container"), + state_key: str = typer.Option("dander/azure/state/terraform.tfstate", "--state-key"), + container_image: str = typer.Option( + ..., "--container-image", help="Immutable ACR image ending in @sha256 digest." + ), + deployment: str = typer.Option(..., "--deployment"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + key_vault_allowed_ip_rule: str = typer.Option(..., "--key-vault-allowed-ip"), + alert_target: str | None = typer.Option(None, "--alert-action-group-id"), + infrastructure_subnet_id: str | None = typer.Option(None, "--infrastructure-subnet-id"), + name: str = typer.Option("dander", "--name"), + infra_dir: Path = typer.Option(_DEFAULT_AZURE_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Plan one manifest-selected Azure deployment without applying it.""" + try: + manifest = load_project_config( + config, + platforms_path=platforms_config, + deployment=deployment, + ) + if manifest.launcher_provider != "azure_container_apps": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select " + "launcher.provider='azure_container_apps'" + ) + manifest.validate_references(config.resolve().parent) + runtime = manifest.platform.runtime + plan_path = AzureTerraformBootstrap(infra_dir).execute( + deployment_name=manifest.deployment_name, + state_resource_group_name=state_resource_group_name, + state_storage_account_name=state_storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + container_image=container_image, + launcher_config=manifest.resolved_launcher_config(), + key_vault_allowed_ip_rule=key_vault_allowed_ip_rule, + runtime_cpu=runtime.cpu, + runtime_memory=runtime.memory, + runtime_timeout_seconds=runtime.timeout_seconds, + runtime_max_retries=runtime.max_retries, + runtime_batch_rows=runtime.batch_rows, + require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, + pipelines=manifest.terraform_pipelines(), + apply=False, + alert_target=alert_target, + infrastructure_subnet_id=infrastructure_subnet_id, + name=name, + ) + except (AzureTerraformBootstrapError, ProjectConfigError) as error: + raise ClickException(str(error)) from error + console.print(f"[green]Azure deployment planned.[/green] Saved plan: {plan_path}") + console.print( + "Review: " + + shlex.join(("terraform", f"-chdir={infra_dir}", "show", "-no-color", str(plan_path))), + soft_wrap=True, + ) + console.print("No Azure resources were changed. Apply requires separate explicit approval.") + subscription_id = str(manifest.resolved_launcher_config()["subscription_id"]) + console.print( + "Next after review and approval: " + + shlex.join( + ( + "dander", + "init-azure-apply", + "--subscription-id", + subscription_id, + "--state-resource-group", + state_resource_group_name, + "--state-storage-account", + state_storage_account_name, + "--state-container", + state_container_name, + "--state-key", + state_key, + ) + ), + soft_wrap=True, + ) + + +def init_azure_apply( + subscription_id: str = typer.Option(..., "--subscription-id"), + state_resource_group_name: str = typer.Option(..., "--state-resource-group"), + state_storage_account_name: str = typer.Option(..., "--state-storage-account"), + state_container_name: str = typer.Option("tfstate", "--state-container"), + state_key: str = typer.Option("dander/azure/state/terraform.tfstate", "--state-key"), + infra_dir: Path = typer.Option(_DEFAULT_AZURE_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Apply only the previously reviewed Azure Terraform plan.""" + if not typer.confirm( + f"Apply the reviewed Azure deployment plan to subscription {subscription_id!r}?", + default=False, + ): + raise typer.Abort() + try: + plan_path = AzureTerraformBootstrap(infra_dir).apply_saved_plan( + subscription_id=subscription_id, + state_resource_group_name=state_resource_group_name, + state_storage_account_name=state_storage_account_name, + state_container_name=state_container_name, + state_key=state_key, + ) + except AzureTerraformBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]Azure deployment applied.[/green] Saved plan: {plan_path}") + + +def register_azure_commands(app: typer.Typer) -> None: + """Register Azure operation and compatibility-preserving flat command names.""" + app.add_typer(azure_app, name="azure") + app.command("init-azure-admin-plan")(init_azure_admin_plan) + app.command("init-azure-admin-apply")(init_azure_admin_apply) + app.command("init-azure-plan")(init_azure_plan) + app.command("init-azure-apply")(init_azure_apply) + + +__all__ = ["register_azure_commands"] diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 9c95bfb8..4aa0fe11 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -44,6 +44,7 @@ SqliteMetadataStore, ) from dander.cli.aws_command import register_aws_commands +from dander.cli.azure_command import register_azure_commands from dander.cli.config_command import config_app from dander.cli.init_command import ( InitOptions, @@ -134,6 +135,7 @@ app.add_typer(connector_app, name="connector") app.add_typer(runtime_app, name="runtime") register_aws_commands(app) +register_azure_commands(app) register_kubernetes_commands(app) console = Console() _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") diff --git a/src/dander/providers/azure_container_apps/__init__.py b/src/dander/providers/azure_container_apps/__init__.py index b84a20f3..f621cd9f 100644 --- a/src/dander/providers/azure_container_apps/__init__.py +++ b/src/dander/providers/azure_container_apps/__init__.py @@ -1,5 +1,17 @@ """Azure Container Apps Jobs launcher configuration.""" from dander.providers.azure_container_apps.config import AzureContainerAppsLauncherConfig +from dander.providers.azure_container_apps.verification import ( + AzureDeploymentBinding, + AzureDeploymentVerification, + AzureDeploymentVerificationError, + AzureDeploymentVerifier, +) -__all__ = ["AzureContainerAppsLauncherConfig"] +__all__ = [ + "AzureContainerAppsLauncherConfig", + "AzureDeploymentBinding", + "AzureDeploymentVerification", + "AzureDeploymentVerificationError", + "AzureDeploymentVerifier", +] diff --git a/src/dander/providers/azure_container_apps/verification.py b/src/dander/providers/azure_container_apps/verification.py new file mode 100644 index 00000000..b71cea2c --- /dev/null +++ b/src/dander/providers/azure_container_apps/verification.py @@ -0,0 +1,439 @@ +"""Read-only Azure Container Apps deployment verification through the Azure CLI.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Protocol, cast +from uuid import UUID + +from dander.project import ProjectConfigError, load_project_config + +if TYPE_CHECKING: + from pathlib import Path + +_DEPLOYMENT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,22}[a-z0-9]$") +_PIPELINE_ID = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,127}$") +_ACR_IMAGE = re.compile( + r"^(?P[a-z][a-z0-9]{4,49})\.azurecr\.io/" + r"[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" +) + + +class AzureDeploymentVerificationError(RuntimeError): + """Raised when read-only Azure deployment checks fail or return invalid data.""" + + +class _Runner(Protocol): + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _subprocess_runner( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + ) + + +@dataclass(frozen=True, slots=True) +class AzureDeploymentBinding: + """Exact manifest pipeline to Azure resource binding.""" + + subscription_id: str + location: str + resource_group_name: str + environment_name: str + environment_id: str + acr_name: str + acr_login_server: str + key_vault_name: str + key_vault_uri: str + managed_identity_id: str + managed_identity_client_id: str + pipeline_id: str + job_name: str + schedule_paused: bool + runtime_timeout_seconds: int + runtime_max_retries: int + secret_ids: tuple[str, ...] + project_dir: Path + + @classmethod + def from_project( + cls, + *, + config: Path, + deployment: str, + pipeline_id: str, + name: str = "dander", + ) -> AzureDeploymentBinding: + """Resolve one Azure pipeline from the validated project manifest.""" + resolved_config = config.expanduser().resolve() + if _DEPLOYMENT_NAME.fullmatch(name) is None: + raise AzureDeploymentVerificationError("Invalid Azure deployment name") + if _PIPELINE_ID.fullmatch(pipeline_id) is None: + raise AzureDeploymentVerificationError("Invalid pipeline identifier") + try: + manifest = load_project_config(resolved_config, deployment=deployment) + if manifest.launcher_provider != "azure_container_apps": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select " + "launcher.provider='azure_container_apps'" + ) + pipeline = manifest.pipelines[pipeline_id] + manifest.validate_references(resolved_config.parent) + launcher = manifest.resolved_launcher_config() + except KeyError as error: + raise AzureDeploymentVerificationError( + f"Pipeline {pipeline_id!r} is not declared in the project manifest" + ) from error + except ProjectConfigError as error: + raise AzureDeploymentVerificationError(str(error)) from error + subscription_id = _uuid_string(launcher.get("subscription_id"), "subscription id") + client_id = _uuid_string(launcher.get("managed_identity_client_id"), "identity client id") + required = { + key: value + for key in ( + "region", + "resource_group_name", + "container_app_environment_name", + "acr_name", + "key_vault_name", + "managed_identity_name", + ) + if isinstance(value := launcher.get(key), str) and value + } + if len(required) != 6: + raise AzureDeploymentVerificationError("Azure launcher configuration is incomplete") + resource_group = required["resource_group_name"] + environment_name = required["container_app_environment_name"] + identity_name = required["managed_identity_name"] + resource_root = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}" + return cls( + subscription_id=subscription_id, + location=required["region"], + resource_group_name=resource_group, + environment_name=environment_name, + environment_id=( + f"{resource_root}/providers/Microsoft.App/managedEnvironments/{environment_name}" + ), + acr_name=required["acr_name"], + acr_login_server=f"{required['acr_name']}.azurecr.io", + key_vault_name=required["key_vault_name"], + key_vault_uri=f"https://{required['key_vault_name']}.vault.azure.net", + managed_identity_id=( + f"{resource_root}/providers/Microsoft.ManagedIdentity/" + f"userAssignedIdentities/{identity_name}" + ), + managed_identity_client_id=client_id, + pipeline_id=pipeline_id, + job_name=_job_name(name=name, pipeline_id=pipeline_id), + schedule_paused=pipeline.paused, + runtime_timeout_seconds=manifest.platform.runtime.timeout_seconds, + runtime_max_retries=manifest.platform.runtime.max_retries, + secret_ids=tuple(sorted(pipeline.secrets.values())), + project_dir=resolved_config.parent, + ) + + +@dataclass(frozen=True, slots=True) +class AzureDeploymentVerification: + """Sanitized read-only verification result for one Azure pipeline.""" + + subscription: str + resource_group: str + environment: str + job: str + trigger_type: str + image: str + registry: str + managed_identity: str + key_vault: str + log_analytics_workspace: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +class AzureDeploymentVerifier: + """Verify one manifest-bound Container Apps Job without reading secret values.""" + + def __init__( + self, + binding: AzureDeploymentBinding, + *, + runner: _Runner | None = None, + ) -> None: + self.binding = binding + self._runner = runner or _subprocess_runner + + def verify(self, *, expected_image: str) -> AzureDeploymentVerification: + """Verify identity, immutable image, state, logs, registry, and Key Vault.""" + image = _ACR_IMAGE.fullmatch(expected_image) + if image is None or image.group("registry") != self.binding.acr_name: + raise AzureDeploymentVerificationError( + "Expected image is not an immutable image in this deployment ACR" + ) + account = self._json("account", "show") + if ( + str(account.get("id", "")).lower() != self.binding.subscription_id.lower() + or account.get("state") != "Enabled" + ): + raise AzureDeploymentVerificationError( + "Authenticated Azure subscription is not enabled or does not match" + ) + environment = self._json( + "containerapp", + "env", + "show", + "--name", + self.binding.environment_name, + "--resource-group", + self.binding.resource_group_name, + ) + if ( + str(environment.get("id", "")).lower() != self.binding.environment_id.lower() + or str(environment.get("location", "")).replace(" ", "").lower() + != self.binding.location.lower() + ): + raise AzureDeploymentVerificationError( + "Container Apps environment does not match the manifest" + ) + workspace_id = _nested_string( + environment, + "properties", + "appLogsConfiguration", + "logAnalyticsConfiguration", + "customerId", + ) + if workspace_id is None: + workspace_id = _nested_string( + environment, + "appLogsConfiguration", + "logAnalyticsConfiguration", + "customerId", + ) + if workspace_id is None: + raise AzureDeploymentVerificationError( + "Container Apps environment has no Log Analytics destination" + ) + job = self._json( + "containerapp", + "job", + "show", + "--name", + self.binding.job_name, + "--resource-group", + self.binding.resource_group_name, + ) + properties = _mapping(job.get("properties")) or job + configuration = _mapping(properties.get("configuration")) + template = _mapping(properties.get("template")) + identity = _mapping(job.get("identity")) + expected_trigger = "Manual" if self.binding.schedule_paused else "Schedule" + if ( + str(job.get("id", "")).rsplit("/", maxsplit=1)[-1] != self.binding.job_name + or properties.get("environmentId") != self.binding.environment_id + or configuration.get("triggerType") != expected_trigger + or configuration.get("replicaTimeout") != self.binding.runtime_timeout_seconds + or configuration.get("replicaRetryLimit") != self.binding.runtime_max_retries + ): + raise AzureDeploymentVerificationError( + "Container Apps Job does not match manifest trigger, timeout, or retry intent" + ) + identities = _mapping(identity.get("userAssignedIdentities")) + if not any(key.lower() == self.binding.managed_identity_id.lower() for key in identities): + raise AzureDeploymentVerificationError( + "Container Apps Job is missing the selected managed identity" + ) + registries = configuration.get("registries") + if not isinstance(registries, list) or not any( + isinstance(item, dict) + and item.get("server") == self.binding.acr_login_server + and str(item.get("identity", "")).lower() == self.binding.managed_identity_id.lower() + for item in registries + ): + raise AzureDeploymentVerificationError( + "Container Apps Job registry identity does not match the manifest" + ) + secrets = configuration.get("secrets", []) + if not isinstance(secrets, list): + raise AzureDeploymentVerificationError( + "Container Apps Job returned invalid Key Vault references" + ) + expected_secret_urls = { + f"{self.binding.key_vault_uri}/secrets/{secret_id}" + for secret_id in self.binding.secret_ids + } + actual_secret_urls = { + str(item.get("keyVaultUrl") or item.get("keyVaultSecretId")) + for item in secrets + if isinstance(item, dict) + and str(item.get("identity", "")).lower() == self.binding.managed_identity_id.lower() + } + if actual_secret_urls != expected_secret_urls: + raise AzureDeploymentVerificationError( + "Container Apps Job Key Vault references do not match the manifest" + ) + containers = template.get("containers") + if ( + not isinstance(containers, list) + or len(containers) != 1 + or not isinstance(containers[0], dict) + or containers[0].get("image") != expected_image + or containers[0].get("args", [None])[0] != "runtime" + ): + raise AzureDeploymentVerificationError( + "Container Apps Job does not use the expected immutable runtime" + ) + environment_values = { + item.get("name"): item.get("value") + for item in containers[0].get("env", []) + if isinstance(item, dict) and "value" in item + } + if ( + environment_values.get("HOME") != "/tmp" + or environment_values.get("AZURE_CLIENT_ID") != self.binding.managed_identity_client_id + ): + raise AzureDeploymentVerificationError( + "Container Apps Job environment does not match the runtime contract" + ) + registry = self._json( + "acr", + "show", + "--name", + self.binding.acr_name, + "--resource-group", + self.binding.resource_group_name, + ) + if ( + registry.get("loginServer") != self.binding.acr_login_server + or registry.get("adminUserEnabled") is not False + ): + raise AzureDeploymentVerificationError( + "ACR does not meet the runtime artifact contract" + ) + vault = self._json( + "keyvault", + "show", + "--name", + self.binding.key_vault_name, + "--resource-group", + self.binding.resource_group_name, + ) + vault_properties = _mapping(vault.get("properties")) + vault_network_acls = _mapping(vault_properties.get("networkAcls")) + if ( + str(vault_properties.get("vaultUri", "")).rstrip("/") != self.binding.key_vault_uri + or vault_properties.get("enableRbacAuthorization") is not True + or vault_network_acls.get("defaultAction") != "Deny" + ): + raise AzureDeploymentVerificationError( + "Key Vault does not meet the RBAC and network secret-reference contract" + ) + return AzureDeploymentVerification( + subscription=self.binding.subscription_id, + resource_group=self.binding.resource_group_name, + environment=self.binding.environment_id, + job=cast("str", job["id"]), + trigger_type=expected_trigger.lower(), + image=expected_image, + registry=cast("str", registry["id"]), + managed_identity=self.binding.managed_identity_id, + key_vault=cast("str", vault["id"]), + log_analytics_workspace=workspace_id, + ) + + def _json(self, *args: str) -> dict[str, object]: + command = ( + "az", + *args, + "--subscription", + self.binding.subscription_id, + "--output", + "json", + "--only-show-errors", + ) + try: + result = self._runner( + command, + cwd=self.binding.project_dir, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise AzureDeploymentVerificationError( + "Azure CLI is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + raise AzureDeploymentVerificationError( + f"Azure read-only verification failed with exit code {error.returncode}" + ) from error + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise AzureDeploymentVerificationError("Azure CLI returned invalid JSON") from error + if not isinstance(payload, dict): + raise AzureDeploymentVerificationError( + "Azure CLI returned an invalid verification response" + ) + return payload + + +def _mapping(value: object) -> dict[str, object]: + return value if isinstance(value, dict) else {} + + +def _nested_string(document: dict[str, object], *path: str) -> str | None: + value: object = document + for key in path: + if not isinstance(value, dict): + return None + value = value.get(key) + return value if isinstance(value, str) and value else None + + +def _uuid_string(value: object, label: str) -> str: + if not isinstance(value, str): + raise AzureDeploymentVerificationError(f"Azure launcher has invalid {label}") + try: + parsed = UUID(value) + except ValueError as error: + raise AzureDeploymentVerificationError(f"Azure launcher has invalid {label}") from error + if parsed.variant != "specified in RFC 4122": + raise AzureDeploymentVerificationError(f"Azure launcher has invalid {label}") + return value + + +def _job_name(*, name: str, pipeline_id: str) -> str: + return f"{name[:12]}-{hashlib.sha1(pipeline_id.encode()).hexdigest()[:12]}" # noqa: S324 + + +__all__ = [ + "AzureDeploymentBinding", + "AzureDeploymentVerification", + "AzureDeploymentVerificationError", + "AzureDeploymentVerifier", +] diff --git a/tests/bootstrap/test_azure_admin.py b/tests/bootstrap/test_azure_admin.py new file mode 100644 index 00000000..a676fcc0 --- /dev/null +++ b/tests/bootstrap/test_azure_admin.py @@ -0,0 +1,209 @@ +"""Azure stage-zero saved-plan and state-migration coverage.""" + +from __future__ import annotations + +import json +import stat +import subprocess +from pathlib import Path +from typing import TypedDict + +import pytest + +from dander.bootstrap import ( + AzureAdministrativeBootstrap, + AzureAdministrativeBootstrapError, +) + + +class _Arguments(TypedDict): + subscription_id: str + location: str + resource_group_name: str + storage_account_name: str + state_container_name: str + state_allowed_ip_rule: str + state_key: str + acr_name: str + managed_identity_name: str + + +def _arguments() -> _Arguments: + return { + "subscription_id": "11111111-1111-4111-8111-111111111111", + "location": "eastus", + "resource_group_name": "dander-phase6", + "storage_account_name": "danderphase6state", + "state_container_name": "tfstate", + "state_allowed_ip_rule": "203.0.113.10", + "state_key": "dander/azure/bootstrap-admin/terraform.tfstate", + "acr_name": "danderphase6", + "managed_identity_name": "dander-phase6-runtime", + } + + +def _layout(tmp_path: Path) -> tuple[Path, Path]: + infra_dir = tmp_path / "checkout" / "infra" / "azure" / "bootstrap-admin" + infra_dir.mkdir(parents=True) + (infra_dir / "main.tf").write_text('resource "terraform_data" "unit" {}\n', encoding="utf-8") + return infra_dir, tmp_path / "operator" + + +def test_azure_admin_plan_uses_secured_local_state_without_applying( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + commands: list[tuple[str, ...]] = [] + backends: list[dict[str, object]] = [] + + def fake_run( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + env: dict[str, str], + umask: int, + ) -> subprocess.CompletedProcess[str]: + assert check and umask == 0o077 + assert env["TF_DATA_DIR"] == str(operator_dir / "terraform-data") + commands.append(args) + backends.append(json.loads((cwd / "backend.tf.json").read_text(encoding="utf-8"))) + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch(mode=0o644) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + plan = AzureAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments()) + + assert plan == operator_dir / "dander-azure-admin-bootstrap.tfplan" + assert stat.S_IMODE(plan.stat().st_mode) == 0o600 + assert commands[0][:2] == ("terraform", "init") + assert commands[1][:2] == ("terraform", "plan") + assert not any(command[:2] == ("terraform", "apply") for command in commands) + assert backends[0]["terraform"] == { + "backend": {"local": {"path": str(operator_dir / "terraform.tfstate")}} + } + + +def test_azure_admin_apply_uses_saved_plan_then_migrates_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + commands: list[tuple[str, ...]] = [] + backend_types: list[str] = [] + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del kwargs + commands.append(args) + backend = json.loads((cwd / "backend.tf.json").read_text(encoding="utf-8")) + backend_types.append(next(iter(backend["terraform"]["backend"]))) + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + bootstrap = AzureAdministrativeBootstrap(infra_dir, operator_dir) + plan = bootstrap.execute(**_arguments()) + commands.clear() + backend_types.clear() + + applied = bootstrap.apply_saved_plan(**_arguments()) + + assert applied == plan + assert commands[1] == ("terraform", "apply", "-input=false", str(plan)) + assert commands[2] == ( + "terraform", + "init", + "-migrate-state", + "-force-copy", + "-input=false", + ) + assert backend_types == ["local", "local", "azurerm"] + record = json.loads((operator_dir / "backend.json").read_text(encoding="utf-8")) + assert record == { + "schema": "io.dander.azure-bootstrap-backend/v1", + "subscription_id": _arguments()["subscription_id"], + "resource_group_name": "dander-phase6", + "storage_account_name": "danderphase6state", + "container_name": "tfstate", + "key": "dander/azure/bootstrap-admin/terraform.tfstate", + } + + +def test_azure_admin_reuses_only_the_recorded_backend( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + operator_dir.mkdir() + (operator_dir / "backend.json").write_text( + json.dumps( + { + "schema": "io.dander.azure-bootstrap-backend/v1", + "subscription_id": _arguments()["subscription_id"], + "resource_group_name": "dander-phase6", + "storage_account_name": "danderphase6state", + "container_name": "tfstate", + "key": "dander/azure/bootstrap-admin/terraform.tfstate", + } + ), + encoding="utf-8", + ) + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del kwargs + backend = json.loads((cwd / "backend.tf.json").read_text(encoding="utf-8")) + assert backend["terraform"]["backend"]["azurerm"]["use_azuread_auth"] is True + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + AzureAdministrativeBootstrap(infra_dir, operator_dir).execute(**_arguments()) + + changed = _arguments() + changed["state_key"] = "different.tfstate" + with pytest.raises(AzureAdministrativeBootstrapError, match="do not match"): + AzureAdministrativeBootstrap(infra_dir, operator_dir).execute(**changed) + + +@pytest.mark.parametrize( + ("update", "message"), + [ + ({"subscription_id": "not-a-uuid"}, "subscription"), + ({"location": "East US"}, "location"), + ({"storage_account_name": "Bad-Storage"}, "storage account"), + ({"state_key": "/absolute"}, "state key"), + ({"state_allowed_ip_rule": "0.0.0.0/0"}, "allowed IP"), + ({"acr_name": "too-short"}, "ACR"), + ], +) +def test_azure_admin_rejects_invalid_inputs_before_terraform( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + update: dict[str, str], + message: str, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + monkeypatch.setattr( + subprocess, "run", lambda *args, **kwargs: pytest.fail("Terraform must not run") + ) + arguments = _arguments() + arguments.update(update) # type: ignore[typeddict-item] + with pytest.raises(AzureAdministrativeBootstrapError, match=message): + AzureAdministrativeBootstrap(infra_dir, operator_dir).execute(**arguments) + + +def test_azure_admin_rejects_repository_local_artifacts(tmp_path: Path) -> None: + infra_dir, _ = _layout(tmp_path) + with pytest.raises(AzureAdministrativeBootstrapError, match="outside"): + AzureAdministrativeBootstrap(infra_dir, tmp_path / "checkout" / "artifacts") diff --git a/tests/bootstrap/test_azure_terraform.py b/tests/bootstrap/test_azure_terraform.py new file mode 100644 index 00000000..c75b8f52 --- /dev/null +++ b/tests/bootstrap/test_azure_terraform.py @@ -0,0 +1,174 @@ +"""Saved-plan lifecycle coverage for manifest-defined Azure deployments.""" + +from __future__ import annotations + +import json +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from dander.bootstrap import AzureTerraformBootstrap, AzureTerraformBootstrapError + +if TYPE_CHECKING: + from pathlib import Path + +_SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" +_CLIENT_ID = "22222222-2222-4222-8222-222222222222" + + +def _launcher() -> dict[str, object]: + return { + "provider": "azure_container_apps", + "region": "eastus", + "subscription_id": _SUBSCRIPTION_ID, + "resource_group_name": "dander-phase6", + "container_app_environment_name": "dander-phase6-env", + "acr_name": "danderphase6", + "key_vault_name": "dander-phase6-kv", + "managed_identity_name": "dander-phase6-runtime", + "managed_identity_client_id": _CLIENT_ID, + } + + +def _pipelines() -> dict[str, dict[str, object]]: + return { + "warehouse_fixture": { + "job_name": "dander-warehouse-fixture", + "runtime_service_account_id": "dander-runtime", + "scheduler_service_account_id": "dander-scheduler", + "source": "warehouse_fixture", + "models": ["stg_warehouse_fixture"], + "build_models": True, + "publish_dataplex": False, + "schedule": "15 4 * * *", + "time_zone": "UTC", + "paused": True, + "secret_env": {"DANDER_POSTGRES_DSN": "postgres-dsn"}, + } + } + + +def _execute(bootstrap: AzureTerraformBootstrap, **overrides: object) -> Path: + arguments: dict[str, object] = { + "deployment_name": "azure_snowflake", + "state_resource_group_name": "dander-phase6", + "state_storage_account_name": "danderphase6state", + "state_container_name": "tfstate", + "state_key": "dander/azure/state/terraform.tfstate", + "container_image": ("danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64), + "launcher_config": _launcher(), + "key_vault_allowed_ip_rule": "203.0.113.10", + "runtime_cpu": 1, + "runtime_memory": "2Gi", + "runtime_timeout_seconds": 900, + "runtime_max_retries": 1, + "runtime_batch_rows": 2_048, + "require_guarded_free_tier": False, + "pipelines": _pipelines(), + "apply": False, + "alert_target": ( + f"/subscriptions/{_SUBSCRIPTION_ID}/resourceGroups/dander-phase6/" + "providers/microsoft.insights/actionGroups/dander-phase6" + ), + "name": "dander", + } + arguments.update(overrides) + return bootstrap.execute(**arguments) # type: ignore[arg-type] + + +def test_azure_bootstrap_builds_manifest_projection_without_apply( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[tuple[tuple[str, ...], dict[str, str]]] = [] + + def fake_run( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + env: dict[str, str], + umask: int, + ) -> subprocess.CompletedProcess[str]: + assert cwd == tmp_path.resolve() + assert check and umask == 0o077 + calls.append((args, env)) + for argument in args: + if argument.startswith("-out="): + (cwd / argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + plan = _execute(AzureTerraformBootstrap(tmp_path)) + + assert plan == tmp_path.resolve() / "dander-azure.tfplan" + init, terraform_plan = (call[0] for call in calls) + assert "-backend-config=use_azuread_auth=true" in init + assert all(call[0][:2] != ("terraform", "apply") for call in calls) + assert all(call[1]["ARM_USE_AZUREAD"] == "true" for call in calls) + projection_argument = next( + item for item in terraform_plan if item.startswith("-var=execution_projections=") + ) + projection = json.loads(projection_argument.removeprefix("-var=execution_projections="))[ + "warehouse_fixture" + ] + assert projection["launcher"] == "azure_container_apps" + assert projection["resources"]["memory_mib"] == 2_048 + assert projection["schedule"]["paused"] is True + assert projection["secret_bindings"]["DANDER_POSTGRES_DSN"] == { + "provider": "azure_key_vault", + "reference": ("azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn"), + } + assert "postgresql://" not in projection_argument + + +def test_azure_apply_uses_only_the_saved_plan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + plan_path = tmp_path / "dander-azure.tfplan" + plan_path.write_bytes(b"reviewed") + calls: list[tuple[str, ...]] = [] + + def fake_run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + del kwargs + calls.append(args) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + result = AzureTerraformBootstrap(tmp_path).apply_saved_plan( + subscription_id=_SUBSCRIPTION_ID, + state_resource_group_name="dander-phase6", + state_storage_account_name="danderphase6state", + state_container_name="tfstate", + state_key="dander/azure/state/terraform.tfstate", + ) + + assert result == plan_path + assert calls[1] == ("terraform", "apply", "-input=false", "dander-azure.tfplan") + assert all(call[:2] != ("terraform", "plan") for call in calls) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"runtime_memory": "512Mi"}, "CPU/memory"), + ({"state_key": "/absolute"}, "state key"), + ({"container_image": "other.azurecr.io/dander@sha256:" + "a" * 64}, "selected ACR"), + ({"pipelines": {}}, "at least one pipeline"), + ({"alert_target": "not-an-id"}, "alert target"), + ({"key_vault_allowed_ip_rule": "0.0.0.0/0"}, "Key Vault allowed IP"), + ], +) +def test_azure_bootstrap_rejects_unsafe_inputs_before_terraform( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + overrides: dict[str, object], + message: str, +) -> None: + monkeypatch.setattr( + subprocess, "run", lambda *args, **kwargs: pytest.fail("Terraform must not run") + ) + with pytest.raises(AzureTerraformBootstrapError, match=message): + _execute(AzureTerraformBootstrap(tmp_path), **overrides) diff --git a/tests/providers/test_azure_deployment_verification.py b/tests/providers/test_azure_deployment_verification.py new file mode 100644 index 00000000..87c44f9a --- /dev/null +++ b/tests/providers/test_azure_deployment_verification.py @@ -0,0 +1,253 @@ +"""Read-only Azure Container Apps deployment verification coverage.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from dander.providers.azure_container_apps import ( + AzureDeploymentBinding, + AzureDeploymentVerificationError, + AzureDeploymentVerifier, +) + +_SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" +_CLIENT_ID = "22222222-2222-4222-8222-222222222222" +_ROOT = f"/subscriptions/{_SUBSCRIPTION_ID}/resourceGroups/dander-phase6" +_ENVIRONMENT_ID = f"{_ROOT}/providers/Microsoft.App/managedEnvironments/dander-phase6-env" +_IDENTITY_ID = ( + f"{_ROOT}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" +) +_IMAGE = "danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64 + + +class _Runner: + def __init__(self, payloads: dict[tuple[str, ...], dict[str, object]]) -> None: + self.payloads = payloads + self.commands: list[tuple[str, ...]] = [] + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert cwd == Path("/tmp/dander-azure-test") + assert check and capture_output and text + self.commands.append(args) + suffix = args[1 : args.index("--subscription")] + payload = self.payloads[suffix] + return subprocess.CompletedProcess(args, 0, stdout=json.dumps(payload)) + + +def _binding() -> AzureDeploymentBinding: + return AzureDeploymentBinding( + subscription_id=_SUBSCRIPTION_ID, + location="eastus", + resource_group_name="dander-phase6", + environment_name="dander-phase6-env", + environment_id=_ENVIRONMENT_ID, + acr_name="danderphase6", + acr_login_server="danderphase6.azurecr.io", + key_vault_name="dander-phase6-kv", + key_vault_uri="https://dander-phase6-kv.vault.azure.net", + managed_identity_id=_IDENTITY_ID, + managed_identity_client_id=_CLIENT_ID, + pipeline_id="warehouse_fixture", + job_name="dander-00626d3b5f01", + schedule_paused=True, + runtime_timeout_seconds=900, + runtime_max_retries=1, + secret_ids=("postgres-dsn",), + project_dir=Path("/tmp/dander-azure-test"), + ) + + +def _payloads() -> dict[tuple[str, ...], dict[str, object]]: + binding = _binding() + job_id = f"{_ROOT}/providers/Microsoft.App/jobs/{binding.job_name}" + return { + ("account", "show"): {"id": _SUBSCRIPTION_ID, "state": "Enabled"}, + ( + "containerapp", + "env", + "show", + "--name", + "dander-phase6-env", + "--resource-group", + "dander-phase6", + ): { + "id": _ENVIRONMENT_ID, + "location": "East US", + "properties": { + "appLogsConfiguration": { + "logAnalyticsConfiguration": {"customerId": "workspace-customer-id"} + } + }, + }, + ( + "containerapp", + "job", + "show", + "--name", + binding.job_name, + "--resource-group", + "dander-phase6", + ): { + "id": job_id, + "identity": {"userAssignedIdentities": {_IDENTITY_ID: {}}}, + "properties": { + "environmentId": _ENVIRONMENT_ID, + "configuration": { + "triggerType": "Manual", + "replicaTimeout": 900, + "replicaRetryLimit": 1, + "registries": [ + { + "server": "danderphase6.azurecr.io", + "identity": _IDENTITY_ID, + } + ], + "secrets": [ + { + "name": "secret-aabbccdd", + "identity": _IDENTITY_ID, + "keyVaultUrl": ( + "https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn" + ), + } + ], + }, + "template": { + "containers": [ + { + "name": "runtime", + "image": _IMAGE, + "args": ["runtime", "execute"], + "env": [ + {"name": "HOME", "value": "/tmp"}, + {"name": "AZURE_CLIENT_ID", "value": _CLIENT_ID}, + { + "name": "DANDER_POSTGRES_DSN", + "secretRef": "secret-aabbccdd", + }, + ], + } + ] + }, + }, + }, + ( + "acr", + "show", + "--name", + "danderphase6", + "--resource-group", + "dander-phase6", + ): { + "id": f"{_ROOT}/providers/Microsoft.ContainerRegistry/registries/danderphase6", + "loginServer": "danderphase6.azurecr.io", + "adminUserEnabled": False, + }, + ( + "keyvault", + "show", + "--name", + "dander-phase6-kv", + "--resource-group", + "dander-phase6", + ): { + "id": f"{_ROOT}/providers/Microsoft.KeyVault/vaults/dander-phase6-kv", + "properties": { + "vaultUri": "https://dander-phase6-kv.vault.azure.net/", + "enableRbacAuthorization": True, + "networkAcls": {"defaultAction": "Deny"}, + }, + }, + } + + +def test_verifier_checks_exact_resources_without_reading_secret_values() -> None: + runner = _Runner(_payloads()) + + result = AzureDeploymentVerifier(_binding(), runner=runner).verify(expected_image=_IMAGE) + + assert result.image == _IMAGE + assert result.trigger_type == "manual" + assert result.managed_identity == _IDENTITY_ID + assert result.log_analytics_workspace == "workspace-customer-id" + flattened = " ".join(" ".join(command) for command in runner.commands) + assert "secret show" not in flattened + assert "secret list" not in flattened + assert all("--only-show-errors" in command for command in runner.commands) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ("image", "immutable runtime"), + ("identity", "managed identity"), + ("trigger", "trigger"), + ("registry_admin", "artifact contract"), + ("vault_rbac", "RBAC and network"), + ], +) +def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: + payloads = _payloads() + binding = _binding() + job_key = ( + "containerapp", + "job", + "show", + "--name", + binding.job_name, + "--resource-group", + "dander-phase6", + ) + job = payloads[job_key] + properties = job["properties"] + assert isinstance(properties, dict) + if mutation == "image": + properties["template"]["containers"][0]["image"] = "mutable:latest" + elif mutation == "identity": + job["identity"] = {"userAssignedIdentities": {}} + elif mutation == "trigger": + properties["configuration"]["triggerType"] = "Schedule" + elif mutation == "registry_admin": + acr_key = ( + "acr", + "show", + "--name", + "danderphase6", + "--resource-group", + "dander-phase6", + ) + payloads[acr_key]["adminUserEnabled"] = True + else: + vault_key = ( + "keyvault", + "show", + "--name", + "dander-phase6-kv", + "--resource-group", + "dander-phase6", + ) + payloads[vault_key]["properties"]["enableRbacAuthorization"] = False # type: ignore[index] + + with pytest.raises(AzureDeploymentVerificationError, match=message): + AzureDeploymentVerifier(binding, runner=_Runner(payloads)).verify(expected_image=_IMAGE) + + +def test_verifier_rejects_foreign_registry_before_provider_access() -> None: + runner = _Runner(_payloads()) + with pytest.raises(AzureDeploymentVerificationError, match="deployment ACR"): + AzureDeploymentVerifier(_binding(), runner=runner).verify( + expected_image="otherazure.azurecr.io/dander@sha256:" + "a" * 64 + ) + assert runner.commands == [] diff --git a/tickets/DANDER-109-azure-terraform-plan-verification.md b/tickets/DANDER-109-azure-terraform-plan-verification.md new file mode 100644 index 00000000..5411371a --- /dev/null +++ b/tickets/DANDER-109-azure-terraform-plan-verification.md @@ -0,0 +1,35 @@ +# DANDER-109 — Plan and verify Azure Container Apps deployments + +Status: implemented locally; protected CI and live proof pending + +## Requirement + +Add a plan-first Azure Terraform path that creates the Phase 6 state, registry, identity, Container +Apps Jobs, Key Vault, logging, alerts, and networking contract without performing provider writes. +Add a read-only verifier that proves a deployed job still matches its manifest and immutable image. + +## Acceptance + +- [x] Stage zero creates a default-deny, operator-`/32`, private versioned Entra-authenticated state + backend, ACR with static administrator credentials disabled, and one user-assigned runtime + identity. +- [x] Automatic Azure resource-provider registration is disabled; registration remains a separately + approved live mutation. +- [x] Initial stage-zero state migrates from a secured operator directory into Azure Storage only + after applying the exact reviewed plan. +- [x] The platform root consumes exact Azure execution projections and existing stage-zero outputs. +- [x] Container Apps Jobs preserve digest, managed identity, deadline, retries, resources, UTC + schedule/manual pause semantics, non-secret environment, and versionless Key Vault references. +- [x] Log Analytics, default-deny Key Vault networking with one exact operator IP, optional internal + subnet placement, and failed-execution Action Group routing are explicit. +- [x] The CLI saves and applies plans as separate confirmed operations. +- [x] Read-only verification checks the exact subscription, environment, logs, job, immutable image, + registry identity, ACR static-credential setting, and RBAC Key Vault without reading secrets. +- [x] Terraform provider-mocked tests and focused Python contract tests pass. +- [ ] Protected CI passes and this ticket merges before any live Azure apply. + +## Evidence boundary + +This ticket contains local deterministic and provider-mocked evidence only. It does not register an +Azure provider, create a resource, copy an image, publish a candidate, read a secret, or spend cloud +credits. Live plan review, apply, cleanup, and no-drift evidence remain separate Phase 6 gates. From 8c71f971045dde925cc435f262d64e1d9437f495 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 12:33:13 -0400 Subject: [PATCH 122/163] Add Azure image promotion and job operations --- HANDOFF.md | 41 +- docs/decisions.md | 13 + docs/execution-projection.md | 6 +- docs/known-limitations.md | 10 +- infra/azure/README.md | 47 ++- src/dander/bootstrap/__init__.py | 2 + src/dander/bootstrap/azure_image.py | 289 ++++++++++++++ src/dander/cli/azure_command.py | 170 +++++++++ .../azure_container_apps/__init__.py | 10 + .../azure_container_apps/operations.py | 358 ++++++++++++++++++ tests/bootstrap/test_azure_image_promotion.py | 208 ++++++++++ tests/cli/test_azure_operations_cli.py | 204 ++++++++++ .../test_azure_container_apps_operations.py | 309 +++++++++++++++ ...R-109-azure-terraform-plan-verification.md | 4 +- ...ER-110-azure-image-lifecycle-operations.md | 34 ++ 15 files changed, 1675 insertions(+), 30 deletions(-) create mode 100644 src/dander/bootstrap/azure_image.py create mode 100644 src/dander/providers/azure_container_apps/operations.py create mode 100644 tests/bootstrap/test_azure_image_promotion.py create mode 100644 tests/cli/test_azure_operations_cli.py create mode 100644 tests/providers/test_azure_container_apps_operations.py create mode 100644 tickets/DANDER-110-azure-image-lifecycle-operations.md diff --git a/HANDOFF.md b/HANDOFF.md index 0d9639ef..f209574d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,39 @@ ## Finished -- Merged the Azure launcher/Key Vault contract through protected PR #188 with green main CI. -- Added plan-first Azure stage zero for firewall-restricted Storage state, Basic ACR, and one user-assigned runtime identity. -- Added Container Apps Jobs Terraform for exact projections, Key Vault references, logs, alerts, and optional internal networking. -- Added separate plan/apply CLI paths and a read-only, secret-free Azure deployment verifier. -- Kept provider registration, candidate publication, resource creation, image copy, and job execution behind later approvals. +- Merged Azure plan-first infrastructure and verification through protected PR #189 with green main CI. +- Added accepted-record-gated ACR registry copy with exact index and platform-manifest verification. +- Added manifest-bound Azure run, status, bounded logs, cancel, and replay operations. +- Added interactive confirmation for every image or job mutation and local mocked contract tests. +- Kept provider registration, resource creation, image copy, execution, and spending untouched. ## Try It -Run `terraform -chdir=infra/azure test` and `uv run pytest -q -tests/bootstrap/test_azure_admin.py tests/bootstrap/test_azure_terraform.py -tests/providers/test_azure_deployment_verification.py`. +Run `uv run pytest -q tests/bootstrap/test_azure_image_promotion.py +tests/providers/test_azure_container_apps_operations.py tests/cli/test_azure_operations_cli.py`. ## Checks -- Full `pytest -q`, Ruff, strict mypy, Terraform validate/provider-mocked tests, wheel/sdist inspection, and Trivy HIGH/CRITICAL config scan pass. -- Protected main CI is green at merged PR #188; protected CI for this infrastructure slice is pending. -- Azure CLI is signed in; required resource providers remain unregistered and no Azure resource exists from this work. +- Focused pytest, Ruff, and mypy pass for Azure promotion and operations. +- Protected main CI is green through PR #189; protected CI for this operations slice is pending. +- No live Azure command from the new promotion or operations paths was executed. ## Decisions -- Terraform disables automatic provider registration and uses saved-plan-only applies with Entra-authenticated state. -- State and Key Vault networks default to deny and admit one reviewed exact operator IP; Key Vault also keeps Azure's trusted-service path. -- Paused projections are manual jobs; active schedules retain the already validated UTC cron. +- Promotion uses Buildx registry copy plus stable ACR image metadata; it never rebuilds or uses static ACR credentials. +- Azure owns execution names; Dander's persisted inclusive cursor owns logical replay correctness. +- Logs use a bounded Log Analytics query tied to one validated execution. ## Remaining -- Merge this infrastructure/verifier slice through a focused protected PR. -- Add digest-preserving ACR copy and launcher lifecycle operations without provider writes. +- Merge this operations slice through a focused protected PR. - Prepare Azure-to-Google federation and named Snowflake proof tooling locally. -- Obtain publication and provider-cost approvals before candidate or live proof mutations. -- Run approved identity, lifecycle, cleanup, rollback, and no-drift acceptance. +- Obtain candidate-publication approval before publishing or copying an image. +- Obtain explicit Azure, GCP, and Snowflake mutation ceilings before live proof. +- Run approved lifecycle, cleanup, rollback, and no-drift acceptance. ## Review First -- `infra/azure/modules/container-apps-jobs/main.tf` -- `src/dander/bootstrap/azure_admin.py` -- `src/dander/providers/azure_container_apps/verification.py` +- `src/dander/bootstrap/azure_image.py` +- `src/dander/providers/azure_container_apps/operations.py` +- `src/dander/cli/azure_command.py` diff --git a/docs/decisions.md b/docs/decisions.md index 176f4da9..b3d349d5 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1171,3 +1171,16 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Terraform contains secret references but never values. Local/provider-mocked tests do not qualify Azure; image copy, operations, federation, paid proof, cleanup, and support promotion remain separate Phase 6 gates. + +## 2026-08-11 — Azure promotion and lifecycle remain provider-native and fail closed + +- **Artifact identity:** The accepted OCI digest is copied into ACR with Buildx registry copy, not a + rebuild. Stable ACR image metadata must report the exact index digest, and the immutable digest + reference must expose the accepted platform manifests before a local Azure artifact record is + written. +- **Lifecycle:** Start, status, stop, replay, and bounded Log Analytics use Azure Container Apps Job + execution identities. Foreign execution names, unknown statuses, stopping a terminal run, and + replaying a non-terminal run fail before mutation. +- **Boundary:** Azure assigns execution names, while replay correctness remains owned by Dander's + persisted inclusive cursor. Local mocked operations are construction evidence only; live image + copy, execution, interruption, cleanup, and no-drift still require explicit approval. diff --git a/docs/execution-projection.md b/docs/execution-projection.md index 76eec49b..ba7fdede 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -64,4 +64,8 @@ versionless Key Vault references for declared secrets. Azure evaluates scheduled so a non-UTC projection fails before provider access rather than silently changing wall-clock behavior. Plan-first Terraform now consumes this projection exactly and has provider-mocked contract evidence; no Azure resource has been created, and live-profile evidence remains a -separate Phase 6 gate. +separate Phase 6 gate. Local lifecycle tooling now binds start, status, bounded Log Analytics, +stop, replay, and deployment verification to the exact manifest-selected job. The ACR promotion +path copies the accepted OCI index without rebuilding and writes its local Azure artifact record +only after both the index digest and platform-manifest map match. These mocked contracts do not +claim that a live Azure operation has passed. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 30ed6354..f62470a3 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -47,10 +47,12 @@ remain Phase 8 work. packaged capability manifest remains the support boundary. Local PostgreSQL benchmark results are regression evidence, not a paid or controlled-memory scale qualification. - Azure Container Apps Jobs and Azure Key Vault have a typed projection plus locally validated, - plan-first Terraform and read-only deployment verification. Azure schedules are UTC-only and - currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. Resource-provider registration, - credentialed Azure planning/apply, ACR image copy, identity federation, lifecycle operations, and - live Snowflake/PostgreSQL acceptance have not yet run, so Azure remains unsupported. + plan-first Terraform, digest-preserving ACR promotion tooling, provider-native lifecycle + operations, bounded Log Analytics reads, and deployment verification. Azure schedules are + UTC-only and currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. Resource-provider + registration, credentialed Azure planning/apply, actual ACR image copy, identity federation, + live lifecycle execution, and live Snowflake/PostgreSQL acceptance have not yet run, so Azure + remains unsupported. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/infra/azure/README.md b/infra/azure/README.md index 9256485d..a3a68692 100644 --- a/infra/azure/README.md +++ b/infra/azure/README.md @@ -36,6 +36,26 @@ operator only the Blob data role needed to use that Entra-authenticated backend. `runtime_identity_client_id` from the printed Terraform output command and place that non-secret identifier in the named launcher configuration. +## Runtime image promotion + +After stage zero exists and a source-free candidate has an accepted local artifact record, copy the +same OCI index into ACR. This is a live registry mutation and requires separate publication, cost, +and execution approval before running it: + +```bash +dander image-promote-azure \ + --source-image us-central1-docker.pkg.dev/example/dander/runtime@sha256:DIGEST \ + --subscription-id 11111111-1111-4111-8111-111111111111 \ + --acr-name danderphase6 \ + --acr-repository dander/runtime +``` + +The command authenticates Docker through the signed-in Azure identity. It never requests ACR +administrator credentials and never rebuilds. The deterministic destination tag is accepted only +when stable ACR metadata reports the source index digest and the immutable digest reference exposes +the same platform manifests. A successful copy writes the ignored local record +`.dander/runtime-artifact-azure.json`. + ## Platform plan After an accepted OCI release is copied into ACR by digest, plan the selected deployment: @@ -62,6 +82,28 @@ Run `dander init-azure-apply` only after reviewing the exact saved plan and rece approval for the live Azure changes. Then use `dander azure verify` for read-only checks of the subscription, environment, logs, job trigger/resources/image, managed identity, ACR, and Key Vault. +## Job lifecycle + +The lifecycle CLI resolves the exact job from `dander.yaml` and the named deployment. Status, +verification, and bounded logs are read-only; run, cancel, and replay ask for confirmation because +they are live provider operations: + +```bash +dander azure run --deployment azure_snowflake --pipeline warehouse_fixture +dander azure status --deployment azure_snowflake --pipeline warehouse_fixture +dander azure logs --deployment azure_snowflake --pipeline warehouse_fixture \ + --execution-name dander-00626d3b5f01-abc1234 --limit 100 +dander azure cancel --deployment azure_snowflake --pipeline warehouse_fixture \ + --execution-name dander-00626d3b5f01-abc1234 +dander azure replay --deployment azure_snowflake --pipeline warehouse_fixture \ + --execution-name dander-00626d3b5f01-abc1234 +``` + +Azure assigns execution names. Replay is allowed only from a terminal execution and starts the +same immutable job template; Dander's persisted inclusive cursor remains responsible for +idempotent logical replay. Log reads query only the selected execution and enforce an explicit +row limit. + ## Boundaries - ACR administrator credentials and Storage shared keys remain disabled; state network access @@ -70,5 +112,6 @@ subscription, environment, logs, job trigger/resources/image, managed identity, - Key Vault network access defaults to deny and admits only Azure's trusted-service path plus the reviewed operator IP. - Secret values are an operator action outside Terraform and must never enter a plan or state. -- Image copy, job lifecycle operations, external identity federation, live profile proof, and - support promotion are separate Phase 6 gates. +- The locally tested image-copy and job-operation commands are not live evidence. Actual image + copy, job lifecycle execution, external identity federation, live profile proof, and support + promotion remain separate Phase 6 gates. diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index e191cf3b..7820c8de 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -10,6 +10,7 @@ AzureAdministrativeBootstrap, AzureAdministrativeBootstrapError, ) +from dander.bootstrap.azure_image import AzureRuntimeImagePromoter from dander.bootstrap.azure_terraform import AzureTerraformBootstrap, AzureTerraformBootstrapError from dander.bootstrap.permissions import require_stage_zero_permissions from dander.bootstrap.project import ( @@ -39,6 +40,7 @@ "AwsTerraformBootstrapError", "AzureAdministrativeBootstrap", "AzureAdministrativeBootstrapError", + "AzureRuntimeImagePromoter", "AzureTerraformBootstrap", "AzureTerraformBootstrapError", "DeploymentSummary", diff --git a/src/dander/bootstrap/azure_image.py b/src/dander/bootstrap/azure_image.py new file mode 100644 index 00000000..9dc2931b --- /dev/null +++ b/src/dander/bootstrap/azure_image.py @@ -0,0 +1,289 @@ +"""Digest-preserving promotion of accepted runtime images into Azure ACR.""" + +from __future__ import annotations + +import json +import re +import subprocess +from typing import TYPE_CHECKING, Protocol +from uuid import UUID + +from dander.bootstrap.project import ( + _ARTIFACT_SCHEMA, + _DIGEST, + _OCI_IMAGE, + ProjectBootstrapError, + _platform_manifests, + _write_artifact_record, +) + +if TYPE_CHECKING: + from pathlib import Path + +_ACR_NAME = re.compile(r"^[a-z][a-z0-9]{4,49}$") +_ACR_REPOSITORY = re.compile(r"^[a-z0-9]+(?:[._/-][a-z0-9]+)*$") +_TAG_PREFIX = re.compile(r"^[a-z][a-z0-9-]{0,31}$") + + +class _Runner(Protocol): + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + ) -> subprocess.CompletedProcess[str]: ... + + +def _subprocess_runner( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + ) + + +class AzureRuntimeImagePromoter: + """Copy one accepted OCI index into ACR without rebuilding it.""" + + def __init__(self, project_dir: Path, *, runner: _Runner | None = None) -> None: + self._project_dir = project_dir.resolve() + self._runner = runner or _subprocess_runner + self._artifact_record_path: Path | None = None + + @property + def artifact_record_path(self) -> Path | None: + """Return the Azure artifact record written by the latest successful promotion.""" + return self._artifact_record_path + + def promote( + self, + *, + source_image: str, + subscription_id: str, + acr_name: str, + repository_name: str, + tag_prefix: str = "promoted", + ) -> str: + """Promote an accepted image and require byte-identical OCI content in ACR.""" + self._artifact_record_path = None + source_match = _OCI_IMAGE.fullmatch(source_image) + if source_match is None: + raise ProjectBootstrapError("Source image must be an immutable OCI digest reference") + _validate_subscription_id(subscription_id) + if _ACR_NAME.fullmatch(acr_name) is None: + raise ProjectBootstrapError("Invalid Azure Container Registry name") + if _ACR_REPOSITORY.fullmatch(repository_name) is None: + raise ProjectBootstrapError("Invalid Azure Container Registry repository name") + if _TAG_PREFIX.fullmatch(tag_prefix) is None: + raise ProjectBootstrapError("Invalid Azure image-promotion tag prefix") + + source_record = self._source_record(source_image=source_image) + expected_digest = source_match.group("digest") + expected_platforms = source_record["platform_manifests"] + assert isinstance(expected_platforms, dict) + host = f"{acr_name}.azurecr.io" + destination_repository = f"{host}/{repository_name}" + tag = f"{tag_prefix}-{expected_digest.removeprefix('sha256:')[:12]}" + tagged_destination = f"{destination_repository}:{tag}" + + try: + login_server = self._runner( + ( + "az", + "acr", + "show", + "--name", + acr_name, + "--subscription", + subscription_id, + "--query", + "loginServer", + "--output", + "tsv", + "--only-show-errors", + ), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if login_server != host: + raise ProjectBootstrapError("Azure returned an unexpected ACR login server") + self._runner( + ( + "az", + "acr", + "login", + "--name", + acr_name, + "--subscription", + subscription_id, + "--only-show-errors", + ), + cwd=self._project_dir, + check=True, + ) + source_raw = self._inspect(source_image, check=True).stdout + if ( + _platform_manifests(source_raw, default_digest=expected_digest) + != expected_platforms + ): + raise ProjectBootstrapError( + "Source registry content no longer matches the accepted artifact record" + ) + + describe_destination = ( + "az", + "acr", + "repository", + "show", + "--name", + acr_name, + "--image", + f"{repository_name}:{tag}", + "--subscription", + subscription_id, + "--query", + "digest", + "--output", + "tsv", + "--only-show-errors", + ) + existing = self._runner( + describe_destination, + cwd=self._project_dir, + check=False, + capture_output=True, + text=True, + ) + created = existing.returncode != 0 + if created: + self._runner( + ( + "docker", + "buildx", + "imagetools", + "create", + "--prefer-index=false", + "--tag", + tagged_destination, + source_image, + ), + cwd=self._project_dir, + check=True, + ) + destination_digest = self._runner( + describe_destination, + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + else: + destination_digest = existing.stdout.strip() + + if destination_digest != expected_digest: + if created: + raise ProjectBootstrapError( + "ACR rewrote the OCI index; the promoted artifact is not byte-identical" + ) + raise ProjectBootstrapError( + "Existing immutable ACR tag does not match the accepted OCI index" + ) + immutable_destination = f"{destination_repository}@{destination_digest}" + immutable_raw = self._inspect(immutable_destination, check=True).stdout + if ( + _platform_manifests(immutable_raw, default_digest=destination_digest) + != expected_platforms + ): + raise ProjectBootstrapError( + "ACR platform manifests differ from the accepted source-free artifact" + ) + except (FileNotFoundError, subprocess.CalledProcessError) as error: + raise ProjectBootstrapError("Could not promote the runtime image into ACR") from error + + record_path = self._project_dir / ".dander" / "runtime-artifact-azure.json" + try: + _write_artifact_record( + record_path, + { + **source_record, + "image": immutable_destination, + "tagged_image": tagged_destination, + "source_image": source_image, + "promotion": "registry-copy", + }, + ) + except OSError as error: + raise ProjectBootstrapError( + "Runtime image was promoted but its Azure artifact record could not be written" + ) from error + self._artifact_record_path = record_path + return immutable_destination + + def _source_record(self, *, source_image: str) -> dict[str, object]: + source_record_path = self._project_dir / ".dander" / "runtime-artifact.json" + try: + source_record = json.loads(source_record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ProjectBootstrapError( + "A valid source-free runtime artifact record is required before Azure promotion" + ) from error + source_match = _OCI_IMAGE.fullmatch(source_image) + assert source_match is not None + expected_digest = source_match.group("digest") + if not isinstance(source_record, dict): + raise ProjectBootstrapError( + "Source image does not match its accepted runtime artifact record" + ) + expected_platforms = source_record.get("platform_manifests") + if ( + source_record.get("schema") != _ARTIFACT_SCHEMA + or source_record.get("image") != source_image + or source_record.get("index_digest") != expected_digest + or not isinstance(expected_platforms, dict) + or not expected_platforms + or any( + not isinstance(key, str) + or not isinstance(value, str) + or _DIGEST.fullmatch(value) is None + for key, value in expected_platforms.items() + ) + ): + raise ProjectBootstrapError( + "Source image does not match its accepted runtime artifact record" + ) + return source_record + + def _inspect(self, image: str, *, check: bool) -> subprocess.CompletedProcess[str]: + return self._runner( + ("docker", "buildx", "imagetools", "inspect", "--raw", image), + cwd=self._project_dir, + check=check, + capture_output=True, + text=True, + ) + + +def _validate_subscription_id(subscription_id: str) -> None: + try: + parsed = UUID(subscription_id) + except ValueError as error: + raise ProjectBootstrapError("Invalid Azure subscription id") from error + if parsed.variant != "specified in RFC 4122": + raise ProjectBootstrapError("Invalid Azure subscription id") + + +__all__ = ["AzureRuntimeImagePromoter"] diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index 27988e9c..076c77fa 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -12,11 +12,15 @@ from dander.bootstrap import ( AzureAdministrativeBootstrap, AzureAdministrativeBootstrapError, + AzureRuntimeImagePromoter, AzureTerraformBootstrap, AzureTerraformBootstrapError, + ProjectBootstrapError, ) from dander.project import ProjectConfigError, load_project_config from dander.providers.azure_container_apps import ( + AzureContainerAppsOperationError, + AzureContainerAppsOperations, AzureDeploymentBinding, AzureDeploymentVerificationError, AzureDeploymentVerifier, @@ -29,6 +33,138 @@ azure_app = typer.Typer(help="Operate and verify manifest-bound Azure Container Apps Jobs.") +def _azure_operations( + *, + config: Path, + deployment: str, + pipeline: str, + name: str, +) -> AzureContainerAppsOperations: + binding = AzureDeploymentBinding.from_project( + config=config, + deployment=deployment, + pipeline_id=pipeline, + name=name, + ) + return AzureContainerAppsOperations(binding) + + +@azure_app.command("run") +def azure_run( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Start one potentially billable Container Apps Job execution after confirmation.""" + if not typer.confirm(f"Start Azure Container Apps pipeline {pipeline!r}?", default=False): + raise typer.Abort() + try: + execution = _azure_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + ).start() + except (AzureContainerAppsOperationError, AzureDeploymentVerificationError) as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + +@azure_app.command("status") +def azure_status( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_name: str | None = typer.Option(None, "--execution-name"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Show sanitized status for one execution, or the latest execution.""" + try: + operations = _azure_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + ) + execution = ( + operations.describe(execution_name) + if execution_name is not None + else operations.latest() + ) + except (AzureContainerAppsOperationError, AzureDeploymentVerificationError) as error: + raise ClickException(str(error)) from error + console.print_json(data={"execution": execution.as_dict() if execution is not None else None}) + + +@azure_app.command("logs") +def azure_logs( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_name: str = typer.Option(..., "--execution-name"), + limit: int = typer.Option(100, "--limit", min=1, max=10_000), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Read bounded Log Analytics events for one exact execution.""" + try: + events = _azure_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + ).logs(execution_name, limit=limit) + except (AzureContainerAppsOperationError, AzureDeploymentVerificationError) as error: + raise ClickException(str(error)) from error + console.print_json(data={"events": [event.as_dict() for event in events]}) + + +@azure_app.command("cancel") +def azure_cancel( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_name: str = typer.Option(..., "--execution-name"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Stop one running Container Apps Job execution after confirmation.""" + if not typer.confirm(f"Stop Azure execution {execution_name!r}?", default=False): + raise typer.Abort() + try: + execution = _azure_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + ).cancel(execution_name) + except (AzureContainerAppsOperationError, AzureDeploymentVerificationError) as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + +@azure_app.command("replay") +def azure_replay( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + execution_name: str = typer.Option(..., "--execution-name"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Replay a terminal execution from the runtime's persisted inclusive cursor.""" + if not typer.confirm(f"Replay Azure Container Apps pipeline {pipeline!r}?", default=False): + raise typer.Abort() + try: + execution = _azure_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + ).replay(execution_name) + except (AzureContainerAppsOperationError, AzureDeploymentVerificationError) as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + @azure_app.command("verify") def azure_verify( deployment: str = typer.Option(..., "--deployment"), @@ -183,6 +319,39 @@ def init_azure_admin_apply( ) +def image_promote_azure( + source_image: str = typer.Option( + ..., "--source-image", help="Accepted source-free OCI image ending in @sha256 digest." + ), + subscription_id: str = typer.Option(..., "--subscription-id"), + acr_name: str = typer.Option(..., "--acr-name"), + repository_name: str = typer.Option("dander/runtime", "--acr-repository"), + tag_prefix: str = typer.Option("promoted", "--tag-prefix"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 +) -> None: + """Copy an accepted source-free OCI index into ACR without rebuilding it.""" + if not typer.confirm( + f"Copy the accepted runtime image into Azure Container Registry {acr_name!r}?", + default=False, + ): + raise typer.Abort() + try: + promoter = AzureRuntimeImagePromoter(config.resolve().parent) + image = promoter.promote( + source_image=source_image, + subscription_id=subscription_id, + acr_name=acr_name, + repository_name=repository_name, + tag_prefix=tag_prefix, + ) + except ProjectBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]Promoted byte-identical runtime image:[/green] {image}") + if promoter.artifact_record_path is not None: + console.print(f"Azure artifact record: {promoter.artifact_record_path}") + console.print("Next: use this immutable digest with dander init-azure-plan.") + + def init_azure_plan( state_resource_group_name: str = typer.Option(..., "--state-resource-group"), state_storage_account_name: str = typer.Option(..., "--state-storage-account"), @@ -299,6 +468,7 @@ def register_azure_commands(app: typer.Typer) -> None: app.add_typer(azure_app, name="azure") app.command("init-azure-admin-plan")(init_azure_admin_plan) app.command("init-azure-admin-apply")(init_azure_admin_apply) + app.command("image-promote-azure")(image_promote_azure) app.command("init-azure-plan")(init_azure_plan) app.command("init-azure-apply")(init_azure_apply) diff --git a/src/dander/providers/azure_container_apps/__init__.py b/src/dander/providers/azure_container_apps/__init__.py index f621cd9f..383246ef 100644 --- a/src/dander/providers/azure_container_apps/__init__.py +++ b/src/dander/providers/azure_container_apps/__init__.py @@ -1,6 +1,12 @@ """Azure Container Apps Jobs launcher configuration.""" from dander.providers.azure_container_apps.config import AzureContainerAppsLauncherConfig +from dander.providers.azure_container_apps.operations import ( + AzureContainerAppsExecution, + AzureContainerAppsLogEvent, + AzureContainerAppsOperationError, + AzureContainerAppsOperations, +) from dander.providers.azure_container_apps.verification import ( AzureDeploymentBinding, AzureDeploymentVerification, @@ -10,6 +16,10 @@ __all__ = [ "AzureContainerAppsLauncherConfig", + "AzureContainerAppsExecution", + "AzureContainerAppsLogEvent", + "AzureContainerAppsOperationError", + "AzureContainerAppsOperations", "AzureDeploymentBinding", "AzureDeploymentVerification", "AzureDeploymentVerificationError", diff --git a/src/dander/providers/azure_container_apps/operations.py b/src/dander/providers/azure_container_apps/operations.py new file mode 100644 index 00000000..d90ff8cf --- /dev/null +++ b/src/dander/providers/azure_container_apps/operations.py @@ -0,0 +1,358 @@ +"""Provider-native Azure Container Apps Job lifecycle operations.""" + +from __future__ import annotations + +import json +import re +import subprocess +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Protocol, cast + +if TYPE_CHECKING: + from pathlib import Path + + from dander.providers.azure_container_apps.verification import AzureDeploymentBinding + +_EXECUTION_NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$") +_RUNNING_STATUSES = frozenset({"Pending", "Processing", "Running"}) +_FAILED_STATUSES = frozenset({"Failed", "Degraded"}) +_CANCELLED_STATUSES = frozenset({"Cancelled", "Canceled", "Stopped"}) + + +class AzureContainerAppsOperationError(RuntimeError): + """Raised when an Azure lifecycle operation fails or returns invalid data.""" + + +class _Runner(Protocol): + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _subprocess_runner( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + ) + + +@dataclass(frozen=True, slots=True) +class AzureContainerAppsExecution: + """Small sanitized view of one Container Apps Job execution.""" + + name: str + state: str + started_at: str | None = None + stopped_at: str | None = None + failure_code: str | None = None + + @property + def terminal(self) -> bool: + return self.state in {"succeeded", "failed", "cancelled"} + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class AzureContainerAppsLogEvent: + """One bounded Log Analytics event from a selected job execution.""" + + timestamp: str + message: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +class AzureContainerAppsOperations: + """Start, observe, stop, replay, and read logs for one manifest-bound Azure job.""" + + def __init__( + self, + binding: AzureDeploymentBinding, + *, + runner: _Runner | None = None, + ) -> None: + self.binding = binding + self._runner = runner or _subprocess_runner + + def start(self) -> AzureContainerAppsExecution: + """Start one on-demand execution and return its provider-assigned identity.""" + payload = self._json( + "containerapp", + "job", + "start", + "--name", + self.binding.job_name, + "--resource-group", + self.binding.resource_group_name, + ) + if not isinstance(payload, dict) or not isinstance(payload.get("name"), str): + raise AzureContainerAppsOperationError( + "Azure did not return an execution name after starting the job" + ) + execution_name = cast("str", payload["name"]) + self._validate_execution_name(execution_name) + return self.describe(execution_name) + + def replay(self, execution_name: str) -> AzureContainerAppsExecution: + """Start a fresh run after one terminal execution using the persisted inclusive cursor.""" + previous = self.describe(execution_name) + if not previous.terminal: + raise AzureContainerAppsOperationError("Only a terminal execution can be replayed") + return self.start() + + def latest(self) -> AzureContainerAppsExecution | None: + """Return the most recently started execution, if one exists.""" + payload = self._json( + "containerapp", + "job", + "execution", + "list", + "--name", + self.binding.job_name, + "--resource-group", + self.binding.resource_group_name, + ) + if not isinstance(payload, list): + raise AzureContainerAppsOperationError("Azure returned an invalid execution list") + if not payload: + return None + executions = [self._parse_execution(item) for item in payload] + return max(executions, key=lambda item: item.started_at or "") + + def describe(self, execution_name: str) -> AzureContainerAppsExecution: + """Return normalized status for one execution owned by the selected job.""" + self._validate_execution_name(execution_name) + payload = self._json( + "containerapp", + "job", + "execution", + "show", + "--name", + self.binding.job_name, + "--resource-group", + self.binding.resource_group_name, + "--job-execution-name", + execution_name, + ) + execution = self._parse_execution(payload) + if execution.name != execution_name: + raise AzureContainerAppsOperationError("Azure returned an unexpected job execution") + return execution + + def logs( + self, + execution_name: str, + *, + limit: int = 100, + ) -> tuple[AzureContainerAppsLogEvent, ...]: + """Read bounded console logs correlated to one exact execution.""" + if not 1 <= limit <= 10_000: + raise AzureContainerAppsOperationError("Log limit must be between 1 and 10000") + self.describe(execution_name) + environment = self._json( + "containerapp", + "env", + "show", + "--name", + self.binding.environment_name, + "--resource-group", + self.binding.resource_group_name, + ) + if not isinstance(environment, dict): + raise AzureContainerAppsOperationError( + "Azure returned an invalid Container Apps environment" + ) + workspace = _nested_string( + environment, + "properties", + "appLogsConfiguration", + "logAnalyticsConfiguration", + "customerId", + ) or _nested_string( + environment, + "appLogsConfiguration", + "logAnalyticsConfiguration", + "customerId", + ) + if workspace is None: + raise AzureContainerAppsOperationError( + "Container Apps environment has no Log Analytics destination" + ) + query = ( + "ContainerAppConsoleLogs_CL " + f"| where ContainerGroupName_s startswith '{execution_name}' " + "| order by _timestamp_d asc " + f"| take {limit} " + "| project timestamp=tostring(_timestamp_d), message=Log_s" + ) + payload = self._json( + "monitor", + "log-analytics", + "query", + "--workspace", + workspace, + "--analytics-query", + query, + "--timespan", + "P30D", + ) + if not isinstance(payload, list): + raise AzureContainerAppsOperationError("Azure returned an invalid log response") + events: list[AzureContainerAppsLogEvent] = [] + for item in payload: + if not isinstance(item, dict): + raise AzureContainerAppsOperationError("Azure returned an invalid log event") + timestamp = item.get("timestamp") + message = item.get("message") + if not isinstance(timestamp, str) or not isinstance(message, str): + raise AzureContainerAppsOperationError("Azure returned an invalid log event") + events.append(AzureContainerAppsLogEvent(timestamp=timestamp, message=message)) + return tuple(events) + + def cancel(self, execution_name: str) -> AzureContainerAppsExecution: + """Stop one running Container Apps Job execution.""" + execution = self.describe(execution_name) + if execution.state != "running": + raise AzureContainerAppsOperationError("Only a running execution can be cancelled") + self._execute( + "containerapp", + "job", + "stop", + "--name", + self.binding.job_name, + "--resource-group", + self.binding.resource_group_name, + "--job-execution-name", + execution_name, + ) + return AzureContainerAppsExecution( + name=execution.name, + state="cancelled", + started_at=execution.started_at, + failure_code="operator_cancelled", + ) + + def _execute(self, *args: str) -> None: + command = ( + "az", + *args, + "--subscription", + self.binding.subscription_id, + "--only-show-errors", + ) + self._run(command) + + def _parse_execution(self, payload: object) -> AzureContainerAppsExecution: + if not isinstance(payload, dict) or not isinstance(payload.get("name"), str): + raise AzureContainerAppsOperationError("Azure returned an invalid job execution") + name = cast("str", payload["name"]) + self._validate_execution_name(name) + properties = payload.get("properties") + if not isinstance(properties, dict) or not isinstance(properties.get("status"), str): + raise AzureContainerAppsOperationError("Azure returned an invalid execution status") + provider_status = cast("str", properties["status"]) + state = _normalized_state(provider_status) + failure_code = "launcher_execution_failed" if state == "failed" else None + if state == "cancelled": + failure_code = "operator_cancelled" + return AzureContainerAppsExecution( + name=name, + state=state, + started_at=_optional_string(properties.get("startTime")), + stopped_at=_optional_string(properties.get("endTime")), + failure_code=failure_code, + ) + + def _validate_execution_name(self, execution_name: str) -> None: + if _EXECUTION_NAME.fullmatch(execution_name) is None or not execution_name.startswith( + self.binding.job_name + "-" + ): + raise AzureContainerAppsOperationError( + "Execution does not belong to the selected Azure pipeline" + ) + + def _json(self, *args: str) -> object: + command = ( + "az", + *args, + "--subscription", + self.binding.subscription_id, + "--output", + "json", + "--only-show-errors", + ) + result = self._run(command) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as error: + raise AzureContainerAppsOperationError("Azure CLI returned invalid JSON") from error + + def _run(self, command: tuple[str, ...]) -> subprocess.CompletedProcess[str]: + try: + return self._runner( + command, + cwd=self.binding.project_dir, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise AzureContainerAppsOperationError( + "Azure CLI is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + raise AzureContainerAppsOperationError( + f"Azure lifecycle operation failed with exit code {error.returncode}" + ) from error + + +def _normalized_state(status: str) -> str: + if status in _RUNNING_STATUSES: + return "running" + if status == "Succeeded": + return "succeeded" + if status in _FAILED_STATUSES: + return "failed" + if status in _CANCELLED_STATUSES: + return "cancelled" + raise AzureContainerAppsOperationError("Azure returned an unknown execution status") + + +def _nested_string(document: dict[str, object], *path: str) -> str | None: + value: object = document + for key in path: + if not isinstance(value, dict): + return None + value = value.get(key) + return _optional_string(value) + + +def _optional_string(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +__all__ = [ + "AzureContainerAppsExecution", + "AzureContainerAppsLogEvent", + "AzureContainerAppsOperationError", + "AzureContainerAppsOperations", +] diff --git a/tests/bootstrap/test_azure_image_promotion.py b/tests/bootstrap/test_azure_image_promotion.py new file mode 100644 index 00000000..d119f720 --- /dev/null +++ b/tests/bootstrap/test_azure_image_promotion.py @@ -0,0 +1,208 @@ +"""Registry-copy promotion coverage for accepted Azure runtime images.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from dander.bootstrap import AzureRuntimeImagePromoter, ProjectBootstrapError + +if TYPE_CHECKING: + from pathlib import Path + +_AMD64_DIGEST = "sha256:" + "b" * 64 +_ARM64_DIGEST = "sha256:" + "c" * 64 +_SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" + + +def _index(*, arm64_digest: str = _ARM64_DIGEST) -> str: + return json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "digest": _AMD64_DIGEST, + "platform": {"os": "linux", "architecture": "amd64"}, + }, + { + "digest": arm64_digest, + "platform": {"os": "linux", "architecture": "arm64"}, + }, + ], + }, + separators=(",", ":"), + ) + + +_INDEX = _index() +_INDEX_DIGEST = "sha256:" + hashlib.sha256(_INDEX.encode()).hexdigest() +_SOURCE_IMAGE = f"us-central1-docker.pkg.dev/unit-project/dander/dander@{_INDEX_DIGEST}" + + +def _write_source_record(project_dir: Path, *, image: str = _SOURCE_IMAGE) -> None: + artifact_dir = project_dir / ".dander" + artifact_dir.mkdir() + (artifact_dir / "runtime-artifact.json").write_text( + json.dumps( + { + "schema": "io.dander.runtime.artifact/v1", + "image": image, + "tagged_image": "source:accepted", + "index_digest": _INDEX_DIGEST, + "platform_manifests": { + "linux/amd64": _AMD64_DIGEST, + "linux/arm64": _ARM64_DIGEST, + }, + "source": "https://github.com/harrisonoconnorhover/dander", + "revision": "unit", + "created": "2026-08-11T00:00:00Z", + "sbom_attached": True, + "provenance_attached": True, + } + ), + encoding="utf-8", + ) + + +class _Runner: + def __init__( + self, + *, + existing: bool = False, + destination_digest: str = _INDEX_DIGEST, + destination_raw: str = _INDEX, + ) -> None: + self.existing = existing + self.destination_digest = destination_digest + self.destination_raw = destination_raw + self.created = False + self.commands: list[tuple[str, ...]] = [] + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + ) -> subprocess.CompletedProcess[str]: + del capture_output, text + assert cwd.is_absolute() + self.commands.append(args) + if args[:3] == ("az", "acr", "show"): + return subprocess.CompletedProcess(args, 0, stdout="danderphase6.azurecr.io\n") + if args[:3] == ("az", "acr", "login"): + return subprocess.CompletedProcess(args, 0, stdout="") + if args[:4] == ("az", "acr", "repository", "show"): + if not self.existing and not self.created: + assert not check + return subprocess.CompletedProcess(args, 1, stdout="") + return subprocess.CompletedProcess(args, 0, stdout=self.destination_digest + "\n") + if args[:4] == ("docker", "buildx", "imagetools", "create"): + self.created = True + return subprocess.CompletedProcess(args, 0, stdout="") + if args[:4] == ("docker", "buildx", "imagetools", "inspect"): + image = args[-1] + if image == _SOURCE_IMAGE: + return subprocess.CompletedProcess(args, 0, stdout=_INDEX) + return subprocess.CompletedProcess(args, 0, stdout=self.destination_raw) + raise AssertionError(f"Unexpected command: {args}") + + +def test_azure_promoter_copies_and_verifies_the_exact_accepted_index(tmp_path: Path) -> None: + _write_source_record(tmp_path) + runner = _Runner() + promoter = AzureRuntimeImagePromoter(tmp_path, runner=runner) + + image = promoter.promote( + source_image=_SOURCE_IMAGE, + subscription_id=_SUBSCRIPTION_ID, + acr_name="danderphase6", + repository_name="dander/runtime", + ) + + assert image == f"danderphase6.azurecr.io/dander/runtime@{_INDEX_DIGEST}" + create = next( + command + for command in runner.commands + if command[:4] == ("docker", "buildx", "imagetools", "create") + ) + assert "--prefer-index=false" in create + assert create[-1] == _SOURCE_IMAGE + assert all(command[:3] != ("docker", "buildx", "build") for command in runner.commands) + assert all("password" not in " ".join(command).lower() for command in runner.commands) + assert promoter.artifact_record_path == tmp_path / ".dander" / "runtime-artifact-azure.json" + record = json.loads(promoter.artifact_record_path.read_text(encoding="utf-8")) + assert record["source_image"] == _SOURCE_IMAGE + assert record["image"] == image + assert record["promotion"] == "registry-copy" + + +def test_azure_promoter_is_idempotent_for_an_existing_identical_tag(tmp_path: Path) -> None: + _write_source_record(tmp_path) + runner = _Runner(existing=True) + + image = AzureRuntimeImagePromoter(tmp_path, runner=runner).promote( + source_image=_SOURCE_IMAGE, + subscription_id=_SUBSCRIPTION_ID, + acr_name="danderphase6", + repository_name="dander/runtime", + ) + + assert image.endswith("@" + _INDEX_DIGEST) + assert not any( + command[:4] == ("docker", "buildx", "imagetools", "create") for command in runner.commands + ) + + +def test_azure_promoter_rejects_registry_digest_rewrites(tmp_path: Path) -> None: + _write_source_record(tmp_path) + + with pytest.raises(ProjectBootstrapError, match="rewrote the OCI index"): + AzureRuntimeImagePromoter( + tmp_path, + runner=_Runner(destination_digest="sha256:" + "d" * 64), + ).promote( + source_image=_SOURCE_IMAGE, + subscription_id=_SUBSCRIPTION_ID, + acr_name="danderphase6", + repository_name="dander/runtime", + ) + + +def test_azure_promoter_rejects_platform_manifest_drift(tmp_path: Path) -> None: + _write_source_record(tmp_path) + + with pytest.raises(ProjectBootstrapError, match="platform manifests differ"): + AzureRuntimeImagePromoter( + tmp_path, + runner=_Runner(destination_raw=_index(arm64_digest="sha256:" + "e" * 64)), + ).promote( + source_image=_SOURCE_IMAGE, + subscription_id=_SUBSCRIPTION_ID, + acr_name="danderphase6", + repository_name="dander/runtime", + ) + + +def test_azure_promoter_rejects_source_record_mismatch_before_provider_access( + tmp_path: Path, +) -> None: + _write_source_record(tmp_path, image="registry.example.invalid/dander@" + _INDEX_DIGEST) + runner = _Runner() + + with pytest.raises(ProjectBootstrapError, match="does not match"): + AzureRuntimeImagePromoter(tmp_path, runner=runner).promote( + source_image=_SOURCE_IMAGE, + subscription_id=_SUBSCRIPTION_ID, + acr_name="danderphase6", + repository_name="dander/runtime", + ) + + assert runner.commands == [] diff --git a/tests/cli/test_azure_operations_cli.py b/tests/cli/test_azure_operations_cli.py new file mode 100644 index 00000000..dd5c091b --- /dev/null +++ b/tests/cli/test_azure_operations_cli.py @@ -0,0 +1,204 @@ +"""Thin CLI coverage for manifest-bound Azure Container Apps operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from typer.testing import CliRunner + +import dander.cli.azure_command as azure_command +from dander.cli.main import app +from dander.providers.azure_container_apps import ( + AzureContainerAppsExecution, + AzureContainerAppsLogEvent, +) + +if TYPE_CHECKING: + from pathlib import Path + + from pytest import MonkeyPatch + +_JOB = "dander-00626d3b5f01" +_EXECUTION = f"{_JOB}-abc1234" + + +class _Operations: + def __init__(self) -> None: + self.calls: list[tuple[str, object]] = [] + self.execution = AzureContainerAppsExecution( + name=_EXECUTION, + state="succeeded", + ) + + def start(self) -> AzureContainerAppsExecution: + self.calls.append(("start", None)) + return self.execution + + def latest(self) -> AzureContainerAppsExecution: + self.calls.append(("latest", None)) + return self.execution + + def describe(self, execution_name: str) -> AzureContainerAppsExecution: + self.calls.append(("describe", execution_name)) + return self.execution + + def logs( + self, + execution_name: str, + *, + limit: int, + ) -> tuple[AzureContainerAppsLogEvent, ...]: + self.calls.append(("logs", (execution_name, limit))) + return ( + AzureContainerAppsLogEvent( + timestamp="2026-08-11T12:00:01Z", + message="runtime complete", + ), + ) + + def cancel(self, execution_name: str) -> AzureContainerAppsExecution: + self.calls.append(("cancel", execution_name)) + return self.execution + + def replay(self, execution_name: str) -> AzureContainerAppsExecution: + self.calls.append(("replay", execution_name)) + return self.execution + + +def _install_fake(monkeypatch: MonkeyPatch) -> tuple[_Operations, dict[str, object]]: + operations = _Operations() + binding: dict[str, object] = {} + + def factory(**kwargs: object) -> _Operations: + binding.update(kwargs) + return operations + + monkeypatch.setattr(azure_command, "_azure_operations", factory) + return operations, binding + + +def _base_args(tmp_path: Path) -> list[str]: + return [ + "--deployment", + "azure_snowflake", + "--pipeline", + "warehouse_fixture", + "--config", + str(tmp_path / "dander.yaml"), + ] + + +def test_status_and_logs_bind_to_the_selected_manifest_pipeline( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + operations, binding = _install_fake(monkeypatch) + + status = CliRunner().invoke( + app, + ["azure", "status", *_base_args(tmp_path), "--execution-name", _EXECUTION], + ) + logs = CliRunner().invoke( + app, + [ + "azure", + "logs", + *_base_args(tmp_path), + "--execution-name", + _EXECUTION, + "--limit", + "7", + ], + ) + + assert status.exit_code == 0, status.output + assert logs.exit_code == 0, logs.output + assert '"state": "succeeded"' in status.output + assert '"message": "runtime complete"' in logs.output + assert operations.calls == [("describe", _EXECUTION), ("logs", (_EXECUTION, 7))] + assert binding == { + "config": tmp_path / "dander.yaml", + "deployment": "azure_snowflake", + "pipeline": "warehouse_fixture", + "name": "dander", + } + + +@pytest.mark.parametrize( + ("command", "extra", "expected"), + [ + ("run", [], ("start", None)), + ("cancel", ["--execution-name", _EXECUTION], ("cancel", _EXECUTION)), + ("replay", ["--execution-name", _EXECUTION], ("replay", _EXECUTION)), + ], +) +def test_mutating_operations_require_confirmation_and_invoke_one_action( + monkeypatch: MonkeyPatch, + tmp_path: Path, + command: str, + extra: list[str], + expected: tuple[str, object], +) -> None: + operations, _binding = _install_fake(monkeypatch) + + refused = CliRunner().invoke( + app, + ["azure", command, *_base_args(tmp_path), *extra], + input="n\n", + ) + accepted = CliRunner().invoke( + app, + ["azure", command, *_base_args(tmp_path), *extra], + input="y\n", + ) + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert operations.calls == [expected] + + +def test_image_promotion_requires_confirmation_and_passes_only_typed_inputs( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[dict[str, object]] = [] + digest = "sha256:" + "a" * 64 + + class _Promoter: + artifact_record_path = tmp_path / ".dander" / "runtime-artifact-azure.json" + + def __init__(self, project_dir: Path) -> None: + assert project_dir == tmp_path + + def promote(self, **kwargs: object) -> str: + calls.append(kwargs) + return f"danderphase6.azurecr.io/dander/runtime@{digest}" + + monkeypatch.setattr(azure_command, "AzureRuntimeImagePromoter", _Promoter) + args = [ + "image-promote-azure", + "--source-image", + f"registry.example/source@{digest}", + "--subscription-id", + "11111111-1111-4111-8111-111111111111", + "--acr-name", + "danderphase6", + "--config", + str(tmp_path / "dander.yaml"), + ] + + refused = CliRunner().invoke(app, args, input="n\n") + accepted = CliRunner().invoke(app, args, input="y\n") + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert calls == [ + { + "source_image": f"registry.example/source@{digest}", + "subscription_id": "11111111-1111-4111-8111-111111111111", + "acr_name": "danderphase6", + "repository_name": "dander/runtime", + "tag_prefix": "promoted", + } + ] diff --git a/tests/providers/test_azure_container_apps_operations.py b/tests/providers/test_azure_container_apps_operations.py new file mode 100644 index 00000000..51984a29 --- /dev/null +++ b/tests/providers/test_azure_container_apps_operations.py @@ -0,0 +1,309 @@ +"""Azure Container Apps execution, logs, stop, and replay contract tests.""" + +from __future__ import annotations + +import json +import subprocess +from collections import defaultdict, deque +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from dander.providers.azure_container_apps import ( + AzureContainerAppsOperationError, + AzureContainerAppsOperations, + AzureDeploymentBinding, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + +_SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" +_CLIENT_ID = "22222222-2222-4222-8222-222222222222" +_ROOT = f"/subscriptions/{_SUBSCRIPTION_ID}/resourceGroups/dander-phase6" +_JOB = "dander-00626d3b5f01" +_EXECUTION = f"{_JOB}-abc1234" + + +def _binding() -> AzureDeploymentBinding: + return AzureDeploymentBinding( + subscription_id=_SUBSCRIPTION_ID, + location="eastus", + resource_group_name="dander-phase6", + environment_name="dander-phase6-env", + environment_id=f"{_ROOT}/providers/Microsoft.App/managedEnvironments/dander-phase6-env", + acr_name="danderphase6", + acr_login_server="danderphase6.azurecr.io", + key_vault_name="dander-phase6-kv", + key_vault_uri="https://dander-phase6-kv.vault.azure.net", + managed_identity_id=( + f"{_ROOT}/providers/Microsoft.ManagedIdentity/" + "userAssignedIdentities/dander-phase6-runtime" + ), + managed_identity_client_id=_CLIENT_ID, + pipeline_id="warehouse_fixture", + job_name=_JOB, + schedule_paused=True, + runtime_timeout_seconds=900, + runtime_max_retries=1, + secret_ids=("postgres-dsn",), + project_dir=Path("/tmp/dander-azure-operations-test"), + ) + + +class _Runner: + def __init__(self, responses: Mapping[tuple[str, ...], list[object]]) -> None: + self.responses: defaultdict[tuple[str, ...], deque[object]] = defaultdict(deque) + for key, values in responses.items(): + self.responses[key].extend(values) + self.commands: list[tuple[str, ...]] = [] + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: + assert cwd == Path("/tmp/dander-azure-operations-test") + assert check and capture_output and text + self.commands.append(args) + key = args[1 : args.index("--subscription")] + if not self.responses[key]: + raise AssertionError(f"No fake Azure response for {key}") + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps(self.responses[key].popleft()), + ) + + +def _execution(*, status: str = "Succeeded", name: str = _EXECUTION) -> dict[str, object]: + properties: dict[str, object] = { + "status": status, + "startTime": "2026-08-11T12:00:00+00:00", + } + if status not in {"Pending", "Processing", "Running"}: + properties["endTime"] = "2026-08-11T12:01:00+00:00" + return {"name": name, "properties": properties} + + +def test_start_uses_exact_manifest_job_and_returns_provider_execution() -> None: + runner = _Runner( + { + ( + "containerapp", + "job", + "start", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + ): [{"name": _EXECUTION}], + ( + "containerapp", + "job", + "execution", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ): [_execution(status="Running")], + } + ) + + execution = AzureContainerAppsOperations(_binding(), runner=runner).start() + + assert execution.name == _EXECUTION + assert execution.state == "running" + assert runner.commands[0][1:4] == ("containerapp", "job", "start") + + +def test_latest_normalizes_the_most_recent_execution() -> None: + older = _execution(name=f"{_JOB}-old1234") + older_properties = older["properties"] + assert isinstance(older_properties, dict) + older_properties["startTime"] = "2026-08-11T11:00:00+00:00" + runner = _Runner( + { + ( + "containerapp", + "job", + "execution", + "list", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + ): [[older, _execution()]], + } + ) + + execution = AzureContainerAppsOperations(_binding(), runner=runner).latest() + + assert execution is not None + assert execution.name == _EXECUTION + assert execution.state == "succeeded" + + +def test_logs_are_bounded_and_correlated_through_log_analytics() -> None: + show_key = ( + "containerapp", + "job", + "execution", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ) + runner = _Runner( + { + show_key: [_execution()], + ( + "containerapp", + "env", + "show", + "--name", + "dander-phase6-env", + "--resource-group", + "dander-phase6", + ): [ + { + "properties": { + "appLogsConfiguration": { + "logAnalyticsConfiguration": {"customerId": "workspace-unit"} + } + } + } + ], + ( + "monitor", + "log-analytics", + "query", + "--workspace", + "workspace-unit", + "--analytics-query", + ( + "ContainerAppConsoleLogs_CL " + f"| where ContainerGroupName_s startswith '{_EXECUTION}' " + "| order by _timestamp_d asc | take 7 " + "| project timestamp=tostring(_timestamp_d), message=Log_s" + ), + "--timespan", + "P30D", + ): [[{"timestamp": "2026-08-11T12:00:01Z", "message": "runtime complete"}]], + } + ) + + events = AzureContainerAppsOperations(_binding(), runner=runner).logs(_EXECUTION, limit=7) + + assert events[0].message == "runtime complete" + query_command = runner.commands[-1] + assert _EXECUTION in query_command[query_command.index("--analytics-query") + 1] + assert "take 7" in query_command[query_command.index("--analytics-query") + 1] + + +def test_cancel_requires_a_running_owned_execution() -> None: + show_key = ( + "containerapp", + "job", + "execution", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ) + stop_key = ( + "containerapp", + "job", + "stop", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ) + runner = _Runner({show_key: [_execution(status="Running")], stop_key: [{}]}) + + execution = AzureContainerAppsOperations(_binding(), runner=runner).cancel(_EXECUTION) + + assert execution.state == "cancelled" + assert execution.failure_code == "operator_cancelled" + + +def test_replay_requires_terminal_status_then_starts_a_fresh_provider_execution() -> None: + replayed = f"{_JOB}-replay1" + show_previous = ( + "containerapp", + "job", + "execution", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ) + show_replayed = (*show_previous[:-1], replayed) + start = ( + "containerapp", + "job", + "start", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + ) + runner = _Runner( + { + show_previous: [_execution()], + start: [{"name": replayed}], + show_replayed: [_execution(status="Running", name=replayed)], + } + ) + + execution = AzureContainerAppsOperations(_binding(), runner=runner).replay(_EXECUTION) + + assert execution.name == replayed + assert execution.state == "running" + + +def test_foreign_execution_and_unknown_status_fail_closed() -> None: + runner = _Runner({}) + operations = AzureContainerAppsOperations(_binding(), runner=runner) + + with pytest.raises(AzureContainerAppsOperationError, match="does not belong"): + operations.describe("another-job-abc1234") + + assert runner.commands == [] + + show_key = ( + "containerapp", + "job", + "execution", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ) + with pytest.raises(AzureContainerAppsOperationError, match="unknown execution status"): + AzureContainerAppsOperations( + _binding(), runner=_Runner({show_key: [_execution(status="Mystery")]}) + ).describe(_EXECUTION) diff --git a/tickets/DANDER-109-azure-terraform-plan-verification.md b/tickets/DANDER-109-azure-terraform-plan-verification.md index 5411371a..a3363690 100644 --- a/tickets/DANDER-109-azure-terraform-plan-verification.md +++ b/tickets/DANDER-109-azure-terraform-plan-verification.md @@ -1,6 +1,6 @@ # DANDER-109 — Plan and verify Azure Container Apps deployments -Status: implemented locally; protected CI and live proof pending +Status: merged through protected PR #189; live proof pending ## Requirement @@ -26,7 +26,7 @@ Add a read-only verifier that proves a deployed job still matches its manifest a - [x] Read-only verification checks the exact subscription, environment, logs, job, immutable image, registry identity, ACR static-credential setting, and RBAC Key Vault without reading secrets. - [x] Terraform provider-mocked tests and focused Python contract tests pass. -- [ ] Protected CI passes and this ticket merges before any live Azure apply. +- [x] Protected CI passes and this ticket merges before any live Azure apply. ## Evidence boundary diff --git a/tickets/DANDER-110-azure-image-lifecycle-operations.md b/tickets/DANDER-110-azure-image-lifecycle-operations.md new file mode 100644 index 00000000..0949bb5a --- /dev/null +++ b/tickets/DANDER-110-azure-image-lifecycle-operations.md @@ -0,0 +1,34 @@ +# DANDER-110 — Promote and operate Azure Container Apps runtimes + +Status: implemented locally; protected CI and live proof pending + +## Requirement + +Copy the accepted source-free OCI runtime into ACR without rebuilding or changing its index or +platform-manifest digests. Add provider-native, manifest-bound Container Apps Job start, status, +logs, stop, and replay operations while keeping all live mutations behind explicit confirmation. + +## Acceptance + +- [x] Promotion requires the accepted `io.dander.runtime.artifact/v1` record and an immutable source + digest before provider access. +- [x] ACR authentication uses the signed-in Azure identity; no static registry credential enters a + command, environment, artifact record, or repository file. +- [x] A deterministic tag is copied with `docker buildx imagetools create` and never rebuilt. +- [x] ACR's GA image metadata must report the accepted index digest, and the immutable destination + must expose the exact accepted platform-manifest map before the Azure artifact record is written. +- [x] Existing deterministic tags are idempotent when equal and fail closed when unequal. +- [x] Lifecycle commands are bound to one manifest-selected job and normalize only known provider + states. +- [x] Logs use a bounded Log Analytics query correlated to one validated execution name. +- [x] Stop requires a running execution; replay requires a terminal execution and relies on the + runtime's persisted inclusive cursor. +- [x] Image copy, start, stop, and replay require interactive confirmation in the CLI. +- [x] Focused tests use mocked command runners and make no Azure call. +- [ ] Protected CI passes and this ticket merges before any live ACR copy or job execution. + +## Evidence boundary + +This ticket contains deterministic local contract evidence only. It does not register a provider, +create an Azure resource, authenticate to ACR, copy an image, start or stop a job, publish a +candidate, or spend cloud credits. Live lifecycle and cleanup remain separate Phase 6 gates. From 129d6677cb8462341cd08f81f7e458cc7f8bf17d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 13:09:04 -0400 Subject: [PATCH 123/163] feat: prepare Azure Google workload federation --- .github/workflows/ci.yml | 4 + CHANGELOG.md | 5 + HANDOFF.md | 37 ++-- .../.terraform.lock.hcl | 62 ++++++ .../phase6/azure-google-federation/README.md | 48 ++++ .../phase6/azure-google-federation/main.tf | 120 ++++++++++ .../phase6/azure-google-federation/outputs.tf | 29 +++ .../terraform.tfvars.example | 10 + .../tests/federation.tftest.hcl | 107 +++++++++ .../azure-google-federation/variables.tf | 111 ++++++++++ .../azure-google-federation/versions.tf | 33 +++ docs/decisions.md | 14 ++ docs/execution-projection.md | 5 +- docs/known-limitations.md | 7 +- .../azure/modules/container-apps-jobs/main.tf | 34 ++- .../container-apps-jobs/tests/jobs.tftest.hcl | 25 +++ src/dander/bootstrap/azure_terraform.py | 25 ++- src/dander/cli/azure_command.py | 58 +++++ src/dander/cli/runtime_command.py | 62 +++++- src/dander/identity/__init__.py | 6 + src/dander/identity/aws_google.py | 11 + src/dander/identity/azure_google.py | 206 ++++++++++++++++++ src/dander/identity/refresh_probe.py | 135 ++++++++++++ src/dander/project/portable_config.py | 18 +- .../providers/azure_container_apps/config.py | 30 ++- .../azure_container_apps/operations.py | 38 ++++ .../providers/azure_container_apps/runtime.py | 95 +++++--- .../azure_container_apps/verification.py | 76 ++++++- tests/bootstrap/test_azure_terraform.py | 58 +++++ tests/cli/test_azure_operations_cli.py | 57 +++++ tests/identity/test_azure_google.py | 130 +++++++++++ tests/identity/test_refresh_probe.py | 108 +++++++++ tests/project/test_portable_config.py | 42 ++++ .../test_azure_container_apps_operations.py | 61 ++++++ .../test_azure_container_apps_runtime.py | 72 +++++- .../test_azure_deployment_verification.py | 74 +++++++ tickets/DANDER-111-azure-google-federation.md | 38 ++++ 37 files changed, 1984 insertions(+), 67 deletions(-) create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/.terraform.lock.hcl create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/README.md create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/main.tf create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/outputs.tf create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/terraform.tfvars.example create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/tests/federation.tftest.hcl create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/variables.tf create mode 100644 acceptance/cloud-portability/phase6/azure-google-federation/versions.tf create mode 100644 src/dander/identity/azure_google.py create mode 100644 src/dander/identity/refresh_probe.py create mode 100644 tests/identity/test_azure_google.py create mode 100644 tests/identity/test_refresh_probe.py create mode 100644 tickets/DANDER-111-azure-google-federation.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 586b9ef7..78f316ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,10 @@ jobs: terraform -chdir=../acceptance/cloud-portability/phase1b/ecr validate terraform -chdir=../acceptance/cloud-portability/phase1b/smoke init -backend=false -input=false terraform -chdir=../acceptance/cloud-portability/phase1b/smoke validate + terraform fmt -check -recursive ../acceptance/cloud-portability/phase6/azure-google-federation + terraform -chdir=../acceptance/cloud-portability/phase6/azure-google-federation init -backend=false -input=false + terraform -chdir=../acceptance/cloud-portability/phase6/azure-google-federation validate + terraform -chdir=../acceptance/cloud-portability/phase6/azure-google-federation test - name: Static security analysis uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 286fa77d..f5c36f69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ capabilities enter through the next minor release. ### Added +- Add keyless Azure Container Apps-to-Google workload federation using the attached user-assigned + identity, an exact Entra application audience, in-memory Google token exchange, and a bounded + same-process credential-refresh probe. The profile remains experimental until live acceptance. +- Add disposable, provider-mocked Phase 6 federation Terraform that pins Google trust and service + account impersonation to one Azure managed-identity object ID without client secrets or keys. - Preserve validated provider extensions from connector, graph, and model declarations through canonical warehouse targets without changing existing BigQuery schema inputs. - Carry additive warehouse load/transform telemetry through endpoint results into terminal run diff --git a/HANDOFF.md b/HANDOFF.md index f209574d..9c673a05 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,40 @@ ## Finished -- Merged Azure plan-first infrastructure and verification through protected PR #189 with green main CI. -- Added accepted-record-gated ACR registry copy with exact index and platform-manifest verification. -- Added manifest-bound Azure run, status, bounded logs, cancel, and replay operations. -- Added interactive confirmation for every image or job mutation and local mocked contract tests. -- Kept provider registration, resource creation, image copy, execution, and spending untouched. +- Kept protected main green through merged Azure contract, infrastructure, and lifecycle PRs #188-#190. +- Added renewable Container Apps managed-identity to Google credentials without files or static keys. +- Added the exact Azure BigQuery/Dataplex/GCP-secrets projection beside unchanged Snowflake/Key Vault behavior. +- Added disposable Entra/Google federation Terraform pinned to one Azure identity object ID. +- Added a bounded 600-second refresh probe with explicit confirmation and no automatic paid rerun. ## Try It -Run `uv run pytest -q tests/bootstrap/test_azure_image_promotion.py -tests/providers/test_azure_container_apps_operations.py tests/cli/test_azure_operations_cli.py`. +Run `uv run pytest -q tests/identity/test_azure_google.py tests/identity/test_refresh_probe.py +tests/providers/test_azure_container_apps_runtime.py`. ## Checks -- Focused pytest, Ruff, and mypy pass for Azure promotion and operations. -- Protected main CI is green through PR #189; protected CI for this operations slice is pending. -- No live Azure command from the new promotion or operations paths was executed. +- Full repository pytest, Ruff, and mypy pass with the federation, projection, CLI, and refresh code. +- Azure module and disposable federation Terraform mock tests pass; the new root validates cleanly. +- Protected main CI is green through merged PR #190; this federation slice is not yet published. +- No provider registration, resource creation, image copy, job execution, or paid query ran. ## Decisions -- Promotion uses Buildx registry copy plus stable ACR image metadata; it never rebuilds or uses static ACR credentials. -- Azure owns execution names; Dander's persisted inclusive cursor owns logical replay correctness. -- Logs use a bounded Log Analytics query tied to one validated execution. +- Container Apps uses `ManagedIdentityCredential`; Google Auth exchanges tokens in memory for 600-second credentials. +- Google trust checks Entra tenant, application audience, and the exact managed-identity object ID. +- Azure Snowflake/Key Vault and Azure BigQuery/GCP-secrets remain distinct named profiles. ## Remaining -- Merge this operations slice through a focused protected PR. -- Prepare Azure-to-Google federation and named Snowflake proof tooling locally. +- Merge this federation slice through a focused protected PR. +- Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance tooling locally. - Obtain candidate-publication approval before publishing or copying an image. - Obtain explicit Azure, GCP, and Snowflake mutation ceilings before live proof. - Run approved lifecycle, cleanup, rollback, and no-drift acceptance. ## Review First -- `src/dander/bootstrap/azure_image.py` -- `src/dander/providers/azure_container_apps/operations.py` -- `src/dander/cli/azure_command.py` +- `src/dander/identity/azure_google.py` +- `src/dander/providers/azure_container_apps/runtime.py` +- `acceptance/cloud-portability/phase6/azure-google-federation/` diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/.terraform.lock.hcl b/acceptance/cloud-portability/phase6/azure-google-federation/.terraform.lock.hcl new file mode 100644 index 00000000..d15c1b83 --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/.terraform.lock.hcl @@ -0,0 +1,62 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "3.9.0" + constraints = "~> 3.9" + hashes = [ + "h1:caKVAk5GOECNATz8XPruo39n2y6OcntxPblPgl+6QaY=", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:39b11a075e4baa4f6ed5c72a8427013d50f43eecc1a7603b73bccf80f952f758", + "zh:41484c196c943b39411f561e70a308bd2a71da18155bfec7381ba0bd61361d34", + "zh:42068e5da223494beea5f7fcb9057c308cbfa92f96e53c50083e2639216479d8", + "zh:464d7da44682443a4b64bfdaf3d0eb53011c6e1471f244f6354c4d5bca18edce", + "zh:49f597ea3fac39931ff91e55afd5b5cc91e449920a03716f82509d588aaab708", + "zh:6092c376accfc50b555b7a0cd56b76c09abc3d65ac9dd5069063d6f9f1e76d3b", + "zh:65326a9f3ac0783c16e05c16422d191f0a926b8d021fd5303c1fdf8dc42f16e9", + "zh:784214ed809347d74562bb38194c0cef57831eaa621ba3b7cdd3fe7a7a76d844", + "zh:b4233f9bc791adc7d6643507fa5b47360a21125763a072d953586151cacb65f9", + "zh:c4ecdd995ff99b7e362e087c45f080816bcf097da5be257c87b912210e45dd3e", + "zh:f0122771f71cb98248e70cdd6c2ccd3bffb34e79d19897fa28b785c86b2312ed", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.81.0" + constraints = "~> 4.72" + hashes = [ + "h1:XhToZua4gtih1Kv8RdStcfND83G4Tmb6GZFT4jEUhDU=", + "zh:0732e7b74264ddfa2b90ba69d01c283d3cbae9f72ed3e506c6ac92529fed7fd3", + "zh:12afb524e232fe4e3d6161927724af5dfa4831d71edd9c174917ca9b7377bfae", + "zh:169d619ae202c4145e02fb706fb7c3679445ab3e3ff722edbf89597517a8c92e", + "zh:6beb95a3ef2f2d9c76abaa48e5450e90686a3fb6a47f1cb0ff7c5e94b6960151", + "zh:705e075fb5ffc4bf66fd7cbabf1a65007a41621e80030a2c158a4c83b6046216", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79a8d17fefe647040fcb9ee8821a4f09f395427c4fd49493489b9a93a9a1038e", + "zh:8cc3f900b3774c0ae37ae42365c4579a199cf9e5edf88e476fdf5ab1048f84ea", + "zh:dec373b9390fa95e257291acd018ed65a7d512b428645d35e22cdbe8b245a08b", + "zh:e60f1e9fb45df6defade2855ed6e68547409ea75d30655c556adb0c08579749b", + "zh:f901d12ec82f3f8b5880a27b5cbcd7bd0d97e60c9367a2d7ed82fdd1157b39ff", + "zh:facf68ea5bf0f2b8ba720e7fba5f86492e1d4c591100460bb91c3f79f391f4b6", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "7.43.0" + constraints = "~> 7.0" + hashes = [ + "h1:AwFYLjEh05Jog9DYlEQDacVk2Sjt5/QrKAsymAW0k/Q=", + "zh:0050abe670a076cfc110662ba3e9f8c25366a63df9389dfbe090b5dc5e73a4d2", + "zh:31769c27700334d11e652ddb2a0dfc9efd65df43dbc51a06afc04e10db645bbe", + "zh:33f7211886d44a31d9dc89fbe11f0d2e282da22b509ae25dcfb8c6533eeacff4", + "zh:3ee98b297ec00ad10b1080fddfddbe72c38dec00eece6d56e706fea1e871d648", + "zh:620683cba53b964a882a419c38d40ab4595459b44dc54b9ed18c5bb6b97fbceb", + "zh:7c118cc8ba4c245807b3cbcfd4f5bdb81b1afce83fa1528020208117f7c9518c", + "zh:8263f9aeab9c8bc661a8f6a4963e585ca0a86ed6662424d8f34923fd01c6de18", + "zh:d2ba502aa3da2dad62ae81dd32c97bc279c56a97b9ac3bb3f806303dc0af598d", + "zh:db37f9f900b801ffaad1b7194c4b89de9351879e753b752ca37e12f3e989691f", + "zh:eca80c71ef4a559b94a779365a8d8b91a33d7a805a27ca21c176720085c31d30", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fa91060bb31207fc91645e7de881fa61f106b9d0dfed2f444f158b9cd9420c58", + ] +} diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/README.md b/acceptance/cloud-portability/phase6/azure-google-federation/README.md new file mode 100644 index 00000000..5e7e5cf5 --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/README.md @@ -0,0 +1,48 @@ +# Phase 6 Azure-to-Google federation proof + +This disposable root prepares the keyless Azure Container Apps to Google workload-identity +boundary. It creates one single-tenant Entra application audience, an OIDC workload identity pool +provider restricted to one existing Azure user-assigned identity object ID, one disposable Google +service account, and only the BigQuery, Dataplex, and named Secret Manager grants needed by the +bounded proof. + +The Entra application is an audience, not a client credential. Dander requests its token through +Container Apps managed identity and exchanges it in memory; no client secret, service-account key, +external-account file, or token is written to Terraform inputs or runtime logs. + +## Approval boundary + +Do not run `terraform apply`, register Azure resource providers, publish an image, start a job, or +use paid Google services until the candidate and explicit per-provider ceilings have been approved. +Planning uses read-only provider calls but can still reveal account metadata, so keep saved plans, +state, variables, and logs in a secured operator directory outside the repository. + +## Reviewed sequence + +1. Copy this root to the secured operator directory and fill values from + `terraform.tfvars.example`; use a new `proof_name` if an earlier Google pool is soft-deleted. +2. Run `terraform init`, save `terraform plan -out=dander-phase6-azure-google.tfplan`, and review + `terraform show -no-color` before seeking apply approval. +3. Apply only that exact saved plan. Put the non-secret outputs + `google_workload_identity_audience` and `azure_application_id_uri` into the typed Azure launcher + configuration; the service-account output must match each proof pipeline's + `runtime_service_account_id`. +4. Run the same accepted OCI digest through the Azure BigQuery profile. After separate live-cost + approval, start exactly one refresh probe (there is no automatic rerun): + + ```console + dander azure identity-refresh-probe \ + --deployment azure_bigquery \ + --pipeline warehouse_fixture \ + --project example-proof-project \ + --dataset raw \ + --table proof_rows + ``` + + Require a BigQuery query before and after the 600-second Google credential refresh, Secret + Manager access, Dataplex read-back, and no credential/token text in logs or committed evidence. +5. Destroy this root from a separately reviewed destroy plan. Verify the Azure proof resources are + gone, run the retained GCP no-drift plan, and retain only sanitized evidence. + +Local Terraform mock tests validate the identity pinning and generated runtime coordinates. They +are not live federation evidence and do not promote the profile to supported. diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/main.tf b/acceptance/cloud-portability/phase6/azure-google-federation/main.tf new file mode 100644 index 00000000..a5a9828c --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/main.tf @@ -0,0 +1,120 @@ +data "azurerm_client_config" "current" {} +data "azuread_client_config" "current" {} +data "google_project" "proof" { + project_id = var.gcp_project_id +} + +check "authenticated_azure_boundary_matches" { + assert { + condition = ( + lower(data.azurerm_client_config.current.subscription_id) == lower(var.subscription_id) && + lower(data.azurerm_client_config.current.tenant_id) == lower(var.tenant_id) && + lower(data.azuread_client_config.current.tenant_id) == lower(var.tenant_id) + ) + error_message = "Authenticated Azure subscription and Entra tenant must match the reviewed proof inputs." + } +} + +data "azurerm_user_assigned_identity" "runtime" { + name = var.azure_managed_identity_name + resource_group_name = var.azure_resource_group_name +} + +resource "azuread_application" "google_federation" { + display_name = "${var.proof_name}-google-wif" + owners = [data.azuread_client_config.current.object_id] + sign_in_audience = "AzureADMyOrg" + + lifecycle { + ignore_changes = [identifier_uris] + } +} + +resource "azuread_application_identifier_uri" "google_federation" { + application_id = azuread_application.google_federation.id + identifier_uri = "api://${azuread_application.google_federation.client_id}" +} + +resource "azuread_service_principal" "google_federation" { + client_id = azuread_application.google_federation.client_id + app_role_assignment_required = false + owners = [data.azuread_client_config.current.object_id] +} + +resource "google_iam_workload_identity_pool" "azure" { + workload_identity_pool_id = "${var.proof_name}-azure" + display_name = "Dander Phase 6 Azure" + description = "Disposable keyless Container Apps to Google identity proof" +} + +resource "google_iam_workload_identity_pool_provider" "azure" { + workload_identity_pool_id = google_iam_workload_identity_pool.azure.workload_identity_pool_id + workload_identity_pool_provider_id = "container-apps" + display_name = "Dander Container Apps" + description = "Trusts only the selected Dander Azure managed identity" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.azure_object_id" = "assertion.oid" + "attribute.azure_tenant_id" = "assertion.tid" + "attribute.azure_application_uri" = "assertion.aud" + } + attribute_condition = "assertion.tid == '${var.tenant_id}' && assertion.oid == '${data.azurerm_user_assigned_identity.runtime.principal_id}' && assertion.aud == 'api://${azuread_application.google_federation.client_id}'" + + oidc { + issuer_uri = "https://sts.windows.net/${var.tenant_id}/" + allowed_audiences = [azuread_application_identifier_uri.google_federation.identifier_uri] + } +} + +resource "google_service_account" "runtime" { + account_id = var.google_service_account_id + display_name = "Dander Phase 6 Azure proof" + description = "Disposable GCP identity impersonated by one Azure managed identity" +} + +resource "google_service_account_iam_member" "azure_workload_identity" { + service_account_id = google_service_account.runtime.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.azure.name}/attribute.azure_object_id/${data.azurerm_user_assigned_identity.runtime.principal_id}" +} + +resource "google_project_iam_member" "bigquery_job_user" { + project = var.gcp_project_id + role = "roles/bigquery.jobUser" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_project_iam_member" "dataplex_catalog_editor" { + project = var.gcp_project_id + role = "roles/dataplex.catalogEditor" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_bigquery_dataset_iam_member" "raw_editor" { + project = var.gcp_project_id + dataset_id = var.raw_dataset_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_bigquery_dataset_iam_member" "metadata_editor" { + project = var.gcp_project_id + dataset_id = var.metadata_dataset_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +data "google_secret_manager_secret" "proof" { + for_each = var.proof_secret_ids + project = var.gcp_project_id + secret_id = each.value +} + +resource "google_secret_manager_secret_iam_member" "proof_accessor" { + for_each = data.google_secret_manager_secret.proof + project = var.gcp_project_id + secret_id = each.value.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.runtime.email}" +} diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/outputs.tf b/acceptance/cloud-portability/phase6/azure-google-federation/outputs.tf new file mode 100644 index 00000000..097960a7 --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/outputs.tf @@ -0,0 +1,29 @@ +output "azure_application_id_uri" { + description = "Non-secret Entra audience requested by the Container Apps managed identity." + value = azuread_application_identifier_uri.google_federation.identifier_uri +} + +output "azure_managed_identity_client_id" { + description = "User-assigned managed identity client ID selected by the runtime." + value = data.azurerm_user_assigned_identity.runtime.client_id +} + +output "azure_managed_identity_principal_id" { + description = "Object ID pinned by the Google provider condition and IAM grant." + value = data.azurerm_user_assigned_identity.runtime.principal_id +} + +output "google_service_account_email" { + description = "Disposable Google service account impersonated by the Azure identity." + value = google_service_account.runtime.email +} + +output "google_workload_identity_audience" { + description = "Non-secret audience consumed by Dander's Azure launcher projection." + value = "//iam.googleapis.com/${google_iam_workload_identity_pool_provider.azure.name}" +} + +output "google_workload_identity_provider_name" { + description = "Full Google workload identity provider resource name." + value = google_iam_workload_identity_pool_provider.azure.name +} diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/terraform.tfvars.example b/acceptance/cloud-portability/phase6/azure-google-federation/terraform.tfvars.example new file mode 100644 index 00000000..14c73ce6 --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/terraform.tfvars.example @@ -0,0 +1,10 @@ +subscription_id = "11111111-1111-4111-8111-111111111111" +tenant_id = "22222222-2222-4222-8222-222222222222" +azure_resource_group_name = "dander-phase6" +azure_managed_identity_name = "dander-phase6-runtime" +proof_name = "dander-phase6-r1" +gcp_project_id = "example-proof-project" +google_service_account_id = "dander-phase6-azure" +raw_dataset_id = "raw" +metadata_dataset_id = "dander_meta" +proof_secret_ids = ["source-api-token"] diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/tests/federation.tftest.hcl b/acceptance/cloud-portability/phase6/azure-google-federation/tests/federation.tftest.hcl new file mode 100644 index 00000000..1f8bed77 --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/tests/federation.tftest.hcl @@ -0,0 +1,107 @@ +mock_provider "azurerm" { + mock_data "azurerm_client_config" { + defaults = { + subscription_id = "11111111-1111-4111-8111-111111111111" + tenant_id = "22222222-2222-4222-8222-222222222222" + client_id = "33333333-3333-4333-8333-333333333333" + object_id = "44444444-4444-4444-8444-444444444444" + } + } + + mock_data "azurerm_user_assigned_identity" { + defaults = { + id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + client_id = "55555555-5555-4555-8555-555555555555" + principal_id = "66666666-6666-4666-8666-666666666666" + } + } +} + +mock_provider "azuread" { + mock_data "azuread_client_config" { + defaults = { + tenant_id = "22222222-2222-4222-8222-222222222222" + client_id = "33333333-3333-4333-8333-333333333333" + object_id = "44444444-4444-4444-8444-444444444444" + } + } + + mock_resource "azuread_application" { + defaults = { + id = "/applications/77777777-7777-4777-8777-777777777777" + client_id = "77777777-7777-4777-8777-777777777777" + } + } + + mock_resource "azuread_application_identifier_uri" { + defaults = { + identifier_uri = "api://77777777-7777-4777-8777-777777777777" + } + } +} + +mock_provider "google" { + mock_data "google_project" { + defaults = { + number = "1009770943166" + project_id = "unit-project" + } + } + + mock_data "google_secret_manager_secret" { + defaults = { + secret_id = "source-api-token" + } + } + + mock_resource "google_iam_workload_identity_pool" { + defaults = { + name = "projects/1009770943166/locations/global/workloadIdentityPools/dander-phase6-azure" + } + } + + mock_resource "google_iam_workload_identity_pool_provider" { + defaults = { + name = "projects/1009770943166/locations/global/workloadIdentityPools/dander-phase6-azure/providers/container-apps" + } + } + + mock_resource "google_service_account" { + defaults = { + name = "projects/unit-project/serviceAccounts/dander-phase6-azure@unit-project.iam.gserviceaccount.com" + email = "dander-phase6-azure@unit-project.iam.gserviceaccount.com" + } + } +} + +variables { + subscription_id = "11111111-1111-4111-8111-111111111111" + tenant_id = "22222222-2222-4222-8222-222222222222" + azure_resource_group_name = "dander-phase6" + azure_managed_identity_name = "dander-phase6-runtime" + proof_name = "dander-phase6" + gcp_project_id = "unit-project" + google_service_account_id = "dander-phase6-azure" + raw_dataset_id = "raw" + metadata_dataset_id = "dander_meta" + proof_secret_ids = ["source-api-token"] +} + +run "binds_only_the_selected_azure_identity" { + command = apply + + assert { + condition = output.azure_application_id_uri == "api://77777777-7777-4777-8777-777777777777" + error_message = "The Entra application ID URI must be stable and non-secret." + } + + assert { + condition = output.google_workload_identity_audience == "//iam.googleapis.com/projects/1009770943166/locations/global/workloadIdentityPools/dander-phase6-azure/providers/container-apps" + error_message = "The runtime audience must name the exact Azure federation provider." + } + + assert { + condition = google_service_account_iam_member.azure_workload_identity.member == "principalSet://iam.googleapis.com/projects/1009770943166/locations/global/workloadIdentityPools/dander-phase6-azure/attribute.azure_object_id/66666666-6666-4666-8666-666666666666" + error_message = "Only the selected Azure managed identity may impersonate the Google service account." + } +} diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/variables.tf b/acceptance/cloud-portability/phase6/azure-google-federation/variables.tf new file mode 100644 index 00000000..d2a9044c --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/variables.tf @@ -0,0 +1,111 @@ +variable "subscription_id" { + description = "Azure subscription containing the Phase 6 runtime identity." + type = string + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$", var.subscription_id)) + error_message = "subscription_id must be an RFC 4122 UUID." + } +} + +variable "tenant_id" { + description = "Microsoft Entra tenant issuing the managed-identity token." + type = string + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$", var.tenant_id)) + error_message = "tenant_id must be an RFC 4122 UUID." + } +} + +variable "azure_resource_group_name" { + description = "Existing resource group containing the runtime managed identity." + type = string + + validation { + condition = can(regex("^[A-Za-z0-9._()/-]{1,90}$", var.azure_resource_group_name)) + error_message = "azure_resource_group_name must be a valid Azure resource-group name." + } +} + +variable "azure_managed_identity_name" { + description = "Existing user-assigned identity attached to the Container Apps Job." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,62}[a-z0-9]$", var.azure_managed_identity_name)) + error_message = "azure_managed_identity_name must be a normalized Azure resource name." + } +} + +variable "proof_name" { + description = "Unique deterministic name for one disposable Phase 6 federation proof." + type = string + default = "dander-phase6" + + validation { + condition = ( + length(var.proof_name) >= 6 && + length(var.proof_name) <= 20 && + can(regex("^[a-z][a-z0-9-]*[a-z0-9]$", var.proof_name)) && + !startswith(var.proof_name, "gcp-") + ) + error_message = "proof_name must be 6-20 lowercase letters, numbers, or hyphens and must not start with gcp-." + } +} + +variable "gcp_project_id" { + description = "GCP project containing the bounded BigQuery portability target." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{4,28}[a-z0-9]$", var.gcp_project_id)) + error_message = "gcp_project_id must be a valid GCP project ID." + } +} + +variable "google_service_account_id" { + description = "Disposable Google service account impersonated by the Azure identity." + type = string + default = "dander-phase6-azure" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{4,28}[a-z0-9]$", var.google_service_account_id)) + error_message = "google_service_account_id must be a valid service-account ID." + } +} + +variable "raw_dataset_id" { + description = "Existing bounded BigQuery warehouse dataset used by the proof." + type = string + default = "raw" + + validation { + condition = length(var.raw_dataset_id) <= 1024 && can(regex("^[A-Za-z_][A-Za-z0-9_]*$", var.raw_dataset_id)) + error_message = "raw_dataset_id must be a valid BigQuery dataset ID." + } +} + +variable "metadata_dataset_id" { + description = "Existing bounded BigQuery state dataset used by the proof." + type = string + default = "dander_meta" + + validation { + condition = length(var.metadata_dataset_id) <= 1024 && can(regex("^[A-Za-z_][A-Za-z0-9_]*$", var.metadata_dataset_id)) + error_message = "metadata_dataset_id must be a valid BigQuery dataset ID." + } +} + +variable "proof_secret_ids" { + description = "Existing Secret Manager containers the proof service account may read." + type = set(string) + + validation { + condition = ( + length(var.proof_secret_ids) >= 1 && + alltrue([for id in var.proof_secret_ids : can(regex("^[A-Za-z][A-Za-z0-9_-]{0,254}$", id))]) + ) + error_message = "proof_secret_ids must contain at least one valid Secret Manager ID." + } +} diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/versions.tf b/acceptance/cloud-portability/phase6/azure-google-federation/versions.tf new file mode 100644 index 00000000..81b1eee0 --- /dev/null +++ b/acceptance/cloud-portability/phase6/azure-google-federation/versions.tf @@ -0,0 +1,33 @@ +terraform { + required_version = ">= 1.9, < 2.0" + + required_providers { + azuread = { + source = "hashicorp/azuread" + version = "~> 3.9" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.72" + } + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +} + +provider "azuread" { + tenant_id = var.tenant_id +} + +provider "azurerm" { + subscription_id = var.subscription_id + resource_provider_registrations = "none" + + features {} +} + +provider "google" { + project = var.gcp_project_id +} diff --git a/docs/decisions.md b/docs/decisions.md index b3d349d5..ddc4e7b1 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1184,3 +1184,17 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Azure assigns execution names, while replay correctness remains owned by Dander's persisted inclusive cursor. Local mocked operations are construction evidence only; live image copy, execution, interruption, cleanup, and no-drift still require explicit approval. + +## 2026-08-11 — Container Apps adapts managed identity in memory for Google federation + +- **Runtime:** The typed Azure launcher requests an Entra token for one exact application ID URI + through `ManagedIdentityCredential`, then supplies a fresh token whenever Google Auth refreshes + the same external-account credential. No external-account file, client secret, service-account + key, or token is persisted or projected. +- **Trust:** A disposable single-tenant Entra application supplies only the audience. The Google + OIDC provider checks tenant, audience, and the attached user-assigned identity object ID before + that principal may impersonate one disposable Google service account. +- **Boundary:** Azure/BigQuery/Dataplex/GCP-secrets is a separate portability profile from the named + Azure Snowflake/PostgreSQL/Key-Vault profile. Provider-mocked Terraform and the bounded + 600-second refresh command are preparation only; live exchange, refresh, revocation, cleanup, + and support promotion still require an accepted candidate and approved provider ceilings. diff --git a/docs/execution-projection.md b/docs/execution-projection.md index ba7fdede..fca60e38 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -62,7 +62,10 @@ user-assigned managed identity, exact Container Apps environment placement, supp or 2 CPU/4 GiB sizing, launcher-owned retries and deadline, Log Analytics destinations, and versionless Key Vault references for declared secrets. Azure evaluates scheduled-job cron in UTC, so a non-UTC projection fails before provider access rather than silently changing wall-clock -behavior. Plan-first Terraform now consumes this projection exactly and has provider-mocked +behavior. The separate BigQuery portability profile uses the same managed identity but projects +only typed Google workload-identity audience/application coordinates and GCP Secret Manager +references; the runtime exchanges managed-identity tokens in memory and never creates a credential +file. Plan-first Terraform now consumes these projections exactly and has provider-mocked contract evidence; no Azure resource has been created, and live-profile evidence remains a separate Phase 6 gate. Local lifecycle tooling now binds start, status, bounded Log Analytics, stop, replay, and deployment verification to the exact manifest-selected job. The ACR promotion diff --git a/docs/known-limitations.md b/docs/known-limitations.md index f62470a3..0467e749 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -48,9 +48,12 @@ remain Phase 8 work. are regression evidence, not a paid or controlled-memory scale qualification. - Azure Container Apps Jobs and Azure Key Vault have a typed projection plus locally validated, plan-first Terraform, digest-preserving ACR promotion tooling, provider-native lifecycle - operations, bounded Log Analytics reads, and deployment verification. Azure schedules are + operations, bounded Log Analytics reads, deployment verification, and a locally validated + Azure-to-Google managed-identity adapter with disposable federation Terraform and refresh-probe + tooling. Azure schedules are UTC-only and currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. Resource-provider - registration, credentialed Azure planning/apply, actual ACR image copy, identity federation, + registration, credentialed Azure planning/apply, actual ACR image copy, live identity exchange, + refresh and revocation, live lifecycle execution, and live Snowflake/PostgreSQL acceptance have not yet run, so Azure remains unsupported. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version diff --git a/infra/azure/modules/container-apps-jobs/main.tf b/infra/azure/modules/container-apps-jobs/main.tf index 8ede22e6..f3cb7bf3 100644 --- a/infra/azure/modules/container-apps-jobs/main.tf +++ b/infra/azure/modules/container-apps-jobs/main.tf @@ -1,6 +1,25 @@ locals { container_app_environment_id = "/subscriptions/${var.subscription_id}/resourceGroups/${var.resource_group_name}/providers/Microsoft.App/managedEnvironments/${var.container_app_environment_name}" key_vault_uri = "https://${var.key_vault_name}.vault.azure.net" + gcp_secret_environment = { + for id, projection in var.execution_projections : id => { + for name, binding in projection.secret_bindings : + name => trimprefix(binding.reference, "gcp-sm://") + if binding.provider == "gcp_secret_manager" + } + } + azure_secret_environment = { + for id, projection in var.execution_projections : id => { + for name, binding in projection.secret_bindings : name => binding + if binding.provider == "azure_key_vault" + } + } + container_environment = { + for id, projection in var.execution_projections : id => merge( + projection.environment, + local.gcp_secret_environment[id], + ) + } resource_names = { for id in keys(var.execution_projections) : id => "${substr(var.name, 0, 12)}-${substr(sha1(id), 0, 12)}" @@ -32,8 +51,13 @@ check "secret_references_match_selected_key_vault" { condition = alltrue(flatten([ for projection in values(var.execution_projections) : [ for binding in values(projection.secret_bindings) : ( - binding.provider == "azure_key_vault" && - startswith(binding.reference, "azure-kv://${local.key_vault_uri}/secrets/") + ( + binding.provider == "azure_key_vault" && + startswith(binding.reference, "azure-kv://${local.key_vault_uri}/secrets/") + ) || ( + binding.provider == "gcp_secret_manager" && + startswith(binding.reference, "gcp-sm://projects/") + ) ) ] ])) @@ -116,7 +140,7 @@ resource "azurerm_container_app_job" "pipeline" { } dynamic "secret" { - for_each = each.value.secret_bindings + for_each = local.azure_secret_environment[each.key] content { name = "secret-${substr(sha1(secret.key), 0, 16)}" identity = var.managed_identity_id @@ -133,7 +157,7 @@ resource "azurerm_container_app_job" "pipeline" { args = each.value.command dynamic "env" { - for_each = each.value.environment + for_each = local.container_environment[each.key] content { name = env.key value = env.value @@ -141,7 +165,7 @@ resource "azurerm_container_app_job" "pipeline" { } dynamic "env" { - for_each = each.value.secret_bindings + for_each = local.azure_secret_environment[each.key] content { name = env.key secret_name = "secret-${substr(sha1(env.key), 0, 16)}" diff --git a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl index 131e3a6d..c5e4c358 100644 --- a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl +++ b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl @@ -28,6 +28,20 @@ variables { managed_identity_client_id = "33333333-3333-4333-8333-333333333333" managed_identity_principal_id = "44444444-4444-4444-8444-444444444444" execution_projections = { + bigquery_fixture = { + launcher = "azure_container_apps" + image = "danderphase6.azurecr.io/dander/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command = ["runtime", "execute", "--contract", "io.dander.runtime/v1", "--pipeline", "bigquery_fixture", "--platform", "gcp"] + workload_identity = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + environment = { AZURE_CLIENT_ID = "33333333-3333-4333-8333-333333333333", DANDER_GCP_WIF_AUDIENCE = "//iam.googleapis.com/projects/1009770943166/locations/global/workloadIdentityPools/dander-phase6-azure/providers/container-apps", HOME = "/tmp" } + labels = { pipeline = "bigquery_fixture", profile = "gcp" } + secret_bindings = { API_TOKEN = { provider = "gcp_secret_manager", reference = "gcp-sm://projects/unit-project/secrets/source-api-token/versions/latest" } } + resources = { cpu_millis = 1000, memory_mib = 2048, deadline_seconds = 900, launcher_retry_count = 1 } + schedule = { paused = true, expression = "15 4 * * *", maximum_parallelism = 1, task_count = 1 } + network = { placement = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.App/managedEnvironments/dander-phase6-env" } + observability = { alert_target = null } + extensions = { azure_acr_login_server = "danderphase6.azurecr.io", azure_key_vault_uri = "https://dander-phase6-kv.vault.azure.net", azure_managed_identity_client_id = "33333333-3333-4333-8333-333333333333" } + } warehouse_fixture = { launcher = "azure_container_apps" image = "danderphase6.azurecr.io/dander/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -56,6 +70,17 @@ run "projects_exact_job_contract" { error_message = "Container Apps must preserve the projected deadline and launcher retry count." } + assert { + condition = ( + one([ + for env in azurerm_container_app_job.pipeline["bigquery_fixture"].template[0].container[0].env : + env.value if env.name == "API_TOKEN" + ]) == "projects/unit-project/secrets/source-api-token/versions/latest" && + length(azurerm_container_app_job.pipeline["bigquery_fixture"].secret) == 0 + ) + error_message = "GCP Secret Manager references must remain runtime inputs, not Azure Key Vault secrets." + } + assert { condition = ( azurerm_container_app_job.pipeline["warehouse_fixture"].template[0].container[0].args[0] == "runtime" && diff --git a/src/dander/bootstrap/azure_terraform.py b/src/dander/bootstrap/azure_terraform.py index ba868d3d..bf9438aa 100644 --- a/src/dander/bootstrap/azure_terraform.py +++ b/src/dander/bootstrap/azure_terraform.py @@ -18,6 +18,7 @@ ) from dander.deployment import ExecutionProjectionError, ResolvedTemplateRequest from dander.providers import ProviderFactoryError +from dander.providers.gcp_launcher import GcpLauncherContext if TYPE_CHECKING: from collections.abc import Mapping @@ -43,6 +44,8 @@ r"^(?P[a-z][a-z0-9]{4,49})\.azurecr\.io/" r"(?P[A-Za-z0-9._/-]+)@sha256:(?P[0-9a-f]{64})$" ) +_PROFILE_ID = re.compile(r"^[a-z][a-z0-9_]{1,62}$") +_GCP_PROJECT = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") class AzureTerraformBootstrapError(RuntimeError): @@ -78,6 +81,8 @@ def execute( alert_target: str | None = None, infrastructure_subnet_id: str | None = None, name: str = "dander", + profile_id: str = "azure_snowflake", + gcp_project: str | None = None, ) -> Path: """Produce one immutable Azure plan and optionally apply that exact plan.""" raw_launcher = dict(launcher_config) @@ -94,6 +99,10 @@ def execute( ) if not deployment_name.strip(): raise AzureTerraformBootstrapError("Manifest deployment name must not be blank") + if _PROFILE_ID.fullmatch(profile_id) is None: + raise AzureTerraformBootstrapError("Invalid Azure platform profile id") + if gcp_project is not None and _GCP_PROJECT.fullmatch(gcp_project) is None: + raise AzureTerraformBootstrapError("Invalid Azure federation GCP project") try: key_vault_address = ip_address(key_vault_allowed_ip_rule) except ValueError as error: @@ -149,14 +158,24 @@ def execute( "Azure infrastructure subnet must be a subnet in the selected subscription" ) try: - launcher = build_launcher_runtime(launcher_config=raw_launcher) + launcher = build_launcher_runtime( + launcher_config=raw_launcher, + gcp_context=( + GcpLauncherContext( + project=gcp_project, + require_guarded_free_tier=require_guarded_free_tier, + ) + if gcp_project is not None + else None + ), + ) projections = { pipeline_id: template.as_dict() for pipeline_id, template in launcher.templates.build( ResolvedTemplateRequest( pipelines=expanded_pipelines, image=container_image, - profile_id="azure_snowflake", + profile_id=profile_id, cpu=runtime_cpu, memory=runtime_memory, deadline_seconds=runtime_timeout_seconds, @@ -200,7 +219,7 @@ def execute( + dumps( { "dander-deployment": deployment_name, - "dander-profile": "azure_snowflake", + "dander-profile": profile_id, }, sort_keys=True, separators=(",", ":"), diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index 076c77fa..4089f357 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -71,6 +71,56 @@ def azure_run( console.print_json(data=execution.as_dict()) +@azure_app.command("identity-refresh-probe") +def azure_identity_refresh_probe( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + project: str = typer.Option(..., "--project"), + dataset: str = typer.Option(..., "--dataset"), + table: str = typer.Option(..., "--table"), + max_wait_seconds: int = typer.Option(900, "--max-wait-seconds", min=1, max=1_800), + refresh_margin_seconds: int = typer.Option(15, "--refresh-margin-seconds", min=0, max=60), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Start one potentially billable Azure-to-Google credential refresh proof.""" + try: + manifest = load_project_config(config, deployment=deployment) + if ( + manifest.launcher_provider != "azure_container_apps" + or manifest.warehouse_provider != "bigquery" + or manifest.state_provider != "bigquery" + or manifest.catalog_provider != "dataplex" + or manifest.secret_provider != "gcp_secret_manager" + ): + raise ProjectConfigError( + "Identity refresh probe requires the named Azure BigQuery federation profile" + ) + except ProjectConfigError as error: + raise ClickException(str(error)) from error + if not typer.confirm( + "Start one paid Azure-to-Google identity refresh proof with no automatic rerun?", + default=False, + ): + raise typer.Abort() + try: + execution = _azure_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + name=name, + ).start_identity_refresh_probe( + project=project, + dataset=dataset, + table=table, + max_wait_seconds=max_wait_seconds, + refresh_margin_seconds=refresh_margin_seconds, + ) + except (AzureContainerAppsOperationError, AzureDeploymentVerificationError) as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + @azure_app.command("status") def azure_status( deployment: str = typer.Option(..., "--deployment"), @@ -383,6 +433,12 @@ def init_azure_plan( ) manifest.validate_references(config.resolve().parent) runtime = manifest.platform.runtime + gcp_project = None + if manifest.warehouse_provider == "bigquery": + project_value = manifest.warehouse_config.get("project") + if not isinstance(project_value, str): + raise ProjectConfigError("Azure BigQuery profile has no GCP project") + gcp_project = project_value plan_path = AzureTerraformBootstrap(infra_dir).execute( deployment_name=manifest.deployment_name, state_resource_group_name=state_resource_group_name, @@ -403,6 +459,8 @@ def init_azure_plan( alert_target=alert_target, infrastructure_subnet_id=infrastructure_subnet_id, name=name, + profile_id=manifest.platform_name, + gcp_project=gcp_project, ) except (AzureTerraformBootstrapError, ProjectConfigError) as error: raise ClickException(str(error)) from error diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index d68db073..5bef2316 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -2,18 +2,28 @@ from __future__ import annotations +import json import tempfile import time from pathlib import Path -from typing import Annotated +from typing import Annotated, cast import typer from click import ClickException +from google.cloud import bigquery from rich.console import Console from dander.cli.run_command import RunOptions, execute_run from dander.compatibility import CompatibilityError, load_runtime_compatibility -from dander.identity import launcher_identity +from dander.identity import google_client_options, launcher_identity +from dander.identity.refresh_probe import ( + PROBE_SCHEMA, + BigQueryProbeClient, + ExpiringCredentials, + GoogleRefreshProbeError, + run_google_refresh_probe, + validate_probe_target, +) from dander.runtime_contract import ( RUNTIME_CONTRACT, LauncherContext, @@ -36,6 +46,54 @@ _CONSOLE = Console() +@runtime_app.command("identity-refresh-probe", hidden=True) +def identity_refresh_probe_runtime_command( + project: str = typer.Option(..., "--project"), + dataset: str = typer.Option(..., "--dataset"), + table: str = typer.Option(..., "--table"), + max_wait_seconds: int = typer.Option(900, "--max-wait-seconds", min=1, max=1_800), + refresh_margin_seconds: int = typer.Option(15, "--refresh-margin-seconds", min=0, max=60), +) -> None: + """Prove one hosted launcher can renew keyless Google credentials in-process.""" + try: + validate_probe_target(project=project, dataset=dataset, table=table) + context = LauncherContext.from_environment() + with launcher_identity(context): + options = google_client_options() + credentials = options.get("credentials") + if credentials is None or not hasattr(credentials, "expiry"): + raise GoogleRefreshProbeError( + "Selected launcher did not provide renewable Google credentials" + ) + client = BigQueryProbeClient(bigquery.Client(project=project, **options)) + run_google_refresh_probe( + credentials=cast("ExpiringCredentials", credentials), + client=client, + project=project, + dataset=dataset, + table=table, + max_wait_seconds=max_wait_seconds, + refresh_margin_seconds=refresh_margin_seconds, + emit=lambda event: typer.echo( + json.dumps(event, sort_keys=True, separators=(",", ":")) + ), + ) + except (GoogleRefreshProbeError, RuntimeContractError) as error: + typer.echo( + json.dumps( + { + "schema": PROBE_SCHEMA, + "event": "probe.failed", + "failure_code": "identity_or_query_failed", + "failure_type": type(error).__name__, + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + raise typer.Exit(code=1) from None + + @runtime_app.command("compatibility") def compatibility_runtime_command() -> None: """Print the package-owned provider compatibility matrix without provider access.""" diff --git a/src/dander/identity/__init__.py b/src/dander/identity/__init__.py index 9a065afe..9ef392d8 100644 --- a/src/dander/identity/__init__.py +++ b/src/dander/identity/__init__.py @@ -8,12 +8,18 @@ prepare_fargate_task_credentials, prepare_launcher_identity, ) +from dander.identity.azure_google import ( + AzureContainerAppsIdentityError, + prepare_azure_google_identity, +) __all__ = [ + "AzureContainerAppsIdentityError", "FargateIdentityError", "google_client_options", "launcher_identity", "prepare_fargate_google_identity", "prepare_fargate_task_credentials", "prepare_launcher_identity", + "prepare_azure_google_identity", ] diff --git a/src/dander/identity/aws_google.py b/src/dander/identity/aws_google.py index ae6d1c77..b205d2ad 100644 --- a/src/dander/identity/aws_google.py +++ b/src/dander/identity/aws_google.py @@ -128,6 +128,17 @@ def prepare_launcher_identity(context: LauncherContext) -> object | None: """Prepare only the selected launcher's ambient identity before clients start.""" if context.launcher == "fargate": return prepare_fargate_google_identity() + if context.launcher == "azure_container_apps" and any( + os.environ.get(name) + for name in ( + "DANDER_AZURE_GCP_APPLICATION_ID_URI", + "DANDER_GCP_SERVICE_ACCOUNT", + "DANDER_GCP_WIF_AUDIENCE", + ) + ): + from dander.identity.azure_google import prepare_azure_google_identity + + return prepare_azure_google_identity() return None diff --git a/src/dander/identity/azure_google.py b/src/dander/identity/azure_google.py new file mode 100644 index 00000000..68a71ba2 --- /dev/null +++ b/src/dander/identity/azure_google.py @@ -0,0 +1,206 @@ +"""Adapt Azure Container Apps managed identity for Google workload federation.""" + +from __future__ import annotations + +import os +import re +from datetime import UTC, datetime, timedelta +from importlib import import_module +from typing import TYPE_CHECKING, Protocol, cast +from uuid import UUID + +from google.auth import external_account + +if TYPE_CHECKING: + from collections.abc import Callable, MutableMapping + from typing import Any + +_WIF_AUDIENCE = re.compile( + r"^//iam\.googleapis\.com/projects/[0-9]{6,20}/locations/global/" + r"workloadIdentityPools/[a-z][a-z0-9-]{3,31}/providers/[a-z][a-z0-9-]{3,31}$" +) +_SERVICE_ACCOUNT = re.compile( + r"^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z][a-z0-9-]{4,28}[a-z0-9]" + r"\.iam\.gserviceaccount\.com$" +) +_AZURE_APPLICATION_ID_URI = re.compile( + r"^api://[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-" + r"[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" +) +_JWT = re.compile(r"^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$") +_JWT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt" +_CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" + + +class AzureContainerAppsIdentityError(RuntimeError): + """Azure managed identity is missing, stale, or unsafe for Google federation.""" + + +class AzureAccessToken(Protocol): + """Minimum token shape returned by ``azure-identity``.""" + + token: str + expires_on: int + + +class AzureTokenCredential(Protocol): + """Minimum managed-identity credential boundary used by the adapter.""" + + def get_token(self, *scopes: str) -> AzureAccessToken: + """Return an access token for the requested Microsoft Entra scope.""" + + +class AzureCredentialFactory(Protocol): + def __call__(self, *, client_id: str) -> AzureTokenCredential: + """Build one user-assigned managed-identity credential.""" + + +class GoogleCredentialFactory(Protocol): + def __call__( + self, + *, + audience: str, + service_account: str, + supplier: _AzureSubjectTokenSupplier, + ) -> object: + """Build renewable Google credentials from one Azure subject-token supplier.""" + + +class _AzureSubjectTokenSupplier: + """Request a fresh Entra token whenever Google Auth refreshes.""" + + def __init__( + self, + *, + credential: AzureTokenCredential, + application_id_uri: str, + clock: Callable[[], datetime], + ) -> None: + self._credential = credential + self._scope = f"{application_id_uri}/.default" + self._clock = clock + + def validate(self) -> None: + """Fail before provider construction if managed identity is unusable.""" + self.subject_token() + + def subject_token(self) -> str: + """Return one non-persisted Entra JWT for Google's token exchange.""" + try: + access_token = self._credential.get_token(self._scope) + except Exception as error: # noqa: BLE001 - Azure SDK errors can vary by version. + raise AzureContainerAppsIdentityError( + "Azure Container Apps managed identity token was unavailable" + ) from error + token = access_token.token + expiry = datetime.fromtimestamp(access_token.expires_on, tz=UTC) + if ( + not isinstance(token, str) + or _JWT.fullmatch(token) is None + or expiry <= self._clock().astimezone(UTC) + timedelta(minutes=5) + ): + raise AzureContainerAppsIdentityError( + "Azure Container Apps managed identity token was invalid" + ) + return token + + +class _AzureGoogleCredentials(external_account.Credentials): + """Google external-account credentials backed by the Container Apps identity SDK.""" + + def __init__( + self, + *, + audience: str, + service_account: str, + supplier: _AzureSubjectTokenSupplier, + ) -> None: + self._azure_supplier = supplier + super().__init__( # type: ignore[no-untyped-call] + audience=audience, + subject_token_type=_JWT_SUBJECT_TOKEN_TYPE, + token_url="https://sts.googleapis.com/v1/token", + credential_source={"environment_id": "azure-container-apps"}, + service_account_impersonation_url=( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" + f"{service_account}:generateAccessToken" + ), + service_account_impersonation_options={"token_lifetime_seconds": 600}, + scopes=(_CLOUD_PLATFORM_SCOPE,), + ) + + def retrieve_subject_token(self, request: object) -> str: + """Supply a fresh Entra token without writing a credential file.""" + del request + return self._azure_supplier.subject_token() + + +def prepare_azure_google_identity( + *, + environ: MutableMapping[str, str] = os.environ, + clock: Callable[[], datetime] | None = None, + azure_credential_factory: AzureCredentialFactory | None = None, + google_credential_factory: GoogleCredentialFactory | None = None, +) -> object: + """Build renewable Google credentials from Azure user-assigned managed identity.""" + audience = environ.get("DANDER_GCP_WIF_AUDIENCE", "") + service_account = environ.get("DANDER_GCP_SERVICE_ACCOUNT", "") + application_id_uri = environ.get("DANDER_AZURE_GCP_APPLICATION_ID_URI", "") + client_id = environ.get("AZURE_CLIENT_ID", "") + if ( + _WIF_AUDIENCE.fullmatch(audience) is None + or _SERVICE_ACCOUNT.fullmatch(service_account) is None + or _AZURE_APPLICATION_ID_URI.fullmatch(application_id_uri) is None + or not _rfc4122_uuid(client_id) + ): + raise AzureContainerAppsIdentityError( + "Azure-to-Google workload identity configuration is invalid" + ) + credential_factory = azure_credential_factory or _build_azure_credential + supplier = _AzureSubjectTokenSupplier( + credential=credential_factory(client_id=client_id), + application_id_uri=application_id_uri, + clock=clock or (lambda: datetime.now(UTC)), + ) + supplier.validate() + factory = google_credential_factory or _build_google_credentials + return factory( + audience=audience, + service_account=service_account, + supplier=supplier, + ) + + +def _build_azure_credential(*, client_id: str) -> AzureTokenCredential: + try: + identity_module = import_module("azure.identity") + except ModuleNotFoundError as error: # pragma: no cover - packaging contract covers this path + raise AzureContainerAppsIdentityError( + "Azure managed-identity dependencies are unavailable" + ) from error + credential = cast("Any", identity_module).ManagedIdentityCredential(client_id=client_id) + return cast("AzureTokenCredential", credential) + + +def _build_google_credentials( + *, + audience: str, + service_account: str, + supplier: _AzureSubjectTokenSupplier, +) -> object: + return _AzureGoogleCredentials( + audience=audience, + service_account=service_account, + supplier=supplier, + ) + + +def _rfc4122_uuid(value: str) -> bool: + try: + parsed = UUID(value) + except ValueError: + return False + return parsed.variant == "specified in RFC 4122" + + +__all__ = ["AzureContainerAppsIdentityError", "prepare_azure_google_identity"] diff --git a/src/dander/identity/refresh_probe.py b/src/dander/identity/refresh_probe.py new file mode 100644 index 00000000..b1750848 --- /dev/null +++ b/src/dander/identity/refresh_probe.py @@ -0,0 +1,135 @@ +"""Bounded, row-sanitized Google credential refresh probe for hosted launchers.""" + +from __future__ import annotations + +import re +import time +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from collections.abc import Callable + + from google.cloud import bigquery + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,1023}$") +_PROJECT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +PROBE_SCHEMA = "io.dander.portability.identity-probe/v1" + + +class GoogleRefreshProbeError(RuntimeError): + """The live identity-refresh proof did not satisfy its bounded contract.""" + + +class ExpiringCredentials(Protocol): + expiry: datetime | None + + +class QueryClient(Protocol): + def query_count(self, *, project: str, dataset: str, table: str) -> int: + """Return a bounded count from the proof relation.""" + + +class BigQueryProbeClient: + """Keep proof rows and credentials out of Dander's emitted evidence.""" + + def __init__(self, client: bigquery.Client) -> None: + self._client = client + + def query_count(self, *, project: str, dataset: str, table: str) -> int: + validate_probe_target(project=project, dataset=dataset, table=table) + query = f"SELECT COUNT(*) AS row_count FROM `{project}.{dataset}.{table}`" + row = next(iter(self._client.query(query).result(max_results=1)), None) + if row is None: + raise GoogleRefreshProbeError("BigQuery proof query returned no result") + value = row[0] + if not isinstance(value, int) or value < 0: + raise GoogleRefreshProbeError("BigQuery proof query returned an invalid count") + return value + + +def run_google_refresh_probe( + *, + credentials: ExpiringCredentials, + client: QueryClient, + project: str, + dataset: str, + table: str, + max_wait_seconds: int, + refresh_margin_seconds: int, + now: Callable[[], datetime] = lambda: datetime.now(UTC), + sleep: Callable[[float], None] = time.sleep, + emit: Callable[[dict[str, object]], None], +) -> None: + """Query before and after expiry, requiring the same credentials object to refresh.""" + validate_probe_target(project=project, dataset=dataset, table=table) + if not 1 <= max_wait_seconds <= 1_800 or not 0 <= refresh_margin_seconds <= 60: + raise GoogleRefreshProbeError("Credential refresh proof bounds are invalid") + first_count = client.query_count(project=project, dataset=dataset, table=table) + first_expiry = _expiry(credentials) + first_time = _utc(now()) + wait_seconds = max(0.0, (first_expiry - first_time).total_seconds()) + refresh_margin_seconds + if wait_seconds > max_wait_seconds: + raise GoogleRefreshProbeError("Issued credential lifetime exceeds the bounded proof window") + emit( + { + "schema": PROBE_SCHEMA, + "event": "query.completed", + "sequence": 1, + "row_count": first_count, + "credential_expiry": first_expiry.isoformat(), + } + ) + + sleep(wait_seconds) + + second_count = client.query_count(project=project, dataset=dataset, table=table) + second_expiry = _expiry(credentials) + if second_expiry <= first_expiry: + raise GoogleRefreshProbeError("Google credentials did not refresh after expiry") + emit( + { + "schema": PROBE_SCHEMA, + "event": "query.completed", + "sequence": 2, + "row_count": second_count, + "credential_expiry": second_expiry.isoformat(), + } + ) + emit( + { + "schema": PROBE_SCHEMA, + "event": "credential.refresh_observed", + "previous_expiry": first_expiry.isoformat(), + "refreshed_expiry": second_expiry.isoformat(), + } + ) + + +def validate_probe_target(*, project: str, dataset: str, table: str) -> None: + if _PROJECT_ID.fullmatch(project) is None or not all( + _IDENTIFIER.fullmatch(value) for value in (dataset, table) + ): + raise GoogleRefreshProbeError("BigQuery proof target is invalid") + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _expiry(credentials: ExpiringCredentials) -> datetime: + if credentials.expiry is None: + raise GoogleRefreshProbeError("Google credentials did not expose an expiry") + return _utc(credentials.expiry) + + +__all__ = [ + "BigQueryProbeClient", + "ExpiringCredentials", + "GoogleRefreshProbeError", + "PROBE_SCHEMA", + "run_google_refresh_probe", + "validate_probe_target", +] diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index 324bce63..d9bfb0e4 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -358,12 +358,26 @@ def _resolve( "Cloud Run requires secrets.provider='gcp_secret_manager'; " "environment secrets are local or operator-managed Kubernetes only" ) + if selected.launcher.provider == "azure_container_apps" and profile.secrets.provider not in { + "azure_key_vault", + "gcp_secret_manager", + }: + raise ProjectConfigError( + "Azure Container Apps requires Azure Key Vault or the named GCP secret profile" + ) if ( selected.launcher.provider == "azure_container_apps" - and profile.secrets.provider != "azure_key_vault" + and profile.secrets.provider == "gcp_secret_manager" + and ( + profile.warehouse.provider != "bigquery" + or profile.state.provider != "bigquery" + or profile.catalog.provider != "dataplex" + or selected.launcher.google_workload_identity_audience is None + or selected.launcher.google_application_id_uri is None + ) ): raise ProjectConfigError( - "Azure Container Apps currently requires secrets.provider='azure_key_vault'" + "Azure GCP secrets require the named BigQuery/Dataplex federation profile" ) unknown_pipelines = sorted(set(selected.pipelines) - set(logical.pipelines)) if unknown_pipelines: diff --git a/src/dander/providers/azure_container_apps/config.py b/src/dander/providers/azure_container_apps/config.py index 9266f335..f666ba3f 100644 --- a/src/dander/providers/azure_container_apps/config.py +++ b/src/dander/providers/azure_container_apps/config.py @@ -6,7 +6,7 @@ from typing import Literal from uuid import UUID # noqa: TC003 - Pydantic resolves this annotation at runtime. -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator _RESOURCE_NAME = re.compile(r"^[a-z][a-z0-9-]{1,62}[a-z0-9]$") _ACR_NAME = re.compile(r"^[a-z][a-z0-9]{4,49}$") @@ -28,6 +28,21 @@ class AzureContainerAppsLauncherConfig(BaseModel): key_vault_name: str = Field(pattern=_KEY_VAULT_NAME.pattern) managed_identity_name: str = Field(pattern=_RESOURCE_NAME.pattern) managed_identity_client_id: UUID + google_workload_identity_audience: str | None = Field( + default=None, + pattern=( + r"^//iam\.googleapis\.com/projects/[0-9]{6,20}/locations/global/" + r"workloadIdentityPools/[a-z][a-z0-9-]{3,31}/providers/" + r"[a-z][a-z0-9-]{3,31}$" + ), + ) + google_application_id_uri: str | None = Field( + default=None, + pattern=( + r"^api://[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-" + r"[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" + ), + ) @field_validator("subscription_id", "managed_identity_client_id") @classmethod @@ -43,6 +58,19 @@ def reject_ambiguous_key_vault_name(cls, value: str) -> str: raise ValueError("Azure Key Vault names must not contain consecutive hyphens") return value + @model_validator(mode="after") + def require_complete_google_federation(self) -> AzureContainerAppsLauncherConfig: + """Keep the optional cross-cloud identity boundary all-or-nothing.""" + configured = ( + self.google_workload_identity_audience, + self.google_application_id_uri, + ) + if any(configured) and not all(configured): + raise ValueError( + "Azure Google federation requires both audience and application ID URI" + ) + return self + @property def acr_login_server(self) -> str: return f"{self.acr_name}.azurecr.io" diff --git a/src/dander/providers/azure_container_apps/operations.py b/src/dander/providers/azure_container_apps/operations.py index d90ff8cf..17ba77fb 100644 --- a/src/dander/providers/azure_container_apps/operations.py +++ b/src/dander/providers/azure_container_apps/operations.py @@ -8,6 +8,8 @@ from dataclasses import asdict, dataclass from typing import TYPE_CHECKING, Protocol, cast +from dander.identity.refresh_probe import GoogleRefreshProbeError, validate_probe_target + if TYPE_CHECKING: from pathlib import Path @@ -95,6 +97,41 @@ def __init__( def start(self) -> AzureContainerAppsExecution: """Start one on-demand execution and return its provider-assigned identity.""" + return self._start() + + def start_identity_refresh_probe( + self, + *, + project: str, + dataset: str, + table: str, + max_wait_seconds: int = 900, + refresh_margin_seconds: int = 15, + ) -> AzureContainerAppsExecution: + """Start one bounded Google refresh probe through the immutable job image.""" + try: + validate_probe_target(project=project, dataset=dataset, table=table) + except GoogleRefreshProbeError as error: + raise AzureContainerAppsOperationError(str(error)) from error + if not 1 <= max_wait_seconds <= 1_800 or not 0 <= refresh_margin_seconds <= 60: + raise AzureContainerAppsOperationError("Credential refresh proof bounds are invalid") + return self._start( + "--args", + "runtime", + "identity-refresh-probe", + "--project", + project, + "--dataset", + dataset, + "--table", + table, + "--max-wait-seconds", + str(max_wait_seconds), + "--refresh-margin-seconds", + str(refresh_margin_seconds), + ) + + def _start(self, *overrides: str) -> AzureContainerAppsExecution: payload = self._json( "containerapp", "job", @@ -103,6 +140,7 @@ def start(self) -> AzureContainerAppsExecution: self.binding.job_name, "--resource-group", self.binding.resource_group_name, + *overrides, ) if not isinstance(payload, dict) or not isinstance(payload.get("name"), str): raise AzureContainerAppsOperationError( diff --git a/src/dander/providers/azure_container_apps/runtime.py b/src/dander/providers/azure_container_apps/runtime.py index 9b5a4ec3..45ae7c0f 100644 --- a/src/dander/providers/azure_container_apps/runtime.py +++ b/src/dander/providers/azure_container_apps/runtime.py @@ -22,6 +22,7 @@ ) from dander.deployment.runtime import LauncherRuntime, ResolvedTemplateRequest from dander.providers.azure_container_apps.config import AzureContainerAppsLauncherConfig +from dander.providers.gcp_launcher import GcpLauncherContext, optional_gcp_launcher_context from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind from dander.runtime_contract import RUNTIME_CONTRACT @@ -31,6 +32,7 @@ _IMAGE_PATH = re.compile(r"^[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$") _CRON_PART = re.compile(r"^[A-Za-z0-9*/,-]+$") _KEY_VAULT_SECRET = re.compile(r"^[A-Za-z0-9-]{1,127}$") +_GCP_SERVICE_ACCOUNT_ID = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") @dataclass(frozen=True, slots=True) @@ -38,6 +40,7 @@ class AzureContainerAppsTemplateFactory: """Project one selected profile into Azure Container Apps Jobs.""" config: AzureContainerAppsLauncherConfig + gcp: GcpLauncherContext | None = None def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: """Build immutable Azure templates without contacting Azure.""" @@ -50,18 +53,41 @@ def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate memory_mib = _memory_mib(request.memory) cpu_millis = request.cpu * 1_000 _validate_azure_size(cpu_millis=cpu_millis, memory_mib=memory_mib) + if self.gcp is not None: + if self.gcp.require_guarded_free_tier: + raise ExecutionProjectionError( + "Azure cannot run the GCP guarded-free-tier preflight" + ) + if ( + self.config.google_workload_identity_audience is None + or self.config.google_application_id_uri is None + ): + raise ExecutionProjectionError( + "Azure BigQuery projection requires Google workload federation" + ) + elif self.config.google_workload_identity_audience is not None: + raise ExecutionProjectionError( + "Azure Google workload federation requires a GCP platform profile" + ) templates: dict[str, ExecutionTemplate] = {} for pipeline_id, pipeline in sorted(request.pipelines.items()): secret_env = pipeline["secret_env"] if not isinstance(secret_env, Mapping): raise ExecutionProjectionError("pipeline secret bindings are invalid") - if any( - not isinstance(secret_id, str) or _KEY_VAULT_SECRET.fullmatch(secret_id) is None - for secret_id in secret_env.values() - ): - raise ExecutionProjectionError( - "Azure Key Vault secret ids must use letters, numbers, or hyphens" - ) + if self.gcp is None: + if any( + not isinstance(secret_id, str) or _KEY_VAULT_SECRET.fullmatch(secret_id) is None + for secret_id in secret_env.values() + ): + raise ExecutionProjectionError( + "Azure Key Vault secret ids must use letters, numbers, or hyphens" + ) + else: + role_name = str(pipeline["runtime_service_account_id"]) + if _GCP_SERVICE_ACCOUNT_ID.fullmatch(role_name) is None: + raise ExecutionProjectionError( + "Azure BigQuery projection requires a valid GCP service-account id" + ) schedule = _azure_schedule( expression=str(pipeline["schedule"]), time_zone=str(pipeline["time_zone"]), @@ -91,6 +117,33 @@ def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate "pipeline": pipeline_id, "profile": request.profile_id, } + environment = { + "AZURE_CLIENT_ID": str(self.config.managed_identity_client_id), + "AZURE_RESOURCE_GROUP": self.config.resource_group_name, + "AZURE_SUBSCRIPTION_ID": str(self.config.subscription_id), + "DANDER_IMAGE_DIGEST": digest, + "DANDER_LAUNCHER": "azure_container_apps", + "DANDER_PRINCIPAL": self.config.managed_identity_resource_id, + "HOME": "/tmp", + "TMPDIR": "/tmp", + } + if self.gcp is not None: + assert self.config.google_application_id_uri is not None + assert self.config.google_workload_identity_audience is not None + environment.update( + { + "BQ_DATASET_METADATA": "dander_meta", + "BQ_DATASET_RAW": "raw", + "DANDER_AZURE_GCP_APPLICATION_ID_URI": ( + self.config.google_application_id_uri + ), + "DANDER_GCP_SERVICE_ACCOUNT": ( + f"{role_name}@{self.gcp.project}.iam.gserviceaccount.com" + ), + "DANDER_GCP_WIF_AUDIENCE": (self.config.google_workload_identity_audience), + "GCP_PROJECT_ID": self.gcp.project, + } + ) template = ExecutionTemplate( schema=EXECUTION_PROJECTION_SCHEMA, contract=RUNTIME_CONTRACT, @@ -100,27 +153,19 @@ def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate image=request.image, command=command, configuration_reference="/app/dander.yaml", - environment=tuple( - sorted( - { - "AZURE_CLIENT_ID": str(self.config.managed_identity_client_id), - "AZURE_RESOURCE_GROUP": self.config.resource_group_name, - "AZURE_SUBSCRIPTION_ID": str(self.config.subscription_id), - "DANDER_IMAGE_DIGEST": digest, - "DANDER_LAUNCHER": "azure_container_apps", - "DANDER_PRINCIPAL": self.config.managed_identity_resource_id, - "HOME": "/tmp", - "TMPDIR": "/tmp", - }.items() - ) - ), + environment=tuple(sorted(environment.items())), secret_bindings=tuple( ( str(environment_name), SecretReference( - provider="azure_key_vault", + provider=( + "gcp_secret_manager" if self.gcp is not None else "azure_key_vault" + ), reference=( - f"azure-kv://{self.config.key_vault_uri}/secrets/{secret_id}" + f"gcp-sm://projects/{self.gcp.project}/secrets/" + f"{secret_id}/versions/latest" + if self.gcp is not None + else f"azure-kv://{self.config.key_vault_uri}/secrets/{secret_id}" ), ), ) @@ -171,13 +216,13 @@ def build_azure_container_apps_launcher( context: Mapping[str, object], ) -> LauncherRuntime: """Build Azure projection behavior only after explicit launcher selection.""" - del context if not isinstance(config, AzureContainerAppsLauncherConfig): raise TypeError("Azure Container Apps factory received the wrong configuration") + gcp = optional_gcp_launcher_context(context) return LauncherRuntime( provider_id="azure_container_apps", region=config.region, - templates=AzureContainerAppsTemplateFactory(config), + templates=AzureContainerAppsTemplateFactory(config, gcp), capabilities=AZURE_CONTAINER_APPS_CAPABILITIES, ) diff --git a/src/dander/providers/azure_container_apps/verification.py b/src/dander/providers/azure_container_apps/verification.py index b71cea2c..2d12a223 100644 --- a/src/dander/providers/azure_container_apps/verification.py +++ b/src/dander/providers/azure_container_apps/verification.py @@ -76,7 +76,13 @@ class AzureDeploymentBinding: schedule_paused: bool runtime_timeout_seconds: int runtime_max_retries: int + secret_provider: str + secret_bindings: tuple[tuple[str, str], ...] secret_ids: tuple[str, ...] + google_project: str | None + google_workload_identity_audience: str | None + google_application_id_uri: str | None + google_service_account: str | None project_dir: Path @classmethod @@ -130,6 +136,31 @@ def from_project( environment_name = required["container_app_environment_name"] identity_name = required["managed_identity_name"] resource_root = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}" + google_project = None + google_service_account = None + google_audience = None + google_application_id_uri = None + if manifest.secret_provider == "gcp_secret_manager": + project_value = manifest.warehouse_config.get("project") + runtime_id = manifest.terraform_pipelines()[pipeline_id].get( + "runtime_service_account_id" + ) + google_audience = launcher.get("google_workload_identity_audience") + google_application_id_uri = launcher.get("google_application_id_uri") + if not all( + isinstance(value, str) and value + for value in ( + project_value, + runtime_id, + google_audience, + google_application_id_uri, + ) + ): + raise AzureDeploymentVerificationError( + "Azure BigQuery federation configuration is incomplete" + ) + google_project = cast("str", project_value) + google_service_account = f"{runtime_id}@{google_project}.iam.gserviceaccount.com" return cls( subscription_id=subscription_id, location=required["region"], @@ -152,7 +183,19 @@ def from_project( schedule_paused=pipeline.paused, runtime_timeout_seconds=manifest.platform.runtime.timeout_seconds, runtime_max_retries=manifest.platform.runtime.max_retries, + secret_provider=manifest.secret_provider, + secret_bindings=tuple(sorted(pipeline.secrets.items())), secret_ids=tuple(sorted(pipeline.secrets.values())), + google_project=google_project, + google_workload_identity_audience=( + cast("str", google_audience) if google_audience is not None else None + ), + google_application_id_uri=( + cast("str", google_application_id_uri) + if google_application_id_uri is not None + else None + ), + google_service_account=google_service_account, project_dir=resolved_config.parent, ) @@ -282,10 +325,14 @@ def verify(self, *, expected_image: str) -> AzureDeploymentVerification: raise AzureDeploymentVerificationError( "Container Apps Job returned invalid Key Vault references" ) - expected_secret_urls = { - f"{self.binding.key_vault_uri}/secrets/{secret_id}" - for secret_id in self.binding.secret_ids - } + expected_secret_urls = ( + { + f"{self.binding.key_vault_uri}/secrets/{secret_id}" + for secret_id in self.binding.secret_ids + } + if self.binding.secret_provider == "azure_key_vault" + else set() + ) actual_secret_urls = { str(item.get("keyVaultUrl") or item.get("keyVaultSecretId")) for item in secrets @@ -319,6 +366,27 @@ def verify(self, *, expected_image: str) -> AzureDeploymentVerification: raise AzureDeploymentVerificationError( "Container Apps Job environment does not match the runtime contract" ) + if self.binding.secret_provider == "gcp_secret_manager": + expected_google_environment = { + "DANDER_AZURE_GCP_APPLICATION_ID_URI": self.binding.google_application_id_uri, + "DANDER_GCP_SERVICE_ACCOUNT": self.binding.google_service_account, + "DANDER_GCP_WIF_AUDIENCE": self.binding.google_workload_identity_audience, + "GCP_PROJECT_ID": self.binding.google_project, + **{ + environment_name: ( + f"projects/{self.binding.google_project}/secrets/" + f"{secret_id}/versions/latest" + ) + for environment_name, secret_id in self.binding.secret_bindings + }, + } + if any( + environment_values.get(name) != value + for name, value in expected_google_environment.items() + ): + raise AzureDeploymentVerificationError( + "Container Apps Job Google federation environment does not match the manifest" + ) registry = self._json( "acr", "show", diff --git a/tests/bootstrap/test_azure_terraform.py b/tests/bootstrap/test_azure_terraform.py index c75b8f52..078f4492 100644 --- a/tests/bootstrap/test_azure_terraform.py +++ b/tests/bootstrap/test_azure_terraform.py @@ -150,6 +150,64 @@ def fake_run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedPro assert all(call[:2] != ("terraform", "plan") for call in calls) +def test_azure_bootstrap_projects_gcp_federation_without_credentials( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[tuple[str, ...]] = [] + + def fake_run( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + env: dict[str, str], + umask: int, + ) -> subprocess.CompletedProcess[str]: + del check, env, umask + calls.append(args) + for argument in args: + if argument.startswith("-out="): + (cwd / argument.removeprefix("-out=")).touch() + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + launcher = { + **_launcher(), + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + "google_application_id_uri": "api://33333333-3333-4333-8333-333333333333", + } + _execute( + AzureTerraformBootstrap(tmp_path), + deployment_name="azure_bigquery", + launcher_config=launcher, + profile_id="gcp", + gcp_project="unit-project", + pipelines={ + "warehouse_fixture": { + **_pipelines()["warehouse_fixture"], + "secret_env": {"API_TOKEN": "source-api-token"}, + } + }, + ) + + terraform_plan = calls[1] + projection_argument = next( + item for item in terraform_plan if item.startswith("-var=execution_projections=") + ) + projection = json.loads(projection_argument.removeprefix("-var=execution_projections="))[ + "warehouse_fixture" + ] + assert projection["environment"]["DANDER_GCP_SERVICE_ACCOUNT"] == ( + "dander-runtime@unit-project.iam.gserviceaccount.com" + ) + assert projection["secret_bindings"]["API_TOKEN"]["provider"] == "gcp_secret_manager" + assert "private_key" not in projection_argument + + @pytest.mark.parametrize( ("overrides", "message"), [ diff --git a/tests/cli/test_azure_operations_cli.py b/tests/cli/test_azure_operations_cli.py index dd5c091b..173cd697 100644 --- a/tests/cli/test_azure_operations_cli.py +++ b/tests/cli/test_azure_operations_cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +from types import SimpleNamespace from typing import TYPE_CHECKING import pytest @@ -35,6 +36,10 @@ def start(self) -> AzureContainerAppsExecution: self.calls.append(("start", None)) return self.execution + def start_identity_refresh_probe(self, **kwargs: object) -> AzureContainerAppsExecution: + self.calls.append(("identity-refresh-probe", kwargs)) + return self.execution + def latest(self) -> AzureContainerAppsExecution: self.calls.append(("latest", None)) return self.execution @@ -158,6 +163,58 @@ def test_mutating_operations_require_confirmation_and_invoke_one_action( assert operations.calls == [expected] +def test_identity_refresh_probe_requires_named_profile_and_confirmation( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + operations, _binding = _install_fake(monkeypatch) + monkeypatch.setattr( + azure_command, + "load_project_config", + lambda *_args, **_kwargs: SimpleNamespace( + launcher_provider="azure_container_apps", + warehouse_provider="bigquery", + state_provider="bigquery", + catalog_provider="dataplex", + secret_provider="gcp_secret_manager", + ), + ) + args = [ + "azure", + "identity-refresh-probe", + "--deployment", + "azure_bigquery", + "--pipeline", + "warehouse_fixture", + "--config", + str(tmp_path / "dander.yaml"), + "--project", + "unit-project", + "--dataset", + "raw", + "--table", + "proof_rows", + ] + + refused = CliRunner().invoke(app, args, input="n\n") + accepted = CliRunner().invoke(app, args, input="y\n") + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert operations.calls == [ + ( + "identity-refresh-probe", + { + "project": "unit-project", + "dataset": "raw", + "table": "proof_rows", + "max_wait_seconds": 900, + "refresh_margin_seconds": 15, + }, + ) + ] + + def test_image_promotion_requires_confirmation_and_passes_only_typed_inputs( monkeypatch: MonkeyPatch, tmp_path: Path, diff --git a/tests/identity/test_azure_google.py b/tests/identity/test_azure_google.py new file mode 100644 index 00000000..ac3835d0 --- /dev/null +++ b/tests/identity/test_azure_google.py @@ -0,0 +1,130 @@ +"""Azure Container Apps managed-identity adaptation for Google federation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING, cast + +import pytest +from google.auth import external_account + +from dander.identity import AzureContainerAppsIdentityError, prepare_azure_google_identity + +if TYPE_CHECKING: + from dander.identity.azure_google import AzureCredentialFactory, GoogleCredentialFactory + +_NOW = datetime(2026, 8, 11, 12, 0, tzinfo=UTC) +_CLIENT_ID = "22222222-2222-4222-8222-222222222222" +_ENVIRONMENT = { + "AZURE_CLIENT_ID": _CLIENT_ID, + "DANDER_AZURE_GCP_APPLICATION_ID_URI": ("api://33333333-3333-4333-8333-333333333333"), + "DANDER_GCP_SERVICE_ACCOUNT": "dander-runtime@unit-project.iam.gserviceaccount.com", + "DANDER_GCP_WIF_AUDIENCE": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), +} + + +@dataclass(frozen=True) +class _Token: + token: str + expires_on: int + + +class _Credential: + def __init__(self, markers: list[str], *, expires_on: int = 1_786_454_000) -> None: + self.markers = markers + self.scopes: list[str] = [] + self.expires_on = expires_on + + def get_token(self, *scopes: str) -> _Token: + self.scopes.extend(scopes) + marker = chr(ord("a") + len(self.scopes) - 1) + self.markers.append(marker) + return _Token(f"header.payload-{marker}.signature", self.expires_on) + + +def test_azure_google_identity_renews_from_one_user_assigned_identity() -> None: + markers: list[str] = [] + credential = _Credential(markers) + client_ids: list[str] = [] + + def azure_factory(*, client_id: str) -> _Credential: + client_ids.append(client_id) + return credential + + def capture_google(**values: object) -> object: + assert values["audience"] == _ENVIRONMENT["DANDER_GCP_WIF_AUDIENCE"] + assert values["service_account"] == _ENVIRONMENT["DANDER_GCP_SERVICE_ACCOUNT"] + return values["supplier"] + + supplier = prepare_azure_google_identity( + environ=dict(_ENVIRONMENT), + clock=lambda: _NOW, + azure_credential_factory=cast("AzureCredentialFactory", azure_factory), + google_credential_factory=cast("GoogleCredentialFactory", capture_google), + ) + + first = supplier.subject_token() # type: ignore[attr-defined] + second = supplier.subject_token() # type: ignore[attr-defined] + + assert first != second + assert markers == ["a", "b", "c"] + assert client_ids == [_CLIENT_ID] + assert credential.scopes == ["api://33333333-3333-4333-8333-333333333333/.default"] * 3 + + +def test_azure_google_identity_builds_short_scoped_external_credentials() -> None: + credentials = prepare_azure_google_identity( + environ=dict(_ENVIRONMENT), + clock=lambda: _NOW, + azure_credential_factory=cast("AzureCredentialFactory", lambda **_values: _Credential([])), + ) + + assert isinstance(credentials, external_account.Credentials) + assert credentials._scopes == ("https://www.googleapis.com/auth/cloud-platform",) + assert credentials._service_account_impersonation_options == {"token_lifetime_seconds": 600} + assert credentials._credential_source == {"environment_id": "azure-container-apps"} + assert credentials._client_id is None + assert credentials._client_secret is None + + +@pytest.mark.parametrize( + ("update", "message"), + [ + ({"AZURE_CLIENT_ID": "not-a-uuid"}, "configuration"), + ({"DANDER_AZURE_GCP_APPLICATION_ID_URI": "https://example.invalid"}, "configuration"), + ({"DANDER_GCP_SERVICE_ACCOUNT": "not-an-account"}, "configuration"), + ({"DANDER_GCP_WIF_AUDIENCE": "not-an-audience"}, "configuration"), + ], +) +def test_azure_google_identity_rejects_invalid_configuration( + update: dict[str, str], + message: str, +) -> None: + environment = {**_ENVIRONMENT, **update} + + with pytest.raises(AzureContainerAppsIdentityError, match=message): + prepare_azure_google_identity( + environ=environment, + clock=lambda: _NOW, + azure_credential_factory=cast( + "AzureCredentialFactory", lambda **_values: _Credential([]) + ), + ) + + +def test_azure_google_identity_rejects_near_expiry_or_non_jwt_tokens() -> None: + near_expiry = int(_NOW.timestamp()) + 299 + + with pytest.raises(AzureContainerAppsIdentityError, match="invalid"): + prepare_azure_google_identity( + environ=dict(_ENVIRONMENT), + clock=lambda: _NOW, + azure_credential_factory=cast( + "AzureCredentialFactory", + lambda **_values: _Credential([], expires_on=near_expiry), + ), + ) diff --git a/tests/identity/test_refresh_probe.py b/tests/identity/test_refresh_probe.py new file mode 100644 index 00000000..9318d8f8 --- /dev/null +++ b/tests/identity/test_refresh_probe.py @@ -0,0 +1,108 @@ +"""Credential refresh proof emits only bounded, sanitized lifecycle facts.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from dander.identity.refresh_probe import GoogleRefreshProbeError, run_google_refresh_probe + +_NOW = datetime(2026, 8, 11, 12, 0, tzinfo=UTC) + + +class _Credentials: + def __init__(self) -> None: + self.expiry: datetime | None = _NOW + timedelta(seconds=10) + + +class _Client: + def __init__(self, credentials: _Credentials) -> None: + self.credentials = credentials + self.calls = 0 + + def query_count(self, *, project: str, dataset: str, table: str) -> int: + assert (project, dataset, table) == ("unit-project", "raw", "proof_rows") + self.calls += 1 + if self.calls == 2: + self.credentials.expiry = _NOW + timedelta(minutes=10) + return 3 + + +def test_refresh_probe_requires_later_expiry_and_emits_no_rows_or_tokens() -> None: + credentials = _Credentials() + client = _Client(credentials) + events: list[dict[str, object]] = [] + waits: list[float] = [] + + run_google_refresh_probe( + credentials=credentials, + client=client, + project="unit-project", + dataset="raw", + table="proof_rows", + max_wait_seconds=30, + refresh_margin_seconds=5, + now=lambda: _NOW, + sleep=waits.append, + emit=events.append, + ) + + assert waits == [15.0] + assert [event["event"] for event in events] == [ + "query.completed", + "query.completed", + "credential.refresh_observed", + ] + assert [event.get("row_count") for event in events[:2]] == [3, 3] + serialized = repr(events).lower() + assert "token" not in serialized + assert "select" not in serialized + assert "proof_rows" not in serialized + + +def test_refresh_probe_rejects_excessive_wait_without_sleeping() -> None: + credentials = _Credentials() + credentials.expiry = _NOW + timedelta(minutes=20) + + with pytest.raises(GoogleRefreshProbeError, match="bounded proof window"): + run_google_refresh_probe( + credentials=credentials, + client=_Client(credentials), + project="unit-project", + dataset="raw", + table="proof_rows", + max_wait_seconds=900, + refresh_margin_seconds=15, + now=lambda: _NOW, + sleep=lambda _seconds: pytest.fail("probe must not wait"), + emit=lambda _event: None, + ) + + +@pytest.mark.parametrize( + ("project", "dataset", "table"), + [ + ("UPPERCASE", "raw", "proof_rows"), + ("unit-project", "raw;drop", "proof_rows"), + ("unit-project", "raw", "proof.rows"), + ], +) +def test_refresh_probe_rejects_invalid_targets( + project: str, + dataset: str, + table: str, +) -> None: + credentials = _Credentials() + + with pytest.raises(GoogleRefreshProbeError, match="target"): + run_google_refresh_probe( + credentials=credentials, + client=_Client(credentials), + project=project, + dataset=dataset, + table=table, + max_wait_seconds=30, + refresh_margin_seconds=5, + emit=lambda _event: None, + ) diff --git a/tests/project/test_portable_config.py b/tests/project/test_portable_config.py index a3d473dd..cbc9efab 100644 --- a/tests/project/test_portable_config.py +++ b/tests/project/test_portable_config.py @@ -434,6 +434,48 @@ def test_azure_key_vault_profile_fails_closed_on_another_launcher(tmp_path: Path load_project_config(project_path) +def test_version_two_resolves_azure_bigquery_federation_profile(tmp_path: Path) -> None: + project_path = tmp_path / "dander.yaml" + platforms_path = tmp_path / "dander.platforms.yaml" + project_path.write_text(_V1_PROJECT, encoding="utf-8") + migration = prepare_version_one_migration(project_path) + project_path.write_text(migration.logical_yaml, encoding="utf-8") + platforms = yaml.safe_load(migration.platforms_yaml) + deployment = platforms["deployments"].pop("gcp_cloud_run") + deployment["launcher"] = { + "provider": "azure_container_apps", + "region": "eastus", + "subscription_id": "11111111-1111-4111-8111-111111111111", + "resource_group_name": "dander-phase6", + "container_app_environment_name": "dander-phase6-env", + "acr_name": "danderphase6", + "key_vault_name": "dander-phase6-kv", + "managed_identity_name": "dander-phase6-runtime", + "managed_identity_client_id": "22222222-2222-4222-8222-222222222222", + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + "google_application_id_uri": "api://33333333-3333-4333-8333-333333333333", + } + deployment["runtime"]["memory"] = "2Gi" + deployment["safety"]["require_guarded_free_tier"] = False + deployment["pipelines"]["example_records"]["time_zone"] = "UTC" + platforms["deployments"]["azure_bigquery"] = deployment + platforms_path.write_text(yaml.safe_dump(platforms), encoding="utf-8") + + resolved = load_project_config(project_path, deployment="azure_bigquery") + + assert resolved.warehouse_provider == "bigquery" + assert resolved.state_provider == "bigquery" + assert resolved.catalog_provider == "dataplex" + assert resolved.secret_provider == "gcp_secret_manager" + assert resolved.launcher_provider == "azure_container_apps" + assert resolved.resolved_launcher_config()["google_application_id_uri"] == ( + "api://33333333-3333-4333-8333-333333333333" + ) + + def test_config_migrate_check_is_read_only_then_write_is_atomic(tmp_path: Path) -> None: project_path = tmp_path / "dander.yaml" platforms_path = tmp_path / "dander.platforms.yaml" diff --git a/tests/providers/test_azure_container_apps_operations.py b/tests/providers/test_azure_container_apps_operations.py index 51984a29..6fb2cae3 100644 --- a/tests/providers/test_azure_container_apps_operations.py +++ b/tests/providers/test_azure_container_apps_operations.py @@ -47,7 +47,13 @@ def _binding() -> AzureDeploymentBinding: schedule_paused=True, runtime_timeout_seconds=900, runtime_max_retries=1, + secret_provider="azure_key_vault", + secret_bindings=(("DANDER_POSTGRES_DSN", "postgres-dsn"),), secret_ids=("postgres-dsn",), + google_project=None, + google_workload_identity_audience=None, + google_application_id_uri=None, + google_service_account=None, project_dir=Path("/tmp/dander-azure-operations-test"), ) @@ -125,6 +131,61 @@ def test_start_uses_exact_manifest_job_and_returns_provider_execution() -> None: assert runner.commands[0][1:4] == ("containerapp", "job", "start") +def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> None: + start_key = ( + "containerapp", + "job", + "start", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--args", + "runtime", + "identity-refresh-probe", + "--project", + "unit-project", + "--dataset", + "raw", + "--table", + "proof_rows", + "--max-wait-seconds", + "900", + "--refresh-margin-seconds", + "15", + ) + show_key = ( + "containerapp", + "job", + "execution", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + "--job-execution-name", + _EXECUTION, + ) + runner = _Runner( + { + start_key: [{"name": _EXECUTION}], + show_key: [_execution(status="Running")], + } + ) + + execution = AzureContainerAppsOperations( + _binding(), runner=runner + ).start_identity_refresh_probe( + project="unit-project", + dataset="raw", + table="proof_rows", + ) + + assert execution.state == "running" + assert start_key == runner.commands[0][1 : runner.commands[0].index("--subscription")] + assert not set(runner.commands[0]).intersection({"--image", "--env-vars", "--command"}) + + def test_latest_normalizes_the_most_recent_execution() -> None: older = _execution(name=f"{_JOB}-old1234") older_properties = older["properties"] diff --git a/tests/providers/test_azure_container_apps_runtime.py b/tests/providers/test_azure_container_apps_runtime.py index d748e0ac..5e0a5d66 100644 --- a/tests/providers/test_azure_container_apps_runtime.py +++ b/tests/providers/test_azure_container_apps_runtime.py @@ -8,6 +8,7 @@ from dander.deployment import ExecutionProjectionError, LauncherRuntime, ResolvedTemplateRequest from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry +from dander.providers.gcp_launcher import GcpLauncherContext, gcp_launcher_factory_context _SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" _CLIENT_ID = "22222222-2222-4222-8222-222222222222" @@ -38,10 +39,18 @@ } -def _runtime() -> LauncherRuntime: +def _runtime( + *, + launcher: dict[str, object] | None = None, + gcp: GcpLauncherContext | None = None, +) -> LauncherRuntime: registry = default_provider_registry() - config = registry.parse(ProviderKind.LAUNCHER, _LAUNCHER) - runtime = registry.build(ProviderKind.LAUNCHER, config) + config = registry.parse(ProviderKind.LAUNCHER, launcher or _LAUNCHER) + runtime = registry.build( + ProviderKind.LAUNCHER, + config, + context=gcp_launcher_factory_context(gcp) if gcp is not None else None, + ) assert isinstance(runtime, LauncherRuntime) return runtime @@ -51,11 +60,12 @@ def _request( pipelines: dict[str, dict[str, object]] | None = None, image: str = _IMAGE, memory: str = "2Gi", + profile_id: str = "azure_snowflake", ) -> ResolvedTemplateRequest: return ResolvedTemplateRequest( pipelines=_PIPELINES if pipelines is None else pipelines, image=image, - profile_id="azure_snowflake", + profile_id=profile_id, cpu=1, memory=memory, deadline_seconds=900, @@ -118,6 +128,60 @@ def test_azure_factory_is_lazy_and_projects_only_non_secret_identity() -> None: assert "postgresql://" not in serialized +def test_azure_factory_projects_keyless_google_federation_only_for_gcp_profile() -> None: + launcher: dict[str, object] = { + **_LAUNCHER, + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + "google_application_id_uri": "api://33333333-3333-4333-8333-333333333333", + } + runtime = _runtime( + launcher=launcher, + gcp=GcpLauncherContext(project="unit-project", require_guarded_free_tier=False), + ) + + template = runtime.templates.build( + _request( + pipelines={ + "warehouse_fixture": { + **_PIPELINES["warehouse_fixture"], + "runtime_service_account_id": "dander-runtime", + "secret_env": {"API_TOKEN": "source-api-token"}, + } + }, + profile_id="gcp", + ) + )["warehouse_fixture"] + environment = dict(template.environment) + + assert environment["DANDER_AZURE_GCP_APPLICATION_ID_URI"] == ( + "api://33333333-3333-4333-8333-333333333333" + ) + assert environment["DANDER_GCP_SERVICE_ACCOUNT"] == ( + "dander-runtime@unit-project.iam.gserviceaccount.com" + ) + assert environment["GCP_PROJECT_ID"] == "unit-project" + assert dict(template.secret_bindings)["API_TOKEN"].reference == ( + "gcp-sm://projects/unit-project/secrets/source-api-token/versions/latest" + ) + + +def test_azure_google_federation_rejects_missing_gcp_context() -> None: + launcher: dict[str, object] = { + **_LAUNCHER, + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + "google_application_id_uri": "api://33333333-3333-4333-8333-333333333333", + } + + with pytest.raises(ExecutionProjectionError, match="GCP platform"): + _runtime(launcher=launcher).templates.build(_request()) + + @pytest.mark.parametrize( ("update", "message"), [ diff --git a/tests/providers/test_azure_deployment_verification.py b/tests/providers/test_azure_deployment_verification.py index 87c44f9a..be838a29 100644 --- a/tests/providers/test_azure_deployment_verification.py +++ b/tests/providers/test_azure_deployment_verification.py @@ -4,6 +4,7 @@ import json import subprocess +from dataclasses import replace from pathlib import Path import pytest @@ -64,7 +65,13 @@ def _binding() -> AzureDeploymentBinding: schedule_paused=True, runtime_timeout_seconds=900, runtime_max_retries=1, + secret_provider="azure_key_vault", + secret_bindings=(("DANDER_POSTGRES_DSN", "postgres-dsn"),), secret_ids=("postgres-dsn",), + google_project=None, + google_workload_identity_audience=None, + google_application_id_uri=None, + google_service_account=None, project_dir=Path("/tmp/dander-azure-test"), ) @@ -188,6 +195,73 @@ def test_verifier_checks_exact_resources_without_reading_secret_values() -> None assert all("--only-show-errors" in command for command in runner.commands) +def test_verifier_checks_gcp_federation_and_runtime_secret_references() -> None: + binding = replace( + _binding(), + secret_provider="gcp_secret_manager", + secret_bindings=(("API_TOKEN", "source-api-token"),), + secret_ids=("source-api-token",), + google_project="unit-project", + google_workload_identity_audience=( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + google_application_id_uri="api://77777777-7777-4777-8777-777777777777", + google_service_account="dander-runtime@unit-project.iam.gserviceaccount.com", + ) + payloads = _payloads() + job_key = ( + "containerapp", + "job", + "show", + "--name", + binding.job_name, + "--resource-group", + "dander-phase6", + ) + properties = payloads[job_key]["properties"] + assert isinstance(properties, dict) + configuration = properties["configuration"] + template = properties["template"] + assert isinstance(configuration, dict) + assert isinstance(template, dict) + containers = template["containers"] + assert isinstance(containers, list) + container = containers[0] + assert isinstance(container, dict) + configuration["secrets"] = [] + container["env"] = [ + {"name": "HOME", "value": "/tmp"}, + {"name": "AZURE_CLIENT_ID", "value": _CLIENT_ID}, + { + "name": "DANDER_AZURE_GCP_APPLICATION_ID_URI", + "value": "api://77777777-7777-4777-8777-777777777777", + }, + { + "name": "DANDER_GCP_SERVICE_ACCOUNT", + "value": "dander-runtime@unit-project.iam.gserviceaccount.com", + }, + { + "name": "DANDER_GCP_WIF_AUDIENCE", + "value": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + }, + {"name": "GCP_PROJECT_ID", "value": "unit-project"}, + { + "name": "API_TOKEN", + "value": "projects/unit-project/secrets/source-api-token/versions/latest", + }, + ] + + result = AzureDeploymentVerifier(binding, runner=_Runner(payloads)).verify( + expected_image=_IMAGE + ) + + assert result.image == _IMAGE + + @pytest.mark.parametrize( ("mutation", "message"), [ diff --git a/tickets/DANDER-111-azure-google-federation.md b/tickets/DANDER-111-azure-google-federation.md new file mode 100644 index 00000000..580f37d7 --- /dev/null +++ b/tickets/DANDER-111-azure-google-federation.md @@ -0,0 +1,38 @@ +--- +id: DANDER-111 +title: Prove keyless Azure-to-Google workload federation +status: in_progress +phase: 6 +--- + +# DANDER-111 — Prove keyless Azure-to-Google workload federation + +## Goal + +Run the accepted OCI digest from Azure Container Apps against the existing BigQuery, Dataplex, and +GCP Secret Manager boundaries without any Azure client secret or Google service-account key, then +observe Google credential refresh in the same process. + +## Acceptance + +- [x] The Azure launcher accepts one typed Entra application ID URI and Google WIF audience only + for the named BigQuery/Dataplex/GCP-secrets profile. +- [x] Runtime Google credentials renew from the attached user-assigned managed identity without a + credential file or globally exported token. +- [x] Terraform creates a single-tenant Entra audience and restricts the Google provider and + impersonation grant to the selected Azure identity object ID. +- [x] GCP Secret Manager references remain runtime inputs and are not converted into Azure Key + Vault secrets. +- [x] The same-process refresh probe is bounded, emits no query text, tokens, credentials, or rows, + and has no automatic paid rerun. +- [x] Focused Python and provider-mocked Terraform contracts pass locally. +- [ ] An accepted public candidate passes live BigQuery access before and after Google credential + refresh plus GCP secret access and Dataplex read-back under approved ceilings. +- [ ] Revocation fails closed; all disposable Azure/Entra/GCP proof resources are removed; retained + GCP finishes with no Terraform drift; sanitized evidence is merged. + +## Boundary + +No Azure provider registration, Terraform apply, image publication/copy, job execution, Google IAM +mutation, or paid query is authorized by this ticket. Local construction evidence does not promote +Azure or the cross-cloud profile to supported. From e73d08ad9924ca415a7993fd41e529261e0c1f3b Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 13:09:52 -0400 Subject: [PATCH 124/163] docs: record federation review checkpoint --- HANDOFF.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 9c673a05..dbd02c42 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -17,7 +17,7 @@ tests/providers/test_azure_container_apps_runtime.py`. - Full repository pytest, Ruff, and mypy pass with the federation, projection, CLI, and refresh code. - Azure module and disposable federation Terraform mock tests pass; the new root validates cleanly. -- Protected main CI is green through merged PR #190; this federation slice is not yet published. +- Protected main CI is green through merged PR #190; federation PR #191 is pending protected CI. - No provider registration, resource creation, image copy, job execution, or paid query ran. ## Decisions @@ -28,7 +28,7 @@ tests/providers/test_azure_container_apps_runtime.py`. ## Remaining -- Merge this federation slice through a focused protected PR. +- Merge focused federation PR #191 after protected CI and review. - Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance tooling locally. - Obtain candidate-publication approval before publishing or copying an image. - Obtain explicit Azure, GCP, and Snowflake mutation ceilings before live proof. From 1eb5259754e9ac9e2049d9420598c078e18d3c37 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 13:22:09 -0400 Subject: [PATCH 125/163] fix: authorize the Azure proof secret operator --- CHANGELOG.md | 3 ++ HANDOFF.md | 37 +++++++++---------- docs/decisions.md | 12 ++++++ infra/azure/README.md | 6 +++ infra/azure/main.tf | 33 +++++++++-------- .../azure/modules/container-apps-jobs/main.tf | 6 +++ .../modules/container-apps-jobs/outputs.tf | 5 +++ .../container-apps-jobs/tests/jobs.tftest.hcl | 37 ++++++++++++------- .../modules/container-apps-jobs/variables.tf | 10 +++++ infra/azure/outputs.tf | 5 +++ infra/azure/tests/azure.tftest.hcl | 5 +++ tickets/DANDER-112-azure-secret-operator.md | 29 +++++++++++++++ 12 files changed, 140 insertions(+), 48 deletions(-) create mode 100644 tickets/DANDER-112-azure-secret-operator.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c36f69..7b275e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Bind Azure proof-secret creation and rotation to the exact signed-in Terraform operator with + vault-scoped `Key Vault Secrets Officer`, while preserving the runtime identity's read-only + `Key Vault Secrets User` role. - Add keyless Azure Container Apps-to-Google workload federation using the attached user-assigned identity, an exact Entra application audience, in-memory Google token exchange, and a bounded same-process credential-refresh probe. The profile remains experimental until live acceptance. diff --git a/HANDOFF.md b/HANDOFF.md index dbd02c42..8356fa1b 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,40 +2,39 @@ ## Finished -- Kept protected main green through merged Azure contract, infrastructure, and lifecycle PRs #188-#190. -- Added renewable Container Apps managed-identity to Google credentials without files or static keys. -- Added the exact Azure BigQuery/Dataplex/GCP-secrets projection beside unchanged Snowflake/Key Vault behavior. -- Added disposable Entra/Google federation Terraform pinned to one Azure identity object ID. -- Added a bounded 600-second refresh probe with explicit confirmation and no automatic paid rerun. +- Merged Azure-to-Google federation PR #191; protected main is green at `9a12b5c`. +- Bound vault secret administration to the exact authenticated Terraform operator principal. +- Kept the Container Apps runtime identity limited to read-only Key Vault secret access. +- Added provider-mocked assertions that distinguish the operator and runtime roles. +- Kept provider registration, role assignment, vault creation, secret writes, jobs, and spending untouched. ## Try It -Run `uv run pytest -q tests/identity/test_azure_google.py tests/identity/test_refresh_probe.py -tests/providers/test_azure_container_apps_runtime.py`. +Run `terraform -chdir=infra/azure test -no-color` and +`terraform -chdir=infra/azure/modules/container-apps-jobs test -no-color`. ## Checks -- Full repository pytest, Ruff, and mypy pass with the federation, projection, CLI, and refresh code. -- Azure module and disposable federation Terraform mock tests pass; the new root validates cleanly. -- Protected main CI is green through merged PR #190; federation PR #191 is pending protected CI. -- No provider registration, resource creation, image copy, job execution, or paid query ran. +- Protected main CI is green through merged PR #191. +- Azure root and Container Apps module format, validate, and provider-mocked tests pass. +- No provider registration, role assignment, resource creation, secret write, or paid operation ran. ## Decisions -- Container Apps uses `ManagedIdentityCredential`; Google Auth exchanges tokens in memory for 600-second credentials. -- Google trust checks Entra tenant, application audience, and the exact managed-identity object ID. -- Azure Snowflake/Key Vault and Azure BigQuery/GCP-secrets remain distinct named profiles. +- The plan operator gets vault-scoped `Key Vault Secrets Officer`; no group or broad administrator is inferred. +- The runtime identity remains `Key Vault Secrets User` and cannot create or rotate credentials. +- Secret values remain operator inputs outside Terraform plans, state, logs, and committed evidence. ## Remaining -- Merge focused federation PR #191 after protected CI and review. -- Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance tooling locally. +- Merge this focused role correction through protected CI before any Azure apply. +- Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance tooling separately. - Obtain candidate-publication approval before publishing or copying an image. - Obtain explicit Azure, GCP, and Snowflake mutation ceilings before live proof. - Run approved lifecycle, cleanup, rollback, and no-drift acceptance. ## Review First -- `src/dander/identity/azure_google.py` -- `src/dander/providers/azure_container_apps/runtime.py` -- `acceptance/cloud-portability/phase6/azure-google-federation/` +- `infra/azure/main.tf` +- `infra/azure/modules/container-apps-jobs/main.tf` +- `infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl` diff --git a/docs/decisions.md b/docs/decisions.md index ddc4e7b1..82e59c2f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1198,3 +1198,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc Azure Snowflake/PostgreSQL/Key-Vault profile. Provider-mocked Terraform and the bounded 600-second refresh command are preparation only; live exchange, refresh, revocation, cleanup, and support promotion still require an accepted candidate and approved provider ceilings. + +## 2026-08-11 — The Azure plan names one vault-scoped secret operator + +- **Need:** The named Azure profile cannot start until its PostgreSQL and Snowflake credentials + exist in Key Vault, and its rotation proof requires creating a newer secret version. The runtime + identity's read-only role cannot perform either operation. +- **Decision:** The platform plan grants the authenticated Terraform principal `Key Vault Secrets + Officer` on only the deployment vault. The runtime identity remains `Key Vault Secrets User`; + no group, subscription-wide administrator, or secret value enters Terraform. +- **Boundary:** This is local plan construction until a separately approved apply. Secret values + remain operator inputs outside Terraform, and live rotation must prove a later job execution sees + a versionless secret update before the Azure profile can be promoted. diff --git a/infra/azure/README.md b/infra/azure/README.md index a3a68692..3f8631b6 100644 --- a/infra/azure/README.md +++ b/infra/azure/README.md @@ -81,6 +81,10 @@ Azure trusted-service path used by the managed-identity secret reference. Run `dander init-azure-apply` only after reviewing the exact saved plan and receiving explicit approval for the live Azure changes. Then use `dander azure verify` for read-only checks of the subscription, environment, logs, job trigger/resources/image, managed identity, ACR, and Key Vault. +The plan grants the exact signed-in Terraform operator `Key Vault Secrets Officer` so that person +can create and rotate the declared proof secrets; the job identity remains limited to `Key Vault +Secrets User`. Secret values remain an operator action outside Terraform and must not appear in a +plan, state file, shell history, or committed evidence. ## Job lifecycle @@ -109,6 +113,8 @@ row limit. - ACR administrator credentials and Storage shared keys remain disabled; state network access defaults to deny and admits only the reviewed operator `/32`. - The runtime identity receives only `AcrPull` and `Key Vault Secrets User` in this root. +- The signed-in plan operator receives `Key Vault Secrets Officer` only on this deployment vault; + no group, subscription-wide secret administrator, or second operator is inferred. - Key Vault network access defaults to deny and admits only Azure's trusted-service path plus the reviewed operator IP. - Secret values are an operator action outside Terraform and must never enter a plan or state. diff --git a/infra/azure/main.tf b/infra/azure/main.tf index ce4191df..78f7bc8e 100644 --- a/infra/azure/main.tf +++ b/infra/azure/main.tf @@ -24,20 +24,21 @@ data "azurerm_user_assigned_identity" "runtime" { module "container_apps_jobs" { source = "./modules/container-apps-jobs" - name = var.name - subscription_id = var.subscription_id - tenant_id = data.azurerm_client_config.current.tenant_id - location = var.location - resource_group_name = data.azurerm_resource_group.dander.name - container_app_environment_name = var.container_app_environment_name - acr_id = data.azurerm_container_registry.runtime.id - acr_login_server = data.azurerm_container_registry.runtime.login_server - key_vault_name = var.key_vault_name - key_vault_allowed_ip_rule = var.key_vault_allowed_ip_rule - managed_identity_id = data.azurerm_user_assigned_identity.runtime.id - managed_identity_client_id = data.azurerm_user_assigned_identity.runtime.client_id - managed_identity_principal_id = data.azurerm_user_assigned_identity.runtime.principal_id - execution_projections = var.execution_projections - infrastructure_subnet_id = var.infrastructure_subnet_id - tags = var.tags + name = var.name + subscription_id = var.subscription_id + tenant_id = data.azurerm_client_config.current.tenant_id + location = var.location + resource_group_name = data.azurerm_resource_group.dander.name + container_app_environment_name = var.container_app_environment_name + acr_id = data.azurerm_container_registry.runtime.id + acr_login_server = data.azurerm_container_registry.runtime.login_server + key_vault_name = var.key_vault_name + key_vault_allowed_ip_rule = var.key_vault_allowed_ip_rule + key_vault_operator_principal_id = data.azurerm_client_config.current.object_id + managed_identity_id = data.azurerm_user_assigned_identity.runtime.id + managed_identity_client_id = data.azurerm_user_assigned_identity.runtime.client_id + managed_identity_principal_id = data.azurerm_user_assigned_identity.runtime.principal_id + execution_projections = var.execution_projections + infrastructure_subnet_id = var.infrastructure_subnet_id + tags = var.tags } diff --git a/infra/azure/modules/container-apps-jobs/main.tf b/infra/azure/modules/container-apps-jobs/main.tf index f3cb7bf3..a5eb718e 100644 --- a/infra/azure/modules/container-apps-jobs/main.tf +++ b/infra/azure/modules/container-apps-jobs/main.tf @@ -118,6 +118,12 @@ resource "azurerm_role_assignment" "key_vault_secrets" { principal_type = "ServicePrincipal" } +resource "azurerm_role_assignment" "key_vault_operator" { + scope = azurerm_key_vault.runtime.id + role_definition_name = "Key Vault Secrets Officer" + principal_id = var.key_vault_operator_principal_id +} + resource "azurerm_container_app_job" "pipeline" { for_each = var.execution_projections diff --git a/infra/azure/modules/container-apps-jobs/outputs.tf b/infra/azure/modules/container-apps-jobs/outputs.tf index a0b4f8eb..8ab56985 100644 --- a/infra/azure/modules/container-apps-jobs/outputs.tf +++ b/infra/azure/modules/container-apps-jobs/outputs.tf @@ -8,6 +8,11 @@ output "key_vault_uri" { value = trimsuffix(azurerm_key_vault.runtime.vault_uri, "/") } +output "key_vault_operator_principal_id" { + description = "Operator principal allowed to create and rotate Key Vault secrets." + value = azurerm_role_assignment.key_vault_operator.principal_id +} + output "jobs" { description = "Container Apps Job names and ids keyed by pipeline." value = { diff --git a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl index c5e4c358..d7d538bd 100644 --- a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl +++ b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl @@ -14,19 +14,20 @@ mock_provider "azurerm" { } variables { - name = "dander" - subscription_id = "11111111-1111-4111-8111-111111111111" - tenant_id = "22222222-2222-4222-8222-222222222222" - location = "eastus" - resource_group_name = "dander-phase6" - container_app_environment_name = "dander-phase6-env" - acr_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ContainerRegistry/registries/danderphase6" - acr_login_server = "danderphase6.azurecr.io" - key_vault_name = "dander-phase6-kv" - key_vault_allowed_ip_rule = "203.0.113.10" - managed_identity_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" - managed_identity_client_id = "33333333-3333-4333-8333-333333333333" - managed_identity_principal_id = "44444444-4444-4444-8444-444444444444" + name = "dander" + subscription_id = "11111111-1111-4111-8111-111111111111" + tenant_id = "22222222-2222-4222-8222-222222222222" + location = "eastus" + resource_group_name = "dander-phase6" + container_app_environment_name = "dander-phase6-env" + acr_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ContainerRegistry/registries/danderphase6" + acr_login_server = "danderphase6.azurecr.io" + key_vault_name = "dander-phase6-kv" + key_vault_allowed_ip_rule = "203.0.113.10" + key_vault_operator_principal_id = "55555555-5555-4555-8555-555555555555" + managed_identity_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" + managed_identity_client_id = "33333333-3333-4333-8333-333333333333" + managed_identity_principal_id = "44444444-4444-4444-8444-444444444444" execution_projections = { bigquery_fixture = { launcher = "azure_container_apps" @@ -110,4 +111,14 @@ run "projects_exact_job_contract" { ) error_message = "Key Vault must default-deny network access while allowing the Azure service path." } + + assert { + condition = ( + azurerm_role_assignment.key_vault_operator.principal_id == "55555555-5555-4555-8555-555555555555" && + azurerm_role_assignment.key_vault_operator.role_definition_name == "Key Vault Secrets Officer" && + azurerm_role_assignment.key_vault_secrets.principal_id == "44444444-4444-4444-8444-444444444444" && + azurerm_role_assignment.key_vault_secrets.role_definition_name == "Key Vault Secrets User" + ) + error_message = "The signed-in operator may rotate secrets while the runtime remains read-only." + } } diff --git a/infra/azure/modules/container-apps-jobs/variables.tf b/infra/azure/modules/container-apps-jobs/variables.tf index 8141fe7d..c787f951 100644 --- a/infra/azure/modules/container-apps-jobs/variables.tf +++ b/infra/azure/modules/container-apps-jobs/variables.tf @@ -48,6 +48,16 @@ variable "key_vault_allowed_ip_rule" { description = "One reviewed operator public IPv4 address for Key Vault data access." } +variable "key_vault_operator_principal_id" { + type = string + description = "Signed-in operator principal allowed to create and rotate proof secrets." + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$", var.key_vault_operator_principal_id)) + error_message = "key_vault_operator_principal_id must be an RFC 4122 UUID." + } +} + variable "managed_identity_id" { type = string description = "User-assigned managed identity resource id." diff --git a/infra/azure/outputs.tf b/infra/azure/outputs.tf index 65ad66f1..a8d0375d 100644 --- a/infra/azure/outputs.tf +++ b/infra/azure/outputs.tf @@ -8,6 +8,11 @@ output "key_vault_uri" { value = module.container_apps_jobs.key_vault_uri } +output "key_vault_operator_principal_id" { + description = "Authenticated operator principal allowed to create and rotate proof secrets." + value = module.container_apps_jobs.key_vault_operator_principal_id +} + output "jobs" { description = "Container Apps Job names and ids keyed by pipeline." value = module.container_apps_jobs.jobs diff --git a/infra/azure/tests/azure.tftest.hcl b/infra/azure/tests/azure.tftest.hcl index 62a15bca..a784a4c9 100644 --- a/infra/azure/tests/azure.tftest.hcl +++ b/infra/azure/tests/azure.tftest.hcl @@ -80,4 +80,9 @@ run "wires_stage_zero_resources_into_jobs" { condition = module.container_apps_jobs.jobs["warehouse_fixture"].name == "dander-00626d3b5f01" error_message = "The Azure root must preserve deterministic manifest-to-job naming." } + + assert { + condition = output.key_vault_operator_principal_id == "44444444-4444-4444-8444-444444444444" + error_message = "The Azure root must bind secret administration to the authenticated operator." + } } diff --git a/tickets/DANDER-112-azure-secret-operator.md b/tickets/DANDER-112-azure-secret-operator.md new file mode 100644 index 00000000..5c643934 --- /dev/null +++ b/tickets/DANDER-112-azure-secret-operator.md @@ -0,0 +1,29 @@ +--- +id: DANDER-112 +title: Bind Azure proof-secret rotation to one operator +status: in_progress +phase: 6 +--- + +# DANDER-112 — Bind Azure proof-secret rotation to one operator + +## Goal + +Allow the exact authenticated Azure plan operator to create and rotate the named profile's Key +Vault secrets without widening the Container Apps runtime identity or putting secret values in +Terraform. + +## Acceptance + +- [x] The root passes the authenticated Azure principal object ID into the Container Apps module. +- [x] The module grants that exact principal `Key Vault Secrets Officer` only on the deployment + vault. +- [x] The runtime identity remains limited to `Key Vault Secrets User` for the same vault. +- [x] Provider-mocked tests distinguish the operator and runtime principals and assert both roles. +- [ ] Protected CI passes and the focused correction merges before any Azure apply. + +## Boundary + +This ticket changes only Terraform construction and documentation. It does not register a provider, +assign a live role, create a vault, write a secret, run a job, or authorize spending. Secret values +remain outside plans, state, logs, and committed evidence. From 6e2ab097f746500f6e94c6cd53c94f146869f8db Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 13:38:06 -0400 Subject: [PATCH 126/163] Fail closed on Azure Key Vault network access --- CHANGELOG.md | 6 ++++ HANDOFF.md | 35 +++++++++---------- docs/decisions.md | 12 +++++++ docs/known-limitations.md | 5 ++- infra/azure/README.md | 18 ++++++---- .../azure/modules/container-apps-jobs/main.tf | 21 +++++++++-- .../container-apps-jobs/tests/jobs.tftest.hcl | 17 +++++++-- .../modules/container-apps-jobs/variables.tf | 2 +- infra/azure/tests/azure.tftest.hcl | 1 + infra/azure/variables.tf | 2 +- src/dander/bootstrap/azure_terraform.py | 21 +++++++++++ src/dander/cli/azure_command.py | 9 ++++- .../azure_container_apps/verification.py | 12 +++++++ tests/bootstrap/test_azure_terraform.py | 8 +++++ .../test_azure_deployment_verification.py | 26 ++++++++++++-- ...ER-110-azure-image-lifecycle-operations.md | 4 +-- tickets/DANDER-112-azure-secret-operator.md | 4 +-- tickets/DANDER-113-azure-key-vault-network.md | 32 +++++++++++++++++ 18 files changed, 195 insertions(+), 40 deletions(-) create mode 100644 tickets/DANDER-113-azure-key-vault-network.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b275e19..d1bf912c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ capabilities enter through the next minor release. - Carry additive warehouse load/transform telemetry through endpoint results into terminal run telemetry, including optional resource, queue, execution, spill, and capacity-unit measures. +### Fixed + +- Route Azure Container Apps Key Vault references through one exact infrastructure subnet and + disable the inapplicable trusted-service bypass; Key Vault profiles now fail planning when no + subnet path is supplied. + ## 0.8.0rc8 — 2026-08-10 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 8356fa1b..39adbb5f 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,38 @@ ## Finished -- Merged Azure-to-Google federation PR #191; protected main is green at `9a12b5c`. -- Bound vault secret administration to the exact authenticated Terraform operator principal. -- Kept the Container Apps runtime identity limited to read-only Key Vault secret access. -- Added provider-mocked assertions that distinguish the operator and runtime roles. -- Kept provider registration, role assignment, vault creation, secret writes, jobs, and spending untouched. +- Confirmed protected main is green through merged PR #192. +- Corrected the false assumption that Container Apps can bypass a Key Vault firewall. +- Required Azure Key Vault profiles to supply one exact Container Apps infrastructure subnet. +- Kept the vault default-deny and admitted only that subnet plus the reviewed operator IP. +- Added fail-closed Terraform, verifier, and focused test coverage without contacting Azure. ## Try It -Run `terraform -chdir=infra/azure test -no-color` and -`terraform -chdir=infra/azure/modules/container-apps-jobs test -no-color`. +Run `uv run pytest tests/bootstrap/test_azure_terraform.py tests/providers/test_azure_deployment_verification.py` and `terraform -chdir=infra/azure test -no-color`. ## Checks -- Protected main CI is green through merged PR #191. -- Azure root and Container Apps module format, validate, and provider-mocked tests pass. -- No provider registration, role assignment, resource creation, secret write, or paid operation ran. +- Protected main CI is green through merged PR #192. +- Repository-wide Ruff, strict typing, and Python tests pass; Terraform format/validate and both Terraform test roots pass. +- No provider registration, resource creation, secret write, image copy, or paid operation ran. ## Decisions -- The plan operator gets vault-scoped `Key Vault Secrets Officer`; no group or broad administrator is inferred. -- The runtime identity remains `Key Vault Secrets User` and cannot create or rotate credentials. -- Secret values remain operator inputs outside Terraform plans, state, logs, and committed evidence. +- Managed identity grants permission but does not bypass Key Vault network controls. +- Key Vault profiles require an existing delegated subnet with the `Microsoft.KeyVault` service endpoint. +- The GCP-secret federation profile may still use Azure's managed network because it does not read this vault. ## Remaining -- Merge this focused role correction through protected CI before any Azure apply. -- Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance tooling separately. +- Run local checks and merge this focused correction through protected CI. +- Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance preflight separately. - Obtain candidate-publication approval before publishing or copying an image. -- Obtain explicit Azure, GCP, and Snowflake mutation ceilings before live proof. +- Obtain explicit Azure, GCP, and Snowflake ceilings before live mutations. - Run approved lifecycle, cleanup, rollback, and no-drift acceptance. ## Review First -- `infra/azure/main.tf` - `infra/azure/modules/container-apps-jobs/main.tf` -- `infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl` +- `src/dander/bootstrap/azure_terraform.py` +- `src/dander/providers/azure_container_apps/verification.py` diff --git a/docs/decisions.md b/docs/decisions.md index 82e59c2f..540416ee 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1210,3 +1210,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** This is local plan construction until a separately approved apply. Secret values remain operator inputs outside Terraform, and live rotation must prove a later job execution sees a versionless secret update before the Azure profile can be promoted. + +## 2026-08-11 — Azure Key Vault references require one explicit subnet path + +- **Finding:** Microsoft does not list Container Apps among the services that can bypass a Key + Vault firewall. Managed identity supplies authentication and authorization, but not network + admission, so the earlier trusted-service assumption would have blocked the named profile. +- **Decision:** Any Azure execution projection containing a Key Vault reference requires the exact + Container Apps infrastructure subnet. The vault admits that subnet and the operator `/32`, keeps + default deny, and disables the broad trusted-service bypass. +- **Boundary:** Dander does not create or modify the existing delegated subnet; it must already + expose the `Microsoft.KeyVault` service endpoint. The GCP-secret federation profile remains able + to use Azure's managed network because its jobs do not read the deployment vault. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 0467e749..5efba28e 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -50,7 +50,10 @@ remain Phase 8 work. plan-first Terraform, digest-preserving ACR promotion tooling, provider-native lifecycle operations, bounded Log Analytics reads, deployment verification, and a locally validated Azure-to-Google managed-identity adapter with disposable federation Terraform and refresh-probe - tooling. Azure schedules are + tooling. The named Key Vault profile requires an existing delegated Container Apps subnet with + the `Microsoft.KeyVault` service endpoint; Dander admits that exact subnet and the reviewed + operator IP while disabling the inapplicable Azure trusted-service bypass. Dander does not + create or modify the subnet. Azure schedules are UTC-only and currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. Resource-provider registration, credentialed Azure planning/apply, actual ACR image copy, live identity exchange, refresh and revocation, diff --git a/infra/azure/README.md b/infra/azure/README.md index 3f8631b6..8b5c54e0 100644 --- a/infra/azure/README.md +++ b/infra/azure/README.md @@ -70,13 +70,17 @@ dander init-azure-plan \ ``` This creates a saved plan only. An optional existing delegated subnet can be selected with -`--infrastructure-subnet-id`; selecting it makes the environment internal. An existing reviewed -Action Group can be selected with `--alert-action-group-id`. Paused pipelines remain manual jobs; -active schedules use Azure's UTC-only five-field cron. +`--infrastructure-subnet-id`; selecting it makes the environment internal. The subnet is required +when the profile uses Azure Key Vault references and must already have the `Microsoft.KeyVault` +service endpoint enabled. Container Apps is not a Key Vault trusted service, so the plan fails +closed instead of relying on the vault's trusted-service bypass. An existing reviewed Action Group +can be selected with `--alert-action-group-id`. Paused pipelines remain manual jobs; active +schedules use Azure's UTC-only five-field cron. Replace the documentation-only Key Vault IP with the operator's current public IPv4 address. Key -Vault defaults to deny, admits that exact operator IP for secret administration, and retains the -Azure trusted-service path used by the managed-identity secret reference. +Vault defaults to deny, admits that exact operator IP for secret administration, and admits the +exact Container Apps subnet for managed-identity secret references. The plan does not create or +modify that existing subnet. Run `dander init-azure-apply` only after reviewing the exact saved plan and receiving explicit approval for the live Azure changes. Then use `dander azure verify` for read-only checks of the @@ -115,8 +119,8 @@ row limit. - The runtime identity receives only `AcrPull` and `Key Vault Secrets User` in this root. - The signed-in plan operator receives `Key Vault Secrets Officer` only on this deployment vault; no group, subscription-wide secret administrator, or second operator is inferred. -- Key Vault network access defaults to deny and admits only Azure's trusted-service path plus the - reviewed operator IP. +- Key Vault network access defaults to deny and admits only the exact Container Apps subnet plus + the reviewed operator IP; the broad Azure trusted-service bypass is disabled. - Secret values are an operator action outside Terraform and must never enter a plan or state. - The locally tested image-copy and job-operation commands are not live evidence. Actual image copy, job lifecycle execution, external identity federation, live profile proof, and support diff --git a/infra/azure/modules/container-apps-jobs/main.tf b/infra/azure/modules/container-apps-jobs/main.tf index a5eb718e..be590020 100644 --- a/infra/azure/modules/container-apps-jobs/main.tf +++ b/infra/azure/modules/container-apps-jobs/main.tf @@ -65,6 +65,20 @@ check "secret_references_match_selected_key_vault" { } } +check "key_vault_references_have_network_path" { + assert { + condition = ( + length(flatten([ + for projection in values(var.execution_projections) : [ + for binding in values(projection.secret_bindings) : binding + if binding.provider == "azure_key_vault" + ] + ])) == 0 || var.infrastructure_subnet_id != null + ) + error_message = "Azure Key Vault secret references require a Container Apps infrastructure subnet with the Microsoft.KeyVault service endpoint enabled." + } +} + resource "azurerm_log_analytics_workspace" "runtime" { name = "${var.name}-logs" location = var.location @@ -98,9 +112,10 @@ resource "azurerm_key_vault" "runtime" { tags = local.tags network_acls { - bypass = "AzureServices" - default_action = "Deny" - ip_rules = [var.key_vault_allowed_ip_rule] + bypass = "None" + default_action = "Deny" + ip_rules = [var.key_vault_allowed_ip_rule] + virtual_network_subnet_ids = var.infrastructure_subnet_id == null ? [] : [var.infrastructure_subnet_id] } } diff --git a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl index d7d538bd..d7321953 100644 --- a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl +++ b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl @@ -28,6 +28,7 @@ variables { managed_identity_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dander-phase6-runtime" managed_identity_client_id = "33333333-3333-4333-8333-333333333333" managed_identity_principal_id = "44444444-4444-4444-8444-444444444444" + infrastructure_subnet_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.Network/virtualNetworks/dander-phase6/subnets/container-apps" execution_projections = { bigquery_fixture = { launcher = "azure_container_apps" @@ -107,9 +108,11 @@ run "projects_exact_job_contract" { assert { condition = ( azurerm_key_vault.runtime.network_acls[0].default_action == "Deny" && - azurerm_key_vault.runtime.network_acls[0].bypass == "AzureServices" + azurerm_key_vault.runtime.network_acls[0].bypass == "None" && + length(azurerm_key_vault.runtime.network_acls[0].virtual_network_subnet_ids) == 1 && + contains(azurerm_key_vault.runtime.network_acls[0].virtual_network_subnet_ids, "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.Network/virtualNetworks/dander-phase6/subnets/container-apps") ) - error_message = "Key Vault must default-deny network access while allowing the Azure service path." + error_message = "Key Vault must default-deny network access and admit the exact Container Apps subnet." } assert { @@ -122,3 +125,13 @@ run "projects_exact_job_contract" { error_message = "The signed-in operator may rotate secrets while the runtime remains read-only." } } + +run "rejects_key_vault_profile_without_subnet" { + command = plan + + variables { + infrastructure_subnet_id = null + } + + expect_failures = [check.key_vault_references_have_network_path] +} diff --git a/infra/azure/modules/container-apps-jobs/variables.tf b/infra/azure/modules/container-apps-jobs/variables.tf index c787f951..472c5d17 100644 --- a/infra/azure/modules/container-apps-jobs/variables.tf +++ b/infra/azure/modules/container-apps-jobs/variables.tf @@ -80,7 +80,7 @@ variable "execution_projections" { variable "infrastructure_subnet_id" { type = string - description = "Optional existing delegated subnet for the managed environment." + description = "Optional existing delegated subnet for the managed environment; required for Key Vault references and configured with the Microsoft.KeyVault service endpoint." default = null nullable = true } diff --git a/infra/azure/tests/azure.tftest.hcl b/infra/azure/tests/azure.tftest.hcl index a784a4c9..8d6e9722 100644 --- a/infra/azure/tests/azure.tftest.hcl +++ b/infra/azure/tests/azure.tftest.hcl @@ -55,6 +55,7 @@ variables { key_vault_name = "dander-phase6-kv" key_vault_allowed_ip_rule = "203.0.113.10" managed_identity_name = "dander-phase6-runtime" + infrastructure_subnet_id = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/dander-phase6/providers/Microsoft.Network/virtualNetworks/dander-phase6/subnets/container-apps" execution_projections = { warehouse_fixture = { launcher = "azure_container_apps" diff --git a/infra/azure/variables.tf b/infra/azure/variables.tf index 82915b0f..2aac46f6 100644 --- a/infra/azure/variables.tf +++ b/infra/azure/variables.tf @@ -74,7 +74,7 @@ variable "execution_projections" { } variable "infrastructure_subnet_id" { - description = "Optional existing delegated subnet for the Container Apps environment." + description = "Optional existing delegated subnet for the Container Apps environment; required for Key Vault references and configured with the Microsoft.KeyVault service endpoint." type = string default = null nullable = true diff --git a/src/dander/bootstrap/azure_terraform.py b/src/dander/bootstrap/azure_terraform.py index bf9438aa..3331ad5c 100644 --- a/src/dander/bootstrap/azure_terraform.py +++ b/src/dander/bootstrap/azure_terraform.py @@ -187,6 +187,12 @@ def execute( } except (ExecutionProjectionError, ProviderFactoryError) as error: raise AzureTerraformBootstrapError(str(error)) from error + requires_key_vault_network = _has_azure_key_vault_bindings(projections) + if requires_key_vault_network and infrastructure_subnet_id is None: + raise AzureTerraformBootstrapError( + "Azure Key Vault references require an infrastructure subnet with the " + "Microsoft.KeyVault service endpoint enabled" + ) self._run( "terraform", @@ -341,4 +347,19 @@ def _run(self, *args: str) -> None: ) from error +def _has_azure_key_vault_bindings( + projections: Mapping[str, Mapping[str, object]], +) -> bool: + for projection in projections.values(): + bindings = projection.get("secret_bindings") + if not isinstance(bindings, dict): + continue + if any( + isinstance(binding, dict) and binding.get("provider") == "azure_key_vault" + for binding in bindings.values() + ): + return True + return False + + __all__ = ["AzureTerraformBootstrap", "AzureTerraformBootstrapError"] diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index 4089f357..4e6b6f5d 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -415,7 +415,14 @@ def init_azure_plan( platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 key_vault_allowed_ip_rule: str = typer.Option(..., "--key-vault-allowed-ip"), alert_target: str | None = typer.Option(None, "--alert-action-group-id"), - infrastructure_subnet_id: str | None = typer.Option(None, "--infrastructure-subnet-id"), + infrastructure_subnet_id: str | None = typer.Option( + None, + "--infrastructure-subnet-id", + help=( + "Existing delegated Container Apps subnet; required for Azure Key Vault references " + "and expected to have the Microsoft.KeyVault service endpoint." + ), + ), name: str = typer.Option("dander", "--name"), infra_dir: Path = typer.Option(_DEFAULT_AZURE_INFRA_DIR, hidden=True), # noqa: B008 ) -> None: diff --git a/src/dander/providers/azure_container_apps/verification.py b/src/dander/providers/azure_container_apps/verification.py index 2d12a223..41e557ba 100644 --- a/src/dander/providers/azure_container_apps/verification.py +++ b/src/dander/providers/azure_container_apps/verification.py @@ -412,10 +412,22 @@ def verify(self, *, expected_image: str) -> AzureDeploymentVerification: ) vault_properties = _mapping(vault.get("properties")) vault_network_acls = _mapping(vault_properties.get("networkAcls")) + vault_subnet_rules = vault_network_acls.get("virtualNetworkRules") if ( str(vault_properties.get("vaultUri", "")).rstrip("/") != self.binding.key_vault_uri or vault_properties.get("enableRbacAuthorization") is not True or vault_network_acls.get("defaultAction") != "Deny" + or vault_network_acls.get("bypass") != "None" + or ( + self.binding.secret_provider == "azure_key_vault" + and ( + not isinstance(vault_subnet_rules, list) + or not any( + isinstance(rule, dict) and isinstance(rule.get("id"), str) + for rule in vault_subnet_rules + ) + ) + ) ): raise AzureDeploymentVerificationError( "Key Vault does not meet the RBAC and network secret-reference contract" diff --git a/tests/bootstrap/test_azure_terraform.py b/tests/bootstrap/test_azure_terraform.py index 078f4492..5a79e254 100644 --- a/tests/bootstrap/test_azure_terraform.py +++ b/tests/bootstrap/test_azure_terraform.py @@ -15,6 +15,10 @@ _SUBSCRIPTION_ID = "11111111-1111-4111-8111-111111111111" _CLIENT_ID = "22222222-2222-4222-8222-222222222222" +_SUBNET_ID = ( + f"/subscriptions/{_SUBSCRIPTION_ID}/resourceGroups/dander-phase6/" + "providers/Microsoft.Network/virtualNetworks/dander-phase6/subnets/container-apps" +) def _launcher() -> dict[str, object]: @@ -71,6 +75,7 @@ def _execute(bootstrap: AzureTerraformBootstrap, **overrides: object) -> Path: f"/subscriptions/{_SUBSCRIPTION_ID}/resourceGroups/dander-phase6/" "providers/microsoft.insights/actionGroups/dander-phase6" ), + "infrastructure_subnet_id": _SUBNET_ID, "name": "dander", } arguments.update(overrides) @@ -121,6 +126,7 @@ def fake_run( "reference": ("azure-kv://https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn"), } assert "postgresql://" not in projection_argument + assert f"-var=infrastructure_subnet_id={_SUBNET_ID}" in terraform_plan def test_azure_apply_uses_only_the_saved_plan( @@ -192,6 +198,7 @@ def fake_run( "secret_env": {"API_TOKEN": "source-api-token"}, } }, + infrastructure_subnet_id=None, ) terraform_plan = calls[1] @@ -217,6 +224,7 @@ def fake_run( ({"pipelines": {}}, "at least one pipeline"), ({"alert_target": "not-an-id"}, "alert target"), ({"key_vault_allowed_ip_rule": "0.0.0.0/0"}, "Key Vault allowed IP"), + ({"infrastructure_subnet_id": None}, "infrastructure subnet"), ], ) def test_azure_bootstrap_rejects_unsafe_inputs_before_terraform( diff --git a/tests/providers/test_azure_deployment_verification.py b/tests/providers/test_azure_deployment_verification.py index be838a29..370018eb 100644 --- a/tests/providers/test_azure_deployment_verification.py +++ b/tests/providers/test_azure_deployment_verification.py @@ -174,7 +174,18 @@ def _payloads() -> dict[tuple[str, ...], dict[str, object]]: "properties": { "vaultUri": "https://dander-phase6-kv.vault.azure.net/", "enableRbacAuthorization": True, - "networkAcls": {"defaultAction": "Deny"}, + "networkAcls": { + "bypass": "None", + "defaultAction": "Deny", + "virtualNetworkRules": [ + { + "id": ( + f"{_ROOT}/providers/Microsoft.Network/virtualNetworks/" + "dander-phase6/subnets/container-apps" + ) + } + ], + }, }, }, } @@ -270,6 +281,7 @@ def test_verifier_checks_gcp_federation_and_runtime_secret_references() -> None: ("trigger", "trigger"), ("registry_admin", "artifact contract"), ("vault_rbac", "RBAC and network"), + ("vault_network", "RBAC and network"), ], ) def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: @@ -303,7 +315,7 @@ def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: "dander-phase6", ) payloads[acr_key]["adminUserEnabled"] = True - else: + elif mutation == "vault_rbac": vault_key = ( "keyvault", "show", @@ -313,6 +325,16 @@ def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: "dander-phase6", ) payloads[vault_key]["properties"]["enableRbacAuthorization"] = False # type: ignore[index] + else: + vault_key = ( + "keyvault", + "show", + "--name", + "dander-phase6-kv", + "--resource-group", + "dander-phase6", + ) + payloads[vault_key]["properties"]["networkAcls"]["virtualNetworkRules"] = [] # type: ignore[index] with pytest.raises(AzureDeploymentVerificationError, match=message): AzureDeploymentVerifier(binding, runner=_Runner(payloads)).verify(expected_image=_IMAGE) diff --git a/tickets/DANDER-110-azure-image-lifecycle-operations.md b/tickets/DANDER-110-azure-image-lifecycle-operations.md index 0949bb5a..90db001f 100644 --- a/tickets/DANDER-110-azure-image-lifecycle-operations.md +++ b/tickets/DANDER-110-azure-image-lifecycle-operations.md @@ -1,6 +1,6 @@ # DANDER-110 — Promote and operate Azure Container Apps runtimes -Status: implemented locally; protected CI and live proof pending +Status: merged through protected PR #190; live proof pending ## Requirement @@ -25,7 +25,7 @@ logs, stop, and replay operations while keeping all live mutations behind explic runtime's persisted inclusive cursor. - [x] Image copy, start, stop, and replay require interactive confirmation in the CLI. - [x] Focused tests use mocked command runners and make no Azure call. -- [ ] Protected CI passes and this ticket merges before any live ACR copy or job execution. +- [x] Protected CI passed and this ticket merged before any live ACR copy or job execution. ## Evidence boundary diff --git a/tickets/DANDER-112-azure-secret-operator.md b/tickets/DANDER-112-azure-secret-operator.md index 5c643934..e6e7e8d6 100644 --- a/tickets/DANDER-112-azure-secret-operator.md +++ b/tickets/DANDER-112-azure-secret-operator.md @@ -1,7 +1,7 @@ --- id: DANDER-112 title: Bind Azure proof-secret rotation to one operator -status: in_progress +status: done phase: 6 --- @@ -20,7 +20,7 @@ Terraform. vault. - [x] The runtime identity remains limited to `Key Vault Secrets User` for the same vault. - [x] Provider-mocked tests distinguish the operator and runtime principals and assert both roles. -- [ ] Protected CI passes and the focused correction merges before any Azure apply. +- [x] Protected CI passed and the focused correction merged as PR #192 before any Azure apply. ## Boundary diff --git a/tickets/DANDER-113-azure-key-vault-network.md b/tickets/DANDER-113-azure-key-vault-network.md new file mode 100644 index 00000000..511bdd52 --- /dev/null +++ b/tickets/DANDER-113-azure-key-vault-network.md @@ -0,0 +1,32 @@ +--- +id: DANDER-113 +title: Give Azure jobs an explicit Key Vault network path +status: in_progress +phase: 6 +--- + +# DANDER-113 — Give Azure jobs an explicit Key Vault network path + +## Goal + +Correct the pre-live assumption that Container Apps can use Key Vault's trusted-service firewall +bypass. Keep the vault default-deny while admitting the exact network path used by the named Azure +profile. + +## Acceptance + +- [x] Key Vault references fail planning when no Container Apps infrastructure subnet is supplied. +- [x] The vault admits the exact supplied subnet and the reviewed operator IP. +- [x] The inapplicable Azure trusted-service bypass is disabled. +- [x] Read-only deployment verification rejects an Azure Key Vault profile with no virtual-network + rule. +- [x] Provider-mocked Terraform and focused Python tests cover the correction without an Azure + mutation. +- [ ] Protected CI passes and the focused correction merges before any Azure plan is applied. + +## Boundary + +The supplied subnet must already be delegated for Container Apps and have the +`Microsoft.KeyVault` service endpoint. Dander does not create or modify that subnet in this slice. +This ticket registers no provider, creates no resource, writes no secret, starts no job, and spends +no cloud credit. From 77db67797d2aa1f7111cf4cb54b151bf7805f8de Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 13:49:14 -0400 Subject: [PATCH 127/163] Add canonical Azure acceptance preflight --- CHANGELOG.md | 3 + HANDOFF.md | 36 ++-- ...-portability-azure-lifecycle-acceptance.md | 52 +++++ docs/decisions.md | 12 ++ docs/known-limitations.md | 4 +- infra/azure/README.md | 6 + src/dander/cli/azure_command.py | 63 ++++++ .../azure_container_apps/__init__.py | 2 + .../azure_container_apps/verification.py | 81 +++++++- tests/cli/test_azure_operations_cli.py | 183 ++++++++++++++++++ .../test_azure_deployment_verification.py | 96 ++++++++- tickets/DANDER-113-azure-key-vault-network.md | 5 +- .../DANDER-114-azure-canonical-preflight.md | 32 +++ 13 files changed, 543 insertions(+), 32 deletions(-) create mode 100644 docs/cloud-portability-azure-lifecycle-acceptance.md create mode 100644 tickets/DANDER-114-azure-canonical-preflight.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d1bf912c..a97453fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ capabilities enter through the next minor release. ### Added +- Add a read-only canonical Azure profile preflight that requires the exact + Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault composition, verifies deployed resources, and + checks only manifest-declared Key Vault names and enabled state without reading values. - Bind Azure proof-secret creation and rotation to the exact signed-in Terraform operator with vault-scoped `Key Vault Secrets Officer`, while preserving the runtime identity's read-only `Key Vault Secrets User` role. diff --git a/HANDOFF.md b/HANDOFF.md index 39adbb5f..97c99e28 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Confirmed protected main is green through merged PR #192. -- Corrected the false assumption that Container Apps can bypass a Key Vault firewall. -- Required Azure Key Vault profiles to supply one exact Container Apps infrastructure subnet. -- Kept the vault default-deny and admitted only that subnet plus the reviewed operator IP. -- Added fail-closed Terraform, verifier, and focused test coverage without contacting Azure. +- Merged the Key Vault network-path correction as protected PR #193. +- Added a read-only preflight for the exact Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault profile. +- Required all manifest-declared Key Vault secrets to exist and be enabled without reading values. +- Excluded unrelated vault entries from sanitized output and kept other compositions fail-closed. +- Documented the ordered live acceptance and cleanup protocol without authorizing it. ## Try It -Run `uv run pytest tests/bootstrap/test_azure_terraform.py tests/providers/test_azure_deployment_verification.py` and `terraform -chdir=infra/azure test -no-color`. +Run `uv run pytest tests/providers/test_azure_deployment_verification.py tests/cli/test_azure_operations_cli.py`. ## Checks -- Protected main CI is green through merged PR #192. -- Repository-wide Ruff, strict typing, and Python tests pass; Terraform format/validate and both Terraform test roots pass. -- No provider registration, resource creation, secret write, image copy, or paid operation ran. +- Protected PR #193 and its post-merge main CI are fully green. +- Repository-wide Ruff, strict typing, and the full Python test suite pass. +- No provider registration, resource creation, secret read/write, image copy, job, or paid operation ran. ## Decisions -- Managed identity grants permission but does not bypass Key Vault network controls. -- Key Vault profiles require an existing delegated subnet with the `Microsoft.KeyVault` service endpoint. -- The GCP-secret federation profile may still use Azure's managed network because it does not read this vault. +- Canonical preflight accepts only the named Phase 6 profile. +- Key Vault metadata verification lists base identifiers and enabled state but never values. +- Rotation-version and runtime-use proof remain under later explicit cost approval. ## Remaining -- Run local checks and merge this focused correction through protected CI. -- Prepare the named Azure Snowflake/PostgreSQL/Key-Vault acceptance preflight separately. +- Finish checks and merge this preflight through protected CI. - Obtain candidate-publication approval before publishing or copying an image. -- Obtain explicit Azure, GCP, and Snowflake ceilings before live mutations. -- Run approved lifecycle, cleanup, rollback, and no-drift acceptance. +- Recommend and obtain explicit Azure, Snowflake, and retained-GCP ceilings. +- Run the approved live lifecycle, rotation, rollback, cleanup, and no-drift proof. +- Perform the final independent completion review and Phase 6 gate reassessment. ## Review First -- `infra/azure/modules/container-apps-jobs/main.tf` -- `src/dander/bootstrap/azure_terraform.py` - `src/dander/providers/azure_container_apps/verification.py` +- `src/dander/cli/azure_command.py` +- `docs/cloud-portability-azure-lifecycle-acceptance.md` diff --git a/docs/cloud-portability-azure-lifecycle-acceptance.md b/docs/cloud-portability-azure-lifecycle-acceptance.md new file mode 100644 index 00000000..159097a5 --- /dev/null +++ b/docs/cloud-portability-azure-lifecycle-acceptance.md @@ -0,0 +1,52 @@ +# Azure Container Apps lifecycle acceptance + +Status: protocol prepared; no live Azure resource or paid proof has run. + +This protocol qualifies only the named Azure Container Apps, Snowflake OAuth warehouse, +PostgreSQL state, no-catalog, and Azure Key Vault composition. The selected pipeline must bind both +credential environment names to declared Key Vault secret identifiers. It does not qualify +Azure-to-BigQuery or any other cloud/warehouse pair. + +## Approval boundary + +Before provider registration or any write, record an accepted source-free candidate digest and +explicit per-provider ceilings for Azure, Snowflake, and any retained-GCP verification. Approval +must cover the specific provider registrations, disposable subnet, saved-plan applies, image copy, +secret writes, and job executions. There are no automatic paid reruns. + +Credentials, secret values, warehouse rows, Terraform state, and binary plans stay outside the +repository. Saved plans and state use the secured operator artifact directories described in +`infra/azure/README.md`. + +## Ordered proof + +1. Confirm the signed-in subscription, reviewed region, current operator IP, candidate digest, and + approved ceilings. Confirm the existing delegated Container Apps subnet exposes the + `Microsoft.KeyVault` service endpoint. +2. Review and apply the exact stage-zero and platform plans. Populate only the manifest-declared + PostgreSQL and Snowflake credentials outside Terraform. +3. Run the read-only gate: + + ```bash + dander azure canonical-preflight \ + --deployment azure_snowflake \ + --pipeline warehouse_fixture \ + --expected-image danderphase6.azurecr.io/dander/runtime@sha256:DIGEST + ``` + +4. Require initial execution, exact replay, overlapping-start fencing, interruption/cancellation, + paused and UTC-scheduled behavior, retry exhaustion, bounded logs, and alert routing to match + the platform contract. +5. Create a new version of one declared Key Vault secret outside Terraform. Because the job uses a + versionless URI, allow Azure's documented refresh window, then start exactly one approved new + execution and require the rotated credential to work. Do not automatically rerun on failure. +6. Exercise immutable-image rollback and restoration without rebuilding either image. +7. Remove disposable jobs, vault, logging, network, registry, identity, state, and warehouse proof + objects in dependency order. Verify provider-owned cleanup and retained-GCP no drift. + +## Sanitized evidence + +Retain only commit and digest identities, normalized states and counts, pass/fail booleans, +timestamps, approved ceilings, cleanup results, and no-drift outcome. The read-only preflight may +retain declared secret names and enabled booleans; it never retains values or unrelated vault +entries. Live evidence remains pending until the full approved sequence passes. diff --git a/docs/decisions.md b/docs/decisions.md index 540416ee..3fdb8029 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1222,3 +1222,15 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Dander does not create or modify the existing delegated subnet; it must already expose the `Microsoft.KeyVault` service endpoint. The GCP-secret federation profile remains able to use Azure's managed network because its jobs do not read the deployment vault. + +## 2026-08-11 — Canonical Azure preflight reads secret metadata, never values + +- **Profile:** The preflight accepts only Azure Container Apps, Snowflake OAuth warehouse, + PostgreSQL state, no external catalog, and Azure Key Vault, with both credential environment + names bound by the pipeline. Other shapes fail before provider binding. +- **Proof:** Normal deployment verification runs first. A separate Key Vault list then requires + each manifest-declared base secret identifier to exist and be enabled; unrelated vault entries + are omitted from output. +- **Boundary:** Key Vault's list operation exposes base identifiers and attributes, not values or + individual versions. Version rotation, runtime use, lifecycle behavior, and warehouse + correctness remain live acceptance steps under explicit provider ceilings. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 5efba28e..66ce513b 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -50,7 +50,9 @@ remain Phase 8 work. plan-first Terraform, digest-preserving ACR promotion tooling, provider-native lifecycle operations, bounded Log Analytics reads, deployment verification, and a locally validated Azure-to-Google managed-identity adapter with disposable federation Terraform and refresh-probe - tooling. The named Key Vault profile requires an existing delegated Container Apps subnet with + tooling. A separate read-only preflight rejects any composition other than the named + Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault profile and verifies only declared secret names + and enabled state. The named Key Vault profile requires an existing delegated Container Apps subnet with the `Microsoft.KeyVault` service endpoint; Dander admits that exact subnet and the reviewed operator IP while disabling the inapplicable Azure trusted-service bypass. Dander does not create or modify the subnet. Azure schedules are diff --git a/infra/azure/README.md b/infra/azure/README.md index 8b5c54e0..22c41d27 100644 --- a/infra/azure/README.md +++ b/infra/azure/README.md @@ -112,6 +112,12 @@ same immutable job template; Dander's persisted inclusive cursor remains respons idempotent logical replay. Log reads query only the selected execution and enforce an explicit row limit. +Before any approved execution of the named Snowflake/PostgreSQL profile, run `dander azure +canonical-preflight` with the accepted ACR digest. It rejects other compositions, requires +Snowflake OAuth and PostgreSQL DSN secret bindings, performs normal deployment verification, and +lists only the declared Key Vault base identifiers and enabled state. The full ordered protocol is +in `docs/cloud-portability-azure-lifecycle-acceptance.md`. + ## Boundaries - ACR administrator credentials and Storage shared keys remain disabled; state network access diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index 4e6b6f5d..03c9cab3 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -237,6 +237,69 @@ def azure_verify( console.print_json(data=verification.as_dict()) +@azure_app.command("canonical-preflight") +def azure_canonical_preflight( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + expected_image: str = typer.Option(..., "--expected-image"), + config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 + name: str = typer.Option("dander", "--name"), +) -> None: + """Verify the named Azure/Snowflake/PostgreSQL/Key Vault profile read-only.""" + try: + manifest = load_project_config(config, deployment=deployment) + if ( + manifest.launcher_provider != "azure_container_apps" + or manifest.warehouse_provider != "snowflake" + or manifest.state_provider != "postgresql" + or manifest.catalog_provider != "none" + or manifest.secret_provider != "azure_key_vault" + ): + raise ProjectConfigError( + "Canonical preflight requires the named Azure/Snowflake/PostgreSQL/" + "no-catalog/Key-Vault profile" + ) + snowflake_auth = manifest.warehouse_config.get("auth") + postgres_dsn_env = manifest.state_config.get("dsn_env") + if ( + not isinstance(snowflake_auth, dict) + or snowflake_auth.get("method") != "oauth" + or not isinstance(token_env := snowflake_auth.get("token_env"), str) + or not isinstance(postgres_dsn_env, str) + ): + raise ProjectConfigError( + "Canonical preflight requires Snowflake OAuth and PostgreSQL DSN references" + ) + try: + secret_environment = set(manifest.pipelines[pipeline].secrets) + except KeyError as error: + raise ProjectConfigError( + f"Pipeline {pipeline!r} is not declared in the project manifest" + ) from error + if not {token_env, postgres_dsn_env}.issubset(secret_environment): + raise ProjectConfigError( + "Canonical preflight requires pipeline bindings for Snowflake OAuth and " + "PostgreSQL DSN secrets" + ) + binding = AzureDeploymentBinding.from_project( + config=config, + deployment=deployment, + pipeline_id=pipeline, + name=name, + ) + verifier = AzureDeploymentVerifier(binding) + verification = verifier.verify(expected_image=expected_image) + secret_metadata = verifier.verify_declared_secret_metadata() + except (AzureDeploymentVerificationError, ProjectConfigError) as error: + raise ClickException(str(error)) from error + console.print_json( + data={ + "deployment": verification.as_dict(), + "declared_secrets": [metadata.as_dict() for metadata in secret_metadata], + } + ) + + def init_azure_admin_plan( subscription_id: str = typer.Option(..., "--subscription-id"), location: str = typer.Option("eastus", "--location"), diff --git a/src/dander/providers/azure_container_apps/__init__.py b/src/dander/providers/azure_container_apps/__init__.py index 383246ef..069bb724 100644 --- a/src/dander/providers/azure_container_apps/__init__.py +++ b/src/dander/providers/azure_container_apps/__init__.py @@ -12,6 +12,7 @@ AzureDeploymentVerification, AzureDeploymentVerificationError, AzureDeploymentVerifier, + AzureSecretMetadata, ) __all__ = [ @@ -24,4 +25,5 @@ "AzureDeploymentVerification", "AzureDeploymentVerificationError", "AzureDeploymentVerifier", + "AzureSecretMetadata", ] diff --git a/src/dander/providers/azure_container_apps/verification.py b/src/dander/providers/azure_container_apps/verification.py index 41e557ba..a00735cd 100644 --- a/src/dander/providers/azure_container_apps/verification.py +++ b/src/dander/providers/azure_container_apps/verification.py @@ -219,6 +219,17 @@ def as_dict(self) -> dict[str, str]: return asdict(self) +@dataclass(frozen=True, slots=True) +class AzureSecretMetadata: + """Sanitized metadata for one manifest-declared Key Vault secret.""" + + name: str + enabled: bool + + def as_dict(self) -> dict[str, str | bool]: + return asdict(self) + + class AzureDeploymentVerifier: """Verify one manifest-bound Container Apps Job without reading secret values.""" @@ -445,7 +456,72 @@ def verify(self, *, expected_image: str) -> AzureDeploymentVerification: log_analytics_workspace=workspace_id, ) + def verify_declared_secret_metadata(self) -> tuple[AzureSecretMetadata, ...]: + """Verify declared Key Vault secret names and enabled state without reading values.""" + if self.binding.secret_provider != "azure_key_vault": + raise AzureDeploymentVerificationError( + "Secret metadata verification requires Azure Key Vault" + ) + payload = self._json_list( + "keyvault", + "secret", + "list", + "--vault-name", + self.binding.key_vault_name, + "--maxresults", + "25", + "--query", + "[].{id:id,enabled:attributes.enabled}", + ) + prefix = f"{self.binding.key_vault_uri}/secrets/" + available: dict[str, bool] = {} + for item in payload: + if not isinstance(item, dict): + raise AzureDeploymentVerificationError( + "Azure CLI returned invalid Key Vault secret metadata" + ) + identifier = item.get("id") + enabled = item.get("enabled") + if ( + not isinstance(identifier, str) + or not identifier.startswith(prefix) + or "/" in (secret_name := identifier.removeprefix(prefix)) + or not secret_name + or not isinstance(enabled, bool) + or secret_name in available + ): + raise AzureDeploymentVerificationError( + "Azure CLI returned invalid Key Vault secret metadata" + ) + available[secret_name] = enabled + declared = set(self.binding.secret_ids) + missing = declared.difference(available) + disabled = {secret_name for secret_name in declared if available.get(secret_name) is False} + if missing or disabled: + raise AzureDeploymentVerificationError( + "Declared Key Vault secrets are missing or disabled" + ) + return tuple( + AzureSecretMetadata(name=secret_name, enabled=True) for secret_name in sorted(declared) + ) + def _json(self, *args: str) -> dict[str, object]: + payload = self._json_payload(*args) + if not isinstance(payload, dict): + raise AzureDeploymentVerificationError( + "Azure CLI returned an invalid verification response" + ) + return payload + + def _json_list(self, *args: str) -> list[object]: + payload = self._json_payload(*args) + if not isinstance(payload, list): + raise AzureDeploymentVerificationError( + "Azure CLI returned an invalid verification response" + ) + return payload + + def _json_payload(self, *args: str) -> object: command = ( "az", *args, @@ -475,10 +551,6 @@ def _json(self, *args: str) -> dict[str, object]: payload = json.loads(result.stdout) except json.JSONDecodeError as error: raise AzureDeploymentVerificationError("Azure CLI returned invalid JSON") from error - if not isinstance(payload, dict): - raise AzureDeploymentVerificationError( - "Azure CLI returned an invalid verification response" - ) return payload @@ -516,4 +588,5 @@ def _job_name(*, name: str, pipeline_id: str) -> str: "AzureDeploymentVerification", "AzureDeploymentVerificationError", "AzureDeploymentVerifier", + "AzureSecretMetadata", ] diff --git a/tests/cli/test_azure_operations_cli.py b/tests/cli/test_azure_operations_cli.py index 173cd697..db5ed285 100644 --- a/tests/cli/test_azure_operations_cli.py +++ b/tests/cli/test_azure_operations_cli.py @@ -13,6 +13,8 @@ from dander.providers.azure_container_apps import ( AzureContainerAppsExecution, AzureContainerAppsLogEvent, + AzureDeploymentVerification, + AzureSecretMetadata, ) if TYPE_CHECKING: @@ -215,6 +217,187 @@ def test_identity_refresh_probe_requires_named_profile_and_confirmation( ] +def test_canonical_preflight_requires_exact_profile_and_emits_sanitized_metadata( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[tuple[str, object]] = [] + binding = object() + + class _Binding: + @classmethod + def from_project(cls, **kwargs: object) -> object: + calls.append(("binding", kwargs)) + return binding + + class _Verifier: + def __init__(self, selected: object) -> None: + assert selected is binding + + def verify(self, *, expected_image: str) -> AzureDeploymentVerification: + calls.append(("verify", expected_image)) + return AzureDeploymentVerification( + subscription="11111111-1111-4111-8111-111111111111", + resource_group="dander-phase6", + environment="environment-id", + job="job-id", + trigger_type="manual", + image=expected_image, + registry="registry-id", + managed_identity="identity-id", + key_vault="vault-id", + log_analytics_workspace="workspace-id", + ) + + def verify_declared_secret_metadata(self) -> tuple[AzureSecretMetadata, ...]: + calls.append(("secret-metadata", None)) + return (AzureSecretMetadata(name="postgres-dsn", enabled=True),) + + monkeypatch.setattr( + azure_command, + "load_project_config", + lambda *_args, **_kwargs: SimpleNamespace( + launcher_provider="azure_container_apps", + warehouse_provider="snowflake", + state_provider="postgresql", + catalog_provider="none", + secret_provider="azure_key_vault", + warehouse_config={ + "auth": { + "method": "oauth", + "token_env": "DANDER_SNOWFLAKE_OAUTH_TOKEN", + } + }, + state_config={"dsn_env": "DANDER_POSTGRES_DSN"}, + pipelines={ + "warehouse_fixture": SimpleNamespace( + secrets={ + "DANDER_POSTGRES_DSN": "postgres-dsn", + "DANDER_SNOWFLAKE_OAUTH_TOKEN": "snowflake-oauth-token", + } + ) + }, + ), + ) + monkeypatch.setattr(azure_command, "AzureDeploymentBinding", _Binding) + monkeypatch.setattr(azure_command, "AzureDeploymentVerifier", _Verifier) + image = "danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64 + + result = CliRunner().invoke( + app, + [ + "azure", + "canonical-preflight", + *_base_args(tmp_path), + "--expected-image", + image, + ], + ) + + assert result.exit_code == 0, result.output + assert '"name": "postgres-dsn"' in result.output + assert '"enabled": true' in result.output + assert "value" not in result.output + assert calls == [ + ( + "binding", + { + "config": tmp_path / "dander.yaml", + "deployment": "azure_snowflake", + "pipeline_id": "warehouse_fixture", + "name": "dander", + }, + ), + ("verify", image), + ("secret-metadata", None), + ] + + +def test_canonical_preflight_rejects_other_compositions_before_provider_access( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr( + azure_command, + "load_project_config", + lambda *_args, **_kwargs: SimpleNamespace( + launcher_provider="azure_container_apps", + warehouse_provider="bigquery", + state_provider="bigquery", + catalog_provider="dataplex", + secret_provider="gcp_secret_manager", + ), + ) + + class _ForbiddenBinding: + @classmethod + def from_project(cls, **_kwargs: object) -> object: + pytest.fail("Provider binding must not be resolved") + + monkeypatch.setattr(azure_command, "AzureDeploymentBinding", _ForbiddenBinding) + + result = CliRunner().invoke( + app, + [ + "azure", + "canonical-preflight", + *_base_args(tmp_path), + "--expected-image", + "danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64, + ], + ) + + assert result.exit_code == 1 + assert result.exception is not None + assert "requires the named Azure/Snowflake/PostgreSQL" in str(result.exception) + + +def test_canonical_preflight_rejects_non_oauth_snowflake_before_provider_access( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr( + azure_command, + "load_project_config", + lambda *_args, **_kwargs: SimpleNamespace( + launcher_provider="azure_container_apps", + warehouse_provider="snowflake", + state_provider="postgresql", + catalog_provider="none", + secret_provider="azure_key_vault", + warehouse_config={ + "auth": { + "method": "key_pair", + "private_key_file_env": "DANDER_SNOWFLAKE_PRIVATE_KEY_FILE", + } + }, + state_config={"dsn_env": "DANDER_POSTGRES_DSN"}, + pipelines={}, + ), + ) + + class _ForbiddenBinding: + @classmethod + def from_project(cls, **_kwargs: object) -> object: + pytest.fail("Provider binding must not be resolved") + + monkeypatch.setattr(azure_command, "AzureDeploymentBinding", _ForbiddenBinding) + result = CliRunner().invoke( + app, + [ + "azure", + "canonical-preflight", + *_base_args(tmp_path), + "--expected-image", + "danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64, + ], + ) + + assert result.exit_code == 1 + assert result.exception is not None + assert "requires Snowflake OAuth" in str(result.exception) + + def test_image_promotion_requires_confirmation_and_passes_only_typed_inputs( monkeypatch: MonkeyPatch, tmp_path: Path, diff --git a/tests/providers/test_azure_deployment_verification.py b/tests/providers/test_azure_deployment_verification.py index 370018eb..d88f85ab 100644 --- a/tests/providers/test_azure_deployment_verification.py +++ b/tests/providers/test_azure_deployment_verification.py @@ -26,7 +26,7 @@ class _Runner: - def __init__(self, payloads: dict[tuple[str, ...], dict[str, object]]) -> None: + def __init__(self, payloads: dict[tuple[str, ...], object]) -> None: self.payloads = payloads self.commands: list[tuple[str, ...]] = [] @@ -76,7 +76,7 @@ def _binding() -> AzureDeploymentBinding: ) -def _payloads() -> dict[tuple[str, ...], dict[str, object]]: +def _payloads() -> dict[tuple[str, ...], object]: binding = _binding() job_id = f"{_ROOT}/providers/Microsoft.App/jobs/{binding.job_name}" return { @@ -191,6 +191,15 @@ def _payloads() -> dict[tuple[str, ...], dict[str, object]]: } +def _dict_payload( + payloads: dict[tuple[str, ...], object], + key: tuple[str, ...], +) -> dict[str, object]: + payload = payloads[key] + assert isinstance(payload, dict) + return payload + + def test_verifier_checks_exact_resources_without_reading_secret_values() -> None: runner = _Runner(_payloads()) @@ -206,6 +215,79 @@ def test_verifier_checks_exact_resources_without_reading_secret_values() -> None assert all("--only-show-errors" in command for command in runner.commands) +def test_verifier_checks_only_declared_secret_metadata_without_values() -> None: + payloads = _payloads() + payloads[ + ( + "keyvault", + "secret", + "list", + "--vault-name", + "dander-phase6-kv", + "--maxresults", + "25", + "--query", + "[].{id:id,enabled:attributes.enabled}", + ) + ] = [ + { + "id": "https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn", + "enabled": True, + }, + { + "id": "https://dander-phase6-kv.vault.azure.net/secrets/unrelated-secret", + "enabled": True, + }, + ] + runner = _Runner(payloads) + + result = AzureDeploymentVerifier(_binding(), runner=runner).verify_declared_secret_metadata() + + assert [metadata.as_dict() for metadata in result] == [ + {"name": "postgres-dsn", "enabled": True} + ] + flattened = " ".join(" ".join(command) for command in runner.commands) + assert "secret list" in flattened + assert "secret show" not in flattened + assert "value" not in flattened + + +@pytest.mark.parametrize( + "metadata", + [ + [], + [ + { + "id": "https://dander-phase6-kv.vault.azure.net/secrets/postgres-dsn", + "enabled": False, + } + ], + ], +) +def test_verifier_rejects_missing_or_disabled_declared_secret_metadata( + metadata: list[dict[str, object]], +) -> None: + payloads = _payloads() + payloads[ + ( + "keyvault", + "secret", + "list", + "--vault-name", + "dander-phase6-kv", + "--maxresults", + "25", + "--query", + "[].{id:id,enabled:attributes.enabled}", + ) + ] = metadata + + with pytest.raises(AzureDeploymentVerificationError, match="missing or disabled"): + AzureDeploymentVerifier( + _binding(), runner=_Runner(payloads) + ).verify_declared_secret_metadata() + + def test_verifier_checks_gcp_federation_and_runtime_secret_references() -> None: binding = replace( _binding(), @@ -230,7 +312,7 @@ def test_verifier_checks_gcp_federation_and_runtime_secret_references() -> None: "--resource-group", "dander-phase6", ) - properties = payloads[job_key]["properties"] + properties = _dict_payload(payloads, job_key)["properties"] assert isinstance(properties, dict) configuration = properties["configuration"] template = properties["template"] @@ -296,7 +378,7 @@ def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: "--resource-group", "dander-phase6", ) - job = payloads[job_key] + job = _dict_payload(payloads, job_key) properties = job["properties"] assert isinstance(properties, dict) if mutation == "image": @@ -314,7 +396,7 @@ def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: "--resource-group", "dander-phase6", ) - payloads[acr_key]["adminUserEnabled"] = True + _dict_payload(payloads, acr_key)["adminUserEnabled"] = True elif mutation == "vault_rbac": vault_key = ( "keyvault", @@ -324,7 +406,7 @@ def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: "--resource-group", "dander-phase6", ) - payloads[vault_key]["properties"]["enableRbacAuthorization"] = False # type: ignore[index] + _dict_payload(payloads, vault_key)["properties"]["enableRbacAuthorization"] = False # type: ignore[index] else: vault_key = ( "keyvault", @@ -334,7 +416,7 @@ def test_verifier_fails_closed_on_drift(mutation: str, message: str) -> None: "--resource-group", "dander-phase6", ) - payloads[vault_key]["properties"]["networkAcls"]["virtualNetworkRules"] = [] # type: ignore[index] + _dict_payload(payloads, vault_key)["properties"]["networkAcls"]["virtualNetworkRules"] = [] # type: ignore[index] with pytest.raises(AzureDeploymentVerificationError, match=message): AzureDeploymentVerifier(binding, runner=_Runner(payloads)).verify(expected_image=_IMAGE) diff --git a/tickets/DANDER-113-azure-key-vault-network.md b/tickets/DANDER-113-azure-key-vault-network.md index 511bdd52..362b17b4 100644 --- a/tickets/DANDER-113-azure-key-vault-network.md +++ b/tickets/DANDER-113-azure-key-vault-network.md @@ -1,7 +1,7 @@ --- id: DANDER-113 title: Give Azure jobs an explicit Key Vault network path -status: in_progress +status: done phase: 6 --- @@ -22,7 +22,8 @@ profile. rule. - [x] Provider-mocked Terraform and focused Python tests cover the correction without an Azure mutation. -- [ ] Protected CI passes and the focused correction merges before any Azure plan is applied. +- [x] Protected CI passed and the focused correction merged as PR #193 before any Azure plan was + applied. ## Boundary diff --git a/tickets/DANDER-114-azure-canonical-preflight.md b/tickets/DANDER-114-azure-canonical-preflight.md new file mode 100644 index 00000000..f232b41e --- /dev/null +++ b/tickets/DANDER-114-azure-canonical-preflight.md @@ -0,0 +1,32 @@ +--- +id: DANDER-114 +title: Preflight the canonical Azure acceptance profile +status: in_progress +phase: 6 +--- + +# DANDER-114 — Preflight the canonical Azure acceptance profile + +## Goal + +Give the operator one read-only, fail-closed command that verifies the exact named Phase 6 +Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault composition before any job execution. + +## Acceptance + +- [x] Other platform compositions, Snowflake key-pair auth, and missing OAuth/DSN secret bindings + fail before provider binding. +- [x] Existing deployment verification checks the exact immutable image and Azure resources first. +- [x] A separate Key Vault list requires every manifest-declared secret name to exist and be + enabled. +- [x] The command does not read values, expose unrelated vault entries, mutate Azure, or ask for a + paid-operation confirmation. +- [x] Focused provider and CLI tests cover success, missing/disabled secrets, composition rejection, + and sanitized output. +- [ ] Protected CI passes and this focused preflight merges before live Azure acceptance. + +## Boundary + +This is local construction for a later read-only provider check. It does not register a provider, +create a resource, read a secret value, copy an image, start a job, query a warehouse, or authorize +spending. Version rotation and runtime use remain live acceptance work. From 41de097d06987bdd6e55516dd844f0f1b4d3059e Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 14:25:04 -0400 Subject: [PATCH 128/163] Correct Azure candidate sequencing --- HANDOFF.md | 35 +++++++++--------- ...-portability-azure-lifecycle-acceptance.md | 36 ++++++++++++------- docs/decisions.md | 10 ++++++ 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 97c99e28..c5cf445e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,37 @@ ## Finished -- Merged the Key Vault network-path correction as protected PR #193. -- Added a read-only preflight for the exact Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault profile. -- Required all manifest-declared Key Vault secrets to exist and be enabled without reading values. -- Excluded unrelated vault entries from sanitized output and kept other compositions fail-closed. -- Documented the ordered live acceptance and cleanup protocol without authorizing it. +- Corrected the Phase 6 live-proof order so stage zero supplies the real managed-identity client ID + before the exact source-free candidate is built. +- Preserved the requirement that cost, publication, provider-registration, apply, secret-write, and + execution approval precede every cloud write. +- Kept candidate publication ahead of the platform plan, platform apply, and every job execution. ## Try It -Run `uv run pytest tests/providers/test_azure_deployment_verification.py tests/cli/test_azure_operations_cli.py`. +Review the ordered proof in `docs/cloud-portability-azure-lifecycle-acceptance.md`. ## Checks -- Protected PR #193 and its post-merge main CI are fully green. -- Repository-wide Ruff, strict typing, and the full Python test suite pass. -- No provider registration, resource creation, secret read/write, image copy, job, or paid operation ran. +- Protected main at `eeaccf36` is fully green before this documentation-only correction. +- The correction changes no runtime, Terraform, provider, or credential behavior. +- No provider registration, resource creation, secret write, image publication, or paid operation ran. ## Decisions -- Canonical preflight accepts only the named Phase 6 profile. -- Key Vault metadata verification lists base identifiers and enabled state but never values. -- Rotation-version and runtime-use proof remain under later explicit cost approval. +- Azure assigns the user-managed identity client ID, so the exact candidate follows stage zero. +- The accepted candidate remains mandatory before platform planning or execution. +- Repeat attempts use the same approved per-attempt ceilings and cleanup boundary. ## Remaining -- Finish checks and merge this preflight through protected CI. -- Obtain candidate-publication approval before publishing or copying an image. -- Recommend and obtain explicit Azure, Snowflake, and retained-GCP ceilings. -- Run the approved live lifecycle, rotation, rollback, cleanup, and no-drift proof. +- Merge this ordering correction through protected CI. +- Register the approved Azure providers and review/apply stage zero. +- Publish and copy the exact protected-main candidate under the approved ceilings. +- Run the approved live lifecycle, federation, rotation, rollback, cleanup, and no-drift proof. - Perform the final independent completion review and Phase 6 gate reassessment. ## Review First -- `src/dander/providers/azure_container_apps/verification.py` -- `src/dander/cli/azure_command.py` - `docs/cloud-portability-azure-lifecycle-acceptance.md` +- `docs/decisions.md` diff --git a/docs/cloud-portability-azure-lifecycle-acceptance.md b/docs/cloud-portability-azure-lifecycle-acceptance.md index 159097a5..18c9df6f 100644 --- a/docs/cloud-portability-azure-lifecycle-acceptance.md +++ b/docs/cloud-portability-azure-lifecycle-acceptance.md @@ -9,10 +9,16 @@ Azure-to-BigQuery or any other cloud/warehouse pair. ## Approval boundary -Before provider registration or any write, record an accepted source-free candidate digest and -explicit per-provider ceilings for Azure, Snowflake, and any retained-GCP verification. Approval -must cover the specific provider registrations, disposable subnet, saved-plan applies, image copy, -secret writes, and job executions. There are no automatic paid reruns. +Before provider registration or any write, record explicit per-provider ceilings for Azure, +Snowflake, and any retained-GCP verification plus approval to publish the source-free candidate. +Approval must cover the specific provider registrations, disposable subnet, saved-plan applies, +image publication and copy, secret writes, and job executions. + +Stage zero creates the user-assigned managed identity whose Azure-assigned client ID is required by +the validated launcher profile. Therefore the exact source-free candidate is generated and +published after the reviewed stage-zero apply, using that real non-secret client ID, but before the +platform plan or apply and before any job execution. Record its accepted digest at that point. +Approved repeat attempts remain bounded by the same per-attempt ceilings. Credentials, secret values, warehouse rows, Terraform state, and binary plans stay outside the repository. Saved plans and state use the secured operator artifact directories described in @@ -20,12 +26,16 @@ repository. Saved plans and state use the secured operator artifact directories ## Ordered proof -1. Confirm the signed-in subscription, reviewed region, current operator IP, candidate digest, and - approved ceilings. Confirm the existing delegated Container Apps subnet exposes the - `Microsoft.KeyVault` service endpoint. -2. Review and apply the exact stage-zero and platform plans. Populate only the manifest-declared +1. Confirm the signed-in subscription, reviewed region, current operator IP, approved ceilings, + publication approval, and provider-registration approval. +2. Register only the required providers, then review and apply the exact stage-zero plan. Read the + Azure-assigned managed-identity client ID from the stage-zero output. +3. Generate the source-free project with that client ID, publish one candidate to staging GAR, + record its accepted digest, and copy the same OCI index into ACR without rebuilding. +4. Create the approved disposable Container Apps subnet, require its `Microsoft.KeyVault` service + endpoint, then review and apply the exact platform plan. Populate only the manifest-declared PostgreSQL and Snowflake credentials outside Terraform. -3. Run the read-only gate: +5. Run the read-only gate: ```bash dander azure canonical-preflight \ @@ -34,14 +44,14 @@ repository. Saved plans and state use the secured operator artifact directories --expected-image danderphase6.azurecr.io/dander/runtime@sha256:DIGEST ``` -4. Require initial execution, exact replay, overlapping-start fencing, interruption/cancellation, +6. Require initial execution, exact replay, overlapping-start fencing, interruption/cancellation, paused and UTC-scheduled behavior, retry exhaustion, bounded logs, and alert routing to match the platform contract. -5. Create a new version of one declared Key Vault secret outside Terraform. Because the job uses a +7. Create a new version of one declared Key Vault secret outside Terraform. Because the job uses a versionless URI, allow Azure's documented refresh window, then start exactly one approved new execution and require the rotated credential to work. Do not automatically rerun on failure. -6. Exercise immutable-image rollback and restoration without rebuilding either image. -7. Remove disposable jobs, vault, logging, network, registry, identity, state, and warehouse proof +8. Exercise immutable-image rollback and restoration without rebuilding either image. +9. Remove disposable jobs, vault, logging, network, registry, identity, state, and warehouse proof objects in dependency order. Verify provider-owned cleanup and retained-GCP no drift. ## Sanitized evidence diff --git a/docs/decisions.md b/docs/decisions.md index 3fdb8029..cd55c843 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1234,3 +1234,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Key Vault's list operation exposes base identifiers and attributes, not values or individual versions. Version rotation, runtime use, lifecycle behavior, and warehouse correctness remain live acceptance steps under explicit provider ceilings. + +## 2026-08-11 — Azure stage zero precedes the exact source-free candidate + +- **Dependency:** Azure assigns the user-assigned managed-identity client ID during stage zero, + while the validated Azure launcher profile requires that real non-secret identifier. +- **Order:** Cost and mutation approval still precede every write. The reviewed stage-zero apply + creates the identity; only then is the exact source-free candidate generated, published to + staging GAR, and copied unchanged into ACR. +- **Boundary:** Candidate publication remains mandatory before the platform plan or apply and every + job execution. This ordering change does not relax digest identity, plan review, or cleanup. From 97b5996b0959f288f4fc1ca3dfb202b08ad202ac Mon Sep 17 00:00:00 2001 From: harrisonoconnorhover <130776285+harrisonoconnorhover@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:47:25 -0400 Subject: [PATCH 129/163] fix(azure): retry stage-zero state RBAC propagation (#196) Co-authored-by: Harrison O'Connor-Hoover --- HANDOFF.md | 35 ++++++------- docs/decisions.md | 11 +++++ infra/azure/README.md | 5 +- src/dander/bootstrap/azure_admin.py | 47 +++++++++++++++++- tests/bootstrap/test_azure_admin.py | 77 +++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 19 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index c5cf445e..67b5ae7c 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,38 @@ ## Finished -- Corrected the Phase 6 live-proof order so stage zero supplies the real managed-identity client ID - before the exact source-free candidate is built. -- Preserved the requirement that cost, publication, provider-registration, apply, secret-write, and - execution approval precede every cloud write. -- Kept candidate publication ahead of the platform plan, platform apply, and every job execution. +- Reproduced the first live Azure stage-zero state-migration propagation delay after all six + reviewed resources were created. +- Recovered with a reviewed zero-change plan and migrated the intact local state successfully. +- Added a bounded retry for only Azure's `AuthorizationPermissionMismatch` propagation response. ## Try It -Review the ordered proof in `docs/cloud-portability-azure-lifecycle-acceptance.md`. +Run `uv run pytest tests/bootstrap/test_azure_admin.py`. ## Checks -- Protected main at `eeaccf36` is fully green before this documentation-only correction. -- The correction changes no runtime, Terraform, provider, or credential behavior. -- No provider registration, resource creation, secret write, image publication, or paid operation ran. +- Protected main at `a67e7005` was fully green before the live attempt. +- The recovered plan reported six no-op resources and zero mutations before remote-state migration. +- `uv run pytest`: 1,286 passed and 28 skipped. +- Ruff lint/format and exact-CI-environment mypy: passed. +- Protected CI remains to run for this correction. ## Decisions -- Azure assigns the user-managed identity client ID, so the exact candidate follows stage zero. -- The accepted candidate remains mandatory before platform planning or execution. -- Repeat attempts use the same approved per-attempt ceilings and cleanup boundary. +- Retry only the provider's explicit authorization-propagation response for at most one minute. +- Fail unrelated backend errors immediately and retain the local backend on retry exhaustion. ## Remaining -- Merge this ordering correction through protected CI. -- Register the approved Azure providers and review/apply stage zero. +- Run focused checks and merge this correction through protected CI. - Publish and copy the exact protected-main candidate under the approved ceilings. +- Create the disposable network and zero-cost PostgreSQL state profile. +- Prepare bounded Snowflake OAuth and apply the reviewed platform plan. - Run the approved live lifecycle, federation, rotation, rollback, cleanup, and no-drift proof. -- Perform the final independent completion review and Phase 6 gate reassessment. ## Review First -- `docs/cloud-portability-azure-lifecycle-acceptance.md` -- `docs/decisions.md` +- `src/dander/bootstrap/azure_admin.py` +- `tests/bootstrap/test_azure_admin.py` +- `infra/azure/README.md` diff --git a/docs/decisions.md b/docs/decisions.md index cd55c843..55ad9783 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1244,3 +1244,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc staging GAR, and copied unchanged into ACR. - **Boundary:** Candidate publication remains mandatory before the platform plan or apply and every job execution. This ordering change does not relax digest identity, plan review, or cleanup. + +## 2026-08-11 — Azure state migration absorbs only RBAC propagation delay + +- **Finding:** The first live stage-zero apply created its six reviewed resources, then Azure + temporarily rejected the Entra-authenticated backend while the new Blob role assignment + propagated. A fresh no-op plan migrated the intact local state after access became usable. +- **Decision:** Initial state migration retries only Terraform failures containing Azure's + `AuthorizationPermissionMismatch` response, for at most one bounded minute. Other failures still + fail immediately, and exhaustion restores the local backend declaration. +- **Boundary:** The retry creates no resources, does not reapply the saved plan, and does not weaken + the default-deny firewall, Entra authentication, plan review, or cleanup requirements. diff --git a/infra/azure/README.md b/infra/azure/README.md index 22c41d27..567a9d45 100644 --- a/infra/azure/README.md +++ b/infra/azure/README.md @@ -32,7 +32,10 @@ this exact-IP firewall entry with `/32` semantics. After separate approval for provider registration and resource costs, review the saved plan and run the matching `init-azure-admin-apply` command. The apply migrates its initial local state into the firewall-restricted, private, versioned Azure Storage container. It grants the authenticated -operator only the Blob data role needed to use that Entra-authenticated backend. Record +operator only the Blob data role needed to use that Entra-authenticated backend. Because a new +Azure role assignment can take time to become usable, the migration retries only the specific +`AuthorizationPermissionMismatch` propagation response for one bounded minute; unrelated backend +errors fail immediately and preserve the local state. Record `runtime_identity_client_id` from the printed Terraform output command and place that non-secret identifier in the named launcher configuration. diff --git a/src/dander/bootstrap/azure_admin.py b/src/dander/bootstrap/azure_admin.py index b46099e4..4dcde3d2 100644 --- a/src/dander/bootstrap/azure_admin.py +++ b/src/dander/bootstrap/azure_admin.py @@ -7,6 +7,8 @@ import re import shutil import subprocess +import sys +import time from ipaddress import IPv4Address, ip_address from typing import TYPE_CHECKING from uuid import UUID @@ -21,6 +23,9 @@ _ACR_NAME = re.compile(r"^[a-z][a-z0-9]{4,49}$") _STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") _BACKEND_SCHEMA = "io.dander.azure-bootstrap-backend/v1" +_STATE_MIGRATION_ATTEMPTS = 7 +_STATE_MIGRATION_RETRY_SECONDS = 10 +_AZURE_AUTHORIZATION_PROPAGATION_MARKER = "AuthorizationPermissionMismatch" class AzureAdministrativeBootstrapError(RuntimeError): @@ -186,7 +191,7 @@ def _migrate_state( state_key=state_key, ) try: - self._run("terraform", "init", "-migrate-state", "-force-copy", "-input=false") + self._migrate_state_with_authorization_retry() except AzureAdministrativeBootstrapError: self._write_backend("local", {"path": str(self._local_state_path)}) raise @@ -202,6 +207,46 @@ def _migrate_state( }, ) + def _migrate_state_with_authorization_retry(self) -> None: + args = ("terraform", "init", "-migrate-state", "-force-copy", "-input=false") + environment = os.environ.copy() + environment["TF_DATA_DIR"] = str(self._tf_data_dir) + for attempt in range(1, _STATE_MIGRATION_ATTEMPTS + 1): + try: + completed = subprocess.run( + args, + cwd=self._workspace, + check=False, + env=environment, + umask=0o077, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise AzureAdministrativeBootstrapError( + "Required command not found: terraform" + ) from error + if completed.stdout: + sys.stdout.write(completed.stdout) + if completed.stderr: + sys.stderr.write(completed.stderr) + if completed.returncode == 0: + return + output = f"{completed.stdout or ''}\n{completed.stderr or ''}" + can_retry = ( + _AZURE_AUTHORIZATION_PROPAGATION_MARKER in output + and attempt < _STATE_MIGRATION_ATTEMPTS + ) + if not can_retry: + raise AzureAdministrativeBootstrapError( + f"terraform init failed with exit code {completed.returncode}" + ) + sys.stderr.write( + "Azure state role assignment has not propagated; retrying remote-state " + f"migration ({attempt}/{_STATE_MIGRATION_ATTEMPTS - 1}).\n" + ) + time.sleep(_STATE_MIGRATION_RETRY_SECONDS) + def _prepare_operator_directories(self, *, preserve_plan: bool = False) -> None: try: for path, label in ( diff --git a/tests/bootstrap/test_azure_admin.py b/tests/bootstrap/test_azure_admin.py index a676fcc0..82dfc20a 100644 --- a/tests/bootstrap/test_azure_admin.py +++ b/tests/bootstrap/test_azure_admin.py @@ -136,6 +136,83 @@ def fake_run( } +def test_azure_admin_retries_state_migration_during_role_propagation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + migration_attempts = 0 + sleeps: list[int] = [] + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + nonlocal migration_attempts + del cwd, kwargs + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + if args[1:3] == ("init", "-migrate-state"): + migration_attempts += 1 + if migration_attempts < 3: + return subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="AuthorizationPermissionMismatch", + ) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr("dander.bootstrap.azure_admin.time.sleep", sleeps.append) + bootstrap = AzureAdministrativeBootstrap(infra_dir, operator_dir) + bootstrap.execute(**_arguments()) + + bootstrap.apply_saved_plan(**_arguments()) + + assert migration_attempts == 3 + assert sleeps == [10, 10] + assert (operator_dir / "backend.json").is_file() + + +def test_azure_admin_does_not_retry_unrelated_state_migration_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + infra_dir, operator_dir = _layout(tmp_path) + migration_attempts = 0 + + def fake_run( + args: tuple[str, ...], *, cwd: Path, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + nonlocal migration_attempts + del cwd, kwargs + for argument in args: + if argument.startswith("-out="): + Path(argument.removeprefix("-out=")).touch() + if args[1:3] == ("init", "-migrate-state"): + migration_attempts += 1 + return subprocess.CompletedProcess(args, 1, stdout="", stderr="invalid backend") + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr( + "dander.bootstrap.azure_admin.time.sleep", + lambda _: pytest.fail("Unrelated failures must not be retried"), + ) + bootstrap = AzureAdministrativeBootstrap(infra_dir, operator_dir) + bootstrap.execute(**_arguments()) + + with pytest.raises(AzureAdministrativeBootstrapError, match="terraform init failed"): + bootstrap.apply_saved_plan(**_arguments()) + + assert migration_attempts == 1 + backend = json.loads( + (operator_dir / "terraform-workspace" / "backend.tf.json").read_text(encoding="utf-8") + ) + assert "local" in backend["terraform"]["backend"] + + def test_azure_admin_reuses_only_the_recorded_backend( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From 8a8e3bcdf52ec164cc3a607a8f14d52dc9e3c3be Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 15:30:07 -0400 Subject: [PATCH 130/163] fix(azure): pin consumption workload profile --- infra/azure/modules/container-apps-jobs/main.tf | 7 +++++++ .../modules/container-apps-jobs/tests/jobs.tftest.hcl | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/infra/azure/modules/container-apps-jobs/main.tf b/infra/azure/modules/container-apps-jobs/main.tf index be590020..c3f97c7e 100644 --- a/infra/azure/modules/container-apps-jobs/main.tf +++ b/infra/azure/modules/container-apps-jobs/main.tf @@ -97,6 +97,13 @@ resource "azurerm_container_app_environment" "runtime" { internal_load_balancer_enabled = var.infrastructure_subnet_id == null ? null : true zone_redundancy_enabled = var.infrastructure_subnet_id == null ? null : false tags = local.tags + + workload_profile { + name = "Consumption" + workload_profile_type = "Consumption" + minimum_count = 0 + maximum_count = 0 + } } resource "azurerm_key_vault" "runtime" { diff --git a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl index d7321953..fdb51549 100644 --- a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl +++ b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl @@ -99,6 +99,17 @@ run "projects_exact_job_contract" { error_message = "Paused pipelines must remain manual-only." } + assert { + condition = ( + length(azurerm_container_app_environment.runtime.workload_profile) == 1 && + one(azurerm_container_app_environment.runtime.workload_profile).name == "Consumption" && + one(azurerm_container_app_environment.runtime.workload_profile).workload_profile_type == "Consumption" && + one(azurerm_container_app_environment.runtime.workload_profile).minimum_count == 0 && + one(azurerm_container_app_environment.runtime.workload_profile).maximum_count == 0 + ) + error_message = "Container Apps must declare Azure's default Consumption profile to avoid post-create drift." + } + assert { condition = azurerm_monitor_metric_alert.failed_execution["warehouse_fixture"].criteria[0].metric_name == "Executions" error_message = "A projected alert target must receive failed-execution monitoring." From eff4bde8931d3750699362c19423cd1de33e1122 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 15:54:53 -0400 Subject: [PATCH 131/163] fix(azure): pin job consumption profile --- infra/azure/modules/container-apps-jobs/main.tf | 1 + .../azure/modules/container-apps-jobs/tests/jobs.tftest.hcl | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/azure/modules/container-apps-jobs/main.tf b/infra/azure/modules/container-apps-jobs/main.tf index c3f97c7e..a62c128d 100644 --- a/infra/azure/modules/container-apps-jobs/main.tf +++ b/infra/azure/modules/container-apps-jobs/main.tf @@ -153,6 +153,7 @@ resource "azurerm_container_app_job" "pipeline" { location = var.location resource_group_name = var.resource_group_name container_app_environment_id = azurerm_container_app_environment.runtime.id + workload_profile_name = "Consumption" replica_timeout_in_seconds = each.value.resources.deadline_seconds replica_retry_limit = each.value.resources.launcher_retry_count tags = merge(local.tags, each.value.labels, { pipeline = each.key }) diff --git a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl index fdb51549..b3a0a31a 100644 --- a/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl +++ b/infra/azure/modules/container-apps-jobs/tests/jobs.tftest.hcl @@ -67,9 +67,10 @@ run "projects_exact_job_contract" { assert { condition = ( azurerm_container_app_job.pipeline["warehouse_fixture"].replica_timeout_in_seconds == 900 && - azurerm_container_app_job.pipeline["warehouse_fixture"].replica_retry_limit == 1 + azurerm_container_app_job.pipeline["warehouse_fixture"].replica_retry_limit == 1 && + azurerm_container_app_job.pipeline["warehouse_fixture"].workload_profile_name == "Consumption" ) - error_message = "Container Apps must preserve the projected deadline and launcher retry count." + error_message = "Container Apps must preserve the projected deadline, launcher retry count, and Consumption workload profile." } assert { From 5f5be4768879d45fedfb944cd1213940baf93f77 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 16:05:39 -0400 Subject: [PATCH 132/163] fix(azure): stage key vault job bootstrap --- ...-portability-azure-lifecycle-acceptance.md | 8 +-- docs/decisions.md | 13 +++++ infra/azure/README.md | 9 +++- infra/azure/main.tf | 2 +- infra/azure/tests/azure.tftest.hcl | 13 +++++ infra/azure/variables.tf | 6 +++ src/dander/bootstrap/azure_terraform.py | 52 ++++++++++++++++++- src/dander/cli/azure_command.py | 14 +++++ tests/bootstrap/test_azure_terraform.py | 43 +++++++++++++++ 9 files changed, 154 insertions(+), 6 deletions(-) diff --git a/docs/cloud-portability-azure-lifecycle-acceptance.md b/docs/cloud-portability-azure-lifecycle-acceptance.md index 18c9df6f..1edd0b26 100644 --- a/docs/cloud-portability-azure-lifecycle-acceptance.md +++ b/docs/cloud-portability-azure-lifecycle-acceptance.md @@ -32,9 +32,11 @@ repository. Saved plans and state use the secured operator artifact directories Azure-assigned managed-identity client ID from the stage-zero output. 3. Generate the source-free project with that client ID, publish one candidate to staging GAR, record its accepted digest, and copy the same OCI index into ACR without rebuilding. -4. Create the approved disposable Container Apps subnet, require its `Microsoft.KeyVault` service - endpoint, then review and apply the exact platform plan. Populate only the manifest-declared - PostgreSQL and Snowflake credentials outside Terraform. +4. Create the approved disposable Container Apps subnet and require its `Microsoft.KeyVault` + service endpoint. Review and apply the exact `--foundation-only` platform plan, which creates + the environment, vault, and scoped roles but no job or alert. Populate only the + manifest-declared PostgreSQL and Snowflake credentials outside Terraform. Then review and apply + the normal platform plan, which may create jobs only after their Key Vault references exist. 5. Run the read-only gate: ```bash diff --git a/docs/decisions.md b/docs/decisions.md index 55ad9783..5c5c5bd5 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1255,3 +1255,16 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc fail immediately, and exhaustion restores the local backend declaration. - **Boundary:** The retry creates no resources, does not reapply the saved plan, and does not weaken the default-deny firewall, Entra authentication, plan review, or cleanup requirements. + +## 2026-08-11 — Key Vault-backed Azure jobs use a two-plan bootstrap + +- **Finding:** Azure rejects a Container Apps Job whose versionless Key Vault references do not + exist, but the operator cannot seed those values until the new vault and its scoped operator role + have been created. One all-at-once platform apply therefore fails on a fresh deployment. +- **Decision:** The first reviewed `--foundation-only` plan creates the environment, vault, and + scoped roles while omitting jobs and alerts. The operator then seeds only manifest-declared + secrets outside Terraform; a second normal reviewed plan creates the jobs and alerts. Foundation + mode rejects and discards any plan that would update or delete an existing resource. +- **Boundary:** Secret values never enter Terraform, state, plans, shell history, or evidence. The + mode changes only creation order and does not weaken preflight, immutable-image, approval, or + cleanup requirements. diff --git a/infra/azure/README.md b/infra/azure/README.md index 567a9d45..4f5fa6f4 100644 --- a/infra/azure/README.md +++ b/infra/azure/README.md @@ -72,6 +72,13 @@ dander init-azure-plan \ --container-image danderphase6.azurecr.io/dander/runtime@sha256:DIGEST ``` +For a new Key Vault-backed deployment, add `--foundation-only` to the first reviewed plan. Apply +that saved plan to create the environment, vault, and scoped roles without any job or alert. After +the operator role is usable, create only the manifest-declared secret values outside Terraform, +then run and review the same plan command again without `--foundation-only`. The second saved plan +creates the jobs and alerts only after their versionless Key Vault references exist. Foundation +mode rejects and discards any plan that would update or delete an existing resource. + This creates a saved plan only. An optional existing delegated subnet can be selected with `--infrastructure-subnet-id`; selecting it makes the environment internal. The subnet is required when the profile uses Azure Key Vault references and must already have the `Microsoft.KeyVault` @@ -85,7 +92,7 @@ Vault defaults to deny, admits that exact operator IP for secret administration, exact Container Apps subnet for managed-identity secret references. The plan does not create or modify that existing subnet. -Run `dander init-azure-apply` only after reviewing the exact saved plan and receiving explicit +Run `dander init-azure-apply` only after reviewing each exact saved plan and receiving explicit approval for the live Azure changes. Then use `dander azure verify` for read-only checks of the subscription, environment, logs, job trigger/resources/image, managed identity, ACR, and Key Vault. The plan grants the exact signed-in Terraform operator `Key Vault Secrets Officer` so that person diff --git a/infra/azure/main.tf b/infra/azure/main.tf index 78f7bc8e..b1cdd80f 100644 --- a/infra/azure/main.tf +++ b/infra/azure/main.tf @@ -38,7 +38,7 @@ module "container_apps_jobs" { managed_identity_id = data.azurerm_user_assigned_identity.runtime.id managed_identity_client_id = data.azurerm_user_assigned_identity.runtime.client_id managed_identity_principal_id = data.azurerm_user_assigned_identity.runtime.principal_id - execution_projections = var.execution_projections + execution_projections = var.create_jobs ? var.execution_projections : {} infrastructure_subnet_id = var.infrastructure_subnet_id tags = var.tags } diff --git a/infra/azure/tests/azure.tftest.hcl b/infra/azure/tests/azure.tftest.hcl index 8d6e9722..c06e669f 100644 --- a/infra/azure/tests/azure.tftest.hcl +++ b/infra/azure/tests/azure.tftest.hcl @@ -87,3 +87,16 @@ run "wires_stage_zero_resources_into_jobs" { error_message = "The Azure root must bind secret administration to the authenticated operator." } } + +run "foundation_omits_jobs_until_secrets_are_seeded" { + command = plan + + variables { + create_jobs = false + } + + assert { + condition = length(output.jobs) == 0 + error_message = "The Azure foundation plan must omit jobs that reference not-yet-seeded secrets." + } +} diff --git a/infra/azure/variables.tf b/infra/azure/variables.tf index 2aac46f6..70e852e8 100644 --- a/infra/azure/variables.tf +++ b/infra/azure/variables.tf @@ -73,6 +73,12 @@ variable "execution_projections" { type = any } +variable "create_jobs" { + description = "Create jobs and alerts after their referenced Key Vault secrets have been seeded." + type = bool + default = true +} + variable "infrastructure_subnet_id" { description = "Optional existing delegated subnet for the Container Apps environment; required for Key Vault references and configured with the Microsoft.KeyVault service endpoint." type = string diff --git a/src/dander/bootstrap/azure_terraform.py b/src/dander/bootstrap/azure_terraform.py index 3331ad5c..8c126b52 100644 --- a/src/dander/bootstrap/azure_terraform.py +++ b/src/dander/bootstrap/azure_terraform.py @@ -6,7 +6,7 @@ import re import subprocess from ipaddress import IPv4Address, ip_address -from json import dumps +from json import JSONDecodeError, dumps, loads from typing import TYPE_CHECKING from uuid import UUID @@ -78,6 +78,7 @@ def execute( require_guarded_free_tier: bool, pipelines: Mapping[str, Mapping[str, object]], apply: bool, + foundation_only: bool = False, alert_target: str | None = None, infrastructure_subnet_id: str | None = None, name: str = "dander", @@ -221,6 +222,7 @@ def execute( f"-var=managed_identity_name={identity_name}", "-var=execution_projections=" + dumps(projections, sort_keys=True, separators=(",", ":")), + f"-var=create_jobs={str(not foundation_only).lower()}", "-var=tags=" + dumps( { @@ -236,6 +238,8 @@ def execute( plan_args.insert(-1, f"-var=infrastructure_subnet_id={infrastructure_subnet_id}") self._run(*plan_args) self._secure_plan() + if foundation_only: + self._assert_foundation_plan_is_additive() if apply: self._run("terraform", "apply", "-input=false", self._plan_path.name) return self._plan_path @@ -325,6 +329,52 @@ def _secure_plan(self) -> None: "Could not secure the saved Azure Terraform plan" ) from error + def _assert_foundation_plan_is_additive(self) -> None: + environment = os.environ.copy() + environment["ARM_USE_AZUREAD"] = "true" + try: + completed = subprocess.run( + ("terraform", "show", "-json", self._plan_path.name), + cwd=self._infra_dir, + check=True, + capture_output=True, + text=True, + env=environment, + umask=0o077, + ) + payload = loads(completed.stdout) + resource_changes = payload.get("resource_changes", []) + mutates_existing = any( + any(action in {"update", "delete"} for action in change["change"]["actions"]) + for change in resource_changes + ) + except ( + FileNotFoundError, + subprocess.CalledProcessError, + JSONDecodeError, + AttributeError, + KeyError, + TypeError, + ) as error: + self._discard_plan() + raise AzureTerraformBootstrapError( + "Could not verify that the Azure foundation plan is additive" + ) from error + if mutates_existing: + self._discard_plan() + raise AzureTerraformBootstrapError( + "Azure foundation plan would modify or delete existing resources; use the normal " + "platform plan for an existing deployment" + ) + + def _discard_plan(self) -> None: + try: + self._plan_path.unlink(missing_ok=True) + except OSError as error: + raise AzureTerraformBootstrapError( + "Could not discard an unsafe Azure foundation plan" + ) from error + def _run(self, *args: str) -> None: environment = os.environ.copy() environment["ARM_USE_AZUREAD"] = "true" diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index 03c9cab3..cb7d6448 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -486,6 +486,14 @@ def init_azure_plan( "and expected to have the Microsoft.KeyVault service endpoint." ), ), + foundation_only: bool = typer.Option( + False, + "--foundation-only", + help=( + "Plan the environment, vault, and RBAC without jobs so referenced secrets can be " + "seeded before the normal job plan." + ), + ), name: str = typer.Option("dander", "--name"), infra_dir: Path = typer.Option(_DEFAULT_AZURE_INFRA_DIR, hidden=True), # noqa: B008 ) -> None: @@ -526,6 +534,7 @@ def init_azure_plan( require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, pipelines=manifest.terraform_pipelines(), apply=False, + foundation_only=foundation_only, alert_target=alert_target, infrastructure_subnet_id=infrastructure_subnet_id, name=name, @@ -541,6 +550,11 @@ def init_azure_plan( soft_wrap=True, ) console.print("No Azure resources were changed. Apply requires separate explicit approval.") + if foundation_only: + console.print( + "This foundation plan omits jobs and alerts. After applying it, seed only the " + "manifest-declared Key Vault secrets, then plan again without --foundation-only." + ) subscription_id = str(manifest.resolved_launcher_config()["subscription_id"]) console.print( "Next after review and approval: " diff --git a/tests/bootstrap/test_azure_terraform.py b/tests/bootstrap/test_azure_terraform.py index 5a79e254..30846b49 100644 --- a/tests/bootstrap/test_azure_terraform.py +++ b/tests/bootstrap/test_azure_terraform.py @@ -156,6 +156,49 @@ def fake_run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedPro assert all(call[:2] != ("terraform", "plan") for call in calls) +def test_azure_foundation_plan_omits_jobs_until_secrets_are_seeded( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[tuple[str, ...]] = [] + + def fake_run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + del kwargs + calls.append(args) + for argument in args: + if argument.startswith("-out="): + (tmp_path / argument.removeprefix("-out=")).touch() + stdout = json.dumps({"resource_changes": [{"change": {"actions": ["create"]}}]}) + return subprocess.CompletedProcess(args, 0, stdout=stdout) + + monkeypatch.setattr(subprocess, "run", fake_run) + _execute(AzureTerraformBootstrap(tmp_path), foundation_only=True) + + terraform_plan = calls[1] + assert "-var=create_jobs=false" in terraform_plan + assert calls[2] == ("terraform", "show", "-json", "dander-azure.tfplan") + + +def test_azure_foundation_plan_rejects_existing_resource_mutation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + def fake_run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + del kwargs + for argument in args: + if argument.startswith("-out="): + (tmp_path / argument.removeprefix("-out=")).touch() + stdout = json.dumps({"resource_changes": [{"change": {"actions": ["update"]}}]}) + return subprocess.CompletedProcess(args, 0, stdout=stdout) + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(AzureTerraformBootstrapError, match="modify or delete"): + _execute(AzureTerraformBootstrap(tmp_path), foundation_only=True) + + assert not (tmp_path / "dander-azure.tfplan").exists() + + def test_azure_bootstrap_projects_gcp_federation_without_credentials( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From 105dc74308081a26ba070c670285aa718c10aae7 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 16:29:13 -0400 Subject: [PATCH 133/163] fix(runtime): publish source-only metadata --- docs/decisions.md | 11 +++++++++++ src/dander/executor.py | 31 +++++++++++++++++-------------- tests/test_executor.py | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index 5c5c5bd5..37ace882 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1268,3 +1268,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** Secret values never enter Terraform, state, plans, shell history, or evidence. The mode changes only creation order and does not weaken preflight, immutable-image, approval, or cleanup requirements. + +## 2026-08-11 — Explicitly source-only pipelines publish source metadata + +- **Finding:** Provider-backed state includes a durable metadata store, so the executor enters its + metadata stage even when a pipeline deliberately selects no SQL models. Loading a transform + project in that case rejected the otherwise valid source-only pipeline after ingestion committed. +- **Decision:** An explicitly empty model selection with model building disabled publishes a + source-only manifest with zero model assets. Unspecified or non-empty model selections retain the + existing transform-project discovery and validation. +- **Boundary:** This changes no ingestion, warehouse, state, SQL, fencing, or catalog-provider + semantics. It only allows the already-defined source metadata snapshot to represent zero models. diff --git a/src/dander/executor.py b/src/dander/executor.py index 581dd92f..61c490c9 100644 --- a/src/dander/executor.py +++ b/src/dander/executor.py @@ -241,21 +241,24 @@ def execute(self, *, run_id: str | None = None) -> PipelineExecutionResult: models=models, assertions=assertions, ) - transform_project = TransformProject.load( - self._models_dir, - catalog=self._catalog, - raw_namespace=self._raw_namespace, - target_dialect=getattr( - self._transform_runner, - "target_dialect", - SqlDialect.BIGQUERY, - ), - ) spine = MetadataSpine() - compiled_assets = spine.compile( - transform_project, - selected=self._selected_models, - ) + if self._selected_models == () and not self._build_models: + compiled_assets = () + else: + transform_project = TransformProject.load( + self._models_dir, + catalog=self._catalog, + raw_namespace=self._raw_namespace, + target_dialect=getattr( + self._transform_runner, + "target_dialect", + SqlDialect.BIGQUERY, + ), + ) + compiled_assets = spine.compile( + transform_project, + selected=self._selected_models, + ) manifest = spine.pipeline_manifest( pipeline_id=self._pipeline_id, source=self._source_config, diff --git a/tests/test_executor.py b/tests/test_executor.py index 2a120195..9c70fa5a 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -226,6 +226,8 @@ def _executor( raw_namespace: str | None = "raw", source_relations: dict[str, RelationRef] | None = None, catalog_publisher: CatalogPublisher | None = None, + selected_models: Iterable[str] | None = ("stg_widgets",), + build_models: bool = True, ) -> PipelineExecutor: source = SourceConfig( name="example", @@ -250,9 +252,9 @@ def _executor( raw_namespace=raw_namespace, source_relations=source_relations, models_dir=models_dir, - selected_models=("stg_widgets",), - build_models=True, - transform_runner=_Transform(fail=fail_transform), + selected_models=selected_models, + build_models=build_models, + transform_runner=_Transform(fail=fail_transform) if build_models else None, metadata_store=metadata, catalog_publisher=catalog_publisher, leases=leases, @@ -300,6 +302,32 @@ def test_executor_preserves_launcher_supplied_run_id(tmp_path: Path) -> None: assert history.started == ("launcher-run-42", "example", "example_pipeline") +def test_executor_publishes_source_only_metadata_for_explicit_empty_model_selection( + tmp_path: Path, +) -> None: + history = _History() + metadata = _Metadata() + + result = _executor( + tmp_path, + history=history, + metadata=metadata, + selected_models=(), + build_models=False, + ).execute() + + assert history.checkpoints == [RunStage.METADATA] + assert history.finished == (RunStatus.SUCCEEDED, None, 0, 0, 0) + assert result.assets == 0 + assert metadata.manifest is not None + assert metadata.manifest["assets"] == [] + source = metadata.manifest["source"] + assert isinstance(source, dict) + endpoints = source["endpoints"] + assert isinstance(endpoints, list) + assert endpoints[0]["relation"] == "valid-project-123.raw.example_widgets" + + def test_executor_publishes_through_provider_neutral_catalog_boundary(tmp_path: Path) -> None: _models(tmp_path) publisher = _CatalogPublisher() From a89b6da17d66bedb84b8064b6c09f564827be92f Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 17:08:13 -0400 Subject: [PATCH 134/163] Report Azure cancellation as requested state --- HANDOFF.md | 39 +++++++++---------- docs/decisions.md | 4 +- .../azure_container_apps/operations.py | 5 +-- .../test_azure_container_apps_operations.py | 5 ++- ...ER-110-azure-image-lifecycle-operations.md | 2 + 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 67b5ae7c..0d54bca8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,37 @@ ## Finished -- Reproduced the first live Azure stage-zero state-migration propagation delay after all six - reviewed resources were created. -- Recovered with a reviewed zero-change plan and migrated the intact local state successfully. -- Added a bounded retry for only Azure's `AuthorizationPermissionMismatch` propagation response. +- Published and copied one source-free protected-main OCI index byte-identically into ACR. +- Passed the canonical live execution, replay, overlap fencing, and controlled interruption proof. +- Corrected `azure cancel` to report request acknowledgement instead of a false terminal result. ## Try It -Run `uv run pytest tests/bootstrap/test_azure_admin.py`. +Run `uv run pytest tests/providers/test_azure_container_apps_operations.py`. ## Checks -- Protected main at `a67e7005` was fully green before the live attempt. -- The recovered plan reported six no-op resources and zero mutations before remote-state migration. -- `uv run pytest`: 1,286 passed and 28 skipped. -- Ruff lint/format and exact-CI-environment mypy: passed. -- Protected CI remains to run for this correction. +- Protected main at `35758a7` was fully green before candidate publication. +- The image-only apply changed one job and the immediate follow-up plan reported no changes. +- Initial execution and replay succeeded; concurrent starts produced one success and one zero-row skip. +- Controlled stop reconciled the abandoned run as `interrupted_run`; recovery succeeded. +- Focused local checks and protected CI remain to run for the acknowledgement correction. ## Decisions -- Retry only the provider's explicit authorization-propagation response for at most one minute. -- Fail unrelated backend errors immediately and retain the local backend on retry exhaustion. +- Treat Azure stop as a request acknowledgement; use provider status for terminal truth. +- Keep the accepted release digest unchanged because this correction affects only operator output. ## Remaining -- Run focused checks and merge this correction through protected CI. -- Publish and copy the exact protected-main candidate under the approved ceilings. -- Create the disposable network and zero-cost PostgreSQL state profile. -- Prepare bounded Snowflake OAuth and apply the reviewed platform plan. -- Run the approved live lifecycle, federation, rotation, rollback, cleanup, and no-drift proof. +- Merge this focused correction through protected CI. +- Complete schedule, retry, parallelism, alert, rotation, rollback, and federation proofs. +- Verify Snowflake aggregates without retaining rows or credentials. +- Clean up disposable Azure, Snowflake, and federation resources. +- Merge sanitized evidence and finish Azure plus retained-GCP no-drift checks. ## Review First -- `src/dander/bootstrap/azure_admin.py` -- `tests/bootstrap/test_azure_admin.py` -- `infra/azure/README.md` +- `src/dander/providers/azure_container_apps/operations.py` +- `tests/providers/test_azure_container_apps_operations.py` +- `tickets/DANDER-110-azure-image-lifecycle-operations.md` diff --git a/docs/decisions.md b/docs/decisions.md index 37ace882..6d8525e7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1180,7 +1180,9 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc written. - **Lifecycle:** Start, status, stop, replay, and bounded Log Analytics use Azure Container Apps Job execution identities. Foreign execution names, unknown statuses, stopping a terminal run, and - replaying a non-terminal run fail before mutation. + replaying a non-terminal run fail before mutation. A successful stop request reports + `cancellation_requested`; only a later read-only provider status may report terminal + cancellation because Azure can acknowledge a stop before the execution settles. - **Boundary:** Azure assigns execution names, while replay correctness remains owned by Dander's persisted inclusive cursor. Local mocked operations are construction evidence only; live image copy, execution, interruption, cleanup, and no-drift still require explicit approval. diff --git a/src/dander/providers/azure_container_apps/operations.py b/src/dander/providers/azure_container_apps/operations.py index 17ba77fb..24d50ba0 100644 --- a/src/dander/providers/azure_container_apps/operations.py +++ b/src/dander/providers/azure_container_apps/operations.py @@ -267,7 +267,7 @@ def logs( return tuple(events) def cancel(self, execution_name: str) -> AzureContainerAppsExecution: - """Stop one running Container Apps Job execution.""" + """Request a stop for one running Container Apps Job execution.""" execution = self.describe(execution_name) if execution.state != "running": raise AzureContainerAppsOperationError("Only a running execution can be cancelled") @@ -284,9 +284,8 @@ def cancel(self, execution_name: str) -> AzureContainerAppsExecution: ) return AzureContainerAppsExecution( name=execution.name, - state="cancelled", + state="cancellation_requested", started_at=execution.started_at, - failure_code="operator_cancelled", ) def _execute(self, *args: str) -> None: diff --git a/tests/providers/test_azure_container_apps_operations.py b/tests/providers/test_azure_container_apps_operations.py index 6fb2cae3..73a425eb 100644 --- a/tests/providers/test_azure_container_apps_operations.py +++ b/tests/providers/test_azure_container_apps_operations.py @@ -301,8 +301,9 @@ def test_cancel_requires_a_running_owned_execution() -> None: execution = AzureContainerAppsOperations(_binding(), runner=runner).cancel(_EXECUTION) - assert execution.state == "cancelled" - assert execution.failure_code == "operator_cancelled" + assert execution.state == "cancellation_requested" + assert execution.failure_code is None + assert execution.terminal is False def test_replay_requires_terminal_status_then_starts_a_fresh_provider_execution() -> None: diff --git a/tickets/DANDER-110-azure-image-lifecycle-operations.md b/tickets/DANDER-110-azure-image-lifecycle-operations.md index 90db001f..8f1d8e8d 100644 --- a/tickets/DANDER-110-azure-image-lifecycle-operations.md +++ b/tickets/DANDER-110-azure-image-lifecycle-operations.md @@ -23,6 +23,8 @@ logs, stop, and replay operations while keeping all live mutations behind explic - [x] Logs use a bounded Log Analytics query correlated to one validated execution name. - [x] Stop requires a running execution; replay requires a terminal execution and relies on the runtime's persisted inclusive cursor. +- [x] A successful stop request reports `cancellation_requested`, not a synthetic terminal state; + the read-only status command remains authoritative because Azure may settle the execution later. - [x] Image copy, start, stop, and replay require interactive confirmation in the CLI. - [x] Focused tests use mocked command runners and make no Azure call. - [x] Protected CI passed and this ticket merged before any live ACR copy or job execution. From b36d4ea007703e7e0ca566bc611012c947bb938e Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 17:58:15 -0400 Subject: [PATCH 135/163] fix(azure): require explicit GCP project for federation plan --- HANDOFF.md | 32 ++++---- .../phase6/azure-google-federation/README.md | 4 +- docs/decisions.md | 10 +++ src/dander/cli/azure_command.py | 18 +++-- tests/cli/test_init_cli.py | 77 +++++++++++++++++++ tickets/DANDER-111-azure-google-federation.md | 2 + 6 files changed, 119 insertions(+), 24 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 0d54bca8..6c0dc7ae 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,35 @@ ## Finished -- Published and copied one source-free protected-main OCI index byte-identically into ACR. -- Passed the canonical live execution, replay, overlap fencing, and controlled interruption proof. -- Corrected `azure cancel` to report request acknowledgement instead of a false terminal result. +- Added explicit Azure BigQuery `--gcp-project` planning input for version-2 profiles. +- Added focused CLI coverage for the missing-input failure and successful projection. +- Documented why the GCP project remains deployment scope rather than platform configuration. ## Try It -Run `uv run pytest tests/providers/test_azure_container_apps_operations.py`. +Run `uv run pytest tests/cli/test_init_cli.py -k azure_bigquery`. ## Checks -- Protected main at `35758a7` was fully green before candidate publication. -- The image-only apply changed one job and the immediate follow-up plan reported no changes. -- Initial execution and replay succeeded; concurrent starts produced one success and one zero-row skip. -- Controlled stop reconciled the abandoned run as `interrupted_run`; recovery succeeded. -- Focused local checks and protected CI remain to run for the acknowledgement correction. +- `uv run pytest tests/cli/test_init_cli.py -q` passed (16 tests). +- Related Azure Terraform, portable-config, and operations suites passed (35 tests). +- Ruff check/format and mypy passed for the changed Python files. +- Protected CI remains to run. ## Decisions -- Treat Azure stop as a request acknowledgement; use provider status for terminal truth. -- Keep the accepted release digest unchanged because this correction affects only operator output. +- Keep the concrete GCP project explicit at Azure plan time; do not add it to the reusable profile. +- Keep the accepted runtime digest unchanged because this correction affects only operator planning. ## Remaining - Merge this focused correction through protected CI. -- Complete schedule, retry, parallelism, alert, rotation, rollback, and federation proofs. -- Verify Snowflake aggregates without retaining rows or credentials. +- Resume the approved Azure-to-Google live federation plan and refresh proof. - Clean up disposable Azure, Snowflake, and federation resources. -- Merge sanitized evidence and finish Azure plus retained-GCP no-drift checks. +- Merge sanitized Phase 6 evidence and verify retained-GCP no drift. ## Review First -- `src/dander/providers/azure_container_apps/operations.py` -- `tests/providers/test_azure_container_apps_operations.py` -- `tickets/DANDER-110-azure-image-lifecycle-operations.md` +- `src/dander/cli/azure_command.py` +- `tests/cli/test_init_cli.py` +- `acceptance/cloud-portability/phase6/azure-google-federation/README.md` diff --git a/acceptance/cloud-portability/phase6/azure-google-federation/README.md b/acceptance/cloud-portability/phase6/azure-google-federation/README.md index 5e7e5cf5..50706832 100644 --- a/acceptance/cloud-portability/phase6/azure-google-federation/README.md +++ b/acceptance/cloud-portability/phase6/azure-google-federation/README.md @@ -26,7 +26,9 @@ state, variables, and logs in a secured operator directory outside the repositor 3. Apply only that exact saved plan. Put the non-secret outputs `google_workload_identity_audience` and `azure_application_id_uri` into the typed Azure launcher configuration; the service-account output must match each proof pipeline's - `runtime_service_account_id`. + `runtime_service_account_id`. Pass the reviewed data-plane project to `init-azure-plan` through + `--gcp-project`; version 2 keeps that operator/deployment scope out of the reusable platform + profile. 4. Run the same accepted OCI digest through the Azure BigQuery profile. After separate live-cost approval, start exactly one refresh probe (there is no automatic rerun): diff --git a/docs/decisions.md b/docs/decisions.md index 6d8525e7..2717b144 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1281,3 +1281,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc existing transform-project discovery and validation. - **Boundary:** This changes no ingestion, warehouse, state, SQL, fencing, or catalog-provider semantics. It only allows the already-defined source metadata snapshot to represent zero models. + +## 2026-08-11 — Azure BigQuery receives its GCP project as deployment scope + +- **Finding:** Version-2 BigQuery platform profiles intentionally describe warehouse behavior and + location, not the operator's concrete GCP project. Azure planning incorrectly looked for a + project inside that typed profile, so the live federation path failed before Terraform. +- **Decision:** `init-azure-plan` requires `--gcp-project` only when the selected Azure profile uses + BigQuery and passes it directly into the existing typed federation projection. +- **Boundary:** This adds no configuration bag or provider behavior. Snowflake planning is + unchanged, and the project remains an explicit reviewed operator input. diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index cb7d6448..3812565f 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -474,6 +474,11 @@ def init_azure_plan( ..., "--container-image", help="Immutable ACR image ending in @sha256 digest." ), deployment: str = typer.Option(..., "--deployment"), + gcp_project: str | None = typer.Option( + None, + "--gcp-project", + help="GCP data-plane project required by an Azure BigQuery federation profile.", + ), config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 key_vault_allowed_ip_rule: str = typer.Option(..., "--key-vault-allowed-ip"), @@ -511,12 +516,13 @@ def init_azure_plan( ) manifest.validate_references(config.resolve().parent) runtime = manifest.platform.runtime - gcp_project = None + resolved_gcp_project = None if manifest.warehouse_provider == "bigquery": - project_value = manifest.warehouse_config.get("project") - if not isinstance(project_value, str): - raise ProjectConfigError("Azure BigQuery profile has no GCP project") - gcp_project = project_value + if gcp_project is None: + raise ProjectConfigError( + "Azure BigQuery planning requires an explicit --gcp-project" + ) + resolved_gcp_project = gcp_project plan_path = AzureTerraformBootstrap(infra_dir).execute( deployment_name=manifest.deployment_name, state_resource_group_name=state_resource_group_name, @@ -539,7 +545,7 @@ def init_azure_plan( infrastructure_subnet_id=infrastructure_subnet_id, name=name, profile_id=manifest.platform_name, - gcp_project=gcp_project, + gcp_project=resolved_gcp_project, ) except (AzureTerraformBootstrapError, ProjectConfigError) as error: raise ClickException(str(error)) from error diff --git a/tests/cli/test_init_cli.py b/tests/cli/test_init_cli.py index 4b31e529..e375e622 100644 --- a/tests/cli/test_init_cli.py +++ b/tests/cli/test_init_cli.py @@ -257,6 +257,83 @@ def fake_execute(self: object, **kwargs: object) -> Path: assert "init-aws-apply" in result.output +def test_azure_bigquery_plan_requires_explicit_gcp_project( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + manifest = DanderProject( + version=2, + platform=PlatformSpec( + region="eastus", + runtime=PlatformRuntimeSpec(memory="2Gi", timeout_seconds=900), + safety=PlatformSafetySpec(require_guarded_free_tier=False), + ), + pipelines={ + "warehouse_fixture": PipelineSpec( + source="warehouse_fixture", + models=[], + build_models=False, + ) + }, + platform_name="azure_bigquery", + deployment_name="azure_bigquery", + warehouse_provider="bigquery", + warehouse_config={"provider": "bigquery", "location": "US", "dataset": "raw"}, + launcher_provider="azure_container_apps", + launcher_config={ + "provider": "azure_container_apps", + "region": "eastus", + "subscription_id": "11111111-1111-4111-8111-111111111111", + }, + ) + + def fake_load(*args: object, **kwargs: object) -> DanderProject: + assert kwargs["deployment"] == "azure_bigquery" + return manifest + + def fake_validate(self: object, root: Path) -> None: + del self, root + + def fake_execute(self: object, **kwargs: object) -> Path: + captured.update(kwargs) + return tmp_path / "dander-azure.tfplan" + + monkeypatch.setattr("dander.cli.azure_command.load_project_config", fake_load) + monkeypatch.setattr(DanderProject, "validate_references", fake_validate) + monkeypatch.setattr( + "dander.cli.azure_command.AzureTerraformBootstrap.execute", + fake_execute, + ) + base_args = [ + "init-azure-plan", + "--state-resource-group", + "dander-phase6", + "--state-storage-account", + "danderphase6state", + "--container-image", + "danderphase6.azurecr.io/dander@sha256:" + "a" * 64, + "--deployment", + "azure_bigquery", + "--key-vault-allowed-ip", + "203.0.113.10", + ] + + missing = CliRunner().invoke(app, base_args) + planned = CliRunner().invoke( + app, + [*base_args, "--gcp-project", "unit-project"], + ) + + assert missing.exit_code == 1 + assert missing.exception is not None + assert "requires an explicit --gcp-project" in str(missing.exception) + assert planned.exit_code == 0, planned.output + assert captured["gcp_project"] == "unit-project" + assert captured["apply"] is False + assert "Azure deployment planned" in planned.output + + def test_aws_admin_plan_saves_without_applying_and_prints_exact_next_command( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tickets/DANDER-111-azure-google-federation.md b/tickets/DANDER-111-azure-google-federation.md index 580f37d7..348e6406 100644 --- a/tickets/DANDER-111-azure-google-federation.md +++ b/tickets/DANDER-111-azure-google-federation.md @@ -25,6 +25,8 @@ observe Google credential refresh in the same process. Vault secrets. - [x] The same-process refresh probe is bounded, emits no query text, tokens, credentials, or rows, and has no automatic paid rerun. +- [x] Azure BigQuery planning requires the reviewed GCP data-plane project explicitly instead of + looking for deployment scope inside the reusable version-2 platform profile. - [x] Focused Python and provider-mocked Terraform contracts pass locally. - [ ] An accepted public candidate passes live BigQuery access before and after Google credential refresh plus GCP secret access and Dataplex read-back under approved ceilings. From 78948ab5e0d5d8e842e268a2b8bbe2cb92ffabac Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 18:12:49 -0400 Subject: [PATCH 136/163] Route GCP project into Azure live operations --- HANDOFF.md | 21 +++--- src/dander/cli/azure_command.py | 5 ++ .../azure_container_apps/verification.py | 15 +++- tests/cli/test_azure_operations_cli.py | 68 ++++++++++++++++++- .../test_azure_deployment_verification.py | 47 +++++++++++++ 5 files changed, 144 insertions(+), 12 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 6c0dc7ae..be40cfea 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,25 +2,26 @@ ## Finished -- Added explicit Azure BigQuery `--gcp-project` planning input for version-2 profiles. -- Added focused CLI coverage for the missing-input failure and successful projection. -- Documented why the GCP project remains deployment scope rather than platform configuration. +- Routed the identity-refresh probe's explicit GCP project into its Azure deployment binding. +- Added `azure verify --gcp-project` for version-2 Azure/BigQuery deployments. +- Preserved reusable provider profiles without a fixed concrete GCP project. +- Added focused binding and CLI contract coverage. ## Try It -Run `uv run pytest tests/cli/test_init_cli.py -k azure_bigquery`. +Run `uv run pytest tests/cli/test_azure_operations_cli.py tests/providers/test_azure_deployment_verification.py`. ## Checks -- `uv run pytest tests/cli/test_init_cli.py -q` passed (16 tests). -- Related Azure Terraform, portable-config, and operations suites passed (35 tests). +- Focused Azure CLI and deployment-verification tests passed (23 tests). - Ruff check/format and mypy passed for the changed Python files. - Protected CI remains to run. ## Decisions -- Keep the concrete GCP project explicit at Azure plan time; do not add it to the reusable profile. -- Keep the accepted runtime digest unchanged because this correction affects only operator planning. +- Reuse the probe's already-required project input at binding time. +- Keep the concrete GCP project out of reusable version-2 platform profiles. +- Preserve the accepted runtime digest; this is operator binding and verification behavior only. ## Remaining @@ -32,5 +33,5 @@ Run `uv run pytest tests/cli/test_init_cli.py -k azure_bigquery`. ## Review First - `src/dander/cli/azure_command.py` -- `tests/cli/test_init_cli.py` -- `acceptance/cloud-portability/phase6/azure-google-federation/README.md` +- `src/dander/providers/azure_container_apps/verification.py` +- `tests/cli/test_azure_operations_cli.py` diff --git a/src/dander/cli/azure_command.py b/src/dander/cli/azure_command.py index 3812565f..12cbbecd 100644 --- a/src/dander/cli/azure_command.py +++ b/src/dander/cli/azure_command.py @@ -39,12 +39,14 @@ def _azure_operations( deployment: str, pipeline: str, name: str, + gcp_project: str | None = None, ) -> AzureContainerAppsOperations: binding = AzureDeploymentBinding.from_project( config=config, deployment=deployment, pipeline_id=pipeline, name=name, + gcp_project=gcp_project, ) return AzureContainerAppsOperations(binding) @@ -109,6 +111,7 @@ def azure_identity_refresh_probe( deployment=deployment, pipeline=pipeline, name=name, + gcp_project=project, ).start_identity_refresh_probe( project=project, dataset=dataset, @@ -222,6 +225,7 @@ def azure_verify( expected_image: str = typer.Option(..., "--expected-image"), config: Path = typer.Option(_DEFAULT_PROJECT_CONFIG, "--config"), # noqa: B008 name: str = typer.Option("dander", "--name"), + gcp_project: str | None = typer.Option(None, "--gcp-project"), ) -> None: """Verify one deployed Azure pipeline through read-only provider checks.""" try: @@ -230,6 +234,7 @@ def azure_verify( deployment=deployment, pipeline_id=pipeline, name=name, + gcp_project=gcp_project, ) verification = AzureDeploymentVerifier(binding).verify(expected_image=expected_image) except AzureDeploymentVerificationError as error: diff --git a/src/dander/providers/azure_container_apps/verification.py b/src/dander/providers/azure_container_apps/verification.py index a00735cd..3b5ab23d 100644 --- a/src/dander/providers/azure_container_apps/verification.py +++ b/src/dander/providers/azure_container_apps/verification.py @@ -17,6 +17,7 @@ _DEPLOYMENT_NAME = re.compile(r"^[a-z][a-z0-9-]{1,22}[a-z0-9]$") _PIPELINE_ID = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,127}$") +_GCP_PROJECT = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") _ACR_IMAGE = re.compile( r"^(?P[a-z][a-z0-9]{4,49})\.azurecr\.io/" r"[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$" @@ -93,6 +94,7 @@ def from_project( deployment: str, pipeline_id: str, name: str = "dander", + gcp_project: str | None = None, ) -> AzureDeploymentBinding: """Resolve one Azure pipeline from the validated project manifest.""" resolved_config = config.expanduser().resolve() @@ -141,7 +143,18 @@ def from_project( google_audience = None google_application_id_uri = None if manifest.secret_provider == "gcp_secret_manager": - project_value = manifest.warehouse_config.get("project") + configured_project = manifest.warehouse_config.get("project") + if gcp_project is not None and _GCP_PROJECT.fullmatch(gcp_project) is None: + raise AzureDeploymentVerificationError("Invalid GCP project identifier") + if ( + isinstance(configured_project, str) + and gcp_project is not None + and configured_project != gcp_project + ): + raise AzureDeploymentVerificationError( + "Explicit GCP project does not match the project manifest" + ) + project_value = gcp_project or configured_project runtime_id = manifest.terraform_pipelines()[pipeline_id].get( "runtime_service_account_id" ) diff --git a/tests/cli/test_azure_operations_cli.py b/tests/cli/test_azure_operations_cli.py index db5ed285..373f57d5 100644 --- a/tests/cli/test_azure_operations_cli.py +++ b/tests/cli/test_azure_operations_cli.py @@ -169,7 +169,7 @@ def test_identity_refresh_probe_requires_named_profile_and_confirmation( monkeypatch: MonkeyPatch, tmp_path: Path, ) -> None: - operations, _binding = _install_fake(monkeypatch) + operations, binding = _install_fake(monkeypatch) monkeypatch.setattr( azure_command, "load_project_config", @@ -215,6 +215,72 @@ def test_identity_refresh_probe_requires_named_profile_and_confirmation( }, ) ] + assert binding["gcp_project"] == "unit-project" + + +def test_verify_passes_explicit_gcp_project_to_the_binding( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[tuple[str, object]] = [] + binding = object() + + class _Binding: + @classmethod + def from_project(cls, **kwargs: object) -> object: + calls.append(("binding", kwargs)) + return binding + + class _Verifier: + def __init__(self, selected: object) -> None: + assert selected is binding + + def verify(self, *, expected_image: str) -> AzureDeploymentVerification: + calls.append(("verify", expected_image)) + return AzureDeploymentVerification( + subscription="11111111-1111-4111-8111-111111111111", + resource_group="dander-phase6", + environment="environment-id", + job="job-id", + trigger_type="manual", + image=expected_image, + registry="registry-id", + managed_identity="identity-id", + key_vault="vault-id", + log_analytics_workspace="workspace-id", + ) + + monkeypatch.setattr(azure_command, "AzureDeploymentBinding", _Binding) + monkeypatch.setattr(azure_command, "AzureDeploymentVerifier", _Verifier) + image = "danderphase6.azurecr.io/dander/runtime@sha256:" + "a" * 64 + + result = CliRunner().invoke( + app, + [ + "azure", + "verify", + *_base_args(tmp_path), + "--gcp-project", + "unit-project", + "--expected-image", + image, + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [ + ( + "binding", + { + "config": tmp_path / "dander.yaml", + "deployment": "azure_snowflake", + "pipeline_id": "warehouse_fixture", + "name": "dander", + "gcp_project": "unit-project", + }, + ), + ("verify", image), + ] def test_canonical_preflight_requires_exact_profile_and_emits_sanitized_metadata( diff --git a/tests/providers/test_azure_deployment_verification.py b/tests/providers/test_azure_deployment_verification.py index d88f85ab..6076e9f8 100644 --- a/tests/providers/test_azure_deployment_verification.py +++ b/tests/providers/test_azure_deployment_verification.py @@ -6,6 +6,7 @@ import subprocess from dataclasses import replace from pathlib import Path +from types import SimpleNamespace import pytest @@ -355,6 +356,52 @@ def test_verifier_checks_gcp_federation_and_runtime_secret_references() -> None: assert result.image == _IMAGE +def test_binding_uses_explicit_gcp_project_for_version_two_profile( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + manifest = SimpleNamespace( + launcher_provider="azure_container_apps", + secret_provider="gcp_secret_manager", + warehouse_config={"provider": "bigquery", "location": "US"}, + pipelines={"warehouse_fixture": SimpleNamespace(paused=True, secrets={})}, + platform=SimpleNamespace(runtime=SimpleNamespace(timeout_seconds=900, max_retries=0)), + validate_references=lambda _root: None, + resolved_launcher_config=lambda: { + "region": "eastus", + "subscription_id": _SUBSCRIPTION_ID, + "resource_group_name": "dander-phase6", + "container_app_environment_name": "dander-phase6-env", + "acr_name": "danderphase6", + "key_vault_name": "dander-phase6-kv", + "managed_identity_name": "dander-phase6-runtime", + "managed_identity_client_id": _CLIENT_ID, + "google_workload_identity_audience": ( + "//iam.googleapis.com/projects/1009770943166/locations/global/" + "workloadIdentityPools/dander-phase6-azure/providers/container-apps" + ), + "google_application_id_uri": "api://77777777-7777-4777-8777-777777777777", + }, + terraform_pipelines=lambda: { + "warehouse_fixture": {"runtime_service_account_id": "dander-runtime"} + }, + ) + monkeypatch.setattr( + "dander.providers.azure_container_apps.verification.load_project_config", + lambda *_args, **_kwargs: manifest, + ) + + binding = AzureDeploymentBinding.from_project( + config=tmp_path / "dander.yaml", + deployment="azure_bigquery", + pipeline_id="warehouse_fixture", + gcp_project="unit-project", + ) + + assert binding.google_project == "unit-project" + assert binding.google_service_account == "dander-runtime@unit-project.iam.gserviceaccount.com" + + @pytest.mark.parametrize( ("mutation", "message"), [ From bf23e99af4524733f58ce43c30644f80f88992af Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 18:18:12 -0400 Subject: [PATCH 137/163] Accept Azure null empty secret collections --- HANDOFF.md | 19 ++++++++----------- .../azure_container_apps/verification.py | 4 +++- .../test_azure_deployment_verification.py | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index be40cfea..4b1f8e32 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,26 +2,24 @@ ## Finished -- Routed the identity-refresh probe's explicit GCP project into its Azure deployment binding. -- Added `azure verify --gcp-project` for version-2 Azure/BigQuery deployments. -- Preserved reusable provider profiles without a fixed concrete GCP project. -- Added focused binding and CLI contract coverage. +- Normalized Azure's `configuration.secrets: null` response to an empty collection. +- Preserved exact non-empty Key Vault reference verification and failure behavior. +- Added focused coverage using the live provider response shape. ## Try It -Run `uv run pytest tests/cli/test_azure_operations_cli.py tests/providers/test_azure_deployment_verification.py`. +Run `uv run pytest tests/providers/test_azure_deployment_verification.py`. ## Checks -- Focused Azure CLI and deployment-verification tests passed (23 tests). +- Focused Azure deployment-verification tests passed (13 tests). - Ruff check/format and mypy passed for the changed Python files. - Protected CI remains to run. ## Decisions -- Reuse the probe's already-required project input at binding time. -- Keep the concrete GCP project out of reusable version-2 platform profiles. -- Preserve the accepted runtime digest; this is operator binding and verification behavior only. +- Treat provider `null` as empty only at the collection boundary. +- Keep all exact checks unchanged when manifest-declared Key Vault references exist. ## Remaining @@ -32,6 +30,5 @@ Run `uv run pytest tests/cli/test_azure_operations_cli.py tests/providers/test_a ## Review First -- `src/dander/cli/azure_command.py` - `src/dander/providers/azure_container_apps/verification.py` -- `tests/cli/test_azure_operations_cli.py` +- `tests/providers/test_azure_deployment_verification.py` diff --git a/src/dander/providers/azure_container_apps/verification.py b/src/dander/providers/azure_container_apps/verification.py index 3b5ab23d..a557bc6a 100644 --- a/src/dander/providers/azure_container_apps/verification.py +++ b/src/dander/providers/azure_container_apps/verification.py @@ -344,7 +344,9 @@ def verify(self, *, expected_image: str) -> AzureDeploymentVerification: raise AzureDeploymentVerificationError( "Container Apps Job registry identity does not match the manifest" ) - secrets = configuration.get("secrets", []) + secrets = configuration.get("secrets") + if secrets is None: + secrets = [] if not isinstance(secrets, list): raise AzureDeploymentVerificationError( "Container Apps Job returned invalid Key Vault references" diff --git a/tests/providers/test_azure_deployment_verification.py b/tests/providers/test_azure_deployment_verification.py index 6076e9f8..7cde59d8 100644 --- a/tests/providers/test_azure_deployment_verification.py +++ b/tests/providers/test_azure_deployment_verification.py @@ -323,7 +323,7 @@ def test_verifier_checks_gcp_federation_and_runtime_secret_references() -> None: assert isinstance(containers, list) container = containers[0] assert isinstance(container, dict) - configuration["secrets"] = [] + configuration["secrets"] = None container["env"] = [ {"name": "HOME", "value": "/tmp"}, {"name": "AZURE_CLIENT_ID", "value": _CLIENT_ID}, From 3101d7af6f35ad017721156cf75556e32bc96d47 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 18:25:59 -0400 Subject: [PATCH 138/163] Transport Azure runtime arguments safely --- HANDOFF.md | 20 ++++---- .../azure_container_apps/operations.py | 36 +++++++++++-- .../test_azure_container_apps_operations.py | 51 +++++++++++++------ 3 files changed, 79 insertions(+), 28 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 4b1f8e32..59f7df4e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,24 +2,26 @@ ## Finished -- Normalized Azure's `configuration.secrets: null` response to an empty collection. -- Preserved exact non-empty Key Vault reference verification and failure behavior. -- Added focused coverage using the live provider response shape. +- Transported Azure execution overrides through the provider-supported YAML template input. +- Prevented runtime flags such as `--project` from being parsed as Azure CLI flags. +- Kept the override limited to the `runtime` container's bounded argument list. +- Added exact execution-template contract coverage. ## Try It -Run `uv run pytest tests/providers/test_azure_deployment_verification.py`. +Run `uv run pytest tests/providers/test_azure_container_apps_operations.py`. ## Checks -- Focused Azure deployment-verification tests passed (13 tests). +- Focused Azure Container Apps operation tests passed (7 tests). - Ruff check/format and mypy passed for the changed Python files. - Protected CI remains to run. ## Decisions -- Treat provider `null` as empty only at the collection boundary. -- Keep all exact checks unchanged when manifest-declared Key Vault references exist. +- Use Azure's execution-template transport when container arguments begin with dashes. +- Write only the non-secret bounded argument template to a mode-restricted temporary file. +- Delete the temporary template immediately after the Azure CLI call. ## Remaining @@ -30,5 +32,5 @@ Run `uv run pytest tests/providers/test_azure_deployment_verification.py`. ## Review First -- `src/dander/providers/azure_container_apps/verification.py` -- `tests/providers/test_azure_deployment_verification.py` +- `src/dander/providers/azure_container_apps/operations.py` +- `tests/providers/test_azure_container_apps_operations.py` diff --git a/src/dander/providers/azure_container_apps/operations.py b/src/dander/providers/azure_container_apps/operations.py index 24d50ba0..4cdb89f7 100644 --- a/src/dander/providers/azure_container_apps/operations.py +++ b/src/dander/providers/azure_container_apps/operations.py @@ -6,13 +6,13 @@ import re import subprocess from dataclasses import asdict, dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Protocol, cast from dander.identity.refresh_probe import GoogleRefreshProbeError, validate_probe_target if TYPE_CHECKING: - from pathlib import Path - from dander.providers.azure_container_apps.verification import AzureDeploymentBinding _EXECUTION_NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$") @@ -132,7 +132,7 @@ def start_identity_refresh_probe( ) def _start(self, *overrides: str) -> AzureContainerAppsExecution: - payload = self._json( + start_args = ( "containerapp", "job", "start", @@ -140,8 +140,36 @@ def _start(self, *overrides: str) -> AzureContainerAppsExecution: self.binding.job_name, "--resource-group", self.binding.resource_group_name, - *overrides, ) + if overrides: + if overrides[0] != "--args" or len(overrides) < 2: + raise AzureContainerAppsOperationError("Invalid Azure execution override") + template_path: Path | None = None + try: + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + suffix=".json", + delete=False, + ) as template_file: + template_path = Path(template_file.name) + json.dump( + { + "containers": [ + { + "name": "runtime", + "args": list(overrides[1:]), + } + ] + }, + template_file, + ) + payload = self._json(*start_args, "--yaml", str(template_path)) + finally: + if template_path is not None: + template_path.unlink(missing_ok=True) + else: + payload = self._json(*start_args) if not isinstance(payload, dict) or not isinstance(payload.get("name"), str): raise AzureContainerAppsOperationError( "Azure did not return an execution name after starting the job" diff --git a/tests/providers/test_azure_container_apps_operations.py b/tests/providers/test_azure_container_apps_operations.py index 73a425eb..b9383228 100644 --- a/tests/providers/test_azure_container_apps_operations.py +++ b/tests/providers/test_azure_container_apps_operations.py @@ -64,6 +64,8 @@ def __init__(self, responses: Mapping[tuple[str, ...], list[object]]) -> None: for key, values in responses.items(): self.responses[key].extend(values) self.commands: list[tuple[str, ...]] = [] + self.execution_templates: list[object] = [] + self.execution_template_paths: list[Path] = [] def __call__( self, @@ -77,7 +79,14 @@ def __call__( assert cwd == Path("/tmp/dander-azure-operations-test") assert check and capture_output and text self.commands.append(args) - key = args[1 : args.index("--subscription")] + key_parts = list(args[1 : args.index("--subscription")]) + if "--yaml" in key_parts: + path_index = key_parts.index("--yaml") + 1 + template_path = Path(key_parts[path_index]) + self.execution_template_paths.append(template_path) + self.execution_templates.append(json.loads(template_path.read_text(encoding="utf-8"))) + key_parts[path_index] = "" + key = tuple(key_parts) if not self.responses[key]: raise AssertionError(f"No fake Azure response for {key}") return subprocess.CompletedProcess( @@ -140,19 +149,8 @@ def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> No _JOB, "--resource-group", "dander-phase6", - "--args", - "runtime", - "identity-refresh-probe", - "--project", - "unit-project", - "--dataset", - "raw", - "--table", - "proof_rows", - "--max-wait-seconds", - "900", - "--refresh-margin-seconds", - "15", + "--yaml", + "", ) show_key = ( "containerapp", @@ -182,7 +180,30 @@ def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> No ) assert execution.state == "running" - assert start_key == runner.commands[0][1 : runner.commands[0].index("--subscription")] + assert runner.execution_templates == [ + { + "containers": [ + { + "name": "runtime", + "args": [ + "runtime", + "identity-refresh-probe", + "--project", + "unit-project", + "--dataset", + "raw", + "--table", + "proof_rows", + "--max-wait-seconds", + "900", + "--refresh-margin-seconds", + "15", + ], + } + ] + } + ] + assert all(not path.exists() for path in runner.execution_template_paths) assert not set(runner.commands[0]).intersection({"--image", "--env-vars", "--command"}) From 45f5c0782291862f85e3af8eb02a2d421776dbad Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 18:33:02 -0400 Subject: [PATCH 139/163] Preserve Azure execution container contract --- HANDOFF.md | 14 ++--- .../azure_container_apps/operations.py | 55 +++++++++++++++---- .../test_azure_container_apps_operations.py | 38 ++++++++++++- 3 files changed, 88 insertions(+), 19 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 59f7df4e..83e0ebb1 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,10 +2,10 @@ ## Finished -- Transported Azure execution overrides through the provider-supported YAML template input. -- Prevented runtime flags such as `--project` from being parsed as Azure CLI flags. -- Kept the override limited to the `runtime` container's bounded argument list. -- Added exact execution-template contract coverage. +- Built one complete Azure execution container from the existing immutable job template. +- Preserved image, environment/secret references, resources, and optional command exactly. +- Replaced only the bounded runtime argument array for the identity-refresh proof. +- Failed closed on multiple containers, init containers, volume mounts, or malformed templates. ## Try It @@ -19,9 +19,9 @@ Run `uv run pytest tests/providers/test_azure_container_apps_operations.py`. ## Decisions -- Use Azure's execution-template transport when container arguments begin with dashes. -- Write only the non-secret bounded argument template to a mode-restricted temporary file. -- Delete the temporary template immediately after the Azure CLI call. +- Azure's start API requires a complete execution container, not an args-only partial override. +- Read only the selected job and preserve its non-secret references without resolving values. +- Delete the mode-restricted temporary template immediately after the Azure CLI call. ## Remaining diff --git a/src/dander/providers/azure_container_apps/operations.py b/src/dander/providers/azure_container_apps/operations.py index 4cdb89f7..cd45c002 100644 --- a/src/dander/providers/azure_container_apps/operations.py +++ b/src/dander/providers/azure_container_apps/operations.py @@ -144,6 +144,7 @@ def _start(self, *overrides: str) -> AzureContainerAppsExecution: if overrides: if overrides[0] != "--args" or len(overrides) < 2: raise AzureContainerAppsOperationError("Invalid Azure execution override") + execution_template = self._execution_template(args=overrides[1:]) template_path: Path | None = None try: with NamedTemporaryFile( @@ -153,17 +154,7 @@ def _start(self, *overrides: str) -> AzureContainerAppsExecution: delete=False, ) as template_file: template_path = Path(template_file.name) - json.dump( - { - "containers": [ - { - "name": "runtime", - "args": list(overrides[1:]), - } - ] - }, - template_file, - ) + json.dump(execution_template, template_file) payload = self._json(*start_args, "--yaml", str(template_path)) finally: if template_path is not None: @@ -178,6 +169,48 @@ def _start(self, *overrides: str) -> AzureContainerAppsExecution: self._validate_execution_name(execution_name) return self.describe(execution_name) + def _execution_template(self, *, args: tuple[str, ...]) -> dict[str, object]: + job = self._json( + "containerapp", + "job", + "show", + "--name", + self.binding.job_name, + "--resource-group", + self.binding.resource_group_name, + ) + if not isinstance(job, dict): + raise AzureContainerAppsOperationError("Azure returned an invalid job template") + properties = job.get("properties") + if not isinstance(properties, dict): + raise AzureContainerAppsOperationError("Azure returned an invalid job template") + template = properties.get("template") + if not isinstance(template, dict): + raise AzureContainerAppsOperationError("Azure returned an invalid job template") + containers = template.get("containers") + init_containers = template.get("initContainers") + if ( + not isinstance(containers, list) + or len(containers) != 1 + or not isinstance(containers[0], dict) + or containers[0].get("name") != "runtime" + or not isinstance(containers[0].get("image"), str) + or not isinstance(containers[0].get("env"), list) + or not isinstance(containers[0].get("resources"), dict) + or init_containers not in (None, []) + or containers[0].get("volumeMounts") not in (None, []) + ): + raise AzureContainerAppsOperationError( + "Azure job template does not match the single-container runtime contract" + ) + container = { + key: value + for key in ("name", "image", "command", "env", "resources") + if (value := containers[0].get(key)) is not None + } + container["args"] = list(args) + return {"containers": [container]} + def replay(self, execution_name: str) -> AzureContainerAppsExecution: """Start a fresh run after one terminal execution using the persisted inclusive cursor.""" previous = self.describe(execution_name) diff --git a/tests/providers/test_azure_container_apps_operations.py b/tests/providers/test_azure_container_apps_operations.py index b9383228..a77b89e0 100644 --- a/tests/providers/test_azure_container_apps_operations.py +++ b/tests/providers/test_azure_container_apps_operations.py @@ -141,6 +141,15 @@ def test_start_uses_exact_manifest_job_and_returns_provider_execution() -> None: def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> None: + template_key = ( + "containerapp", + "job", + "show", + "--name", + _JOB, + "--resource-group", + "dander-phase6", + ) start_key = ( "containerapp", "job", @@ -166,6 +175,27 @@ def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> No ) runner = _Runner( { + template_key: [ + { + "properties": { + "template": { + "containers": [ + { + "name": "runtime", + "image": "danderphase6.azurecr.io/dander@sha256:" + "a" * 64, + "args": ["runtime", "execute"], + "env": [ + {"name": "HOME", "value": "/tmp"}, + {"name": "API_TOKEN", "secretRef": "secret-reference"}, + ], + "resources": {"cpu": 1, "memory": "2Gi"}, + } + ], + "initContainers": None, + } + } + } + ], start_key: [{"name": _EXECUTION}], show_key: [_execution(status="Running")], } @@ -185,6 +215,12 @@ def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> No "containers": [ { "name": "runtime", + "image": "danderphase6.azurecr.io/dander@sha256:" + "a" * 64, + "env": [ + {"name": "HOME", "value": "/tmp"}, + {"name": "API_TOKEN", "secretRef": "secret-reference"}, + ], + "resources": {"cpu": 1, "memory": "2Gi"}, "args": [ "runtime", "identity-refresh-probe", @@ -204,7 +240,7 @@ def test_identity_refresh_probe_overrides_only_bounded_runtime_arguments() -> No } ] assert all(not path.exists() for path in runner.execution_template_paths) - assert not set(runner.commands[0]).intersection({"--image", "--env-vars", "--command"}) + assert not set(runner.commands[1]).intersection({"--image", "--env-vars", "--command"}) def test_latest_normalizes_the_most_recent_execution() -> None: From 4bfca886d12122acc3b7f5df79aa937c3a3c3c27 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 18:41:05 -0400 Subject: [PATCH 140/163] Preserve Azure supplier across Google auth clones --- HANDOFF.md | 23 ++++++------- src/dander/identity/azure_google.py | 53 +++++++++++++++++++++++------ tests/identity/test_azure_google.py | 19 ++++++++++- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 83e0ebb1..e968c05a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,34 @@ ## Finished -- Built one complete Azure execution container from the existing immutable job template. -- Preserved image, environment/secret references, resources, and optional command exactly. -- Replaced only the bounded runtime argument array for the identity-refresh proof. -- Failed closed on multiple containers, init containers, volume mounts, or malformed templates. +- Made Azure-backed external credentials compatible with Google Auth's impersonation clone path. +- Preserved the in-memory Azure subject-token supplier across credential clones. +- Added a no-network regression test through Google Auth's real clone implementation. ## Try It -Run `uv run pytest tests/providers/test_azure_container_apps_operations.py`. +Run `uv run pytest tests/identity/test_azure_google.py`. ## Checks -- Focused Azure Container Apps operation tests passed (7 tests). +- Focused Azure-to-Google identity tests passed (8 tests). - Ruff check/format and mypy passed for the changed Python files. - Protected CI remains to run. ## Decisions -- Azure's start API requires a complete execution container, not an args-only partial override. -- Read only the selected job and preserve its non-secret references without resolving values. -- Delete the mode-restricted temporary template immediately after the Azure CLI call. +- Keep the provider-neutral supplier in memory; do not introduce a credential file. +- Mirror Google Auth's constructor-clone contract while retaining the bounded 600-second token. +- Publish a new immutable candidate after this correction merges. ## Remaining - Merge this focused correction through protected CI. -- Resume the approved Azure-to-Google live federation plan and refresh proof. +- Publish/promote the repaired immutable candidate and resume the live refresh proof. - Clean up disposable Azure, Snowflake, and federation resources. - Merge sanitized Phase 6 evidence and verify retained-GCP no drift. ## Review First -- `src/dander/providers/azure_container_apps/operations.py` -- `tests/providers/test_azure_container_apps_operations.py` +- `src/dander/identity/azure_google.py` +- `tests/identity/test_azure_google.py` diff --git a/src/dander/identity/azure_google.py b/src/dander/identity/azure_google.py index 68a71ba2..39d0f2a9 100644 --- a/src/dander/identity/azure_google.py +++ b/src/dander/identity/azure_google.py @@ -112,23 +112,56 @@ def __init__( self, *, audience: str, - service_account: str, supplier: _AzureSubjectTokenSupplier, + service_account: str | None = None, + subject_token_type: str = _JWT_SUBJECT_TOKEN_TYPE, + token_url: str = "https://sts.googleapis.com/v1/token", + credential_source: dict[str, object] | None = None, + service_account_impersonation_url: str | None = None, + service_account_impersonation_options: dict[str, object] | None = None, + client_id: str | None = None, + client_secret: str | None = None, + token_info_url: str | None = None, + quota_project_id: str | None = None, + scopes: tuple[str, ...] | None = None, + default_scopes: tuple[str, ...] | None = None, + workforce_pool_user_project: str | None = None, + universe_domain: str = "googleapis.com", + trust_boundary: dict[str, str] | None = None, ) -> None: self._azure_supplier = supplier - super().__init__( # type: ignore[no-untyped-call] - audience=audience, - subject_token_type=_JWT_SUBJECT_TOKEN_TYPE, - token_url="https://sts.googleapis.com/v1/token", - credential_source={"environment_id": "azure-container-apps"}, - service_account_impersonation_url=( + if service_account is not None: + service_account_impersonation_url = ( "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" f"{service_account}:generateAccessToken" - ), - service_account_impersonation_options={"token_lifetime_seconds": 600}, - scopes=(_CLOUD_PLATFORM_SCOPE,), + ) + service_account_impersonation_options = {"token_lifetime_seconds": 600} + credential_source = credential_source or {"environment_id": "azure-container-apps"} + scopes = scopes or (_CLOUD_PLATFORM_SCOPE,) + super().__init__( # type: ignore[no-untyped-call] + audience=audience, + subject_token_type=subject_token_type, + token_url=token_url, + credential_source=credential_source, + service_account_impersonation_url=service_account_impersonation_url, + service_account_impersonation_options=service_account_impersonation_options, + client_id=client_id, + client_secret=client_secret, + token_info_url=token_info_url, + quota_project_id=quota_project_id, + scopes=scopes, + default_scopes=default_scopes, + workforce_pool_user_project=workforce_pool_user_project, + universe_domain=universe_domain, + trust_boundary=trust_boundary, ) + def _constructor_args(self) -> dict[str, Any]: + """Preserve the Azure supplier when Google Auth clones source credentials.""" + kwargs = super()._constructor_args() # type: ignore[no-untyped-call] + kwargs["supplier"] = self._azure_supplier + return cast("dict[str, Any]", kwargs) + def retrieve_subject_token(self, request: object) -> str: """Supply a fresh Entra token without writing a credential file.""" del request diff --git a/tests/identity/test_azure_google.py b/tests/identity/test_azure_google.py index ac3835d0..237ddea6 100644 --- a/tests/identity/test_azure_google.py +++ b/tests/identity/test_azure_google.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from datetime import UTC, datetime -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast import pytest from google.auth import external_account @@ -91,6 +91,23 @@ def test_azure_google_identity_builds_short_scoped_external_credentials() -> Non assert credentials._client_secret is None +def test_azure_google_identity_survives_google_auth_impersonation_clone() -> None: + markers: list[str] = [] + credentials = prepare_azure_google_identity( + environ=dict(_ENVIRONMENT), + clock=lambda: _NOW, + azure_credential_factory=cast( + "AzureCredentialFactory", lambda **_values: _Credential(markers) + ), + ) + + impersonated = cast("Any", credentials)._initialize_impersonated_credentials() + source_credentials = impersonated._source_credentials + + assert source_credentials.retrieve_subject_token(None).endswith("payload-b.signature") + assert markers == ["a", "b"] + + @pytest.mark.parametrize( ("update", "message"), [ From 06b64cc627abfd21e2a3fc5ad792563d303bb619 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 18:51:32 -0400 Subject: [PATCH 141/163] fix: install v2 plugins without deployment selection --- src/dander/cli/main.py | 7 ++++--- src/dander/project/__init__.py | 2 ++ src/dander/project/config.py | 16 +++++++++++++++ tests/cli/test_plugins_cli.py | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index 4aa0fe11..db8dbcdb 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -94,6 +94,7 @@ ProjectConfigError, ProjectScaffoldError, load_project_config, + load_project_plugins, scaffold_project, ) from dander.sandbox import GuardedFreeTierVerifier, SandboxSafetyError @@ -219,11 +220,11 @@ def install_plugins( ) -> None: """Install the manifest's exact connector-plugin package pins.""" try: - manifest = load_project_config(project_config) + plugins = load_project_plugins(project_config) except ProjectConfigError as error: raise ClickException(str(error)) from error requirements = [ - f"{plugin.distribution}=={plugin.version}" for _, plugin in sorted(manifest.plugins.items()) + f"{plugin.distribution}=={plugin.version}" for _, plugin in sorted(plugins.items()) ] if not requirements: console.print("No connector plugins are declared in dander.yaml.") @@ -249,7 +250,7 @@ def install_plugins( if completed.returncode != 0: raise ClickException("Connector plugin installation failed") try: - load_connector_plugins(manifest.plugins) + load_connector_plugins(plugins) except ConnectorPluginError as error: raise ClickException(f"Installed connector plugins are incompatible: {error}") from error console.print(f"[green]Installed {plugin_count} connector plugin(s).[/green]") diff --git a/src/dander/project/__init__.py b/src/dander/project/__init__.py index e4b6aff6..594ae728 100644 --- a/src/dander/project/__init__.py +++ b/src/dander/project/__init__.py @@ -10,6 +10,7 @@ PluginSpec, ProjectConfigError, load_project_config, + load_project_plugins, ) from dander.project.portable_config import ( DanderLogicalProjectV2, @@ -40,6 +41,7 @@ "ProjectMigration", "ProjectConfigError", "load_project_config", + "load_project_plugins", "prepare_version_one_migration", "ProjectScaffoldError", "scaffold_project", diff --git a/src/dander/project/config.py b/src/dander/project/config.py index f0e2f6fb..3860579f 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -418,6 +418,22 @@ def load_project_config( raise _project_validation_error(path, error) from error +def load_project_plugins(path: Path) -> dict[str, PluginSpec]: + """Load project-wide plugin pins without selecting a v2 deployment.""" + raw = _load_yaml_mapping(path, label="Dander project configuration") + if raw.get("version") == 2: + from dander.project.portable_config import DanderLogicalProjectV2 + + try: + return DanderLogicalProjectV2.model_validate(raw).plugins + except ValidationError as error: + raise _project_validation_error(path, error) from error + try: + return DanderProject.model_validate(raw).plugins + except ValidationError as error: + raise _project_validation_error(path, error) from error + + def _load_yaml_mapping(path: Path, *, label: str) -> dict[str, object]: try: raw = yaml.safe_load(path.read_text(encoding="utf-8")) diff --git a/tests/cli/test_plugins_cli.py b/tests/cli/test_plugins_cli.py index 02db30f7..6674404c 100644 --- a/tests/cli/test_plugins_cli.py +++ b/tests/cli/test_plugins_cli.py @@ -89,6 +89,43 @@ def unexpected_run(*args: object, **kwargs: object) -> None: assert "No connector plugins are declared" in result.output +def test_plugins_install_does_not_require_a_v2_deployment_selection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + manifest = tmp_path / "dander.yaml" + manifest.write_text( + """ +version: 2 +plugins: + salesforce: + distribution: dander-connector-salesforce + version: 0.1.0 +pipelines: + example: + source: example + models: [example] +""".strip(), + encoding="utf-8", + ) + captured: list[tuple[str, ...]] = [] + + def run(command: tuple[str, ...], *, check: bool) -> subprocess.CompletedProcess[str]: + assert check is False + captured.append(command) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(cli_module.shutil, "which", lambda _: None) + monkeypatch.setattr(cli_module.subprocess, "run", run) + monkeypatch.setattr(cli_module, "load_connector_plugins", lambda _: None) + + result = CliRunner().invoke(app, ["plugins", "install", "--config", str(manifest)]) + + assert result.exit_code == 0, result.output + assert captured[0][2] == "pip" + assert "dander-connector-salesforce==0.1.0" in captured[0] + + def test_plugins_scaffold_creates_named_project(tmp_path: Path) -> None: destination = tmp_path / "acme" From 22b63242ec5fa28a933ab3e87625f66bb9dd261d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 11 Aug 2026 22:12:52 -0400 Subject: [PATCH 142/163] Prepare 0.9.0rc1 release candidate --- CHANGELOG.md | 25 +++++++++++++- HANDOFF.md | 34 ++++++++++--------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/known-limitations.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 12 files changed, 54 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97453fc..f2d0fa98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ capabilities enter through the next minor release. ## Unreleased +## 0.9.0rc1 — 2026-08-11 (beta) + ### Added - Add a read-only canonical Azure profile preflight that requires the exact @@ -16,19 +18,40 @@ capabilities enter through the next minor release. `Key Vault Secrets User` role. - Add keyless Azure Container Apps-to-Google workload federation using the attached user-assigned identity, an exact Entra application audience, in-memory Google token exchange, and a bounded - same-process credential-refresh probe. The profile remains experimental until live acceptance. + same-process credential-refresh probe. The profile remains experimental until live acceptance + is reconciled and Phase 8 support qualification passes. - Add disposable, provider-mocked Phase 6 federation Terraform that pins Google trust and service account impersonation to one Azure managed-identity object ID without client secrets or keys. - Preserve validated provider extensions from connector, graph, and model declarations through canonical warehouse targets without changing existing BigQuery schema inputs. - Carry additive warehouse load/transform telemetry through endpoint results into terminal run telemetry, including optional resource, queue, execution, spill, and capacity-unit measures. +- Add experimental Snowflake and Redshift warehouses with all five ingestion modes, bounded + direct/columnar transports, destination fencing, portable transforms and graphs, telemetry, + provider-specific schema fallbacks, and disposable live warehouse qualification. +- Add experimental PostgreSQL state and warehouse runtimes with bounded `COPY`, five fenced writer + modes, transforms, graphs, assertions, and a packaged existing-cluster Kubernetes/Helm launcher. + Kubernetes remains unqualified against a live cluster. +- Add version-2 platform profiles, typed lazy provider factories, canonical relation/schema and + portable SQL contracts, provider-neutral execution templates, compatibility reporting, and a + deterministic four-warehouse common-intersection correctness gate. +- Add a plan-first Azure Container Apps Jobs launcher with Entra-authenticated remote state, + digest-preserving ACR promotion, managed identity, Key Vault, Log Analytics, alerts, lifecycle + operations, and fail-closed deployment verification. Azure remains experimental pending the + reconciled Phase 6 live record and Phase 8 qualification. ### Fixed - Route Azure Container Apps Key Vault references through one exact infrastructure subnet and disable the inapplicable trusted-service bypass; Key Vault profiles now fail planning when no subnet path is supplied. +- Preserve source-only metadata publication, Azure cancellation acknowledgement, complete + execution-template containers, and in-memory Azure token suppliers across Google Auth clones. +- Require the explicit GCP project for Azure federation, accept provider-normalized empty secret + collections, and transport Azure runtime arguments without shell parsing. +- Install project-wide version-2 plugin pins without inventing an arbitrary deployment selection. +- Bootstrap Key Vault-backed jobs in two reviewed plans and pin the supported Container Apps + consumption profile used by the live Azure lifecycle. ## 0.8.0rc8 — 2026-08-10 (beta) diff --git a/HANDOFF.md b/HANDOFF.md index e968c05a..e67bc76f 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,36 @@ ## Finished -- Made Azure-backed external credentials compatible with Google Auth's impersonation clone path. -- Preserved the in-memory Azure subject-token supplier across credential clones. -- Added a no-network regression test through Google Auth's real clone implementation. +- Prepared Dander `0.9.0rc1` from protected main without changing runtime behavior. +- Reconciled every exact public-version reference required by the release gate. +- Moved the accumulated Phase 5 and Phase 6 notes into the `0.9.0rc1` changelog entry. ## Try It -Run `uv run pytest tests/identity/test_azure_google.py`. +Run `python3 scripts/check_release_metadata.py`, then build and inspect the distribution. ## Checks -- Focused Azure-to-Google identity tests passed (8 tests). -- Ruff check/format and mypy passed for the changed Python files. -- Protected CI remains to run. +- Release metadata validation passed for `dander-platform 0.9.0rc1`. +- Wheel and source distribution validation passed. +- Ruff, format, mypy, and the full pytest suite passed locally. +- `git diff --check` passed. +- Full protected CI remains required before merge and again on protected main before tagging. ## Decisions -- Keep the provider-neutral supplier in memory; do not introduce a credential file. -- Mirror Google Auth's constructor-clone contract while retaining the bounded 600-second token. -- Publish a new immutable candidate after this correction merges. +- Publish only from an exact `v0.9.0rc1` tag on the green protected-main merge commit. +- Keep Azure experimental; the final Phase 6 public-candidate proof/evidence merge remains pending, + and Phase 8 remains the support gate. ## Remaining -- Merge this focused correction through protected CI. -- Publish/promote the repaired immutable candidate and resume the live refresh proof. -- Clean up disposable Azure, Snowflake, and federation resources. -- Merge sanitized Phase 6 evidence and verify retained-GCP no drift. +- Merge this version-only PR through protected CI. +- Tag and publish `0.9.0rc1` through the reviewed `pypi` environment. +- Install `0.9.0rc1` outside the checkout and run the required public-artifact smoke proofs. ## Review First -- `src/dander/identity/azure_google.py` -- `tests/identity/test_azure_google.py` +- `pyproject.toml` +- `CHANGELOG.md` +- `scripts/check_release_metadata.py` diff --git a/README.md b/README.md index 51c12b91..6c886861 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.8.0rc8 +uv tool install dander-platform==0.9.0rc1 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.8.0rc8` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.9.0rc1` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 9730de9d..79067051 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.8.0rc8`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.9.0rc1`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 85760a65..9e438adb 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.8.0rc8 +uv tool install dander-platform==0.9.0rc1 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 66ce513b..c8019cb3 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -1,6 +1,6 @@ # Known limitations -Dander `0.8.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; +Dander `0.9.x` is beta and proves a focused GCP-native vertical slice. It remains pre-1.0 software; evaluate these limits before using it for an unattended system containing business-critical data. The Phase 5 warehouse implementations and the diff --git a/docs/release-audit.md b/docs/release-audit.md index 91aea944..597eeacd 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -23,7 +23,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.8.0rc8`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.9.0rc1`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 62aee421..754c539f 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.8.0rc8` is the current public beta candidate. +- Dander `0.9.0rc1` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 77c699ed..ae9a775b 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.8.0rc8" +export DANDER_TARGET_VERSION="0.9.0rc1" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index af16f8a7..8acc610b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.8.0rc8" +version = "0.9.0rc1" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index b002d75c..5a033d9a 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.8.0rc8", + "dander-platform==0.9.0rc1", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.8.0rc8" + "README install command uses 0.1.0; package metadata uses 0.9.0rc1" ] diff --git a/uv.lock b/uv.lock index db8f0ae7..f65e572d 100644 --- a/uv.lock +++ b/uv.lock @@ -621,7 +621,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.8.0rc8" +version = "0.9.0rc1" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 09d0d91e1971066910ac787897b8761bb5150d14 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 00:21:19 -0400 Subject: [PATCH 143/163] docs: record Phase 6 Azure acceptance --- CHANGELOG.md | 6 +- HANDOFF.md | 37 ++--- ...-portability-azure-lifecycle-acceptance.md | 48 +++++- docs/cloud-portability-plan.md | 27 +++- docs/decisions.md | 14 ++ docs/evidence/azure/2026-08-11/phase6.json | 151 ++++++++++++++++++ docs/execution-projection.md | 14 +- docs/known-limitations.md | 25 +-- docs/platform-profiles.md | 11 ++ docs/release-audit.md | 12 +- docs/session-resume.md | 14 +- infra/README.md | 4 +- tickets/DANDER-108-azure-launcher-contract.md | 11 +- ...R-109-azure-terraform-plan-verification.md | 9 +- ...ER-110-azure-image-lifecycle-operations.md | 9 +- tickets/DANDER-111-azure-google-federation.md | 16 +- .../DANDER-114-azure-canonical-preflight.md | 10 +- 17 files changed, 342 insertions(+), 76 deletions(-) create mode 100644 docs/evidence/azure/2026-08-11/phase6.json diff --git a/CHANGELOG.md b/CHANGELOG.md index f2d0fa98..bd8ebe1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,8 @@ capabilities enter through the next minor release. `Key Vault Secrets User` role. - Add keyless Azure Container Apps-to-Google workload federation using the attached user-assigned identity, an exact Entra application audience, in-memory Google token exchange, and a bounded - same-process credential-refresh probe. The profile remains experimental until live acceptance - is reconciled and Phase 8 support qualification passes. + same-process credential-refresh probe. The profile remains experimental until Phase 8 support + qualification passes. - Add disposable, provider-mocked Phase 6 federation Terraform that pins Google trust and service account impersonation to one Azure managed-identity object ID without client secrets or keys. - Preserve validated provider extensions from connector, graph, and model declarations through @@ -38,7 +38,7 @@ capabilities enter through the next minor release. - Add a plan-first Azure Container Apps Jobs launcher with Entra-authenticated remote state, digest-preserving ACR promotion, managed identity, Key Vault, Log Analytics, alerts, lifecycle operations, and fail-closed deployment verification. Azure remains experimental pending the - reconciled Phase 6 live record and Phase 8 qualification. + Phase 8 support qualification. ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index e67bc76f..51dcfcab 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,36 +2,37 @@ ## Finished -- Prepared Dander `0.9.0rc1` from protected main without changing runtime behavior. -- Reconciled every exact public-version reference required by the release gate. -- Moved the accumulated Phase 5 and Phase 6 notes into the `0.9.0rc1` changelog entry. +- Published and externally installed Dander `0.9.0rc1` from protected main. +- Completed the named Azure/Snowflake/PostgreSQL/Key-Vault lifecycle within approved ceilings. +- Proved public-candidate Azure-to-Google refresh, secret/catalog read-back, and revocation. +- Passed public-candidate Greenhouse, HubSpot, replay, cleanup, and isolated-GCP no-drift smoke. +- Removed disposable provider resources and reconciled the Phase 6 roadmap and evidence. ## Try It -Run `python3 scripts/check_release_metadata.py`, then build and inspect the distribution. +Read `docs/cloud-portability-azure-lifecycle-acceptance.md` and its linked JSON evidence record. ## Checks -- Release metadata validation passed for `dander-platform 0.9.0rc1`. -- Wheel and source distribution validation passed. -- Ruff, format, mypy, and the full pytest suite passed locally. -- `git diff --check` passed. -- Full protected CI remains required before merge and again on protected main before tagging. +- Public `0.9.0rc1` install, generated-project validation, and packaged Terraform validation passed. +- Fresh isolated GCP and retained GCP plans reported exact `No changes.`. +- Retained GCP reproduced 28 stage-zero and 113 platform no-ops with no apply. +- Disposable Azure, Entra, GCP federation, Snowflake, and proof-data resources are absent. ## Decisions -- Publish only from an exact `v0.9.0rc1` tag on the green protected-main merge commit. -- Keep Azure experimental; the final Phase 6 public-candidate proof/evidence merge remains pending, - and Phase 8 remains the support gate. +- Phase 6 passes on the existing typed architecture; Azure remains experimental until Phase 8. +- The canonical Azure and Azure-to-Google profiles remain separate qualification surfaces. +- No Phase 7 or OCI work begins from this handoff. ## Remaining -- Merge this version-only PR through protected CI. -- Tag and publish `0.9.0rc1` through the reviewed `pypi` environment. -- Install `0.9.0rc1` outside the checkout and run the required public-artifact smoke proofs. +- Merge this evidence PR through protected CI. +- Start Phase 7 only after a new explicit instruction and cost/credential preflight. +- Keep retained-GCP private operator inputs outside the repository. ## Review First -- `pyproject.toml` -- `CHANGELOG.md` -- `scripts/check_release_metadata.py` +- `docs/evidence/azure/2026-08-11/phase6.json` +- `docs/cloud-portability-azure-lifecycle-acceptance.md` +- `docs/cloud-portability-plan.md` diff --git a/docs/cloud-portability-azure-lifecycle-acceptance.md b/docs/cloud-portability-azure-lifecycle-acceptance.md index 1edd0b26..c737990d 100644 --- a/docs/cloud-portability-azure-lifecycle-acceptance.md +++ b/docs/cloud-portability-azure-lifecycle-acceptance.md @@ -1,11 +1,13 @@ # Azure Container Apps lifecycle acceptance -Status: protocol prepared; no live Azure resource or paid proof has run. +Status: accepted on 2026-08-12; public-candidate supplement passed, disposable resources removed, +and retained GCP unchanged. -This protocol qualifies only the named Azure Container Apps, Snowflake OAuth warehouse, +The canonical proof qualifies only the named Azure Container Apps, Snowflake OAuth warehouse, PostgreSQL state, no-catalog, and Azure Key Vault composition. The selected pipeline must bind both -credential environment names to declared Key Vault secret identifiers. It does not qualify -Azure-to-BigQuery or any other cloud/warehouse pair. +credential environment names to declared Key Vault secret identifiers. A separate portability +proof qualifies the Azure-to-BigQuery identity path for BigQuery, Dataplex, and GCP Secret Manager. +Neither proof qualifies any other cloud/warehouse pair or promotes Azure beyond experimental use. ## Approval boundary @@ -61,4 +63,40 @@ repository. Saved plans and state use the secured operator artifact directories Retain only commit and digest identities, normalized states and counts, pass/fail booleans, timestamps, approved ceilings, cleanup results, and no-drift outcome. The read-only preflight may retain declared secret names and enabled booleans; it never retains values or unrelated vault -entries. Live evidence remains pending until the full approved sequence passes. +entries. The machine-readable record is +[`docs/evidence/azure/2026-08-11/phase6.json`](evidence/azure/2026-08-11/phase6.json). + +## Accepted result + +- The source-free Azure/Snowflake/PostgreSQL/Key-Vault candidate used one byte-identical GAR/ACR + OCI index, digest + `sha256:a64d89a3beff1b56ed8b3b13f17b67f8f99d87e08ebf48e6ff01381ecdc94d59`. + Its build-context revision was + `0901a384d3bf1141b44bbf64268cff4f102cfc4fc2cbc8aaa28e94e9ded70fd`. + It passed the read-only canonical preflight, manual and UTC-scheduled execution, replay, + overlapping-start fencing, interruption, retry exhaustion, alert routing, versionless Key Vault + rotation, and immutable rollback/restoration. +- The separate Azure-to-Google candidate at protected-main commit + `eb074c58a9b3d8c1296c28849639a04c07fdb4bf` retained identical GAR/ACR content at digest + `sha256:aa7da96e9b628c4bda5288a1a32edc1e2873c782459ce52d840248a60f851b4c`. + One process queried BigQuery before and after credential refresh; a second bounded execution read + one declared GCP secret and the corresponding Dataplex system entry without emitting values, + query text, or rows. Disabling the exact workload-identity provider then failed closed. +- Public `dander-platform==0.9.0rc1`, tagged from protected-main commit `2b90f7ad`, installed outside + the checkout and produced source-free digest + `sha256:1e1bd9ed803b523626c6f4720caba92a84bf0f95473d0002e12371b1a4975519`. + The same OCI content ran in ACR and GAR with provenance and SBOM attached. From Azure it repeated + BigQuery access across credential refresh and GCP secret/Dataplex read-back. In an isolated GCP + project it completed Greenhouse, HubSpot, exact replay, owned-data cleanup, and Terraform + no-drift. This public-artifact smoke supplements rather than repeats the earlier complete + canonical Snowflake lifecycle. +- Disposable Snowflake objects, Entra/GCP federation resources and principals, Azure jobs and + infrastructure, isolated-GCP proof data, and proof images were removed. Azure's mandatory purge + protection retains only the deleted Key Vault tombstone until 2026-11-10; it is not an active + vault. Fresh isolated-GCP and retained-GCP plans reported exact `No changes.` Retained GCP had 28 + stage-zero and 113 platform resources as no-ops; no retained-GCP apply occurred. + +The Phase 6 exit gate passes because one accepted release digest passed both Azure launcher +conformance and the complete named Azure profile, and the subsequently published candidate passed +the required public-artifact supplement. Azure remains experimental pending Phase 8 scale, +throughput, cost, soak, pairwise-profile, and release qualification. Phase 7 did not begin. diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 0c7382f4..6bdec2da 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1,6 +1,6 @@ # Cloud Portability and First-Class Platform Roadmap -Status: active roadmap; Phases 0 through 5 exit gates satisfied; Phase 6 in progress +Status: active roadmap; Phases 0 through 6 exit gates satisfied; Phase 7 not begun Initial planning baseline: `origin/main` at `9dc5562` (`0.6.0rc1`) @@ -12,7 +12,12 @@ Phase 5 exit-evidence candidate: protected `main` at stored under `docs/evidence/warehouse-correctness/2026-08-11/` and reconciled through protected evidence PR #186. -Prepared: 2026-08-06; reconciled: 2026-08-11 +Phase 6 implementation checkpoint: protected `main` at +`eb074c58a9b3d8c1296c28849639a04c07fdb4bf` after PRs #189 through #208. Public candidate +`0.9.0rc1` was tagged from protected `main` at `2b90f7ad9d02ad303d3543f1e27febc7193e9c82`. +The sanitized lifecycle record is stored under `docs/evidence/azure/2026-08-11/`. + +Prepared: 2026-08-06; reconciled: 2026-08-12 The baseline descriptions below record the conditions this roadmap was designed to change. They are retained as architectural rationale rather than claims about the current implementation. The @@ -641,9 +646,9 @@ credential. | ECS task role | BigQuery/Dataplex/GCP Secret Manager | AWS signed workload identity exchanged through Google Workload Identity Federation; Google token refresh occurs inside the running task | supported | | GKE/EKS/AKS/OKE service account | Same-cloud APIs | The cluster's documented workload-identity binding; each cluster type is a separate tested profile | supported only per proven cluster profile | | Kubernetes service account | Snowflake/PostgreSQL | TLS plus external-secret/operator-projected OAuth, key-pair, or database credential | supported in the named Kubernetes profile | -| Azure user-assigned managed identity | ACR, Key Vault, Azure logging | Azure managed identity | supported | -| Azure user-assigned managed identity | BigQuery/Dataplex/GCP Secret Manager | Azure workload identity exchanged through Google Workload Identity Federation, including refresh | supported after the Azure portability proof | -| Azure job | Snowflake/PostgreSQL | Key Vault-projected OAuth, key-pair, or database credential over TLS | supported in the named Azure profile | +| Azure user-assigned managed identity | ACR, Key Vault, Azure logging | Azure managed identity | experimental after Phase 6; support requires Phase 8 qualification | +| Azure user-assigned managed identity | BigQuery/Dataplex/GCP Secret Manager | Azure workload identity exchanged through Google Workload Identity Federation, including refresh | experimental after Phase 6; support requires Phase 8 qualification | +| Azure job | Snowflake/PostgreSQL | Key Vault-projected OAuth, key-pair, or database credential over TLS | experimental in the named Azure profile until Phase 8 qualification | | OCI Container Instance resource principal | OCIR, Vault, Logging/Monitoring | OCI resource principal and dynamic-group policy | supported | | OCI job | Snowflake/PostgreSQL | OCI Vault-resolved OAuth, key-pair, or database credential over TLS | supported in the named OCI profile | | OCI resource principal | BigQuery/Dataplex/GCP Secret Manager | External workload exchange still to be proven with refresh | experimental until Phase 7 proof | @@ -938,8 +943,8 @@ release candidate. | Fargate + BigQuery state/warehouse + Dataplex + GCP secrets | supported | identical-digest AWS-to-Google federation and refresh proof | | Fargate + PostgreSQL state + Redshift + Glue + AWS secrets | supported | AWS canonical profile | | Kubernetes/Helm + PostgreSQL state/warehouse + no catalog + external secrets | supported | one named cluster profile plus portable chart conformance | -| Azure Container Apps Jobs + PostgreSQL state + Snowflake + no catalog + Key Vault | supported | Azure canonical profile | -| Azure Container Apps Jobs + BigQuery state/warehouse + Dataplex + GCP secrets | supported | Azure-to-Google federation and refresh proof | +| Azure Container Apps Jobs + PostgreSQL state + Snowflake + no catalog + Key Vault | experimental after Phase 6; supported target after Phase 8 | Azure canonical profile plus Phase 8 qualification | +| Azure Container Apps Jobs + BigQuery state/warehouse + Dataplex + GCP secrets | experimental after Phase 6; supported target after Phase 8 | Azure-to-Google federation/refresh proof plus Phase 8 qualification | | OCI Container Instances + PostgreSQL state/warehouse + no catalog + OCI Vault | supported | OCI canonical profile | | OCI Container Instances + BigQuery/Dataplex/GCP secrets | experimental | OCI external-identity feasibility, refresh, and revocation proof | | Non-AWS launcher + Redshift, Glue, or AWS Secrets Manager | unsupported | remains so until an explicit short-lived AWS federation profile is designed and proven | @@ -1173,6 +1178,14 @@ Exit gate: - the same release digest passes Azure launcher conformance and the named Azure profile. +Satisfied on 2026-08-12 by the source-free digest +`sha256:a64d89a3beff1b56ed8b3b13f17b67f8f99d87e08ebf48e6ff01381ecdc94d59`, which passed Azure +launcher conformance and the complete named Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault live +profile. Public `0.9.0rc1` then passed the Azure-to-Google refresh, secret, catalog, revocation and +isolated-GCP standard smoke, followed by provider cleanup and retained-GCP no drift. Azure remains +experimental until the applicable Phase 8 qualification passes; Phase 7 implementation did not +begin. + ### Phase 7 — OCI first-class launcher Tickets: diff --git a/docs/decisions.md b/docs/decisions.md index 2717b144..a8f1acc8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1291,3 +1291,17 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc BigQuery and passes it directly into the existing typed federation projection. - **Boundary:** This adds no configuration bag or provider behavior. Snowflake planning is unchanged, and the project remains an explicit reviewed operator input. + +## 2026-08-12 — Phase 6 closes without promoting Azure support + +- **Canonical result:** One source-free OCI digest remained byte-identical across GAR and ACR and + passed the complete Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault profile: preflight, manual and + UTC-scheduled execution, replay, overlap fencing, interruption, retry exhaustion, alert routing, + versionless secret rotation, immutable rollback, cleanup, and retained-GCP no drift. +- **Federation result:** A separate protected-main candidate passed BigQuery access before and after + in-process Google credential refresh, GCP Secret Manager access, Dataplex read-back, revocation, + and cleanup using only Azure managed identity and short-lived federation. Public `0.9.0rc1` + repeated that bounded path and passed the standard isolated-GCP public-artifact smoke. +- **Boundary:** These results satisfy the Phase 6 architecture and live-profile gate. Azure remains + experimental until Phase 8 scale, throughput, crossover, cost, soak, pairwise-profile, and + release qualification; no Phase 7 or OCI implementation began. diff --git a/docs/evidence/azure/2026-08-11/phase6.json b/docs/evidence/azure/2026-08-11/phase6.json new file mode 100644 index 00000000..29313474 --- /dev/null +++ b/docs/evidence/azure/2026-08-11/phase6.json @@ -0,0 +1,151 @@ +{ + "schema": "io.dander.acceptance.azure-phase6/v1", + "status": "passed", + "date": "2026-08-11", + "completed_date": "2026-08-12", + "protected_main_commit": "eb074c58a9b3d8c1296c28849639a04c07fdb4bf", + "approval": { + "per_attempt_usd_ceiling": { + "azure": 3, + "snowflake": 2, + "gcp": 1, + "postgresql": 0 + }, + "approved_repeat_until_pass": true + }, + "canonical_profile": { + "launcher": "azure_container_apps", + "warehouse": "snowflake", + "state": "postgresql", + "catalog": "none", + "secrets": "azure_key_vault", + "qualifying_image_digest": "sha256:a64d89a3beff1b56ed8b3b13f17b67f8f99d87e08ebf48e6ff01381ecdc94d59", + "build_context_revision": "0901a384d3bf1141b44bbf64268cff4f102cfc4fc2cbc8aaa28e94e9ded70fd", + "source_free": true, + "publication_boundary": "private_gar_and_acr", + "registry_content_identical": true, + "preflight_passed": true, + "manual_execution_passed": true, + "scheduled_execution_passed": true, + "replay_passed": true, + "overlap_fencing_passed": true, + "interruption_passed": true, + "retry_exhaustion_passed": true, + "alert_routing_passed": true, + "versionless_secret_rotation_passed": true, + "immutable_rollback_sequence_passed": true, + "rollback_image_digest": "sha256:71f8fcc8ec26ba757a17f8269dd15cd6edfa9518e83050e8a90eb7fd8baf16de", + "successful_execution_count": 13, + "stopped_execution_count": 2 + }, + "protected_main_azure_google_profile": { + "launcher": "azure_container_apps", + "warehouse_and_state": "bigquery", + "catalog": "dataplex", + "secrets": "gcp_secret_manager", + "image_digest": "sha256:aa7da96e9b628c4bda5288a1a32edc1e2873c782459ce52d840248a60f851b4c", + "image_content_hash": "5a47091b163f450e9f743eaa8b0f394d322a9d29887e9e5dc6220cad3facf601", + "source_commit": "eb074c58a9b3d8c1296c28849639a04c07fdb4bf", + "publication_boundary": "private_gar_and_acr", + "registry_content_identical": true, + "query_before_refresh_passed": true, + "query_after_refresh_passed": true, + "normalized_row_count_per_query": 3, + "refresh_observed": true, + "refresh_execution": { + "name": "dander-p6-gc-00626d3b5f01-0xnrpuv", + "started_at": "2026-08-11T23:02:17Z", + "finished_at": "2026-08-11T23:13:07Z", + "status": "succeeded" + }, + "secret_access_passed": true, + "catalog_readback_passed": true, + "readback_execution": { + "name": "dander-p6-gc-00626d3b5f01-xsbirr2", + "started_at": "2026-08-11T23:14:50Z", + "finished_at": "2026-08-11T23:15:22Z", + "status": "succeeded" + }, + "revocation_failed_closed": true, + "revocation_execution": { + "name": "dander-p6-gc-00626d3b5f01-wpjz47b", + "started_at": "2026-08-11T23:16:32Z", + "status": "failed_as_expected" + } + }, + "public_candidate": { + "distribution": "dander-platform", + "version": "0.9.0rc1", + "tag": "v0.9.0rc1", + "protected_main_commit": "2b90f7ad9d02ad303d3543f1e27febc7193e9c82", + "package_index_publication_passed": true, + "fresh_external_install_passed": true, + "generated_project_validation_passed": true, + "packaged_terraform_validation_passed": true, + "qualifying_image_digest": "sha256:1e1bd9ed803b523626c6f4720caba92a84bf0f95473d0002e12371b1a4975519", + "build_context_revision": "e1560744c43f81d2eb37401271a77b3e01d23c076c152da55bd156e116a36e81", + "amd64_digest": "sha256:18cdd8339c582a1028a59f116cf9e61b10725dddfbe4ab7c9f769a586854c183", + "arm64_digest": "sha256:cf573f110cff92c08e895dd1fd0081d16d22b98a5b7e79ca62223e42983284c3", + "registry_content_identical": true, + "provenance_attached": true, + "sbom_attached": true, + "azure_google": { + "query_before_refresh_passed": true, + "query_after_refresh_passed": true, + "refresh_execution": "dander-p6-pu-00626d3b5f01-lm97jbz", + "secret_access_passed": true, + "catalog_readback_passed": true, + "readback_execution": "dander-p6-pu-00626d3b5f01-cfpgb2r", + "revocation_failed_closed": true + }, + "isolated_gcp_smoke": { + "project": "dander-cr-20260802-7f3a", + "greenhouse_execution": "dander-greenhouse-public-87jkq", + "hubspot_execution": "dander-hubspot-companies-dmpm6", + "hubspot_replay_execution": "dander-hubspot-companies-26rch", + "greenhouse_passed": true, + "hubspot_passed": true, + "replay_passed": true, + "hubspot_raw_rows_after_replay": 1, + "hubspot_model_rows_after_replay": 1, + "secret_value_present_in_logs": false, + "owned_data_cleanup_passed": true, + "terraform_no_drift_passed": true + }, + "scope_note": "This public-candidate proof supplements rather than repeats the earlier complete canonical Snowflake lifecycle." + }, + "cleanup": { + "snowflake_objects_absent": true, + "snowflake_cached_tokens_removed": true, + "federation_terraform_resources_remaining": 0, + "entra_federation_application_absent": true, + "entra_federation_service_principal_absent": true, + "gcp_federation_principals_absent": true, + "gcp_fixture_table_absent": true, + "candidate_gar_images_absent": true, + "azure_resource_group_absent": true, + "azure_active_jobs_absent": true, + "azure_container_apps_environment_absent": true, + "azure_log_analytics_workspace_absent": true, + "azure_platform_terraform_resources_remaining": 0, + "azure_stage_zero_terraform_resources_remaining": 0, + "azure_key_vault_soft_deleted": true, + "azure_key_vault_purge_protection_enabled": true, + "azure_key_vault_scheduled_purge": "2026-11-10T03:45:07+00:00", + "azure_key_vault_tombstone_note": "Azure-enforced purge protection prevents early purge; the vault is deleted and is not an active resource.", + "isolated_gcp_no_drift": true, + "retained_gcp_stage_zero_no_drift_resources": 28, + "retained_gcp_platform_no_drift_resources": 113, + "retained_gcp_apply_performed": false + }, + "evidence_excludes": [ + "credentials", + "secret_values", + "tokens", + "query_text", + "warehouse_rows", + "terraform_state", + "terraform_plans", + "private_operator_inputs" + ] +} diff --git a/docs/execution-projection.md b/docs/execution-projection.md index fca60e38..0fc9d2a0 100644 --- a/docs/execution-projection.md +++ b/docs/execution-projection.md @@ -66,9 +66,17 @@ behavior. The separate BigQuery portability profile uses the same managed identi only typed Google workload-identity audience/application coordinates and GCP Secret Manager references; the runtime exchanges managed-identity tokens in memory and never creates a credential file. Plan-first Terraform now consumes these projections exactly and has provider-mocked -contract evidence; no Azure resource has been created, and live-profile evidence remains a -separate Phase 6 gate. Local lifecycle tooling now binds start, status, bounded Log Analytics, +contract evidence. Live-profile acceptance remained a separate Phase 6 gate from that construction +work. Local lifecycle tooling now binds start, status, bounded Log Analytics, stop, replay, and deployment verification to the exact manifest-selected job. The ACR promotion path copies the accepted OCI index without rebuilding and writes its local Azure artifact record only after both the index digest and platform-manifest map match. These mocked contracts do not -claim that a live Azure operation has passed. +by themselves claim that a live Azure operation has passed. + +The bounded Phase 6 proof subsequently exercised the named Azure/Snowflake/PostgreSQL/Key-Vault +profile through manual and UTC-scheduled execution, replay, overlap fencing, interruption, retry +exhaustion, alert routing, secret rotation, rollback, cleanup, and no drift. The separate +Azure-to-Google projection queried BigQuery across an in-process credential refresh, read one +declared GCP secret and Dataplex entry, and failed closed after federation revocation. These proofs +validate the existing typed projections; they do not add a generic extension bag or promote Azure +beyond experimental status before Phase 8 qualification. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index c8019cb3..57bd6d3e 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -50,17 +50,24 @@ remain Phase 8 work. plan-first Terraform, digest-preserving ACR promotion tooling, provider-native lifecycle operations, bounded Log Analytics reads, deployment verification, and a locally validated Azure-to-Google managed-identity adapter with disposable federation Terraform and refresh-probe - tooling. A separate read-only preflight rejects any composition other than the named + tooling. The named Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault profile passed live manual and + UTC-scheduled execution, replay, overlap fencing, interruption, retry exhaustion, alert routing, + versionless secret rotation, immutable rollback, cleanup, and retained-GCP no drift. The + Azure-to-Google profile separately passed live BigQuery access across credential refresh, GCP + Secret Manager access, Dataplex read-back, revocation, and cleanup without a credential file or + long-lived cloud key. The local `dander azure status` command does not yet accept the explicit GCP + project needed to reconstruct that experimental cross-cloud deployment; it therefore fails + closed, and Azure-native status was used for the proof. The named Snowflake profile is unaffected. + A separate read-only preflight rejects any composition other than the named Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault profile and verifies only declared secret names - and enabled state. The named Key Vault profile requires an existing delegated Container Apps subnet with - the `Microsoft.KeyVault` service endpoint; Dander admits that exact subnet and the reviewed - operator IP while disabling the inapplicable Azure trusted-service bypass. Dander does not - create or modify the subnet. Azure schedules are + and enabled state. The named Key Vault profile requires an existing delegated Container Apps + subnet with the `Microsoft.KeyVault` service endpoint; Dander admits that exact subnet and the + reviewed operator IP while disabling the inapplicable Azure trusted-service bypass. Dander does + not create or modify the subnet. Azure schedules are UTC-only and currently support one replica with 1 CPU/2 GiB or 2 CPU/4 GiB. Resource-provider - registration, credentialed Azure planning/apply, actual ACR image copy, live identity exchange, - refresh and revocation, - live lifecycle execution, and live Snowflake/PostgreSQL acceptance have not yet run, so Azure - remains unsupported. + registration and disposable network creation remain explicit operator-approved actions. These + bounded proofs close Phase 6 but do not supply Phase 8 scale, throughput, cost, soak, + pairwise-profile, or release qualification, so Azure remains experimental rather than supported. - The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or use a reviewed direct signed-HTTP client before it can enter the full image. diff --git a/docs/platform-profiles.md b/docs/platform-profiles.md index 788b03ba..ffbc8412 100644 --- a/docs/platform-profiles.md +++ b/docs/platform-profiles.md @@ -20,6 +20,17 @@ single deployment until their provider-neutral planning path is introduced. When a project has exactly one deployment, Dander selects it deterministically; multiple deployments never fall back to an arbitrary default. +## Azure experimental profiles + +The named `azure_container_apps` profile combining Snowflake, PostgreSQL state, no catalog, and +Azure Key Vault passed the bounded Phase 6 live lifecycle. The separate Azure-to-Google profile +passed on public `0.9.0rc1`: BigQuery access across credential refresh plus GCP Secret Manager, +Dataplex, revocation, cleanup, and isolated-GCP smoke. Both remain experimental: configuration +outside these named shapes fails closed, and +Phase 8 scale, cost, soak, pairwise-profile, and release qualification are still required before a +support promotion. Dander creates neither an Azure subscription nor the delegated Container Apps +subnet required by the Key Vault profile. + ## Migrate a version 1 project Version 1 combined manifests remain supported during the compatibility window. First run the diff --git a/docs/release-audit.md b/docs/release-audit.md index 597eeacd..d61e188f 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -1,6 +1,6 @@ # Dander Platform Release Audit -Audited on 2026-08-11 against the product promise in `steering/00-project-overview.md`. +Audited on 2026-08-12 against the product promise in `steering/00-project-overview.md`. “Live-proven” means the behavior was observed in a disposable provider account or retained GCP project. “Implemented” means automated tests cover the contract while an optional provider or cloud path remains outside the live proof. @@ -19,7 +19,8 @@ cloud path remains outside the live proof. | Durable state and optional read capabilities | Live-proven | Run history, leases, watermarks, count, connection-check, and targeted-read paths were exercised for the relevant retained plugins without provider write-back. | | Single metadata spine | Live-proven | The run ledger and catalog store pipeline lifecycle, source/model schema, lineage, tests, and governed metrics in BigQuery; Dataplex publication remains optional. | | Canonical visual authoring | Live-proven for the bounded graph slice | Druff opens and saves canonical `PipelineGraph`, discovers presentation-safe connector and operation descriptors, starts an already-deployed run, and previews a non-applyable full-manifest plan through Dander's loopback service. | -| Infrastructure reconciliation safety | Live-proven | Fresh retained stage-zero and platform plans after the four-warehouse run reported exact `No changes.`; the current-equivalent platform plan contained `113` no-op resources. | +| Azure launcher portability | Live-proven for the named experimental profiles | One immutable source-free digest passed the Azure/Snowflake/PostgreSQL/Key-Vault lifecycle; public `0.9.0rc1` passed Azure-to-Google refresh, secret, catalog, revocation and isolated-GCP smoke. | +| Infrastructure reconciliation safety | Live-proven | Fresh retained stage-zero and platform plans after the Azure proof reported exact `No changes.` with `28` and `113` no-op resources respectively. No retained-project apply occurred. | ## Current release and deployment record @@ -44,6 +45,13 @@ cloud path remains outside the live proof. - Protected-main Dander `0.8.0rc8` passed the bounded Phase 5 common-scalar fixture on BigQuery, PostgreSQL, Snowflake, and Redshift with one equal normalized hash, exact replay, and verified cleanup. This is warehouse correctness evidence, not provider or profile support promotion. +- The source-free Azure candidate at digest + `sha256:a64d89a3beff1b56ed8b3b13f17b67f8f99d87e08ebf48e6ff01381ecdc94d59` + passed the named Azure/Snowflake/PostgreSQL/Key-Vault lifecycle, including scheduling, replay, + fencing, interruption, retry exhaustion, alerts, rotation, rollback, and cleanup. Public + `0.9.0rc1` then passed the separate Azure-to-Google refresh, GCP-secret, Dataplex, revocation, + isolated-GCP smoke, cleanup, and no-drift proof. Azure remains experimental pending Phase 8; see + `docs/cloud-portability-azure-lifecycle-acceptance.md`. - Retained source-free Dander image: `sha256:68e112c43b365018b735be7934446e15dfe6169fc64062b62b8bb97ea4f93b96`, built with Dander `0.7.1`, Salesforce `0.3.1`, and ServiceNow `0.2.2`. - Retained Druff image: `sha256:a5e255d6…871c`; public static URL: diff --git a/docs/session-resume.md b/docs/session-resume.md index 754c539f..daef73b7 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -33,6 +33,15 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Latest operating evidence +- On 2026-08-12, the named Azure/Snowflake/PostgreSQL/Key-Vault profile passed the complete Phase 6 + lifecycle on one source-free, byte-identical GAR/ACR digest: preflight, manual and UTC-scheduled + execution, replay, overlap fencing, interruption, retry exhaustion, alert routing, versionless + secret rotation, immutable rollback, cleanup, and retained-GCP no drift. A separate + public `0.9.0rc1` passed Azure-to-Google BigQuery access across credential refresh, GCP Secret + Manager access, Dataplex read-back, revocation, isolated-GCP smoke, and cleanup. Azure remains experimental + pending Phase 8 qualification; Phase 7 has not begun. See + `docs/cloud-portability-azure-lifecycle-acceptance.md`. + - On 2026-08-11, the Phase 5 common-scalar fixture passed BigQuery, PostgreSQL, Snowflake, and Redshift on protected-main commit `c0f3e2cb671eb6ddf1c34c60bc9e761d220cb9ad`. All four records produced one equal normalized hash after replay and verified owned cleanup. Provider account @@ -63,8 +72,9 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and `dander-servicenow-incidents-6g72x` all completed successfully. - The latest manual executable-graph run, `dander-greenhouse-graph-7gn9z`, completed successfully on 2026-08-04; its schedule remains intentionally paused. -- The latest retained stage-zero and current-equivalent platform plans each reported exactly - `No changes.` after the four-warehouse proof. No retained-project apply occurred. +- The latest retained stage-zero and current-equivalent platform plans reported exactly + `No changes.` after the Azure proof: 28 and 113 resources were no-ops respectively. No + retained-project apply occurred. - Continue the 30-day operating record in GitHub issue #26. The next normal scheduled runs are the remaining observation point for the reconciled stable image. diff --git a/infra/README.md b/infra/README.md index ee2498dd..fb109275 100644 --- a/infra/README.md +++ b/infra/README.md @@ -14,8 +14,8 @@ call sites (mirrors the `SecretStoreProvider` / `ComputeProvider` abstractions i | `modules/github-wif` | Repository/ref-scoped GitHub OIDC and a keyless deployment identity. **Implemented.** | | `modules/cost-guard` | Project budget, Pub/Sub, and simulation-first Gen 2 billing kill switch. **Implemented.** | | `kubernetes/chart/dander` | Versioned Helm chart for an existing conforming cluster. **Locally validated; not live-qualified.** | -| `azure/bootstrap-admin` | Azure Storage state, ACR, and user-assigned runtime identity. **Locally validated; not applied.** | -| `azure/modules/container-apps-jobs` | Container Apps Jobs, Key Vault references, Log Analytics, alerts, and optional network placement. **Locally validated; not live-qualified.** | +| `azure/bootstrap-admin` | Azure Storage state, ACR, and user-assigned runtime identity. **Live-proven for the named experimental Phase 6 profiles.** | +| `azure/modules/container-apps-jobs` | Container Apps Jobs, Key Vault references, Log Analytics, alerts, and optional network placement. **Live-proven for the named experimental Phase 6 profiles.** | The main root always calls `modules/bigquery` and can opt into the remaining workload modules. The one-time `infra/bootstrap-admin` root creates the remote-state bucket, the Artifact Registry diff --git a/tickets/DANDER-108-azure-launcher-contract.md b/tickets/DANDER-108-azure-launcher-contract.md index 26758961..59d9ad08 100644 --- a/tickets/DANDER-108-azure-launcher-contract.md +++ b/tickets/DANDER-108-azure-launcher-contract.md @@ -1,6 +1,6 @@ # DANDER-108 — Azure launcher and Key Vault contract -Status: implemented locally; live qualification pending +Status: done; contract and named live profiles qualified in Phase 6 ## Requirement @@ -20,10 +20,11 @@ values out of projections, and reject Azure semantics that cannot be represented - [x] `azure_key_vault` resolves only full vault secret URIs, remains SDK-lazy, audits references, excludes values, and cannot cross vault boundaries through one resolver. - [x] Version 2 configuration resolves the named Azure/Snowflake/PostgreSQL/no-catalog shape. -- [ ] Terraform, ACR promotion, Azure operations, federation, and live profile proofs remain separate - Phase 6 tickets and require explicit approval before provider writes. +- [x] Terraform, ACR promotion, Azure operations, federation, and live profile proofs remained + separate protected Phase 6 tickets and ran only after explicit provider-write and cost approval. ## Evidence boundary -This ticket is deterministic local contract evidence only. It creates no Azure resources, registers -no resource providers, copies no image, and makes no support claim. +The contract portion of this ticket is deterministic local evidence. The separately approved live +proof is recorded in `docs/cloud-portability-azure-lifecycle-acceptance.md`; it closes Phase 6 but +does not promote Azure beyond experimental status before Phase 8 qualification. diff --git a/tickets/DANDER-109-azure-terraform-plan-verification.md b/tickets/DANDER-109-azure-terraform-plan-verification.md index a3363690..ef86a756 100644 --- a/tickets/DANDER-109-azure-terraform-plan-verification.md +++ b/tickets/DANDER-109-azure-terraform-plan-verification.md @@ -1,6 +1,6 @@ # DANDER-109 — Plan and verify Azure Container Apps deployments -Status: merged through protected PR #189; live proof pending +Status: done; merged through protected PR #189 and live-qualified on 2026-08-11 ## Requirement @@ -30,6 +30,7 @@ Add a read-only verifier that proves a deployed job still matches its manifest a ## Evidence boundary -This ticket contains local deterministic and provider-mocked evidence only. It does not register an -Azure provider, create a resource, copy an image, publish a candidate, read a secret, or spend cloud -credits. Live plan review, apply, cleanup, and no-drift evidence remain separate Phase 6 gates. +The implementation PR contained only deterministic and provider-mocked evidence. The later +separately approved run reviewed and applied the saved plans, verified the deployed jobs, removed +the disposable resources, and ended with retained-GCP no drift. Sanitized evidence is linked from +`docs/cloud-portability-azure-lifecycle-acceptance.md`. diff --git a/tickets/DANDER-110-azure-image-lifecycle-operations.md b/tickets/DANDER-110-azure-image-lifecycle-operations.md index 8f1d8e8d..742f1772 100644 --- a/tickets/DANDER-110-azure-image-lifecycle-operations.md +++ b/tickets/DANDER-110-azure-image-lifecycle-operations.md @@ -1,6 +1,6 @@ # DANDER-110 — Promote and operate Azure Container Apps runtimes -Status: merged through protected PR #190; live proof pending +Status: done; merged through protected PR #190 and live-qualified on 2026-08-11 ## Requirement @@ -31,6 +31,7 @@ logs, stop, and replay operations while keeping all live mutations behind explic ## Evidence boundary -This ticket contains deterministic local contract evidence only. It does not register a provider, -create an Azure resource, authenticate to ACR, copy an image, start or stop a job, publish a -candidate, or spend cloud credits. Live lifecycle and cleanup remain separate Phase 6 gates. +The implementation PR contained deterministic local contract evidence only. The separately +approved Phase 6 run preserved registry digest identity and exercised start, schedule, status, +bounded logs, stop, replay, rollback, cleanup, and no drift. Azure remains experimental pending +Phase 8 qualification. diff --git a/tickets/DANDER-111-azure-google-federation.md b/tickets/DANDER-111-azure-google-federation.md index 348e6406..2fcbfc9f 100644 --- a/tickets/DANDER-111-azure-google-federation.md +++ b/tickets/DANDER-111-azure-google-federation.md @@ -1,7 +1,7 @@ --- id: DANDER-111 title: Prove keyless Azure-to-Google workload federation -status: in_progress +status: done phase: 6 --- @@ -28,13 +28,15 @@ observe Google credential refresh in the same process. - [x] Azure BigQuery planning requires the reviewed GCP data-plane project explicitly instead of looking for deployment scope inside the reusable version-2 platform profile. - [x] Focused Python and provider-mocked Terraform contracts pass locally. -- [ ] An accepted public candidate passes live BigQuery access before and after Google credential - refresh plus GCP secret access and Dataplex read-back under approved ceilings. -- [ ] Revocation fails closed; all disposable Azure/Entra/GCP proof resources are removed; retained +- [x] An accepted public candidate passes live BigQuery access before and after Google credential + refresh plus GCP secret access and Dataplex read-back under approved ceilings. Public + `dander-platform==0.9.0rc1` and its source-free digest were installed outside the checkout and + exercised from Azure Container Apps. +- [x] Revocation fails closed; all disposable Azure/Entra/GCP proof resources are removed; retained GCP finishes with no Terraform drift; sanitized evidence is merged. ## Boundary -No Azure provider registration, Terraform apply, image publication/copy, job execution, Google IAM -mutation, or paid query is authorized by this ticket. Local construction evidence does not promote -Azure or the cross-cloud profile to supported. +Provider writes and paid queries ran only under the operator's explicit per-attempt approval. The +sanitized live record excludes credentials, tokens, query text, rows, state, and plans. This proof +does not promote Azure or the cross-cloud profile beyond experimental status. diff --git a/tickets/DANDER-114-azure-canonical-preflight.md b/tickets/DANDER-114-azure-canonical-preflight.md index f232b41e..bf29ab2c 100644 --- a/tickets/DANDER-114-azure-canonical-preflight.md +++ b/tickets/DANDER-114-azure-canonical-preflight.md @@ -1,7 +1,7 @@ --- id: DANDER-114 title: Preflight the canonical Azure acceptance profile -status: in_progress +status: done phase: 6 --- @@ -23,10 +23,10 @@ Azure/Snowflake/PostgreSQL/no-catalog/Key-Vault composition before any job execu paid-operation confirmation. - [x] Focused provider and CLI tests cover success, missing/disabled secrets, composition rejection, and sanitized output. -- [ ] Protected CI passes and this focused preflight merges before live Azure acceptance. +- [x] Protected CI passed and this focused preflight merged as PR #194 before live Azure acceptance. ## Boundary -This is local construction for a later read-only provider check. It does not register a provider, -create a resource, read a secret value, copy an image, start a job, query a warehouse, or authorize -spending. Version rotation and runtime use remain live acceptance work. +The command itself remains read-only and does not authorize spending. It passed against the live +named profile before execution; the separately approved lifecycle then proved version rotation and +runtime use without retaining secret values. From 1145b07bb6af05f1663dfe0d626d527583c16f55 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 00:28:54 -0400 Subject: [PATCH 144/163] docs: finalize Phase 6 handoff --- HANDOFF.md | 4 ++-- docs/cloud-portability-plan.md | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 51dcfcab..2b0ecca6 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -17,7 +17,7 @@ Read `docs/cloud-portability-azure-lifecycle-acceptance.md` and its linked JSON - Public `0.9.0rc1` install, generated-project validation, and packaged Terraform validation passed. - Fresh isolated GCP and retained GCP plans reported exact `No changes.`. - Retained GCP reproduced 28 stage-zero and 113 platform no-ops with no apply. -- Disposable Azure, Entra, GCP federation, Snowflake, and proof-data resources are absent. +- PR #210 and protected-main CI run `31563211592` passed; disposable proof resources are absent. ## Decisions @@ -27,9 +27,9 @@ Read `docs/cloud-portability-azure-lifecycle-acceptance.md` and its linked JSON ## Remaining -- Merge this evidence PR through protected CI. - Start Phase 7 only after a new explicit instruction and cost/credential preflight. - Keep retained-GCP private operator inputs outside the repository. +- Azure remains experimental until its applicable Phase 8 qualification passes. ## Review First diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 6bdec2da..a057f664 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -15,7 +15,9 @@ evidence PR #186. Phase 6 implementation checkpoint: protected `main` at `eb074c58a9b3d8c1296c28849639a04c07fdb4bf` after PRs #189 through #208. Public candidate `0.9.0rc1` was tagged from protected `main` at `2b90f7ad9d02ad303d3543f1e27febc7193e9c82`. -The sanitized lifecycle record is stored under `docs/evidence/azure/2026-08-11/`. +The sanitized lifecycle record was merged through PR #210 at protected `main` +`0197d931ad1b86f3101a5f2d51170a77c23fe1b7` and is stored under +`docs/evidence/azure/2026-08-11/`. Prepared: 2026-08-06; reconciled: 2026-08-12 From 75eb05281ec614fce4a596fcfd0ff452d9e8a1bc Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 00:50:43 -0400 Subject: [PATCH 145/163] Add OCI Vault provider runtime --- HANDOFF.md | 38 +++--- docs/cloud-portability-plan.md | 2 +- docs/decisions.md | 13 ++ docs/known-limitations.md | 7 +- docs/oci-artifact-contract.md | 8 +- pyproject.toml | 5 +- src/dander/providers/defaults.py | 7 + src/dander/providers/dependencies.py | 2 +- src/dander/providers/oci_vault/__init__.py | 5 + src/dander/providers/oci_vault/config.py | 13 ++ src/dander/providers/oci_vault/runtime.py | 50 +++++++ src/dander/security/__init__.py | 2 + src/dander/security/oci_vault.py | 121 ++++++++++++++++ tests/providers/test_oci_vault_runtime.py | 152 +++++++++++++++++++++ uv.lock | 88 ++++++++++++ 15 files changed, 484 insertions(+), 29 deletions(-) create mode 100644 src/dander/providers/oci_vault/__init__.py create mode 100644 src/dander/providers/oci_vault/config.py create mode 100644 src/dander/providers/oci_vault/runtime.py create mode 100644 src/dander/security/oci_vault.py create mode 100644 tests/providers/test_oci_vault_runtime.py diff --git a/HANDOFF.md b/HANDOFF.md index 2b0ecca6..afc0e096 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,37 @@ ## Finished -- Published and externally installed Dander `0.9.0rc1` from protected main. -- Completed the named Azure/Snowflake/PostgreSQL/Key-Vault lifecycle within approved ceilings. -- Proved public-candidate Azure-to-Google refresh, secret/catalog read-back, and revocation. -- Passed public-candidate Greenhouse, HubSpot, replay, cleanup, and isolated-GCP no-drift smoke. -- Removed disposable provider resources and reconciled the Phase 6 roadmap and evidence. +- Preserved the completed Phase 6 evidence and protected-main baseline. +- Began Phase 7 with the independently reviewable OCI Vault provider slice. +- Added Oracle SDK `2.184.1` to the `oci` and full-runtime dependency sets. +- Added resource-principal-only, versionless OCI Vault resolution with lazy SDK imports. ## Try It -Read `docs/cloud-portability-azure-lifecycle-acceptance.md` and its linked JSON evidence record. +Run `uv run pytest tests/providers/test_oci_vault_runtime.py tests/providers/test_dependencies.py`. ## Checks -- Public `0.9.0rc1` install, generated-project validation, and packaged Terraform validation passed. -- Fresh isolated GCP and retained GCP plans reported exact `No changes.`. -- Retained GCP reproduced 28 stage-zero and 113 platform no-ops with no apply. -- PR #210 and protected-main CI run `31563211592` passed; disposable proof resources are absent. +- Full pytest passed: 1,303 passed and 28 skipped. +- Ruff and repository-wide strict Mypy passed. +- `uv lock` retained cryptography 50 and added OCI SDK `2.184.1`. +- The locked full-runtime dependency audit found no known vulnerabilities. ## Decisions -- Phase 6 passes on the existing typed architecture; Azure remains experimental until Phase 8. -- The canonical Azure and Azure-to-Google profiles remain separate qualification surfaces. -- No Phase 7 or OCI work begins from this handoff. +- Vault and launcher remain separate protected PRs. +- OCI Vault always reads the current version through an ambient resource principal. +- Live OCI writes remain at a $0 ceiling until credential preflight and explicit approval. ## Remaining -- Start Phase 7 only after a new explicit instruction and cost/credential preflight. -- Keep retained-GCP private operator inputs outside the repository. -- Azure remains experimental until its applicable Phase 8 qualification passes. +- Merge the Vault slice after complete CI and review. +- Implement the typed OCI launcher, then infrastructure and lifecycle operations in separate PRs. +- Build one new post-merge candidate and run the complete OCI live gate after account preflight. +- Complete Phase 8 only after Phase 7 passes. ## Review First -- `docs/evidence/azure/2026-08-11/phase6.json` -- `docs/cloud-portability-azure-lifecycle-acceptance.md` -- `docs/cloud-portability-plan.md` +- `src/dander/security/oci_vault.py` +- `src/dander/providers/oci_vault/runtime.py` +- `tests/providers/test_oci_vault_runtime.py` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index a057f664..77110324 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1,6 +1,6 @@ # Cloud Portability and First-Class Platform Roadmap -Status: active roadmap; Phases 0 through 6 exit gates satisfied; Phase 7 not begun +Status: active roadmap; Phases 0 through 6 exit gates satisfied; Phase 7 in progress Initial planning baseline: `origin/main` at `9dc5562` (`0.6.0rc1`) diff --git a/docs/decisions.md b/docs/decisions.md index a8f1acc8..5ddb26fa 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1305,3 +1305,16 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** These results satisfy the Phase 6 architecture and live-profile gate. Azure remains experimental until Phase 8 scale, throughput, crossover, cost, soak, pairwise-profile, and release qualification; no Phase 7 or OCI implementation began. + +## 2026-08-12 — OCI Vault enters before the OCI launcher + +- **Dependency:** Oracle SDK `2.184.1` widens its `cryptography` constraint to admit Dander's + audited version 50 line. The `oci` and `runtime-all` extras therefore include that SDK without a + dependency downgrade; older incompatible SDK releases remain excluded. +- **Secret boundary:** The OCI Vault provider resolves only an exact secret OCID or an exact + vault-OCID/name pair at the `CURRENT` stage. It uses the ambient resource principal, decodes one + base64 UTF-8 value, audits the reference but never the value, and imports the SDK only when + selected. +- **Sequencing:** Vault is independently reviewable before the Container Instances launcher, as + required by the provider PR boundary. This slice makes no launcher, live-profile, or support + claim; paid OCI mutation remains disabled until credential preflight and a per-attempt ceiling. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 57bd6d3e..2b765dba 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -68,9 +68,10 @@ remain Phase 8 work. registration and disposable network creation remain explicit operator-approved actions. These bounded proofs close Phase 6 but do not supply Phase 8 scale, throughput, cost, soak, pairwise-profile, or release qualification, so Azure remains experimental rather than supported. -- The reserved `oci` extra is empty because Oracle's current SDK requires a `cryptography` version - below Dander's audited fixed line. OCI implementation must resolve that dependency boundary or - use a reviewed direct signed-HTTP client before it can enter the full image. +- The `oci` extra now installs Oracle's SDK `2.184.1` or newer without downgrading Dander's audited + `cryptography` line. OCI Vault has a lazy, resource-principal-only resolver for exact secret OCIDs + and versionless vault/name references, but no OCI launcher or live OCI profile is yet qualified. + SDK presence and local Vault tests do not create an OCI support claim. ## Ingestion and schemas diff --git a/docs/oci-artifact-contract.md b/docs/oci-artifact-contract.md index d1d97ecf..1c167103 100644 --- a/docs/oci-artifact-contract.md +++ b/docs/oci-artifact-contract.md @@ -28,14 +28,14 @@ adapter rather than writing into the image filesystem. Python installations expose `bigquery`, `snowflake`, `redshift`, `postgres`, `gcp`, `aws`, `azure`, and `oci` extras. These install provider SDKs only; an extra is not an adapter or a support claim. Provider implementation imports remain lazy until that provider is selected. The `oci` -extra is currently an empty reserved name: Oracle's available SDK constrains `cryptography` below -the audited fixed version, so it is excluded until a compatible SDK or direct signed-HTTP adapter -passes its own review. +extra requires Oracle's SDK `2.184.1` or newer because that release admits Dander's audited +`cryptography` 50 line. Earlier SDK releases remain incompatible with the full runtime. The `runtime-all` extra is the deterministic union of those dependency sets. Repository and generated source-free Dockerfiles install it, then validate every required distribution from package metadata without importing an SDK. This catches an incomplete build before an image can be published. The packaged capability manifest remains authoritative about adapters actually present -and supported in that image; it continues to list only the proven GCP/BigQuery composition today. +and supported in that image. Adding the OCI SDK and Vault resolver does not add an OCI launcher or +profile before their separate implementation and live gates pass. The image uses the maintained Debian `libpq5` package with pure-Python Psycopg instead of bundling an opaque database client library inside a wheel. diff --git a/pyproject.toml b/pyproject.toml index 8acc610b..44a89895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,9 @@ azure = [ "azure-identity>=1.19,<2", "azure-keyvault-secrets>=4.9,<5", ] -oci = [] +oci = [ + "oci>=2.184.1,<3", +] runtime-all = [ "azure-identity>=1.19,<2", "azure-keyvault-secrets>=4.9,<5", @@ -90,6 +92,7 @@ runtime-all = [ "google-cloud-bigquery-storage>=2.30", "google-cloud-dataplex>=2.20,<3", "google-cloud-secret-manager>=2.20", + "oci>=2.184.1,<3", "psycopg[pool]>=3.2,<4; sys_platform == 'linux'", "psycopg[binary,pool]>=3.2,<4; sys_platform != 'linux'", "pyarrow>=19,<26", diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 3371903b..1d2eb630 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -12,6 +12,7 @@ from dander.providers.glue import GlueCatalogConfig from dander.providers.kubernetes import KubernetesLauncherConfig from dander.providers.no_catalog import NoCatalogConfig +from dander.providers.oci_vault import OciVaultConfig from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig from dander.providers.redshift import RedshiftWarehouseConfig from dander.providers.registry import ProviderKind, ProviderRegistry, lazy_provider_factory @@ -123,6 +124,12 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.azure_key_vault.runtime:AZURE_KEY_VAULT_FACTORY" ), ) + registry.register( + kind=ProviderKind.SECRETS, + provider_id="oci_vault", + config_model=OciVaultConfig, + load_factory=lazy_provider_factory("dander.providers.oci_vault.runtime:OCI_VAULT_FACTORY"), + ) registry.register( kind=ProviderKind.LAUNCHER, provider_id="cloud_run", diff --git a/src/dander/providers/dependencies.py b/src/dander/providers/dependencies.py index 0f420a8b..8299fcba 100644 --- a/src/dander/providers/dependencies.py +++ b/src/dander/providers/dependencies.py @@ -49,7 +49,7 @@ class ProviderDependencySet: extra="azure", distributions=("azure-identity", "azure-keyvault-secrets"), ), - ProviderDependencySet(extra="oci", distributions=()), + ProviderDependencySet(extra="oci", distributions=("oci",)), ) FULL_RUNTIME_DISTRIBUTIONS = tuple( diff --git a/src/dander/providers/oci_vault/__init__.py b/src/dander/providers/oci_vault/__init__.py new file mode 100644 index 00000000..6348cafa --- /dev/null +++ b/src/dander/providers/oci_vault/__init__.py @@ -0,0 +1,5 @@ +"""OCI Vault provider configuration.""" + +from dander.providers.oci_vault.config import OciVaultConfig + +__all__ = ["OciVaultConfig"] diff --git a/src/dander/providers/oci_vault/config.py b/src/dander/providers/oci_vault/config.py new file mode 100644 index 00000000..ecbd3f0a --- /dev/null +++ b/src/dander/providers/oci_vault/config.py @@ -0,0 +1,13 @@ +"""Dependency-light OCI Vault configuration.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +class OciVaultConfig(BaseModel): + """Select launcher-native OCI Vault secret resolution.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["oci_vault"] diff --git a/src/dander/providers/oci_vault/runtime.py b/src/dander/providers/oci_vault/runtime.py new file mode 100644 index 00000000..daeeceec --- /dev/null +++ b/src/dander/providers/oci_vault/runtime.py @@ -0,0 +1,50 @@ +"""OCI Vault runtime selected through the provider registry.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from dander.providers.oci_vault.config import OciVaultConfig +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.security.oci_vault import OciVaultSecretStore +from dander.security.runtime import SecretCapabilities, SecretRuntime + +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + + from pydantic import BaseModel + + +def build_oci_vault( + config: BaseModel, + context: Mapping[str, object], +) -> SecretRuntime: + """Build a lazy resource-principal Vault resolver after explicit selection.""" + if not isinstance(config, OciVaultConfig): + raise TypeError("OCI Vault factory received the wrong configuration") + return SecretRuntime( + provider_id="oci_vault", + store=OciVaultSecretStore(client=cast("Any", context.get("client"))), + capabilities=SecretCapabilities( + provider_id="oci_vault", + reference_forms=frozenset( + { + "oci_vault_secret_ocid", + "oci_vault_secret_name", + } + ), + environment_indirection=False, + audited_access=True, + ), + ) + + +OCI_VAULT_FACTORY: ProviderFactory[SecretRuntime] = ProviderFactory( + kind=ProviderKind.SECRETS, + provider_id="oci_vault", + api_version=PROVIDER_API_VERSION, + build=build_oci_vault, +) + +__all__ = ["OCI_VAULT_FACTORY"] diff --git a/src/dander/security/__init__.py b/src/dander/security/__init__.py index dfbd281c..3cdd1efd 100644 --- a/src/dander/security/__init__.py +++ b/src/dander/security/__init__.py @@ -11,6 +11,7 @@ ) from dander.security.oauth1 import OAuth1TBA from dander.security.oauth_jwt import OAuth2JWT +from dander.security.oci_vault import OciVaultSecretStore from dander.security.runtime import SecretCapabilities, SecretRuntime from dander.security.secret_manager import ( DefaultSecretStore, @@ -32,6 +33,7 @@ "OAuth2JWT", "OAuth1TBA", "OAuthTokenError", + "OciVaultSecretStore", "SecretResolutionError", "SecretCapabilities", "SecretRuntime", diff --git a/src/dander/security/oci_vault.py b/src/dander/security/oci_vault.py new file mode 100644 index 00000000..347d9871 --- /dev/null +++ b/src/dander/security/oci_vault.py @@ -0,0 +1,121 @@ +"""Secret resolution from versionless OCI Vault references.""" + +from __future__ import annotations + +import base64 +import binascii +import re +from importlib import import_module +from typing import TYPE_CHECKING, Protocol, cast + +from dander.security.secret_manager import SecretResolutionError, audit_secret_access + +if TYPE_CHECKING: + from typing import Any + +_OCID_SUFFIX = r"[A-Za-z0-9]+" +_REALM = r"oc[0-9]+" +_REGION = r"[a-z0-9-]+" +_SECRET_OCID = rf"ocid1\.vaultsecret\.{_REALM}\.{_REGION}\.{_OCID_SUFFIX}" +_VAULT_OCID = rf"ocid1\.vault\.{_REALM}\.{_REGION}\.{_OCID_SUFFIX}" +_REFERENCE = re.compile( + rf"^oci-vault://(?:" + rf"(?P{_SECRET_OCID})|" + rf"(?P{_VAULT_OCID})/secrets/" + rf"(?P[A-Za-z][A-Za-z0-9_-]{{0,254}})" + rf")$" +) + + +class _SecretBundleContent(Protocol): + content: str | None + content_type: str | None + + +class _SecretBundle(Protocol): + secret_bundle_content: _SecretBundleContent + + +class _SecretResponse(Protocol): + data: _SecretBundle + + +class _SecretsClient(Protocol): + def get_secret_bundle(self, *, secret_id: str, stage: str) -> object: + """Read the current bundle for one exact secret OCID.""" + + def get_secret_bundle_by_name( + self, + *, + secret_name: str, + vault_id: str, + stage: str, + ) -> object: + """Read the current bundle for one name in one exact vault.""" + + +class OciVaultSecretStore: + """Resolve OCI Vault values with the ambient resource principal.""" + + def __init__(self, client: _SecretsClient | None = None) -> None: + self._client = client + self._vault_id: str | None = None + + def get_secret(self, reference: str) -> str: + """Return one UTF-8 secret while always selecting the current version.""" + match = _REFERENCE.fullmatch(reference) + if match is None: + raise SecretResolutionError( + "OCI Vault references must use a secret OCID or exact vault/name URI" + ) + client = self._client_for() + secret_id = match.group("secret_id") + vault_id = match.group("vault_id") + if secret_id is not None: + response = client.get_secret_bundle(secret_id=secret_id, stage="CURRENT") + else: + assert vault_id is not None + secret_name = match.group("secret_name") + assert secret_name is not None + if self._vault_id is None: + self._vault_id = vault_id + elif self._vault_id != vault_id: + raise SecretResolutionError("One OCI Vault resolver cannot cross vaults") + response = client.get_secret_bundle_by_name( + secret_name=secret_name, + vault_id=vault_id, + stage="CURRENT", + ) + value = _decode_secret(cast("_SecretResponse", response)) + audit_secret_access(reference, "oci_vault") + return value + + def _client_for(self) -> _SecretsClient: + if self._client is not None: + return self._client + try: + oci_module = cast("Any", import_module("oci")) + signer = oci_module.auth.signers.get_resource_principals_signer() + client = oci_module.secrets.SecretsClient({}, signer=signer) + except (AttributeError, ImportError, OSError) as error: + raise SecretResolutionError( + "OCI Vault requires the OCI SDK and an ambient resource principal" + ) from error + self._client = cast("_SecretsClient", client) + return self._client + + +def _decode_secret(response: _SecretResponse) -> str: + content = response.data.secret_bundle_content + if content.content_type != "BASE64" or not isinstance(content.content, str): + raise SecretResolutionError("OCI Vault secret is not base64 text") + try: + value = base64.b64decode(content.content, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError) as error: + raise SecretResolutionError("OCI Vault secret is not valid UTF-8 text") from error + if not value: + raise SecretResolutionError("OCI Vault secret is empty") + return value + + +__all__ = ["OciVaultSecretStore"] diff --git a/tests/providers/test_oci_vault_runtime.py b/tests/providers/test_oci_vault_runtime.py new file mode 100644 index 00000000..36f1ac83 --- /dev/null +++ b/tests/providers/test_oci_vault_runtime.py @@ -0,0 +1,152 @@ +"""OCI Vault provider selection and resource-principal-safe resolution.""" + +from __future__ import annotations + +import base64 +import logging +import sys +from dataclasses import dataclass + +import pytest + +from dander.providers import ProviderKind, default_provider_registry +from dander.security import SecretResolutionError, SecretRuntime + +_VAULT_ID = "ocid1.vault.oc1.iad." + "a" * 32 +_SECRET_ID = "ocid1.vaultsecret.oc1.iad." + "b" * 32 + + +@dataclass +class _Content: + content: str | None + content_type: str | None = "BASE64" + + +@dataclass +class _Bundle: + secret_bundle_content: _Content + + +@dataclass +class _Response: + data: _Bundle + + +class _Client: + def __init__(self, value: bytes = b"resolved-value") -> None: + self.content = base64.b64encode(value).decode("ascii") + self.requests: list[tuple[str, dict[str, str]]] = [] + + def get_secret_bundle(self, *, secret_id: str, stage: str) -> _Response: + self.requests.append(("id", {"secret_id": secret_id, "stage": stage})) + return _Response(_Bundle(_Content(self.content))) + + def get_secret_bundle_by_name( + self, + *, + secret_name: str, + vault_id: str, + stage: str, + ) -> _Response: + self.requests.append( + ( + "name", + {"secret_name": secret_name, "vault_id": vault_id, "stage": stage}, + ) + ) + return _Response(_Bundle(_Content(self.content))) + + +def _runtime(client: _Client) -> SecretRuntime: + registry = default_provider_registry() + config = registry.parse(ProviderKind.SECRETS, {"provider": "oci_vault"}) + runtime = registry.build(ProviderKind.SECRETS, config, context={"client": client}) + assert isinstance(runtime, SecretRuntime) + return runtime + + +def test_oci_vault_provider_is_lazy_and_reads_current_secret_by_name( + caplog: pytest.LogCaptureFixture, +) -> None: + runtime_module = "dander.providers.oci_vault.runtime" + sys.modules.pop(runtime_module, None) + sys.modules.pop("oci", None) + client = _Client() + registry = default_provider_registry() + config = registry.parse(ProviderKind.SECRETS, {"provider": "oci_vault"}) + + assert runtime_module not in sys.modules + runtime = registry.build(ProviderKind.SECRETS, config, context={"client": client}) + assert isinstance(runtime, SecretRuntime) + assert runtime_module in sys.modules + assert "oci" not in sys.modules + reference = f"oci-vault://{_VAULT_ID}/secrets/postgres-dsn" + with caplog.at_level(logging.INFO): + assert runtime.store.get_secret(reference) == "resolved-value" + + assert client.requests == [ + ( + "name", + {"secret_name": "postgres-dsn", "vault_id": _VAULT_ID, "stage": "CURRENT"}, + ) + ] + assert caplog.records[-1].__dict__["secret_backend"] == "oci_vault" + assert "resolved-value" not in caplog.text + assert runtime.capabilities.reference_forms == frozenset( + {"oci_vault_secret_ocid", "oci_vault_secret_name"} + ) + + +def test_oci_vault_reads_current_secret_by_ocid() -> None: + client = _Client() + + assert _runtime(client).store.get_secret(f"oci-vault://{_SECRET_ID}") == "resolved-value" + assert client.requests == [("id", {"secret_id": _SECRET_ID, "stage": "CURRENT"})] + + +@pytest.mark.parametrize( + ("reference", "message"), + [ + ("oci-vault://short-name", "secret OCID"), + (f"oci-vault://{_VAULT_ID}/secrets/not/a/name", "secret OCID"), + ], +) +def test_oci_vault_rejects_ambiguous_references(reference: str, message: str) -> None: + with pytest.raises(SecretResolutionError, match=message): + _runtime(_Client()).store.get_secret(reference) + + +@pytest.mark.parametrize( + ("content", "content_type", "message"), + [ + ("not base64", "BASE64", "valid UTF-8"), + (None, "BASE64", "base64 text"), + (base64.b64encode(b"").decode("ascii"), "BASE64", "empty"), + (base64.b64encode(b"value").decode("ascii"), "PLAIN_TEXT", "base64 text"), + ], +) +def test_oci_vault_rejects_invalid_or_empty_content( + content: str | None, + content_type: str, + message: str, +) -> None: + client = _Client() + client.content = content # type: ignore[assignment] + response = client.get_secret_bundle(secret_id=_SECRET_ID, stage="CURRENT") + response.data.secret_bundle_content.content_type = content_type + + class InvalidClient(_Client): + def get_secret_bundle(self, *, secret_id: str, stage: str) -> _Response: + return response + + with pytest.raises(SecretResolutionError, match=message): + _runtime(InvalidClient()).store.get_secret(f"oci-vault://{_SECRET_ID}") + + +def test_oci_vault_resolver_cannot_cross_vaults() -> None: + runtime = _runtime(_Client()) + runtime.store.get_secret(f"oci-vault://{_VAULT_ID}/secrets/postgres-dsn") + other = "ocid1.vault.oc1.iad." + "c" * 32 + + with pytest.raises(SecretResolutionError, match="cross vault"): + runtime.store.get_secret(f"oci-vault://{other}/secrets/postgres-dsn") diff --git a/uv.lock b/uv.lock index f65e572d..649ee5d5 100644 --- a/uv.lock +++ b/uv.lock @@ -479,6 +479,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, ] +[[package]] +name = "circuitbreaker" +version = "2.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/ac/de7a92c4ed39cba31fe5ad9203b76a25ca67c530797f6bb420fff5f65ccb/circuitbreaker-2.1.3.tar.gz", hash = "sha256:1a4baee510f7bea3c91b194dcce7c07805fe96c4423ed5594b75af438531d084", size = 10787, upload-time = "2025-03-31T08:12:08.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/34/15f08edd4628f65217de1fc3c1a27c82e46fe357d60c217fc9881e12ebcc/circuitbreaker-2.1.3-py3-none-any.whl", hash = "sha256:87ba6a3ed03fdc7032bc175561c2b04d52ade9d5faf94ca2b035fbdc5e6b1dd1", size = 7737, upload-time = "2025-03-31T08:12:07.802Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -569,6 +578,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" }, ] +[[package]] +name = "crc32c" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/36/fd18ef23c42926b79c7003e16cb0f79043b5b179c633521343d3b499e996/crc32c-2.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:572ffb1b78cce3d88e8d4143e154d31044a44be42cb3f6fbbf77f1e7a941c5ab", size = 66379, upload-time = "2025-10-17T06:19:10.115Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b8/c584958e53f7798dd358f5bdb1bbfc97483134f053ee399d3eeb26cca075/crc32c-2.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf827b3758ee0c4aacd21ceca0e2da83681f10295c38a10bfeb105f7d98f7a68", size = 63042, upload-time = "2025-10-17T06:19:10.946Z" }, + { url = "https://files.pythonhosted.org/packages/62/e6/6f2af0ec64a668a46c861e5bc778ea3ee42171fedfc5440f791f470fd783/crc32c-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:106fbd79013e06fa92bc3b51031694fcc1249811ed4364ef1554ee3dd2c7f5a2", size = 61528, upload-time = "2025-10-17T06:19:11.768Z" }, + { url = "https://files.pythonhosted.org/packages/17/8b/4a04bd80a024f1a23978f19ae99407783e06549e361ab56e9c08bba3c1d3/crc32c-2.8-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6dde035f91ffbfe23163e68605ee5a4bb8ceebd71ed54bb1fb1d0526cdd125a2", size = 80028, upload-time = "2025-10-17T06:19:12.554Z" }, + { url = "https://files.pythonhosted.org/packages/21/8f/01c7afdc76ac2007d0e6a98e7300b4470b170480f8188475b597d1f4b4c6/crc32c-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e41ebe7c2f0fdcd9f3a3fd206989a36b460b4d3f24816d53e5be6c7dba72c5e1", size = 81531, upload-time = "2025-10-17T06:19:13.406Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/8f78c5a8cc66486be5f51b6f038fc347c3ba748d3ea68be17a014283c331/crc32c-2.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ecf66cf90266d9c15cea597d5cc86c01917cd1a238dc3c51420c7886fa750d7e", size = 80608, upload-time = "2025-10-17T06:19:14.223Z" }, + { url = "https://files.pythonhosted.org/packages/db/86/fad1a94cdeeeb6b6e2323c87f970186e74bfd6fbfbc247bf5c88ad0873d5/crc32c-2.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:59eee5f3a69ad0793d5fa9cdc9b9d743b0cd50edf7fccc0a3988a821fef0208c", size = 79886, upload-time = "2025-10-17T06:19:15.345Z" }, + { url = "https://files.pythonhosted.org/packages/d5/db/1a7cb6757a1e32376fa2dfce00c815ea4ee614a94f9bff8228e37420c183/crc32c-2.8-cp312-cp312-win32.whl", hash = "sha256:a73d03ce3604aa5d7a2698e9057a0eef69f529c46497b27ee1c38158e90ceb76", size = 64896, upload-time = "2025-10-17T06:19:16.457Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8e/2024de34399b2e401a37dcb54b224b56c747b0dc46de4966886827b4d370/crc32c-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:56b3b7d015247962cf58186e06d18c3d75a1a63d709d3233509e1c50a2d36aa2", size = 66645, upload-time = "2025-10-17T06:19:17.235Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, + { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, + { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, + { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, + { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, + { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, + { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, + { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, + { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, + { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, + { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, + { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, + { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, + { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, + { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, + { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, + { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, +] + [[package]] name = "cryptography" version = "50.0.0" @@ -670,6 +732,9 @@ gcp = [ { name = "google-cloud-dataplex" }, { name = "google-cloud-secret-manager" }, ] +oci = [ + { name = "oci" }, +] postgres = [ { name = "psycopg", extra = ["binary"], marker = "sys_platform != 'linux'" }, { name = "psycopg", extra = ["pool"] }, @@ -689,6 +754,7 @@ runtime-all = [ { name = "google-cloud-bigquery-storage" }, { name = "google-cloud-dataplex" }, { name = "google-cloud-secret-manager" }, + { name = "oci" }, { name = "psycopg", extra = ["binary"], marker = "sys_platform != 'linux'" }, { name = "psycopg", extra = ["pool"] }, { name = "pyarrow" }, @@ -731,6 +797,8 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27" }, { name = "jinja2", specifier = ">=3.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11" }, + { name = "oci", marker = "extra == 'oci'", specifier = ">=2.184.1,<3" }, + { name = "oci", marker = "extra == 'runtime-all'", specifier = ">=2.184.1,<3" }, { name = "packaging", specifier = ">=24.0" }, { name = "psycopg", extras = ["binary", "pool"], marker = "sys_platform != 'linux' and extra == 'postgres'", specifier = ">=3.2,<4" }, { name = "psycopg", extras = ["binary", "pool"], marker = "sys_platform != 'linux' and extra == 'runtime-all'", specifier = ">=3.2,<4" }, @@ -1864,6 +1932,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, ] +[[package]] +name = "oci" +version = "2.184.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "circuitbreaker" }, + { name = "crc32c" }, + { name = "cryptography" }, + { name = "pyjwt" }, + { name = "pyopenssl" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/2d/fa5368cfabb868f4111c6978e8b5f66aa3a55076c40c1a59ac3081b0227b/oci-2.184.1.tar.gz", hash = "sha256:617dad69caf8dd6e521d224dbc3e8a8bc289906943a0214fd2c3419094e26435", size = 17990631, upload-time = "2026-08-11T11:01:26.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/63/5ae22e42aaf96a5da74dc2b9de449c78b4d7418cce621d5da723b3e49f32/oci-2.184.1-py3-none-any.whl", hash = "sha256:bd814e38a70da2190e721937455a08689ab13c0750bd2ef8dd0c98b2dc5a38ea", size = 36628063, upload-time = "2026-08-11T11:01:18.178Z" }, +] + [[package]] name = "orjson" version = "3.11.9" From 788a30daf828fa6ff0981f04f662d62724016026 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 00:55:22 -0400 Subject: [PATCH 146/163] Document OCI Vault test fixture --- .gitleaksignore | 5 +++++ tests/providers/test_oci_vault_runtime.py | 12 +++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.gitleaksignore b/.gitleaksignore index c0f1ab42..67b71989 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -13,3 +13,8 @@ b5ea9389a35354a3da154238abba2446f42069cf:tests/security/test_enterprise_auth.py: # rk_live_ prefixes gitleaks's stripe-access-token rule matches on, but the historical commit still # contains the original wording and can't be edited after the fact without rewriting pushed history. 60dc73b90458055b5d8a9d7c7d3e830d283199be:tickets/DANDER-72-row-level-operations.md:stripe-access-token:345 + +# tests/providers/test_oci_vault_runtime.py — a synthetic OCI Vault resource OCID assembled +# from a fixed prefix plus repeated test characters. The scanner matched the original constant +# name, not credential material; the current tree uses a neutral fixture name as well. +75eb05281ec614fce4a596fcfd0ff452d9e8a1bc:tests/providers/test_oci_vault_runtime.py:generic-api-key:16 diff --git a/tests/providers/test_oci_vault_runtime.py b/tests/providers/test_oci_vault_runtime.py index 36f1ac83..7c398dc1 100644 --- a/tests/providers/test_oci_vault_runtime.py +++ b/tests/providers/test_oci_vault_runtime.py @@ -13,7 +13,7 @@ from dander.security import SecretResolutionError, SecretRuntime _VAULT_ID = "ocid1.vault.oc1.iad." + "a" * 32 -_SECRET_ID = "ocid1.vaultsecret.oc1.iad." + "b" * 32 +_TEST_RESOURCE_OCID = "ocid1.vaultsecret.oc1.iad." + "b" * 32 @dataclass @@ -100,8 +100,10 @@ def test_oci_vault_provider_is_lazy_and_reads_current_secret_by_name( def test_oci_vault_reads_current_secret_by_ocid() -> None: client = _Client() - assert _runtime(client).store.get_secret(f"oci-vault://{_SECRET_ID}") == "resolved-value" - assert client.requests == [("id", {"secret_id": _SECRET_ID, "stage": "CURRENT"})] + assert ( + _runtime(client).store.get_secret(f"oci-vault://{_TEST_RESOURCE_OCID}") == "resolved-value" + ) + assert client.requests == [("id", {"secret_id": _TEST_RESOURCE_OCID, "stage": "CURRENT"})] @pytest.mark.parametrize( @@ -132,7 +134,7 @@ def test_oci_vault_rejects_invalid_or_empty_content( ) -> None: client = _Client() client.content = content # type: ignore[assignment] - response = client.get_secret_bundle(secret_id=_SECRET_ID, stage="CURRENT") + response = client.get_secret_bundle(secret_id=_TEST_RESOURCE_OCID, stage="CURRENT") response.data.secret_bundle_content.content_type = content_type class InvalidClient(_Client): @@ -140,7 +142,7 @@ def get_secret_bundle(self, *, secret_id: str, stage: str) -> _Response: return response with pytest.raises(SecretResolutionError, match=message): - _runtime(InvalidClient()).store.get_secret(f"oci-vault://{_SECRET_ID}") + _runtime(InvalidClient()).store.get_secret(f"oci-vault://{_TEST_RESOURCE_OCID}") def test_oci_vault_resolver_cannot_cross_vaults() -> None: From 410b64c7c288b1dd427655612a5278d3c5c8171e Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 01:15:39 -0400 Subject: [PATCH 147/163] Add OCI Container Instances projection --- HANDOFF.md | 34 +-- docs/cloud-portability-plan.md | 8 + docs/compatibility-matrix.md | 6 + docs/decisions.md | 14 + docs/known-limitations.md | 11 +- docs/oci-artifact-contract.md | 7 +- src/dander/deployment/__init__.py | 2 + src/dander/deployment/projection.py | 29 +++ src/dander/project/config.py | 20 +- src/dander/project/portable_config.py | 26 +- src/dander/providers/defaults.py | 10 + .../oci_container_instances/__init__.py | 7 + .../oci_container_instances/config.py | 62 +++++ .../oci_container_instances/runtime.py | 244 ++++++++++++++++++ tests/project/test_oci_portable_config.py | 155 +++++++++++ .../test_oci_container_instances_runtime.py | 189 ++++++++++++++ 16 files changed, 794 insertions(+), 30 deletions(-) create mode 100644 src/dander/providers/oci_container_instances/__init__.py create mode 100644 src/dander/providers/oci_container_instances/config.py create mode 100644 src/dander/providers/oci_container_instances/runtime.py create mode 100644 tests/project/test_oci_portable_config.py create mode 100644 tests/providers/test_oci_container_instances_runtime.py diff --git a/HANDOFF.md b/HANDOFF.md index afc0e096..1ab54813 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,37 +2,37 @@ ## Finished -- Preserved the completed Phase 6 evidence and protected-main baseline. -- Began Phase 7 with the independently reviewable OCI Vault provider slice. -- Added Oracle SDK `2.184.1` to the `oci` and full-runtime dependency sets. -- Added resource-principal-only, versionless OCI Vault resolution with lazy SDK imports. +- Merged the OCI SDK/Vault provider slice through protected PR #212. +- Added a separate lazy, typed OCI Container Instances launcher projection. +- Restricted OCI to the PostgreSQL/PostgreSQL/no-catalog/OCI-Vault profile. +- Added exact OCIR digest, flex-shape, UTC schedule, network, and resource-principal validation. ## Try It -Run `uv run pytest tests/providers/test_oci_vault_runtime.py tests/providers/test_dependencies.py`. +Run `uv run pytest tests/providers/test_oci_container_instances_runtime.py`. ## Checks -- Full pytest passed: 1,303 passed and 28 skipped. -- Ruff and repository-wide strict Mypy passed. -- `uv lock` retained cryptography 50 and added OCI SDK `2.184.1`. -- The locked full-runtime dependency audit found no known vulnerabilities. +- Focused launcher/profile regressions passed: 51 tests. +- The complete pytest suite, Ruff, and repository-wide strict Mypy passed. +- Wheel and source distribution built with the OCI launcher modules included. +- Protected-main CI for the merged Vault slice passed all five jobs. ## Decisions -- Vault and launcher remain separate protected PRs. -- OCI Vault always reads the current version through an ambient resource principal. -- Live OCI writes remain at a $0 ceiling until credential preflight and explicit approval. +- OCI retries are whole-task attempts owned by the later reconciler; native restart is `NEVER`. +- OCI OCPUs and fixed ephemeral storage remain explicit provider semantics. +- Live OCI writes remain at $0 pending preflight and a numeric per-attempt ceiling. ## Remaining -- Merge the Vault slice after complete CI and review. -- Implement the typed OCI launcher, then infrastructure and lifecycle operations in separate PRs. +- Merge this launcher contract through a protected PR. +- Implement Terraform, then scheduling/lifecycle operations in separate PRs. - Build one new post-merge candidate and run the complete OCI live gate after account preflight. - Complete Phase 8 only after Phase 7 passes. ## Review First -- `src/dander/security/oci_vault.py` -- `src/dander/providers/oci_vault/runtime.py` -- `tests/providers/test_oci_vault_runtime.py` +- `src/dander/providers/oci_container_instances/runtime.py` +- `src/dander/providers/oci_container_instances/config.py` +- `tests/providers/test_oci_container_instances_runtime.py` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 77110324..ec7da168 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1201,6 +1201,14 @@ Tickets: 6. PostgreSQL canonical live profile and external-identity feasibility report; 7. schedule, retry, parallelism, interruption, rollback, and reconciliation proof. +Progress on 2026-08-12: the OCI SDK/Vault provider and the separate typed Container Instances +launcher projection are implemented behind lazy provider selection. The launcher accepts only the +named PostgreSQL/PostgreSQL/no-catalog/OCI-Vault profile and exact digest-qualified OCIR images; +infrastructure, promotion, scheduling Function, operations, reconciliation, credential-refresh, +live-profile, cleanup, and no-drift evidence remain open. This progress is not an exit-gate or +support claim, and paid OCI mutation remains disabled pending credential preflight and an explicit +per-attempt ceiling. + Exit gate: - the same release digest passes OCI launcher conformance and the named OCI profile; unsupported diff --git a/docs/compatibility-matrix.md b/docs/compatibility-matrix.md index 3e5b273c..8a11b9a6 100644 --- a/docs/compatibility-matrix.md +++ b/docs/compatibility-matrix.md @@ -30,6 +30,12 @@ Catalog selection is independent from this state/warehouse matrix. Dataplex is s the released GCP profile, `none` is a complete no-cloud-mutation provider, and AWS Glue is an experimental direct API projection with local create/update/readback conformance only. +Launcher/profile selection has an additional fail-closed boundary. The experimental OCI Container +Instances projection currently accepts only PostgreSQL state plus PostgreSQL warehouse, no external +catalog, and OCI Vault. That typed combination is local construction evidence, not a live-profile or +support promotion; all other OCI state, warehouse, catalog, secret, and cross-cloud identity +combinations remain unsupported. + ## Warehouse capabilities The same command publishes the exact implemented warehouse surface. These declarations are diff --git a/docs/decisions.md b/docs/decisions.md index 5ddb26fa..bf1d17ee 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1318,3 +1318,17 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Sequencing:** Vault is independently reviewable before the Container Instances launcher, as required by the provider PR boundary. This slice makes no launcher, live-profile, or support claim; paid OCI mutation remains disabled until credential preflight and a per-attempt ceiling. + +## 2026-08-12 — OCI Container Instances starts with one fail-closed native profile + +- **Selection:** The lazy OCI launcher accepts only the named PostgreSQL state/PostgreSQL + warehouse/no-catalog/OCI-Vault profile. BigQuery and other cross-cloud identities remain + unsupported until a separate keyless refresh proof exists; static cloud keys are never a + fallback. +- **Projection:** Images must use the exact selected OCIR repository and immutable digest. OCI + resource principals are mandatory, schedules are UTC-only, flex-shape memory uses whole GiB, + public IP assignment defaults off, and provider restart policy is `NEVER` so the later lifecycle + reconciler owns bounded whole-task attempts. +- **Boundary:** OCI OCPUs and fixed ephemeral storage remain explicit provider semantics rather + than false cross-provider parity. This contract adds no Terraform, Function, API mutation, live + evidence, packaged support claim, or paid authorization. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 2b765dba..abc558c0 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -70,8 +70,15 @@ remain Phase 8 work. pairwise-profile, or release qualification, so Azure remains experimental rather than supported. - The `oci` extra now installs Oracle's SDK `2.184.1` or newer without downgrading Dander's audited `cryptography` line. OCI Vault has a lazy, resource-principal-only resolver for exact secret OCIDs - and versionless vault/name references, but no OCI launcher or live OCI profile is yet qualified. - SDK presence and local Vault tests do not create an OCI support claim. + and versionless vault/name references. OCI Container Instances now has a typed, lazy execution + projection for only the PostgreSQL/PostgreSQL/no-catalog/OCI-Vault profile. It requires an exact + digest in the selected OCIR repository, a resource-principal dynamic group, UTC five-field cron, + whole-GiB memory, and one container attempt with provider restart policy `NEVER`; unsupported + profiles and inputs fail before OCI access. OCI OCPUs are provider billing/resource units rather + than a cross-provider performance claim, and the fixed 15 GB ephemeral-storage allocation is not + configurable through this projection. Terraform, the launch Function, lifecycle reconciliation, + CLI operations, digest promotion, and the live OCI profile remain incomplete. No OCI launcher + support claim exists until those separate gates pass. ## Ingestion and schemas diff --git a/docs/oci-artifact-contract.md b/docs/oci-artifact-contract.md index 1c167103..c519124c 100644 --- a/docs/oci-artifact-contract.md +++ b/docs/oci-artifact-contract.md @@ -35,7 +35,10 @@ The `runtime-all` extra is the deterministic union of those dependency sets. Rep generated source-free Dockerfiles install it, then validate every required distribution from package metadata without importing an SDK. This catches an incomplete build before an image can be published. The packaged capability manifest remains authoritative about adapters actually present -and supported in that image. Adding the OCI SDK and Vault resolver does not add an OCI launcher or -profile before their separate implementation and live gates pass. +and supported in that image. The OCI SDK and Vault resolver alone did not add a launcher. The +separately registered OCI Container Instances projection remains experimental and outside the +packaged support manifest until its infrastructure, operations, same-digest live profile, and +Phase 8 qualification gates pass. + The image uses the maintained Debian `libpq5` package with pure-Python Psycopg instead of bundling an opaque database client library inside a wheel. diff --git a/src/dander/deployment/__init__.py b/src/dander/deployment/__init__.py index 1d1ef477..24fd64d3 100644 --- a/src/dander/deployment/__init__.py +++ b/src/dander/deployment/__init__.py @@ -6,6 +6,7 @@ EXECUTION_PROJECTION_SCHEMA, FARGATE_CAPABILITIES, KUBERNETES_CAPABILITIES, + OCI_CONTAINER_INSTANCES_CAPABILITIES, ExecutionProjectionError, ExecutionRequest, ExecutionTemplate, @@ -35,6 +36,7 @@ "ExecutionTemplate", "ExecutionTemplateFactory", "KUBERNETES_CAPABILITIES", + "OCI_CONTAINER_INSTANCES_CAPABILITIES", "LauncherCapabilities", "LauncherRuntime", "NetworkPlacement", diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index ef4c86f2..2a6d23b4 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -403,6 +403,35 @@ class LauncherCapabilities: ) +OCI_CONTAINER_INSTANCES_CAPABILITIES = LauncherCapabilities( + launcher="oci_container_instances", + cpu_millis=frozenset({1_000, 2_000, 4_000, 6_000, 8_000}), + minimum_memory_mib=1_024, + maximum_memory_mib=1_540_096, + maximum_deadline_seconds=86_400, + maximum_launcher_retries=10, + maximum_task_count=1, + maximum_parallelism=1, + supports_ephemeral_storage=False, + supports_schedules=True, + supports_time_zones=True, + supports_network_placement=True, + extension_names=frozenset( + { + "oci_assign_public_ip", + "oci_availability_domain", + "oci_compartment_id", + "oci_graceful_shutdown_seconds", + "oci_registry_endpoint", + "oci_restart_policy", + "oci_shape", + "oci_tenancy_id", + "oci_vault_id", + } + ), +) + + def validate_launcher_projection( template: ExecutionTemplate, capabilities: LauncherCapabilities, diff --git a/src/dander/project/config.py b/src/dander/project/config.py index 3860579f..0274b0b0 100644 --- a/src/dander/project/config.py +++ b/src/dander/project/config.py @@ -240,13 +240,19 @@ class DanderProject(BaseModel): default_factory=_default_catalog_config, exclude=True, ) - secret_provider: Literal["gcp_secret_manager", "environment", "azure_key_vault"] = Field( - default="gcp_secret_manager", - exclude=True, - ) - launcher_provider: Literal["cloud_run", "fargate", "kubernetes", "azure_container_apps"] = ( - Field(default="cloud_run", exclude=True) - ) + secret_provider: Literal[ + "gcp_secret_manager", + "environment", + "azure_key_vault", + "oci_vault", + ] = Field(default="gcp_secret_manager", exclude=True) + launcher_provider: Literal[ + "cloud_run", + "fargate", + "kubernetes", + "azure_container_apps", + "oci_container_instances", + ] = Field(default="cloud_run", exclude=True) launcher_config: dict[str, object] = Field(default_factory=dict, exclude=True) deployed_pipeline_ids: tuple[str, ...] | None = Field(default=None, exclude=True) diff --git a/src/dander/project/portable_config.py b/src/dander/project/portable_config.py index d9bfb0e4..697628d3 100644 --- a/src/dander/project/portable_config.py +++ b/src/dander/project/portable_config.py @@ -36,6 +36,10 @@ from dander.providers.glue import GlueCatalogConfig # noqa: TC001 from dander.providers.kubernetes import KubernetesLauncherConfig # noqa: TC001 from dander.providers.no_catalog import NoCatalogConfig # noqa: TC001 +from dander.providers.oci_container_instances import ( # noqa: TC001 + OciContainerInstancesLauncherConfig, +) +from dander.providers.oci_vault import OciVaultConfig # noqa: TC001 from dander.providers.postgresql import ( # noqa: TC001 PostgreSQLStateConfig, PostgreSQLWarehouseConfig, @@ -138,7 +142,7 @@ def validate_pipeline_ids( SecretProviderSpec = Annotated[ - GcpSecretManagerConfig | EnvironmentSecretConfig | AzureKeyVaultConfig, + GcpSecretManagerConfig | EnvironmentSecretConfig | AzureKeyVaultConfig | OciVaultConfig, Field(discriminator="provider"), ] @@ -147,7 +151,8 @@ def validate_pipeline_ids( CloudRunLauncherConfig | FargateLauncherConfig | KubernetesLauncherConfig - | AzureContainerAppsLauncherConfig, + | AzureContainerAppsLauncherConfig + | OciContainerInstancesLauncherConfig, Field(discriminator="provider"), ] @@ -350,6 +355,13 @@ def _resolve( raise ProjectConfigError( "Azure Key Vault projection currently requires launcher.provider='azure_container_apps'" ) + if ( + profile.secrets.provider == "oci_vault" + and selected.launcher.provider != "oci_container_instances" + ): + raise ProjectConfigError( + "OCI Vault projection currently requires launcher.provider='oci_container_instances'" + ) if ( selected.launcher.provider == "cloud_run" and profile.secrets.provider != "gcp_secret_manager" @@ -379,6 +391,16 @@ def _resolve( raise ProjectConfigError( "Azure GCP secrets require the named BigQuery/Dataplex federation profile" ) + if selected.launcher.provider == "oci_container_instances" and ( + profile.secrets.provider != "oci_vault" + or profile.warehouse.provider != "postgresql" + or profile.state.provider != "postgresql" + or profile.catalog.provider != "none" + ): + raise ProjectConfigError( + "OCI Container Instances currently requires the named " + "PostgreSQL/PostgreSQL/no-catalog/OCI-Vault profile" + ) unknown_pipelines = sorted(set(selected.pipelines) - set(logical.pipelines)) if unknown_pipelines: raise ProjectConfigError( diff --git a/src/dander/providers/defaults.py b/src/dander/providers/defaults.py index 1d2eb630..ef6d5b24 100644 --- a/src/dander/providers/defaults.py +++ b/src/dander/providers/defaults.py @@ -12,6 +12,7 @@ from dander.providers.glue import GlueCatalogConfig from dander.providers.kubernetes import KubernetesLauncherConfig from dander.providers.no_catalog import NoCatalogConfig +from dander.providers.oci_container_instances import OciContainerInstancesLauncherConfig from dander.providers.oci_vault import OciVaultConfig from dander.providers.postgresql import PostgreSQLStateConfig, PostgreSQLWarehouseConfig from dander.providers.redshift import RedshiftWarehouseConfig @@ -162,4 +163,13 @@ def default_provider_registry() -> ProviderRegistry: "dander.providers.azure_container_apps.runtime:AZURE_CONTAINER_APPS_LAUNCHER_FACTORY" ), ) + registry.register( + kind=ProviderKind.LAUNCHER, + provider_id="oci_container_instances", + config_model=OciContainerInstancesLauncherConfig, + load_factory=lazy_provider_factory( + "dander.providers.oci_container_instances.runtime:" + "OCI_CONTAINER_INSTANCES_LAUNCHER_FACTORY" + ), + ) return registry diff --git a/src/dander/providers/oci_container_instances/__init__.py b/src/dander/providers/oci_container_instances/__init__.py new file mode 100644 index 00000000..a0d5638a --- /dev/null +++ b/src/dander/providers/oci_container_instances/__init__.py @@ -0,0 +1,7 @@ +"""OCI Container Instances launcher configuration.""" + +from dander.providers.oci_container_instances.config import ( + OciContainerInstancesLauncherConfig, +) + +__all__ = ["OciContainerInstancesLauncherConfig"] diff --git a/src/dander/providers/oci_container_instances/config.py b/src/dander/providers/oci_container_instances/config.py new file mode 100644 index 00000000..40f87c85 --- /dev/null +++ b/src/dander/providers/oci_container_instances/config.py @@ -0,0 +1,62 @@ +"""Validated OCI Container Instances configuration without SDK imports.""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +_REGION = re.compile(r"^[a-z]{2}(?:-[a-z0-9]+)+-[1-9][0-9]*$") +_TENANCY_OCID = re.compile(r"^ocid1\.tenancy\.oc[0-9]+\.\.[A-Za-z0-9]+$") +_COMPARTMENT_OCID = re.compile(r"^ocid1\.compartment\.oc[0-9]+\.\.[A-Za-z0-9]+$") +_SUBNET_OCID = re.compile(r"^ocid1\.subnet\.oc[0-9]+\.[a-z0-9-]+\.[A-Za-z0-9]+$") +_VAULT_OCID = re.compile(r"^ocid1\.vault\.oc[0-9]+\.[a-z0-9-]+\.[A-Za-z0-9]+$") +_AVAILABILITY_DOMAIN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9:-]{0,254}$") +_NAMESPACE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,99}$") +_REPOSITORY = re.compile(r"^[a-z0-9][a-z0-9._/-]{0,254}$") +_DYNAMIC_GROUP = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,99}$") + + +class OciContainerInstancesLauncherConfig(BaseModel): + """Select one reviewed OCI compartment, network, registry, and identity.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["oci_container_instances"] + region: str = Field(pattern=_REGION.pattern) + tenancy_id: str = Field(pattern=_TENANCY_OCID.pattern) + compartment_id: str = Field(pattern=_COMPARTMENT_OCID.pattern) + availability_domain: str = Field(pattern=_AVAILABILITY_DOMAIN.pattern) + subnet_id: str = Field(pattern=_SUBNET_OCID.pattern) + registry_namespace: str = Field(pattern=_NAMESPACE.pattern) + repository_name: str = Field(pattern=_REPOSITORY.pattern) + vault_id: str = Field(pattern=_VAULT_OCID.pattern) + dynamic_group_name: str = Field(pattern=_DYNAMIC_GROUP.pattern) + shape: Literal[ + "CI.Standard.E4.Flex", + "CI.Standard.E5.Flex", + "CI.Standard.A1.Flex", + ] = "CI.Standard.E4.Flex" + assign_public_ip: bool = False + graceful_shutdown_seconds: int = Field(default=120, ge=0) + + @field_validator("repository_name") + @classmethod + def reject_ambiguous_repository_path(cls, value: str) -> str: + parts = value.split("/") + if any(part in {"", ".", ".."} for part in parts): + raise ValueError("OCI repository_name must contain unambiguous path segments") + return value + + @property + def registry_endpoint(self) -> str: + return f"ocir.{self.region}.oci.oraclecloud.com" + + @property + def repository(self) -> str: + return f"{self.registry_endpoint}/{self.registry_namespace}/{self.repository_name}" + + @property + def resource_principal_identity(self) -> str: + return f"oci-resource-principal://dynamic-group/{self.dynamic_group_name}" diff --git a/src/dander/providers/oci_container_instances/runtime.py b/src/dander/providers/oci_container_instances/runtime.py new file mode 100644 index 00000000..5ec6cfbb --- /dev/null +++ b/src/dander/providers/oci_container_instances/runtime.py @@ -0,0 +1,244 @@ +"""OCI Container Instances execution projection.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from dander import __version__ +from dander.deployment.projection import ( + EXECUTION_PROJECTION_SCHEMA, + OCI_CONTAINER_INSTANCES_CAPABILITIES, + ExecutionProjectionError, + ExecutionTemplate, + NetworkPlacement, + ObservabilityProjection, + ResourceProjection, + ScheduleProjection, + SecretReference, + validate_launcher_projection, +) +from dander.deployment.runtime import LauncherRuntime, ResolvedTemplateRequest +from dander.providers.oci_container_instances.config import ( + OciContainerInstancesLauncherConfig, +) +from dander.providers.registry import PROVIDER_API_VERSION, ProviderFactory, ProviderKind +from dander.runtime_contract import RUNTIME_CONTRACT + +if TYPE_CHECKING: + from pydantic import BaseModel + +_CRON_PART = re.compile(r"^[A-Za-z0-9*/,-]+$") +_VAULT_SECRET = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,254}$") + + +@dataclass(frozen=True, slots=True) +class OciContainerInstancesTemplateFactory: + """Project one named PostgreSQL profile into OCI Container Instances.""" + + config: OciContainerInstancesLauncherConfig + + def build(self, request: ResolvedTemplateRequest) -> dict[str, ExecutionTemplate]: + """Build immutable OCI templates without contacting OCI.""" + expected_prefix = f"{self.config.repository}@" + image_digest = request.image.removeprefix(expected_prefix) + if ( + not request.image.startswith(expected_prefix) + or re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest) is None + ): + raise ExecutionProjectionError( + "OCI Container Instances requires an immutable image " + "in the selected OCIR repository" + ) + memory_mib = _memory_mib(request.memory) + cpu_millis = request.cpu * 1_000 + _validate_oci_size( + shape=self.config.shape, + cpu_millis=cpu_millis, + memory_mib=memory_mib, + ) + templates: dict[str, ExecutionTemplate] = {} + for pipeline_id, pipeline in sorted(request.pipelines.items()): + secret_env = pipeline["secret_env"] + if not isinstance(secret_env, Mapping): + raise ExecutionProjectionError("pipeline secret bindings are invalid") + if any( + not isinstance(secret_name, str) or _VAULT_SECRET.fullmatch(secret_name) is None + for secret_name in secret_env.values() + ): + raise ExecutionProjectionError( + "OCI Vault secret names must use letters, numbers, underscores, or hyphens" + ) + schedule = _oci_schedule( + expression=str(pipeline["schedule"]), + time_zone=str(pipeline["time_zone"]), + ) + command: tuple[str, ...] = ( + "runtime", + "execute", + "--contract", + RUNTIME_CONTRACT, + "--pipeline", + pipeline_id, + "--platform", + request.profile_id, + "--config", + "/app/dander.yaml", + "--models-dir", + "/app/models", + "--batch-rows", + str(request.batch_rows), + ) + if bool(pipeline["build_models"]): + command = (*command, "--catalog-output", "/tmp/dander-catalog.json") + digest = request.image.rsplit("@", maxsplit=1)[-1] + identity = self.config.resource_principal_identity + template = ExecutionTemplate( + schema=EXECUTION_PROJECTION_SCHEMA, + contract=RUNTIME_CONTRACT, + pipeline_id=pipeline_id, + profile_id=request.profile_id, + launcher="oci_container_instances", + image=request.image, + command=command, + configuration_reference="/app/dander.yaml", + environment=tuple( + sorted( + { + "DANDER_IMAGE_DIGEST": digest, + "DANDER_LAUNCHER": "oci_container_instances", + "DANDER_OCI_REGION": self.config.region, + "DANDER_PRINCIPAL": identity, + "HOME": "/tmp", + "TMPDIR": "/tmp", + }.items() + ) + ), + secret_bindings=tuple( + ( + str(environment_name), + SecretReference( + provider="oci_vault", + reference=(f"oci-vault://{self.config.vault_id}/secrets/{secret_name}"), + ), + ) + for environment_name, secret_name in sorted(secret_env.items()) + ), + workload_identity=identity, + resources=ResourceProjection( + cpu_millis=cpu_millis, + memory_mib=memory_mib, + ephemeral_storage_mib=None, + deadline_seconds=request.deadline_seconds, + runtime_retry_count=0, + launcher_retry_count=request.launcher_retry_count, + ), + schedule=ScheduleProjection( + task_count=1, + maximum_parallelism=1, + expression=schedule, + time_zone="UTC", + paused=bool(pipeline["paused"]), + ), + network=NetworkPlacement( + placement=self.config.subnet_id, + extensions=( + ("oci_assign_public_ip", str(self.config.assign_public_ip).lower()), + ("oci_availability_domain", self.config.availability_domain), + ), + ), + labels=( + ("dander_version", __version__), + ("image_digest", digest), + ("pipeline", pipeline_id), + ("profile", request.profile_id), + ), + observability=ObservabilityProjection( + log_destination="oci_logging", + metric_namespace="oci_computecontainerinstance", + alert_target=request.alert_target, + retention_days=30, + ), + extensions=( + ("oci_compartment_id", self.config.compartment_id), + ( + "oci_graceful_shutdown_seconds", + str(self.config.graceful_shutdown_seconds), + ), + ("oci_registry_endpoint", self.config.registry_endpoint), + ("oci_restart_policy", "NEVER"), + ("oci_shape", self.config.shape), + ("oci_tenancy_id", self.config.tenancy_id), + ("oci_vault_id", self.config.vault_id), + ), + ) + validate_launcher_projection(template, OCI_CONTAINER_INSTANCES_CAPABILITIES) + templates[pipeline_id] = template + return templates + + +def build_oci_container_instances_launcher( + config: BaseModel, + context: Mapping[str, object], +) -> LauncherRuntime: + """Build OCI projection behavior only after explicit launcher selection.""" + del context + if not isinstance(config, OciContainerInstancesLauncherConfig): + raise TypeError("OCI Container Instances factory received the wrong configuration") + return LauncherRuntime( + provider_id="oci_container_instances", + region=config.region, + templates=OciContainerInstancesTemplateFactory(config), + capabilities=OCI_CONTAINER_INSTANCES_CAPABILITIES, + ) + + +def _memory_mib(value: str) -> int: + match = re.fullmatch(r"([1-9][0-9]*)(Mi|Gi)", value) + if match is None: + raise ExecutionProjectionError("runtime memory must use Mi or Gi") + quantity = int(match.group(1)) + memory_mib = quantity if match.group(2) == "Mi" else quantity * 1_024 + if memory_mib % 1_024: + raise ExecutionProjectionError("OCI Container Instances requires whole-GiB memory") + return memory_mib + + +def _validate_oci_size(*, shape: str, cpu_millis: int, memory_mib: int) -> None: + ocpus = cpu_millis // 1_000 + memory_gib = memory_mib // 1_024 + if cpu_millis % 1_000 or not 1 <= ocpus <= 8: + raise ExecutionProjectionError("OCI Container Instances requires 1 through 8 OCPUs") + if shape == "CI.Standard.E5.Flex": + valid = 1 <= memory_gib <= 1_504 + elif shape == "CI.Standard.A1.Flex": + valid = ocpus <= memory_gib <= min(64 * ocpus, 488) + else: + valid = ocpus <= memory_gib <= min(64 * ocpus, 1_024) + if not valid: + raise ExecutionProjectionError( + "OCI Container Instances does not support the requested shape CPU/memory pair" + ) + + +def _oci_schedule(*, expression: str, time_zone: str) -> str: + if time_zone != "UTC": + raise ExecutionProjectionError("OCI scheduled Functions require UTC") + parts = expression.split() + if len(parts) != 5 or any(_CRON_PART.fullmatch(part) is None for part in parts): + raise ExecutionProjectionError( + "OCI scheduled Functions require a valid five-field cron schedule" + ) + return " ".join(parts) + + +OCI_CONTAINER_INSTANCES_LAUNCHER_FACTORY: ProviderFactory[LauncherRuntime] = ProviderFactory( + kind=ProviderKind.LAUNCHER, + provider_id="oci_container_instances", + api_version=PROVIDER_API_VERSION, + build=build_oci_container_instances_launcher, +) + +__all__ = ["OCI_CONTAINER_INSTANCES_LAUNCHER_FACTORY"] diff --git a/tests/project/test_oci_portable_config.py b/tests/project/test_oci_portable_config.py new file mode 100644 index 00000000..432ff059 --- /dev/null +++ b/tests/project/test_oci_portable_config.py @@ -0,0 +1,155 @@ +"""Named OCI profile selection and unsupported-combination rejection.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +import yaml + +from dander.project import ProjectConfigError, load_project_config + +if TYPE_CHECKING: + from pathlib import Path + +_TENANCY_OCID = "ocid1.tenancy.oc1.." + "a" * 32 +_COMPARTMENT_OCID = "ocid1.compartment.oc1.." + "b" * 32 +_SUBNET_OCID = "ocid1.subnet.oc1.iad." + "c" * 32 +_VAULT_OCID = "ocid1.vault.oc1.iad." + "d" * 32 + + +def _documents() -> tuple[dict[str, object], dict[str, object]]: + logical: dict[str, object] = { + "version": 2, + "pipelines": { + "warehouse_fixture": { + "source": "warehouse_fixture", + "models": ["fixture"], + } + }, + } + platforms: dict[str, object] = { + "version": 1, + "platforms": { + "oci_postgresql": { + "warehouse": { + "provider": "postgresql", + "database": "dander", + "schema": "raw", + }, + "state": { + "provider": "postgresql", + "authority_id": "postgresql:oci-phase7", + }, + "catalog": {"provider": "none"}, + "secrets": {"provider": "oci_vault"}, + } + }, + "deployments": { + "oci_container_instances": { + "platform": "oci_postgresql", + "launcher": { + "provider": "oci_container_instances", + "region": "us-ashburn-1", + "tenancy_id": _TENANCY_OCID, + "compartment_id": _COMPARTMENT_OCID, + "availability_domain": "Unit:US-ASHBURN-AD-1", + "subnet_id": _SUBNET_OCID, + "registry_namespace": "unitnamespace", + "repository_name": "dander/runtime", + "vault_id": _VAULT_OCID, + "dynamic_group_name": "dander_phase7_runtime", + }, + "runtime": { + "cpu": 1, + "memory": "2Gi", + "timeout_seconds": 900, + "max_retries": 1, + }, + "safety": {"require_guarded_free_tier": False}, + "pipelines": { + "warehouse_fixture": { + "schedule": "15 4 * * *", + "time_zone": "UTC", + "paused": True, + "secret_bindings": {"DANDER_POSTGRES_DSN": "postgres-dsn"}, + } + }, + } + }, + } + return logical, platforms + + +def _write_documents( + tmp_path: Path, + *, + platforms: dict[str, object], + logical: dict[str, object], +) -> Path: + project_path = tmp_path / "dander.yaml" + project_path.write_text(yaml.safe_dump(logical), encoding="utf-8") + (tmp_path / "dander.platforms.yaml").write_text( + yaml.safe_dump(platforms), + encoding="utf-8", + ) + return project_path + + +def test_version_two_resolves_complete_oci_postgresql_profile(tmp_path: Path) -> None: + logical, platforms = _documents() + resolved = load_project_config( + _write_documents(tmp_path, platforms=platforms, logical=logical), + deployment="oci_container_instances", + ) + + assert resolved.platform_name == "oci_postgresql" + assert resolved.warehouse_provider == "postgresql" + assert resolved.state_provider == "postgresql" + assert resolved.catalog_provider == "none" + assert resolved.secret_provider == "oci_vault" + assert resolved.launcher_provider == "oci_container_instances" + assert resolved.resolved_launcher_config()["vault_id"] == _VAULT_OCID + assert resolved.platform.runtime.memory == "2Gi" + + +@pytest.mark.parametrize( + ("profile_update", "launcher_provider", "message"), + [ + ( + {"warehouse": {"provider": "bigquery", "location": "US"}}, + "oci_container_instances", + "named PostgreSQL", + ), + ({}, "cloud_run", "OCI Vault projection"), + ], +) +def test_oci_profile_fails_closed_on_unsupported_combinations( + tmp_path: Path, + profile_update: dict[str, object], + launcher_provider: str, + message: str, +) -> None: + logical, platforms = _documents() + profiles = platforms["platforms"] + assert isinstance(profiles, dict) + profile = profiles["oci_postgresql"] + assert isinstance(profile, dict) + profile.update(profile_update) + deployments = platforms["deployments"] + assert isinstance(deployments, dict) + deployment = deployments["oci_container_instances"] + assert isinstance(deployment, dict) + launcher = deployment["launcher"] + assert isinstance(launcher, dict) + if launcher_provider == "cloud_run": + deployment["launcher"] = { + "provider": "cloud_run", + "region": "us-central1", + } + + with pytest.raises(ProjectConfigError, match=message): + load_project_config( + _write_documents(tmp_path, platforms=platforms, logical=logical), + deployment="oci_container_instances", + ) diff --git a/tests/providers/test_oci_container_instances_runtime.py b/tests/providers/test_oci_container_instances_runtime.py new file mode 100644 index 00000000..32becab1 --- /dev/null +++ b/tests/providers/test_oci_container_instances_runtime.py @@ -0,0 +1,189 @@ +"""OCI Container Instances launcher projection and fail-closed limits.""" + +from __future__ import annotations + +import sys + +import pytest + +from dander.deployment import ExecutionProjectionError, LauncherRuntime, ResolvedTemplateRequest +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry + +_TENANCY_OCID = "ocid1.tenancy.oc1.." + "a" * 32 +_COMPARTMENT_OCID = "ocid1.compartment.oc1.." + "b" * 32 +_SUBNET_OCID = "ocid1.subnet.oc1.iad." + "c" * 32 +_VAULT_OCID = "ocid1.vault.oc1.iad." + "d" * 32 +_IMAGE = "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime@sha256:" + "e" * 64 +_LAUNCHER = { + "provider": "oci_container_instances", + "region": "us-ashburn-1", + "tenancy_id": _TENANCY_OCID, + "compartment_id": _COMPARTMENT_OCID, + "availability_domain": "Unit:US-ASHBURN-AD-1", + "subnet_id": _SUBNET_OCID, + "registry_namespace": "unitnamespace", + "repository_name": "dander/runtime", + "vault_id": _VAULT_OCID, + "dynamic_group_name": "dander_phase7_runtime", +} +_PIPELINES: dict[str, dict[str, object]] = { + "warehouse_fixture": { + "runtime_service_account_id": "unused-on-oci", + "build_models": True, + "schedule": "15 4 * * *", + "time_zone": "UTC", + "paused": True, + "secret_env": {"DANDER_POSTGRES_DSN": "postgres-dsn"}, + } +} + + +def _runtime(*, launcher: dict[str, object] | None = None) -> LauncherRuntime: + registry = default_provider_registry() + config = registry.parse(ProviderKind.LAUNCHER, launcher or _LAUNCHER) + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + return runtime + + +def _request( + *, + pipelines: dict[str, dict[str, object]] | None = None, + image: str = _IMAGE, + cpu: int = 1, + memory: str = "2Gi", +) -> ResolvedTemplateRequest: + return ResolvedTemplateRequest( + pipelines=_PIPELINES if pipelines is None else pipelines, + image=image, + profile_id="oci_postgresql", + cpu=cpu, + memory=memory, + deadline_seconds=900, + launcher_retry_count=1, + batch_rows=1_000, + alert_target="oci_notifications", + ) + + +def test_oci_factory_is_lazy_and_projects_resource_principal_without_gcp_context() -> None: + module_name = "dander.providers.oci_container_instances.runtime" + sys.modules.pop(module_name, None) + registry = default_provider_registry() + config = registry.parse(ProviderKind.LAUNCHER, _LAUNCHER) + + assert module_name not in sys.modules + runtime = registry.build(ProviderKind.LAUNCHER, config) + assert isinstance(runtime, LauncherRuntime) + assert module_name in sys.modules + + template = runtime.templates.build(_request())["warehouse_fixture"] + environment = dict(template.environment) + secrets = dict(template.secret_bindings) + + assert runtime.provider_id == "oci_container_instances" + assert runtime.region == "us-ashburn-1" + assert template.image == _IMAGE + assert template.profile_id == "oci_postgresql" + assert template.resources.cpu_millis == 1_000 + assert template.resources.memory_mib == 2_048 + assert template.resources.ephemeral_storage_mib is None + assert template.schedule.expression == "15 4 * * *" + assert template.schedule.time_zone == "UTC" + assert template.schedule.paused is True + assert template.network.placement == _SUBNET_OCID + assert dict(template.network.extensions) == { + "oci_assign_public_ip": "false", + "oci_availability_domain": "Unit:US-ASHBURN-AD-1", + } + assert template.workload_identity == ( + "oci-resource-principal://dynamic-group/dander_phase7_runtime" + ) + assert environment["DANDER_OCI_REGION"] == "us-ashburn-1" + assert environment["HOME"] == "/tmp" + assert "DANDER_POSTGRES_DSN" not in environment + assert secrets["DANDER_POSTGRES_DSN"].provider == "oci_vault" + assert secrets["DANDER_POSTGRES_DSN"].reference == ( + f"oci-vault://{_VAULT_OCID}/secrets/postgres-dsn" + ) + assert dict(template.extensions) == { + "oci_compartment_id": _COMPARTMENT_OCID, + "oci_graceful_shutdown_seconds": "120", + "oci_registry_endpoint": "ocir.us-ashburn-1.oci.oraclecloud.com", + "oci_restart_policy": "NEVER", + "oci_shape": "CI.Standard.E4.Flex", + "oci_tenancy_id": _TENANCY_OCID, + "oci_vault_id": _VAULT_OCID, + } + serialized = repr(template.as_dict()) + assert "postgresql://" not in serialized + assert "private_key" not in serialized + + +@pytest.mark.parametrize( + ("update", "message"), + [ + ({"time_zone": "America/New_York"}, "require UTC"), + ({"schedule": "0 9 * * ?"}, "five-field"), + ({"secret_env": {"TOKEN": "not.a.vault.name"}}, "secret names"), + ], +) +def test_oci_projection_rejects_unhonored_schedule_or_secret_intent( + update: dict[str, object], + message: str, +) -> None: + pipeline = {**_PIPELINES["warehouse_fixture"], **update} + with pytest.raises(ExecutionProjectionError, match=message): + _runtime().templates.build(_request(pipelines={"warehouse_fixture": pipeline})) + + +@pytest.mark.parametrize( + ("image", "cpu", "memory", "launcher_update", "message"), + [ + ( + "ocir.us-phoenix-1.oci.oraclecloud.com/unitnamespace/dander/runtime@sha256:" + "e" * 64, + 1, + "2Gi", + {}, + "selected OCIR repository", + ), + (_IMAGE, 1, "1536Mi", {}, "whole-GiB"), + ( + _IMAGE, + 4, + "2Gi", + {"shape": "CI.Standard.A1.Flex"}, + "CPU/memory pair", + ), + ], +) +def test_oci_projection_rejects_unowned_images_and_invalid_resource_pairs( + image: str, + cpu: int, + memory: str, + launcher_update: dict[str, object], + message: str, +) -> None: + with pytest.raises(ExecutionProjectionError, match=message): + _runtime(launcher={**_LAUNCHER, **launcher_update}).templates.build( + _request(image=image, cpu=cpu, memory=memory) + ) + + +@pytest.mark.parametrize( + ("update", "field"), + [ + ({"tenancy_id": "ocid1.tenancy.oc1.iad.invalid"}, "tenancy_id"), + ({"repository_name": "dander//runtime"}, "repository_name"), + ({"subnet_id": "ocid1.vcn.oc1.iad.invalid"}, "subnet_id"), + ], +) +def test_oci_launcher_configuration_rejects_ambiguous_identifiers( + update: dict[str, object], + field: str, +) -> None: + with pytest.raises(ProviderFactoryError, match=field): + default_provider_registry().parse( + ProviderKind.LAUNCHER, + {**_LAUNCHER, **update}, + ) From 2f0c3b6212781c7738aa5037346b02c6b04cae99 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 01:36:29 -0400 Subject: [PATCH 148/163] Add OCI Terraform foundation --- .github/workflows/ci.yml | 12 +- .github/workflows/live-proof.yml | 4 +- HANDOFF.md | 30 +- docs/cloud-portability-plan.md | 19 +- docs/decisions.md | 13 + infra/README.md | 4 + infra/oci/.terraform.lock.hcl | 24 + infra/oci/README.md | 18 + infra/oci/bootstrap-admin/.terraform.lock.hcl | 24 + infra/oci/bootstrap-admin/README.md | 14 + infra/oci/bootstrap-admin/main.tf | 32 + infra/oci/bootstrap-admin/outputs.tf | 19 + .../tests/bootstrap.tftest.hcl | 36 + infra/oci/bootstrap-admin/variables.tf | 67 ++ infra/oci/bootstrap-admin/versions.tf | 17 + infra/oci/main.tf | 149 ++++ infra/oci/outputs.tf | 39 ++ infra/oci/terraform.tfvars.example | 7 + infra/oci/tests/oci.tftest.hcl | 49 ++ infra/oci/variables.tf | 84 +++ infra/oci/versions.tf | 19 + pyproject.toml | 15 + src/dander/bootstrap/__init__.py | 8 + src/dander/bootstrap/oci_terraform.py | 639 ++++++++++++++++++ src/dander/cli/main.py | 2 + src/dander/cli/oci_command.py | 276 ++++++++ tests/bootstrap/test_oci_terraform.py | 192 ++++++ tests/cli/test_oci_cli.py | 137 ++++ 28 files changed, 1924 insertions(+), 25 deletions(-) create mode 100644 infra/oci/.terraform.lock.hcl create mode 100644 infra/oci/README.md create mode 100644 infra/oci/bootstrap-admin/.terraform.lock.hcl create mode 100644 infra/oci/bootstrap-admin/README.md create mode 100644 infra/oci/bootstrap-admin/main.tf create mode 100644 infra/oci/bootstrap-admin/outputs.tf create mode 100644 infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl create mode 100644 infra/oci/bootstrap-admin/variables.tf create mode 100644 infra/oci/bootstrap-admin/versions.tf create mode 100644 infra/oci/main.tf create mode 100644 infra/oci/outputs.tf create mode 100644 infra/oci/terraform.tfvars.example create mode 100644 infra/oci/tests/oci.tftest.hcl create mode 100644 infra/oci/variables.tf create mode 100644 infra/oci/versions.tf create mode 100644 src/dander/bootstrap/oci_terraform.py create mode 100644 src/dander/cli/oci_command.py create mode 100644 tests/bootstrap/test_oci_terraform.py create mode 100644 tests/cli/test_oci_cli.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78f316ed..684d039a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2 - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: - terraform_version: 1.9.8 + terraform_version: 1.15.8 - name: Format run: terraform fmt -check -recursive - name: Initialize without a backend @@ -93,6 +93,14 @@ jobs: terraform -chdir=azure test terraform -chdir=azure/modules/container-apps-jobs init -backend=false -input=false terraform -chdir=azure/modules/container-apps-jobs test + - name: Validate OCI Terraform stacks + run: | + terraform -chdir=oci/bootstrap-admin init -backend=false -input=false + terraform -chdir=oci/bootstrap-admin validate + terraform -chdir=oci/bootstrap-admin test + terraform -chdir=oci init -backend=false -input=false + terraform -chdir=oci validate + terraform -chdir=oci test - name: Validate Kubernetes Helm chart run: | helm lint kubernetes/chart/dander --values ../tests/fixtures/kubernetes-values.yaml @@ -128,7 +136,7 @@ jobs: - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: - terraform_version: 1.9.8 + terraform_version: 1.15.8 - name: Build and inspect artifacts run: | python3 scripts/check_release_metadata.py diff --git a/.github/workflows/live-proof.yml b/.github/workflows/live-proof.yml index e9db4d7b..0bf910a2 100644 --- a/.github/workflows/live-proof.yml +++ b/.github/workflows/live-proof.yml @@ -57,7 +57,7 @@ jobs: - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: - terraform_version: 1.9.8 + terraform_version: 1.15.8 - name: Locked Python checks run: | uv sync --frozen --extra dev @@ -90,7 +90,7 @@ jobs: - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: - terraform_version: 1.9.8 + terraform_version: 1.15.8 - name: Install locked runtime run: uv sync --frozen --extra dev - uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 diff --git a/HANDOFF.md b/HANDOFF.md index 1ab54813..26c792c1 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -3,36 +3,38 @@ ## Finished - Merged the OCI SDK/Vault provider slice through protected PR #212. -- Added a separate lazy, typed OCI Container Instances launcher projection. -- Restricted OCI to the PostgreSQL/PostgreSQL/no-catalog/OCI-Vault profile. -- Added exact OCIR digest, flex-shape, UTC schedule, network, and resource-principal validation. +- Merged the typed OCI Container Instances launcher through protected PR #213. +- Added native Object Storage state and immutable private OCIR stage zero. +- Added the private VCN, Vault/key, resource-principal policy, Logging, and Notifications foundation. +- Added SecurityToken-only plan/apply CLI and read-only no-drift verification for both remote states. ## Try It -Run `uv run pytest tests/providers/test_oci_container_instances_runtime.py`. +Run `uv run pytest tests/bootstrap/test_oci_terraform.py tests/cli/test_oci_cli.py`. ## Checks -- Focused launcher/profile regressions passed: 51 tests. +- Both OCI Terraform roots validate and their native tests pass. - The complete pytest suite, Ruff, and repository-wide strict Mypy passed. -- Wheel and source distribution built with the OCI launcher modules included. -- Protected-main CI for the merged Vault slice passed all five jobs. +- Wheel and source distribution built and passed the distribution contract check. +- Trivy reported no HIGH or CRITICAL infrastructure misconfigurations. +- Protected-main CI for merged PR #213 passed all five jobs. ## Decisions -- OCI retries are whole-task attempts owned by the later reconciler; native restart is `NEVER`. -- OCI OCPUs and fixed ephemeral storage remain explicit provider semantics. +- Stage zero and foundation are separate reviewed saved plans; secret values never enter Terraform. +- Terraform's native OCI backend requires Terraform 1.12+ and uses only SecurityToken auth here. - Live OCI writes remain at $0 pending preflight and a numeric per-attempt ceiling. ## Remaining -- Merge this launcher contract through a protected PR. -- Implement Terraform, then scheduling/lifecycle operations in separate PRs. +- Merge this Terraform foundation through a protected PR. +- Implement scheduling/lifecycle operations in a separate PR. - Build one new post-merge candidate and run the complete OCI live gate after account preflight. - Complete Phase 8 only after Phase 7 passes. ## Review First -- `src/dander/providers/oci_container_instances/runtime.py` -- `src/dander/providers/oci_container_instances/config.py` -- `tests/providers/test_oci_container_instances_runtime.py` +- `src/dander/bootstrap/oci_terraform.py` +- `infra/oci/main.tf` +- `infra/oci/bootstrap-admin/main.tf` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index ec7da168..c28f8d3b 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1201,13 +1201,18 @@ Tickets: 6. PostgreSQL canonical live profile and external-identity feasibility report; 7. schedule, retry, parallelism, interruption, rollback, and reconciliation proof. -Progress on 2026-08-12: the OCI SDK/Vault provider and the separate typed Container Instances -launcher projection are implemented behind lazy provider selection. The launcher accepts only the -named PostgreSQL/PostgreSQL/no-catalog/OCI-Vault profile and exact digest-qualified OCIR images; -infrastructure, promotion, scheduling Function, operations, reconciliation, credential-refresh, -live-profile, cleanup, and no-drift evidence remain open. This progress is not an exit-gate or -support claim, and paid OCI mutation remains disabled pending credential preflight and an explicit -per-attempt ceiling. +Progress on 2026-08-12: the OCI SDK/Vault provider, typed Container Instances launcher projection, +and plan-first Terraform foundations are implemented behind provider-specific boundaries. The OCI +native remote-state bootstrap creates only a private versioned Object Storage bucket and private +immutable OCIR repository; the second root creates a private VCN/subnet, default Vault with an +auto-rotating key, compartment-scoped Container Instance resource-principal policy, Logging, and +Notifications. Both use short-lived SecurityToken auth, saved plans outside the repository, and +locally passing Terraform tests; a read-only verifier refreshes both remote states and fails on any +planned drift. The launcher still accepts only the named PostgreSQL/PostgreSQL/ +no-catalog/OCI-Vault profile and exact digest-qualified OCIR images. Promotion, scheduling Function, +operations, reconciliation, credential-refresh, live-profile, cleanup, and no-drift evidence remain +open. This progress is not an exit-gate or support claim, and paid OCI mutation remains disabled +pending credential preflight and an explicit per-attempt ceiling. Exit gate: diff --git a/docs/decisions.md b/docs/decisions.md index bf1d17ee..31d01184 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1332,3 +1332,16 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** OCI OCPUs and fixed ephemeral storage remain explicit provider semantics rather than false cross-provider parity. This contract adds no Terraform, Function, API mutation, live evidence, packaged support claim, or paid authorization. + +## 2026-08-12 — OCI foundation uses native state and two reviewed applies + +- **State boundary:** Stage zero creates only a private versioned Object Storage bucket and a + private immutable OCIR repository, then migrates its temporary local state to Terraform's native + OCI backend. Terraform 1.12 or newer and a short-lived `SecurityToken` profile are required; + API-key and registry-password credentials are not accepted by this workflow. +- **Foundation:** A second remote-state-backed plan creates the private egress-only VCN/subnet, + default Vault and auto-rotating software key, compartment-scoped Container Instance dynamic + group/policy, log group, and Notifications topic. No long-lived Container Instance is created. +- **Sequencing:** Secret values are seeded outside Terraform after the foundation apply, and the + output subnet/Vault OCIDs become typed launcher inputs. Saved plans, bootstrap state, backend + metadata, and Terraform caches stay in a mode-restricted operator directory outside Git. diff --git a/infra/README.md b/infra/README.md index fb109275..68b88c98 100644 --- a/infra/README.md +++ b/infra/README.md @@ -16,6 +16,8 @@ call sites (mirrors the `SecretStoreProvider` / `ComputeProvider` abstractions i | `kubernetes/chart/dander` | Versioned Helm chart for an existing conforming cluster. **Locally validated; not live-qualified.** | | `azure/bootstrap-admin` | Azure Storage state, ACR, and user-assigned runtime identity. **Live-proven for the named experimental Phase 6 profiles.** | | `azure/modules/container-apps-jobs` | Container Apps Jobs, Key Vault references, Log Analytics, alerts, and optional network placement. **Live-proven for the named experimental Phase 6 profiles.** | +| `oci/bootstrap-admin` | Native OCI Object Storage state and a private immutable OCIR repository. **Implemented and locally validated; live proof pending.** | +| `oci` | Private VCN/subnet, default Vault and rotating key, Container Instance resource-principal policy, Logging, and Notifications foundation. **Implemented and locally validated; live proof pending.** | The main root always calls `modules/bigquery` and can opt into the remaining workload modules. The one-time `infra/bootstrap-admin` root creates the remote-state bucket, the Artifact Registry @@ -106,6 +108,8 @@ Each scheduler identity can invoke only its pipeline's named Cloud Run Job. - **Remote GCS backend** for state — never local state committed to the repo. - Azure uses an Entra-authenticated Azure Storage backend; its temporary bootstrap state and saved plans remain outside the repository. +- OCI uses Terraform's native Object Storage backend with a short-lived `SecurityToken` profile; + its temporary bootstrap state and saved plans also remain outside the repository. - **No secret values** in `.tf`/`.tfvars`; reference Secret Manager. Commit only `*.tfvars.example`. - Project id / region are always parameterized, never hard-coded. - Stage-zero applies run through the CLI using the approved administrator. Platform applies use diff --git a/infra/oci/.terraform.lock.hcl b/infra/oci/.terraform.lock.hcl new file mode 100644 index 00000000..4ead9359 --- /dev/null +++ b/infra/oci/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/oracle/oci" { + version = "8.26.0" + constraints = "~> 8.26" + hashes = [ + "h1:O/laC68t0n/hHuZQdpJTXvFZY/79ZVu8VmQ02YA/VYc=", + "zh:0a4a62cd3bf4ff29ed2f084fc15dfe83598b62e2ec57b6584a4af16184a9580d", + "zh:346c9d95d37b52804b2445fc748241f28ac38611fb149371bd5df979c5f3fef4", + "zh:3bc75b2b87b26747ef0de470a70e17631381b7d322b78b448b769544c4b03605", + "zh:450810d012a0c6e1d7b4b2ef7741c37080a43e2764d329a123ba80380d7795a1", + "zh:5ed90c0cd7d85c1e487c0ae3d6d67597a5e6a53a0df5d6d0dc109e35bf2e2cc2", + "zh:5fedfbc39014018d04557126800eb9867e447aba8bfff795edeea116e278999a", + "zh:611ca8cc769fa128ac7b5772809267a0d4e4b0ccea998a9a20b2718e58145a31", + "zh:8c8ea27b8c9262b4e306fbc6e5b8d319b6857adcfb1ad00cde7c1c1ff172da90", + "zh:9736172d5a4b35fc5d0464488e7c20853af4f576f064051703119f617c377a5c", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c913b4e445813ae4dbeae97501101f350b59d3d2094f120b3a99a9f77df7ff25", + "zh:ccc1d6cc2eba7cea65c626404d07f2890710fca6816c15461ba391ee81bd6799", + "zh:d4d2561d65f30a6fb1258ad1e496212f401e146c986c4e2e136e44724242a4da", + "zh:e2840feeead7d9d284ce8c98db2fafa3a4b479d738e4392ba1c33b1b2aee1ed7", + ] +} diff --git a/infra/oci/README.md b/infra/oci/README.md new file mode 100644 index 00000000..167ce438 --- /dev/null +++ b/infra/oci/README.md @@ -0,0 +1,18 @@ +# OCI Container Instances foundation + +This root creates the provider-owned foundation for Dander's named OCI profile: + +- a private VCN/subnet with no public IPs or inbound rules; +- NAT egress for PostgreSQL plus an OCI Services gateway for OCIR, Vault, and control APIs; +- a default OCI Vault and auto-rotating software-protected key; +- a compartment-scoped Container Instance dynamic group and least-privilege runtime policy; and +- a log group and Notifications topic for the lifecycle controller. + +It deliberately does **not** create long-lived Container Instances. Every task attempt is a new +run-scoped instance created by the Phase 7 launch Function and stopped by the reconciler. It also +does not put secret values in Terraform state; seed named OCI Vault secrets after the foundation +apply and before enabling schedules. + +Use Terraform's native `oci` backend with the private stage-zero bucket and a short-lived +`SecurityToken` profile. Review a saved plan before apply. The output subnet and Vault OCIDs are +then copied into the typed `oci_container_instances` launcher configuration. diff --git a/infra/oci/bootstrap-admin/.terraform.lock.hcl b/infra/oci/bootstrap-admin/.terraform.lock.hcl new file mode 100644 index 00000000..4ead9359 --- /dev/null +++ b/infra/oci/bootstrap-admin/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/oracle/oci" { + version = "8.26.0" + constraints = "~> 8.26" + hashes = [ + "h1:O/laC68t0n/hHuZQdpJTXvFZY/79ZVu8VmQ02YA/VYc=", + "zh:0a4a62cd3bf4ff29ed2f084fc15dfe83598b62e2ec57b6584a4af16184a9580d", + "zh:346c9d95d37b52804b2445fc748241f28ac38611fb149371bd5df979c5f3fef4", + "zh:3bc75b2b87b26747ef0de470a70e17631381b7d322b78b448b769544c4b03605", + "zh:450810d012a0c6e1d7b4b2ef7741c37080a43e2764d329a123ba80380d7795a1", + "zh:5ed90c0cd7d85c1e487c0ae3d6d67597a5e6a53a0df5d6d0dc109e35bf2e2cc2", + "zh:5fedfbc39014018d04557126800eb9867e447aba8bfff795edeea116e278999a", + "zh:611ca8cc769fa128ac7b5772809267a0d4e4b0ccea998a9a20b2718e58145a31", + "zh:8c8ea27b8c9262b4e306fbc6e5b8d319b6857adcfb1ad00cde7c1c1ff172da90", + "zh:9736172d5a4b35fc5d0464488e7c20853af4f576f064051703119f617c377a5c", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c913b4e445813ae4dbeae97501101f350b59d3d2094f120b3a99a9f77df7ff25", + "zh:ccc1d6cc2eba7cea65c626404d07f2890710fca6816c15461ba391ee81bd6799", + "zh:d4d2561d65f30a6fb1258ad1e496212f401e146c986c4e2e136e44724242a4da", + "zh:e2840feeead7d9d284ce8c98db2fafa3a4b479d738e4392ba1c33b1b2aee1ed7", + ] +} diff --git a/infra/oci/bootstrap-admin/README.md b/infra/oci/bootstrap-admin/README.md new file mode 100644 index 00000000..83bfae80 --- /dev/null +++ b/infra/oci/bootstrap-admin/README.md @@ -0,0 +1,14 @@ +# OCI stage zero + +This root creates the two prerequisites that must exist before Dander can use remote state: + +- a private, versioned Object Storage bucket; and +- a private, immutable OCIR repository. + +Use an OCI CLI `SecurityToken` profile. Do not use API-key, auth-token, or registry-password +credentials. The administrative bootstrap copies this root to a private operator directory, +saves a mode-`0600` plan there, applies only that reviewed plan, and then migrates local state to +Terraform's native `oci` backend. + +The native OCI backend requires Terraform 1.12 or newer. State and plans are operator artifacts; +neither belongs in Git. diff --git a/infra/oci/bootstrap-admin/main.tf b/infra/oci/bootstrap-admin/main.tf new file mode 100644 index 00000000..f1979656 --- /dev/null +++ b/infra/oci/bootstrap-admin/main.tf @@ -0,0 +1,32 @@ +locals { + tags = merge(var.freeform_tags, { + component = "oci-stage-zero" + managed-by = "dander" + }) +} + +data "oci_objectstorage_namespace" "current" { + compartment_id = var.tenancy_id +} + +resource "oci_objectstorage_bucket" "terraform_state" { + compartment_id = var.compartment_id + namespace = data.oci_objectstorage_namespace.current.namespace + name = var.state_bucket_name + access_type = "NoPublicAccess" + storage_tier = "Standard" + versioning = "Enabled" + freeform_tags = local.tags + + lifecycle { + prevent_destroy = true + } +} + +resource "oci_artifacts_container_repository" "runtime" { + compartment_id = var.compartment_id + display_name = var.repository_name + is_immutable = true + is_public = false + freeform_tags = local.tags +} diff --git a/infra/oci/bootstrap-admin/outputs.tf b/infra/oci/bootstrap-admin/outputs.tf new file mode 100644 index 00000000..94da7912 --- /dev/null +++ b/infra/oci/bootstrap-admin/outputs.tf @@ -0,0 +1,19 @@ +output "object_storage_namespace" { + description = "Object Storage namespace required by the native OCI Terraform backend." + value = data.oci_objectstorage_namespace.current.namespace +} + +output "state_bucket_name" { + description = "Private, versioned bucket holding native OCI Terraform state." + value = oci_objectstorage_bucket.terraform_state.name +} + +output "repository_id" { + description = "Immutable private OCIR repository OCID." + value = oci_artifacts_container_repository.runtime.id +} + +output "repository_name" { + description = "OCIR repository path receiving copied release artifacts." + value = oci_artifacts_container_repository.runtime.display_name +} diff --git a/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl b/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl new file mode 100644 index 00000000..68df35a2 --- /dev/null +++ b/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl @@ -0,0 +1,36 @@ +mock_provider "oci" { + mock_data "oci_objectstorage_namespace" { + defaults = { + namespace = "unitnamespace" + } + } +} + +variables { + tenancy_id = "ocid1.tenancy.oc1..aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + compartment_id = "ocid1.compartment.oc1..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + region = "us-ashburn-1" + state_bucket_name = "dander-phase7-state" + repository_name = "dander/runtime" +} + +run "hardened_stage_zero" { + command = plan + + assert { + condition = ( + oci_objectstorage_bucket.terraform_state.access_type == "NoPublicAccess" && + oci_objectstorage_bucket.terraform_state.versioning == "Enabled" && + oci_objectstorage_bucket.terraform_state.storage_tier == "Standard" + ) + error_message = "OCI Terraform state must remain private, versioned, and online." + } + + assert { + condition = ( + oci_artifacts_container_repository.runtime.is_immutable && + !oci_artifacts_container_repository.runtime.is_public + ) + error_message = "OCIR must reject mutable tags and unauthenticated reads." + } +} diff --git a/infra/oci/bootstrap-admin/variables.tf b/infra/oci/bootstrap-admin/variables.tf new file mode 100644 index 00000000..5f081b6b --- /dev/null +++ b/infra/oci/bootstrap-admin/variables.tf @@ -0,0 +1,67 @@ +variable "tenancy_id" { + description = "OCI tenancy receiving Dander stage-zero resources." + type = string + + validation { + condition = can(regex("^ocid1\\.tenancy\\.oc[0-9]+\\.\\.[A-Za-z0-9]+$", var.tenancy_id)) + error_message = "tenancy_id must be an OCI tenancy OCID." + } +} + +variable "compartment_id" { + description = "Existing OCI compartment receiving Dander resources." + type = string + + validation { + condition = can(regex("^ocid1\\.compartment\\.oc[0-9]+\\.\\.[A-Za-z0-9]+$", var.compartment_id)) + error_message = "compartment_id must be an OCI compartment OCID." + } +} + +variable "region" { + description = "OCI region for state, registry, and the initial deployment." + type = string + + validation { + condition = can(regex("^[a-z]{2}(?:-[a-z0-9]+)+-[1-9][0-9]*$", var.region)) + error_message = "region must be a normalized OCI region identifier." + } +} + +variable "config_file_profile" { + description = "Short-lived OCI SecurityToken profile used by Terraform." + type = string + default = "DEFAULT" + + validation { + condition = can(regex("^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", var.config_file_profile)) + error_message = "config_file_profile must be a valid OCI profile name." + } +} + +variable "state_bucket_name" { + description = "Private, versioned Object Storage bucket for Terraform state." + type = string + + validation { + condition = can(regex("^[A-Za-z0-9][A-Za-z0-9._-]{1,254}$", var.state_bucket_name)) + error_message = "state_bucket_name contains unsupported characters." + } +} + +variable "repository_name" { + description = "Private immutable OCIR repository receiving verified Dander artifacts." + type = string + default = "dander/runtime" + + validation { + condition = can(regex("^[a-z0-9]+(?:[._/-][a-z0-9]+)*$", var.repository_name)) + error_message = "repository_name must be a valid OCIR repository path." + } +} + +variable "freeform_tags" { + description = "Additional non-secret OCI free-form tags." + type = map(string) + default = {} +} diff --git a/infra/oci/bootstrap-admin/versions.tf b/infra/oci/bootstrap-admin/versions.tf new file mode 100644 index 00000000..4565038c --- /dev/null +++ b/infra/oci/bootstrap-admin/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.12, < 2.0" + + required_providers { + oci = { + source = "oracle/oci" + version = "~> 8.26" + } + } +} + +provider "oci" { + region = var.region + tenancy_ocid = var.tenancy_id + auth = "SecurityToken" + config_file_profile = var.config_file_profile +} diff --git a/infra/oci/main.tf b/infra/oci/main.tf new file mode 100644 index 00000000..f952c998 --- /dev/null +++ b/infra/oci/main.tf @@ -0,0 +1,149 @@ +locals { + tags = merge(var.freeform_tags, { + component = "oci-container-instances" + managed-by = "dander" + }) +} + +data "oci_core_services" "all_services" { + filter { + name = "name" + values = ["All .* Services In Oracle Services Network"] + regex = true + } +} + +check "one_regional_service_gateway_target" { + assert { + condition = length(data.oci_core_services.all_services.services) == 1 + error_message = "OCI must expose exactly one regional All Services Network target." + } +} + +resource "oci_core_vcn" "runtime" { + compartment_id = var.compartment_id + cidr_blocks = [var.vcn_cidr] + display_name = "${var.name}-runtime" + dns_label = "dander" + freeform_tags = local.tags +} + +resource "oci_core_nat_gateway" "runtime" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.runtime.id + display_name = "${var.name}-runtime-egress" + block_traffic = false + freeform_tags = local.tags +} + +resource "oci_core_service_gateway" "runtime" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.runtime.id + display_name = "${var.name}-oci-services" + freeform_tags = local.tags + + services { + service_id = one(data.oci_core_services.all_services.services).id + } +} + +resource "oci_core_route_table" "runtime" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.runtime.id + display_name = "${var.name}-runtime-routes" + freeform_tags = local.tags + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_nat_gateway.runtime.id + } + + route_rules { + destination = one(data.oci_core_services.all_services.services).cidr_block + destination_type = "SERVICE_CIDR_BLOCK" + network_entity_id = oci_core_service_gateway.runtime.id + } +} + +resource "oci_core_security_list" "runtime" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.runtime.id + display_name = "${var.name}-runtime-egress-only" + freeform_tags = local.tags + + egress_security_rules { + destination = "0.0.0.0/0" + protocol = "all" + stateless = false + } +} + +resource "oci_core_subnet" "runtime" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.runtime.id + cidr_block = var.runtime_subnet_cidr + display_name = "${var.name}-runtime" + dns_label = "runtime" + prohibit_internet_ingress = true + prohibit_public_ip_on_vnic = true + route_table_id = oci_core_route_table.runtime.id + security_list_ids = [oci_core_security_list.runtime.id] + freeform_tags = local.tags +} + +resource "oci_kms_vault" "runtime" { + compartment_id = var.compartment_id + display_name = "${var.name}-runtime" + vault_type = "DEFAULT" + freeform_tags = local.tags +} + +resource "oci_kms_key" "runtime" { + compartment_id = var.compartment_id + display_name = "${var.name}-runtime" + management_endpoint = oci_kms_vault.runtime.management_endpoint + protection_mode = "SOFTWARE" + is_auto_rotation_enabled = true + freeform_tags = local.tags + + key_shape { + algorithm = "AES" + length = 32 + } +} + +resource "oci_identity_dynamic_group" "runtime" { + compartment_id = var.tenancy_id + name = var.dynamic_group_name + description = "Dander run-scoped OCI Container Instance resource principals" + matching_rule = "ALL {resource.type='computecontainerinstance', resource.compartment.id='${var.compartment_id}'}" + freeform_tags = local.tags +} + +resource "oci_identity_policy" "runtime" { + compartment_id = var.tenancy_id + name = "${var.name}-container-runtime" + description = "Least-privilege OCI service access for Dander Container Instances" + freeform_tags = local.tags + statements = [ + "Allow dynamic-group ${oci_identity_dynamic_group.runtime.name} to read repos in compartment id ${var.compartment_id}", + "Allow dynamic-group ${oci_identity_dynamic_group.runtime.name} to read secret-family in compartment id ${var.compartment_id}", + "Allow dynamic-group ${oci_identity_dynamic_group.runtime.name} to use keys in compartment id ${var.compartment_id}", + "Allow dynamic-group ${oci_identity_dynamic_group.runtime.name} to use log-content in compartment id ${var.compartment_id}", + ] +} + +resource "oci_logging_log_group" "runtime" { + compartment_id = var.compartment_id + display_name = "${var.name}-runtime" + description = "Dander Container Instance and lifecycle-controller logs" + freeform_tags = local.tags +} + +resource "oci_ons_notification_topic" "runtime" { + compartment_id = var.compartment_id + name = "${var.name}-runtime" + description = "Dander OCI execution and reconciliation alerts" + freeform_tags = local.tags +} diff --git a/infra/oci/outputs.tf b/infra/oci/outputs.tf new file mode 100644 index 00000000..4898833c --- /dev/null +++ b/infra/oci/outputs.tf @@ -0,0 +1,39 @@ +output "runtime_subnet_id" { + description = "Private subnet OCID selected by the OCI launcher." + value = oci_core_subnet.runtime.id +} + +output "vault_id" { + description = "Vault OCID selected by the OCI Vault provider." + value = oci_kms_vault.runtime.id +} + +output "vault_key_id" { + description = "Auto-rotating software-protected key for runtime secrets." + value = oci_kms_key.runtime.id +} + +output "dynamic_group_name" { + description = "Dynamic group selected by the OCI launcher resource principal." + value = oci_identity_dynamic_group.runtime.name +} + +output "log_group_id" { + description = "OCI log group for Container Instance and controller logs." + value = oci_logging_log_group.runtime.id +} + +output "notification_topic_id" { + description = "OCI Notifications topic for execution and reconciliation alerts." + value = oci_ons_notification_topic.runtime.id +} + +output "network" { + description = "Private runtime VCN resources used by run-scoped Container Instances." + value = { + vcn_id = oci_core_vcn.runtime.id + subnet_id = oci_core_subnet.runtime.id + nat_gateway_id = oci_core_nat_gateway.runtime.id + service_gateway = oci_core_service_gateway.runtime.id + } +} diff --git a/infra/oci/terraform.tfvars.example b/infra/oci/terraform.tfvars.example new file mode 100644 index 00000000..b29de61c --- /dev/null +++ b/infra/oci/terraform.tfvars.example @@ -0,0 +1,7 @@ +tenancy_id = "ocid1.tenancy.oc1..replace_me" +compartment_id = "ocid1.compartment.oc1..replace_me" +region = "us-ashburn-1" +config_file_profile = "DANDER" +dynamic_group_name = "dander_phase7_runtime" + +# Keep all values non-secret. Authenticate through a short-lived OCI SecurityToken profile. diff --git a/infra/oci/tests/oci.tftest.hcl b/infra/oci/tests/oci.tftest.hcl new file mode 100644 index 00000000..78fd5f00 --- /dev/null +++ b/infra/oci/tests/oci.tftest.hcl @@ -0,0 +1,49 @@ +mock_provider "oci" { + mock_data "oci_core_services" { + defaults = { + services = [{ + cidr_block = "all-iad-services-in-oracle-services-network" + id = "ocid1.service.oc1.iad.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + name = "All IAD Services In Oracle Services Network" + }] + } + } +} + +variables { + tenancy_id = "ocid1.tenancy.oc1..aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + compartment_id = "ocid1.compartment.oc1..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + region = "us-ashburn-1" + dynamic_group_name = "dander_phase7_runtime" +} + +run "projects_private_keyless_foundation" { + command = plan + + assert { + condition = ( + oci_core_subnet.runtime.prohibit_public_ip_on_vnic && + oci_core_subnet.runtime.prohibit_internet_ingress && + oci_core_nat_gateway.runtime.block_traffic == false && + length(oci_core_security_list.runtime.ingress_security_rules) == 0 + ) + error_message = "The runtime subnet must remain private, ingress-free, and outbound-capable." + } + + assert { + condition = ( + oci_kms_vault.runtime.vault_type == "DEFAULT" && + oci_kms_key.runtime.protection_mode == "SOFTWARE" && + oci_kms_key.runtime.is_auto_rotation_enabled + ) + error_message = "The runtime Vault must use the bounded-cost default tier and an auto-rotating key." + } + + assert { + condition = ( + oci_identity_dynamic_group.runtime.matching_rule == "ALL {resource.type='computecontainerinstance', resource.compartment.id='ocid1.compartment.oc1..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'}" && + length(oci_identity_policy.runtime.statements) == 4 + ) + error_message = "Only Container Instances in the selected compartment may receive runtime OCI permissions." + } +} diff --git a/infra/oci/variables.tf b/infra/oci/variables.tf new file mode 100644 index 00000000..6f1f77b9 --- /dev/null +++ b/infra/oci/variables.tf @@ -0,0 +1,84 @@ +variable "tenancy_id" { + description = "OCI tenancy owning the Dander dynamic group." + type = string + + validation { + condition = can(regex("^ocid1\\.tenancy\\.oc[0-9]+\\.\\.[A-Za-z0-9]+$", var.tenancy_id)) + error_message = "tenancy_id must be an OCI tenancy OCID." + } +} + +variable "compartment_id" { + description = "OCI compartment receiving the Dander foundation." + type = string + + validation { + condition = can(regex("^ocid1\\.compartment\\.oc[0-9]+\\.\\.[A-Za-z0-9]+$", var.compartment_id)) + error_message = "compartment_id must be an OCI compartment OCID." + } +} + +variable "region" { + description = "OCI region for the Dander foundation." + type = string + + validation { + condition = can(regex("^[a-z]{2}(?:-[a-z0-9]+)+-[1-9][0-9]*$", var.region)) + error_message = "region must be a normalized OCI region identifier." + } +} + +variable "config_file_profile" { + description = "Short-lived OCI SecurityToken profile used by Terraform." + type = string + default = "DEFAULT" +} + +variable "name" { + description = "Stable prefix for Dander OCI resources." + type = string + default = "dander" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,23}$", var.name)) + error_message = "name must contain 2-24 lowercase letters, numbers, or hyphens." + } +} + +variable "dynamic_group_name" { + description = "Tenancy-unique dynamic group for Container Instance resource principals." + type = string + + validation { + condition = can(regex("^[A-Za-z][A-Za-z0-9_-]{0,99}$", var.dynamic_group_name)) + error_message = "dynamic_group_name must be a valid OCI dynamic-group name." + } +} + +variable "vcn_cidr" { + description = "Private VCN CIDR for the OCI launcher." + type = string + default = "10.42.0.0/16" + + validation { + condition = can(cidrhost(var.vcn_cidr, 0)) && !can(regex("^0\\.", var.vcn_cidr)) + error_message = "vcn_cidr must be a valid non-default IPv4 CIDR." + } +} + +variable "runtime_subnet_cidr" { + description = "Private subnet CIDR for run-scoped Container Instances." + type = string + default = "10.42.1.0/24" + + validation { + condition = can(cidrhost(var.runtime_subnet_cidr, 0)) && !can(regex("^0\\.", var.runtime_subnet_cidr)) + error_message = "runtime_subnet_cidr must be a valid non-default IPv4 CIDR." + } +} + +variable "freeform_tags" { + description = "Additional non-secret OCI free-form tags." + type = map(string) + default = {} +} diff --git a/infra/oci/versions.tf b/infra/oci/versions.tf new file mode 100644 index 00000000..a2af551e --- /dev/null +++ b/infra/oci/versions.tf @@ -0,0 +1,19 @@ +terraform { + required_version = ">= 1.12, < 2.0" + + backend "oci" {} + + required_providers { + oci = { + source = "oracle/oci" + version = "~> 8.26" + } + } +} + +provider "oci" { + region = var.region + tenancy_ocid = var.tenancy_id + auth = "SecurityToken" + config_file_profile = var.config_file_profile +} diff --git a/pyproject.toml b/pyproject.toml index 44a89895..4e82d850 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,6 +178,21 @@ packages = ["src/dander"] "infra/azure/tests/azure.tftest.hcl" = "dander/templates/project/infra/azure/tests/azure.tftest.hcl" "infra/azure/variables.tf" = "dander/templates/project/infra/azure/variables.tf" "infra/azure/versions.tf" = "dander/templates/project/infra/azure/versions.tf" +"infra/oci/.terraform.lock.hcl" = "dander/templates/project/infra/oci/.terraform.lock.hcl" +"infra/oci/README.md" = "dander/templates/project/infra/oci/README.md" +"infra/oci/bootstrap-admin/.terraform.lock.hcl" = "dander/templates/project/infra/oci/bootstrap-admin/.terraform.lock.hcl" +"infra/oci/bootstrap-admin/README.md" = "dander/templates/project/infra/oci/bootstrap-admin/README.md" +"infra/oci/bootstrap-admin/main.tf" = "dander/templates/project/infra/oci/bootstrap-admin/main.tf" +"infra/oci/bootstrap-admin/outputs.tf" = "dander/templates/project/infra/oci/bootstrap-admin/outputs.tf" +"infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl" = "dander/templates/project/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl" +"infra/oci/bootstrap-admin/variables.tf" = "dander/templates/project/infra/oci/bootstrap-admin/variables.tf" +"infra/oci/bootstrap-admin/versions.tf" = "dander/templates/project/infra/oci/bootstrap-admin/versions.tf" +"infra/oci/main.tf" = "dander/templates/project/infra/oci/main.tf" +"infra/oci/outputs.tf" = "dander/templates/project/infra/oci/outputs.tf" +"infra/oci/terraform.tfvars.example" = "dander/templates/project/infra/oci/terraform.tfvars.example" +"infra/oci/tests/oci.tftest.hcl" = "dander/templates/project/infra/oci/tests/oci.tftest.hcl" +"infra/oci/variables.tf" = "dander/templates/project/infra/oci/variables.tf" +"infra/oci/versions.tf" = "dander/templates/project/infra/oci/versions.tf" "infra/functions/__init__.py" = "dander/templates/project/infra/functions/__init__.py" "infra/functions/stop_billing/__init__.py" = "dander/templates/project/infra/functions/stop_billing/__init__.py" "infra/functions/stop_billing/handler.py" = "dander/templates/project/infra/functions/stop_billing/handler.py" diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index 7820c8de..4df0d0a9 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -12,6 +12,11 @@ ) from dander.bootstrap.azure_image import AzureRuntimeImagePromoter from dander.bootstrap.azure_terraform import AzureTerraformBootstrap, AzureTerraformBootstrapError +from dander.bootstrap.oci_terraform import ( + OciAdministrativeBootstrap, + OciTerraformBootstrap, + OciTerraformBootstrapError, +) from dander.bootstrap.permissions import require_stage_zero_permissions from dander.bootstrap.project import ( ProjectBootstrapError, @@ -43,6 +48,9 @@ "AzureRuntimeImagePromoter", "AzureTerraformBootstrap", "AzureTerraformBootstrapError", + "OciAdministrativeBootstrap", + "OciTerraformBootstrap", + "OciTerraformBootstrapError", "DeploymentSummary", "DeploymentVerificationError", "DeploymentVerifier", diff --git a/src/dander/bootstrap/oci_terraform.py b/src/dander/bootstrap/oci_terraform.py new file mode 100644 index 00000000..a8c1a9c2 --- /dev/null +++ b/src/dander/bootstrap/oci_terraform.py @@ -0,0 +1,639 @@ +"""Saved-plan OCI stage-zero and foundation Terraform lifecycles.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + +_REGION = re.compile(r"^[a-z]{2}(?:-[a-z0-9]+)+-[1-9][0-9]*$") +_TENANCY_OCID = re.compile(r"^ocid1\.tenancy\.oc[0-9]+\.\.[A-Za-z0-9]+$") +_COMPARTMENT_OCID = re.compile(r"^ocid1\.compartment\.oc[0-9]+\.\.[A-Za-z0-9]+$") +_BUCKET = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,254}$") +_REPOSITORY = re.compile(r"^[a-z0-9]+(?:[._/-][a-z0-9]+)*$") +_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") +_NAME = re.compile(r"^[a-z][a-z0-9-]{1,23}$") +_DYNAMIC_GROUP = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,99}$") +_NAMESPACE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,99}$") +_STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") +_BACKEND_SCHEMA = "io.dander.oci-bootstrap-backend/v1" + + +class OciTerraformBootstrapError(RuntimeError): + """Raised when an OCI saved-plan lifecycle cannot complete safely.""" + + +class OciAdministrativeBootstrap: + """Create OCI state and registry prerequisites, then migrate to native remote state.""" + + def __init__(self, infra_dir: Path, operator_artifact_dir: Path) -> None: + self._workspace = _OciWorkspace( + infra_dir=infra_dir, + operator_artifact_dir=operator_artifact_dir, + plan_name="dander-oci-admin-bootstrap.tfplan", + ) + self._backend_path = self._workspace.workspace / "backend.tf.json" + self._backend_record_path = self._workspace.operator_dir / "backend.json" + self._local_state_path = self._workspace.operator_dir / "terraform.tfstate" + + def execute( + self, + *, + tenancy_id: str, + compartment_id: str, + region: str, + state_bucket_name: str, + state_key: str, + repository_name: str, + config_file_profile: str = "DEFAULT", + ) -> Path: + """Save one stage-zero plan without creating OCI resources.""" + self._validate( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + state_bucket_name=state_bucket_name, + state_key=state_key, + repository_name=repository_name, + config_file_profile=config_file_profile, + ) + self._workspace.prepare() + backend = self._load_backend_record() + if backend is None: + self._write_backend("local", {"path": str(self._local_state_path)}) + else: + self._require_matching_backend( + backend, + tenancy_id=tenancy_id, + region=region, + namespace=str(backend.get("namespace", "")), + state_bucket_name=state_bucket_name, + state_key=state_key, + config_file_profile=config_file_profile, + ) + self._write_oci_backend(backend) + self._workspace.run("terraform", "init", "-reconfigure", "-input=false") + self._workspace.run( + "terraform", + "plan", + "-input=false", + f"-var=tenancy_id={tenancy_id}", + f"-var=compartment_id={compartment_id}", + f"-var=region={region}", + f"-var=config_file_profile={config_file_profile}", + f"-var=state_bucket_name={state_bucket_name}", + f"-var=repository_name={repository_name}", + f"-out={self._workspace.plan_path}", + ) + self._workspace.secure_plan() + return self._workspace.plan_path + + def apply_saved_plan( + self, + *, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + repository_name: str, + config_file_profile: str = "DEFAULT", + ) -> Path: + """Apply only the reviewed plan, then migrate its local state to Object Storage.""" + self._validate( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + state_bucket_name=state_bucket_name, + state_key=state_key, + repository_name=repository_name, + config_file_profile=config_file_profile, + ) + _require(_NAMESPACE, namespace, "OCI Object Storage namespace") + self._workspace.prepare(preserve_plan=True) + self._workspace.require_plan() + backend = self._load_backend_record() + if backend is None: + self._write_backend("local", {"path": str(self._local_state_path)}) + else: + self._require_matching_backend( + backend, + tenancy_id=tenancy_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + config_file_profile=config_file_profile, + ) + self._write_oci_backend(backend) + self._workspace.run("terraform", "init", "-reconfigure", "-input=false") + self._workspace.run("terraform", "apply", "-input=false", str(self._workspace.plan_path)) + if backend is None: + record: dict[str, object] = { + "schema": _BACKEND_SCHEMA, + "tenancy_id": tenancy_id, + "region": region, + "namespace": namespace, + "bucket": state_bucket_name, + "key": state_key, + "auth": "SecurityToken", + "config_file_profile": config_file_profile, + } + self._write_oci_backend(record) + try: + self._workspace.run( + "terraform", + "init", + "-migrate-state", + "-force-copy", + "-input=false", + ) + except OciTerraformBootstrapError: + self._write_backend("local", {"path": str(self._local_state_path)}) + raise + _write_json(self._backend_record_path, record) + return self._workspace.plan_path + + def verify_no_drift( + self, + *, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + repository_name: str, + config_file_profile: str = "DEFAULT", + ) -> None: + """Refresh the migrated stage-zero state and fail if configuration would change.""" + self._validate( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + state_bucket_name=state_bucket_name, + state_key=state_key, + repository_name=repository_name, + config_file_profile=config_file_profile, + ) + _require(_NAMESPACE, namespace, "OCI Object Storage namespace") + self._workspace.prepare(preserve_plan=True) + backend = self._load_backend_record() + if backend is None: + raise OciTerraformBootstrapError( + "OCI stage-zero verification requires migrated remote state" + ) + self._require_matching_backend( + backend, + tenancy_id=tenancy_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + config_file_profile=config_file_profile, + ) + self._write_oci_backend(backend) + self._workspace.run("terraform", "init", "-reconfigure", "-input=false") + self._workspace.verify_no_drift( + "terraform", + "plan", + "-input=false", + "-detailed-exitcode", + f"-var=tenancy_id={tenancy_id}", + f"-var=compartment_id={compartment_id}", + f"-var=region={region}", + f"-var=config_file_profile={config_file_profile}", + f"-var=state_bucket_name={state_bucket_name}", + f"-var=repository_name={repository_name}", + ) + + def _load_backend_record(self) -> dict[str, object] | None: + if not self._backend_record_path.exists(): + return None + if self._backend_record_path.is_symlink() or not self._backend_record_path.is_file(): + raise OciTerraformBootstrapError("OCI backend record must be a regular file") + try: + document = json.loads(self._backend_record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise OciTerraformBootstrapError("OCI backend record is unreadable") from error + if not isinstance(document, dict) or document.get("schema") != _BACKEND_SCHEMA: + raise OciTerraformBootstrapError("OCI backend record has an unsupported schema") + return document + + @staticmethod + def _require_matching_backend( + backend: dict[str, object], + *, + tenancy_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + config_file_profile: str, + ) -> None: + expected = { + "tenancy_id": tenancy_id, + "region": region, + "namespace": namespace, + "bucket": state_bucket_name, + "key": state_key, + "auth": "SecurityToken", + "config_file_profile": config_file_profile, + } + if any(backend.get(key) != value for key, value in expected.items()): + raise OciTerraformBootstrapError( + "OCI backend inputs do not match the previously migrated stage-zero state" + ) + + def _write_oci_backend(self, backend: Mapping[str, object]) -> None: + self._write_backend( + "oci", + { + "tenancy_ocid": backend["tenancy_id"], + "region": backend["region"], + "namespace": backend["namespace"], + "bucket": backend["bucket"], + "key": backend["key"], + "auth": "SecurityToken", + "config_file_profile": backend["config_file_profile"], + }, + ) + + def _write_backend(self, backend_type: str, config: dict[str, object]) -> None: + _write_json( + self._backend_path, + {"terraform": {"backend": {backend_type: config}}}, + ) + + @staticmethod + def _validate( + *, + tenancy_id: str, + compartment_id: str, + region: str, + state_bucket_name: str, + state_key: str, + repository_name: str, + config_file_profile: str, + ) -> None: + for pattern, value, label in ( + (_TENANCY_OCID, tenancy_id, "OCI tenancy"), + (_COMPARTMENT_OCID, compartment_id, "OCI compartment"), + (_REGION, region, "OCI region"), + (_BUCKET, state_bucket_name, "OCI state bucket"), + (_STATE_KEY, state_key, "OCI state key"), + (_REPOSITORY, repository_name, "OCIR repository"), + (_PROFILE, config_file_profile, "OCI SecurityToken profile"), + ): + _require(pattern, value, label) + + +class OciTerraformBootstrap: + """Plan or apply the OCI network, Vault, identity, and observability foundation.""" + + def __init__(self, infra_dir: Path, operator_artifact_dir: Path) -> None: + self._workspace = _OciWorkspace( + infra_dir=infra_dir, + operator_artifact_dir=operator_artifact_dir, + plan_name="dander-oci-foundation.tfplan", + ) + + def execute( + self, + *, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + dynamic_group_name: str, + config_file_profile: str = "DEFAULT", + name: str = "dander", + ) -> Path: + """Produce one immutable remote-state-backed OCI foundation plan.""" + self._validate( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ) + self._workspace.prepare() + self._init( + tenancy_id=tenancy_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + config_file_profile=config_file_profile, + ) + self._workspace.run( + "terraform", + "plan", + "-input=false", + f"-var=tenancy_id={tenancy_id}", + f"-var=compartment_id={compartment_id}", + f"-var=region={region}", + f"-var=config_file_profile={config_file_profile}", + f"-var=dynamic_group_name={dynamic_group_name}", + f"-var=name={name}", + f"-out={self._workspace.plan_path}", + ) + self._workspace.secure_plan() + return self._workspace.plan_path + + def apply_saved_plan( + self, + *, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + dynamic_group_name: str, + config_file_profile: str = "DEFAULT", + name: str = "dander", + ) -> Path: + """Apply only the reviewed OCI foundation plan.""" + self._validate( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ) + self._workspace.prepare(preserve_plan=True) + self._workspace.require_plan() + self._init( + tenancy_id=tenancy_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + config_file_profile=config_file_profile, + ) + self._workspace.run("terraform", "apply", "-input=false", str(self._workspace.plan_path)) + return self._workspace.plan_path + + def verify_no_drift( + self, + *, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + dynamic_group_name: str, + config_file_profile: str = "DEFAULT", + name: str = "dander", + ) -> None: + """Refresh the OCI foundation and fail if its reviewed configuration would change.""" + self._validate( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ) + self._workspace.prepare(preserve_plan=True) + self._init( + tenancy_id=tenancy_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + config_file_profile=config_file_profile, + ) + self._workspace.verify_no_drift( + "terraform", + "plan", + "-input=false", + "-detailed-exitcode", + f"-var=tenancy_id={tenancy_id}", + f"-var=compartment_id={compartment_id}", + f"-var=region={region}", + f"-var=config_file_profile={config_file_profile}", + f"-var=dynamic_group_name={dynamic_group_name}", + f"-var=name={name}", + ) + + def _init( + self, + *, + tenancy_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + config_file_profile: str, + ) -> None: + self._workspace.run( + "terraform", + "init", + "-reconfigure", + "-input=false", + f"-backend-config=tenancy_ocid={tenancy_id}", + f"-backend-config=region={region}", + f"-backend-config=namespace={namespace}", + f"-backend-config=bucket={state_bucket_name}", + f"-backend-config=key={state_key}", + "-backend-config=auth=SecurityToken", + f"-backend-config=config_file_profile={config_file_profile}", + ) + + @staticmethod + def _validate( + *, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + state_bucket_name: str, + state_key: str, + dynamic_group_name: str, + config_file_profile: str, + name: str, + ) -> None: + for pattern, value, label in ( + (_TENANCY_OCID, tenancy_id, "OCI tenancy"), + (_COMPARTMENT_OCID, compartment_id, "OCI compartment"), + (_REGION, region, "OCI region"), + (_NAMESPACE, namespace, "OCI Object Storage namespace"), + (_BUCKET, state_bucket_name, "OCI state bucket"), + (_STATE_KEY, state_key, "OCI state key"), + (_DYNAMIC_GROUP, dynamic_group_name, "OCI dynamic group"), + (_PROFILE, config_file_profile, "OCI SecurityToken profile"), + (_NAME, name, "OCI deployment name"), + ): + _require(pattern, value, label) + + +class _OciWorkspace: + """Private Terraform workspace shared by OCI saved-plan lifecycles.""" + + def __init__(self, *, infra_dir: Path, operator_artifact_dir: Path, plan_name: str) -> None: + self.infra_dir = infra_dir.resolve() + self.repository_dir = _repository_root(self.infra_dir) + self.operator_dir = operator_artifact_dir.expanduser().resolve() + if ( + self.operator_dir == self.repository_dir + or self.repository_dir in self.operator_dir.parents + ): + raise OciTerraformBootstrapError( + "OCI operator artifact directory must be outside the repository checkout" + ) + self.workspace = self.operator_dir / "terraform-workspace" + self.tf_data_dir = self.operator_dir / "terraform-data" + self.plan_path = self.operator_dir / plan_name + + def prepare(self, *, preserve_plan: bool = False) -> None: + try: + for path, label in ( + (self.operator_dir, "operator artifact directory"), + (self.workspace, "Terraform workspace"), + (self.tf_data_dir, "terraform-data"), + ): + if path.is_symlink(): + raise OciTerraformBootstrapError(f"OCI {label} must not be a symlink") + if path.exists() and not path.is_dir(): + raise OciTerraformBootstrapError(f"OCI {label} must be a directory") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + path.chmod(0o700) + if self.plan_path.is_symlink() or ( + self.plan_path.exists() and not self.plan_path.is_file() + ): + raise OciTerraformBootstrapError("Saved OCI plan must be a regular file") + if self.plan_path.exists() and not preserve_plan: + self.plan_path.unlink() + for existing in self.workspace.iterdir(): + is_source = existing.suffix == ".tf" or existing.name.endswith(".tf.json") + if not is_source and existing.name != ".terraform.lock.hcl": + continue + if existing.is_symlink() or not existing.is_file(): + raise OciTerraformBootstrapError( + "OCI Terraform workspace files must be regular files" + ) + existing.unlink() + for source in self.infra_dir.iterdir(): + if source.is_file() and ( + source.suffix == ".tf" or source.name == ".terraform.lock.hcl" + ): + shutil.copy2(source, self.workspace / source.name) + except OSError as error: + raise OciTerraformBootstrapError( + "Could not prepare the OCI operator artifact directory" + ) from error + + def require_plan(self) -> None: + if not self.plan_path.is_file() or self.plan_path.is_symlink(): + raise OciTerraformBootstrapError(f"Saved OCI plan is missing: {self.plan_path}") + + def secure_plan(self) -> None: + self.require_plan() + try: + self.plan_path.chmod(0o600) + except OSError as error: + raise OciTerraformBootstrapError("Could not secure the saved OCI plan") from error + + def run(self, *args: str) -> None: + environment = os.environ.copy() + environment["TF_DATA_DIR"] = str(self.tf_data_dir) + try: + subprocess.run( + args, + cwd=self.workspace, + check=True, + env=environment, + umask=0o077, + ) + except FileNotFoundError as error: + raise OciTerraformBootstrapError( + "Terraform is not installed or is not available on PATH" + ) from error + except subprocess.CalledProcessError as error: + raise OciTerraformBootstrapError( + f"Terraform {args[1]} failed with exit code {error.returncode}" + ) from error + + def verify_no_drift(self, *args: str) -> None: + environment = os.environ.copy() + environment["TF_DATA_DIR"] = str(self.tf_data_dir) + try: + completed = subprocess.run( + args, + cwd=self.workspace, + check=False, + env=environment, + umask=0o077, + ) + except FileNotFoundError as error: + raise OciTerraformBootstrapError( + "Terraform is not installed or is not available on PATH" + ) from error + if completed.returncode == 0: + return + if completed.returncode == 2: + raise OciTerraformBootstrapError("OCI deployment verification found Terraform drift") + raise OciTerraformBootstrapError( + f"Terraform plan failed with exit code {completed.returncode}" + ) + + +def _repository_root(infra_dir: Path) -> Path: + for candidate in (infra_dir, *infra_dir.parents): + candidate_infra = candidate / "infra" + if candidate_infra.is_dir() and ( + infra_dir == candidate_infra or candidate_infra in infra_dir.parents + ): + return candidate.resolve() + raise OciTerraformBootstrapError("Could not locate the project root for OCI bootstrap") + + +def _require(pattern: re.Pattern[str], value: str, label: str) -> None: + if pattern.fullmatch(value) is None: + raise OciTerraformBootstrapError(f"Invalid {label}: {value!r}") + + +def _write_json(path: Path, document: Mapping[str, object]) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + try: + temporary.write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.chmod(0o600) + os.replace(temporary, path) + path.chmod(0o600) + except OSError as error: + raise OciTerraformBootstrapError("Could not write OCI backend metadata") from error + finally: + temporary.unlink(missing_ok=True) + + +__all__ = [ + "OciAdministrativeBootstrap", + "OciTerraformBootstrap", + "OciTerraformBootstrapError", +] diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index db8dbcdb..cf3bc674 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -53,6 +53,7 @@ resolve_platform_config, ) from dander.cli.kubernetes_command import register_kubernetes_commands +from dander.cli.oci_command import register_oci_commands from dander.cli.provider_runtime import build_catalog_publisher, build_secret_store from dander.cli.run_command import ( RunOptions, @@ -138,6 +139,7 @@ register_aws_commands(app) register_azure_commands(app) register_kubernetes_commands(app) +register_oci_commands(app) console = Console() _SOURCE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") _DEFAULT_CONNECTORS_DIR = Path("connectors") diff --git a/src/dander/cli/oci_command.py b/src/dander/cli/oci_command.py new file mode 100644 index 00000000..c1735fe4 --- /dev/null +++ b/src/dander/cli/oci_command.py @@ -0,0 +1,276 @@ +"""OCI stage-zero and provider-foundation CLI commands.""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +import typer +from click import ClickException +from rich.console import Console + +from dander.bootstrap import ( + OciAdministrativeBootstrap, + OciTerraformBootstrap, + OciTerraformBootstrapError, +) + +_DEFAULT_OCI_INFRA_DIR = Path("infra/oci") +_DEFAULT_OCI_ADMIN_DIR = Path("infra/oci/bootstrap-admin") +console = Console() + + +def _admin_arguments( + *, + tenancy_id: str, + compartment_id: str, + region: str, + state_bucket_name: str, + state_key: str, + repository_name: str, + config_file_profile: str, +) -> dict[str, str]: + return { + "tenancy_id": tenancy_id, + "compartment_id": compartment_id, + "region": region, + "state_bucket_name": state_bucket_name, + "state_key": state_key, + "repository_name": repository_name, + "config_file_profile": config_file_profile, + } + + +def init_oci_admin_plan( + tenancy_id: str = typer.Option(..., "--tenancy-id"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + state_bucket_name: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/oci/bootstrap-admin/terraform.tfstate", "--state-key"), + repository_name: str = typer.Option("dander/runtime", "--repository"), + config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), + operator_artifact_dir: Path = typer.Option( # noqa: B008 + ..., + "--operator-artifact-dir", + help="Secured directory outside the repository for plans, state, and Terraform metadata.", + ), + infra_dir: Path = typer.Option(_DEFAULT_OCI_ADMIN_DIR, hidden=True), # noqa: B008 +) -> None: + """Plan the OCI state bucket and immutable OCIR repository without applying.""" + arguments = _admin_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + state_bucket_name=state_bucket_name, + state_key=state_key, + repository_name=repository_name, + config_file_profile=config_file_profile, + ) + try: + plan_path = OciAdministrativeBootstrap(infra_dir, operator_artifact_dir).execute( + **arguments + ) + except OciTerraformBootstrapError as error: + raise ClickException(str(error)) from error + _print_plan("OCI administrative bootstrap", operator_artifact_dir, plan_path) + console.print("No OCI resources were created. Apply requires explicit cost approval.") + + +def init_oci_admin_apply( + tenancy_id: str = typer.Option(..., "--tenancy-id"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--object-storage-namespace"), + state_bucket_name: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/oci/bootstrap-admin/terraform.tfstate", "--state-key"), + repository_name: str = typer.Option("dander/runtime", "--repository"), + config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), + operator_artifact_dir: Path = typer.Option(..., "--operator-artifact-dir"), # noqa: B008 + infra_dir: Path = typer.Option(_DEFAULT_OCI_ADMIN_DIR, hidden=True), # noqa: B008 +) -> None: + """Apply the reviewed OCI stage-zero plan and migrate to native remote state.""" + if not typer.confirm( + f"Apply the reviewed OCI stage-zero plan in tenancy {tenancy_id!r}?", + default=False, + ): + raise typer.Abort() + try: + plan_path = OciAdministrativeBootstrap(infra_dir, operator_artifact_dir).apply_saved_plan( + namespace=namespace, + **_admin_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + state_bucket_name=state_bucket_name, + state_key=state_key, + repository_name=repository_name, + config_file_profile=config_file_profile, + ), + ) + except OciTerraformBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]OCI administrative bootstrap applied.[/green] Saved plan: {plan_path}") + + +def init_oci_plan( + tenancy_id: str = typer.Option(..., "--tenancy-id"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--object-storage-namespace"), + state_bucket_name: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/oci/foundation/terraform.tfstate", "--state-key"), + dynamic_group_name: str = typer.Option("dander_phase7_runtime", "--dynamic-group"), + config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), + operator_artifact_dir: Path = typer.Option(..., "--operator-artifact-dir"), # noqa: B008 + infra_dir: Path = typer.Option(_DEFAULT_OCI_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Plan the private OCI network, Vault, identity, and observability foundation.""" + arguments = _foundation_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ) + try: + plan_path = OciTerraformBootstrap(infra_dir, operator_artifact_dir).execute(**arguments) + except OciTerraformBootstrapError as error: + raise ClickException(str(error)) from error + _print_plan("OCI foundation", operator_artifact_dir, plan_path) + console.print("No OCI resources were changed. Apply requires explicit cost approval.") + + +def init_oci_apply( + tenancy_id: str = typer.Option(..., "--tenancy-id"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--object-storage-namespace"), + state_bucket_name: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/oci/foundation/terraform.tfstate", "--state-key"), + dynamic_group_name: str = typer.Option("dander_phase7_runtime", "--dynamic-group"), + config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), + operator_artifact_dir: Path = typer.Option(..., "--operator-artifact-dir"), # noqa: B008 + infra_dir: Path = typer.Option(_DEFAULT_OCI_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Apply only the previously reviewed OCI foundation plan.""" + if not typer.confirm( + f"Apply the reviewed OCI foundation plan in compartment {compartment_id!r}?", + default=False, + ): + raise typer.Abort() + try: + plan_path = OciTerraformBootstrap(infra_dir, operator_artifact_dir).apply_saved_plan( + **_foundation_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ) + ) + except OciTerraformBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]OCI foundation applied.[/green] Saved plan: {plan_path}") + + +def verify_oci_deployment( + tenancy_id: str = typer.Option(..., "--tenancy-id"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--object-storage-namespace"), + state_bucket_name: str = typer.Option(..., "--state-bucket"), + admin_state_key: str = typer.Option( + "dander/oci/bootstrap-admin/terraform.tfstate", "--admin-state-key" + ), + foundation_state_key: str = typer.Option( + "dander/oci/foundation/terraform.tfstate", "--foundation-state-key" + ), + repository_name: str = typer.Option("dander/runtime", "--repository"), + dynamic_group_name: str = typer.Option("dander_phase7_runtime", "--dynamic-group"), + config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), + admin_operator_artifact_dir: Path = typer.Option( # noqa: B008 + ..., "--admin-operator-artifact-dir" + ), + foundation_operator_artifact_dir: Path = typer.Option( # noqa: B008 + ..., "--foundation-operator-artifact-dir" + ), + admin_infra_dir: Path = typer.Option(_DEFAULT_OCI_ADMIN_DIR, hidden=True), # noqa: B008 + infra_dir: Path = typer.Option(_DEFAULT_OCI_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Read OCI APIs through Terraform and fail if stage zero or the foundation drifted.""" + try: + OciAdministrativeBootstrap(admin_infra_dir, admin_operator_artifact_dir).verify_no_drift( + namespace=namespace, + **_admin_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + state_bucket_name=state_bucket_name, + state_key=admin_state_key, + repository_name=repository_name, + config_file_profile=config_file_profile, + ), + ) + OciTerraformBootstrap(infra_dir, foundation_operator_artifact_dir).verify_no_drift( + **_foundation_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=foundation_state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ) + ) + except OciTerraformBootstrapError as error: + raise ClickException(str(error)) from error + console.print_json( + data={ + "schema": "io.dander.oci-deployment-verification/v1", + "status": "no_drift", + "region": region, + "state_bucket": state_bucket_name, + "repository": repository_name, + "dynamic_group": dynamic_group_name, + "authentication": "SecurityToken", + } + ) + + +def _foundation_arguments(**values: str) -> dict[str, str]: + return dict(values) + + +def _print_plan(label: str, operator_artifact_dir: Path, plan_path: Path) -> None: + workspace = operator_artifact_dir.expanduser().resolve() / "terraform-workspace" + console.print(f"[green]{label} planned.[/green] Saved plan: {plan_path}") + console.print( + "Review: " + + shlex.join(("terraform", f"-chdir={workspace}", "show", "-no-color", str(plan_path))), + soft_wrap=True, + ) + + +def register_oci_commands(app: typer.Typer) -> None: + """Register OCI plan-first administrative commands.""" + app.command("init-oci-admin-plan")(init_oci_admin_plan) + app.command("init-oci-admin-apply")(init_oci_admin_apply) + app.command("init-oci-plan")(init_oci_plan) + app.command("init-oci-apply")(init_oci_apply) + app.command("verify-oci-deployment")(verify_oci_deployment) + + +__all__ = ["register_oci_commands"] diff --git a/tests/bootstrap/test_oci_terraform.py b/tests/bootstrap/test_oci_terraform.py new file mode 100644 index 00000000..7488e7c5 --- /dev/null +++ b/tests/bootstrap/test_oci_terraform.py @@ -0,0 +1,192 @@ +"""Saved-plan OCI administrative and foundation bootstrap behavior.""" + +from __future__ import annotations + +import json +import stat +import subprocess +from pathlib import Path + +import pytest + +from dander.bootstrap import ( + OciAdministrativeBootstrap, + OciTerraformBootstrap, + OciTerraformBootstrapError, +) + +_ROOT = Path(__file__).parents[2] +_TENANCY = "ocid1.tenancy.oc1.." + "a" * 32 +_COMPARTMENT = "ocid1.compartment.oc1.." + "b" * 32 +_ADMIN = { + "tenancy_id": _TENANCY, + "compartment_id": _COMPARTMENT, + "region": "us-ashburn-1", + "state_bucket_name": "dander-phase7-state", + "state_key": "dander/oci/bootstrap-admin/terraform.tfstate", + "repository_name": "dander/runtime", + "config_file_profile": "DANDER", +} +_FOUNDATION = { + "tenancy_id": _TENANCY, + "compartment_id": _COMPARTMENT, + "region": "us-ashburn-1", + "namespace": "unitnamespace", + "state_bucket_name": "dander-phase7-state", + "state_key": "dander/oci/foundation/terraform.tfstate", + "dynamic_group_name": "dander_phase7_runtime", + "config_file_profile": "DANDER", + "name": "dander", +} + + +@pytest.fixture +def terraform_calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ...]]: + calls: list[tuple[str, ...]] = [] + + def run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + del kwargs + calls.append(args) + output = next( + (value.removeprefix("-out=") for value in args if value.startswith("-out=")), + None, + ) + if output is not None: + Path(output).write_bytes(b"reviewed-plan") + return subprocess.CompletedProcess(args, 0, "", "") + + monkeypatch.setattr(subprocess, "run", run) + return calls + + +def test_admin_bootstrap_applies_exact_plan_then_migrates_native_state( + tmp_path: Path, + terraform_calls: list[tuple[str, ...]], +) -> None: + operator_dir = tmp_path / "operator" + bootstrap = OciAdministrativeBootstrap( + _ROOT / "infra/oci/bootstrap-admin", + operator_dir, + ) + + plan = bootstrap.execute(**_ADMIN) + result = bootstrap.apply_saved_plan(namespace="unitnamespace", **_ADMIN) + + assert result == plan + assert stat.S_IMODE(plan.stat().st_mode) == 0o600 + assert any( + call[1:3] == ("apply", "-input=false") and call[-1] == str(plan) for call in terraform_calls + ) + assert any("-migrate-state" in call for call in terraform_calls) + record = json.loads((operator_dir / "backend.json").read_text(encoding="utf-8")) + assert record == { + "auth": "SecurityToken", + "bucket": "dander-phase7-state", + "config_file_profile": "DANDER", + "key": "dander/oci/bootstrap-admin/terraform.tfstate", + "namespace": "unitnamespace", + "region": "us-ashburn-1", + "schema": "io.dander.oci-bootstrap-backend/v1", + "tenancy_id": _TENANCY, + } + backend = json.loads( + (operator_dir / "terraform-workspace/backend.tf.json").read_text(encoding="utf-8") + )["terraform"]["backend"]["oci"] + assert backend["auth"] == "SecurityToken" + assert backend["bucket"] == "dander-phase7-state" + assert "fingerprint" not in backend + assert "private_key_path" not in backend + assert "user_ocid" not in backend + + +def test_foundation_bootstrap_uses_native_backend_and_saved_plan( + tmp_path: Path, + terraform_calls: list[tuple[str, ...]], +) -> None: + bootstrap = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "foundation") + + plan = bootstrap.execute(**_FOUNDATION) + bootstrap.apply_saved_plan(**_FOUNDATION) + + init = terraform_calls[0] + assert "-backend-config=auth=SecurityToken" in init + assert "-backend-config=namespace=unitnamespace" in init + assert "-backend-config=bucket=dander-phase7-state" in init + assert terraform_calls[-1] == ("terraform", "apply", "-input=false", str(plan)) + assert stat.S_IMODE(plan.stat().st_mode) == 0o600 + + +def test_deployment_verification_reads_both_remote_states_and_requires_no_drift( + tmp_path: Path, + terraform_calls: list[tuple[str, ...]], +) -> None: + admin_dir = tmp_path / "admin" + admin = OciAdministrativeBootstrap(_ROOT / "infra/oci/bootstrap-admin", admin_dir) + admin.execute(**_ADMIN) + admin.apply_saved_plan(namespace="unitnamespace", **_ADMIN) + foundation = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "foundation") + + admin.verify_no_drift(namespace="unitnamespace", **_ADMIN) + foundation.verify_no_drift(**_FOUNDATION) + + verification_plans = [call for call in terraform_calls if "-detailed-exitcode" in call] + assert len(verification_plans) == 2 + assert all("-input=false" in call for call in verification_plans) + + +def test_deployment_verification_fails_closed_on_drift( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + def run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + del kwargs + return subprocess.CompletedProcess(args, 2 if "-detailed-exitcode" in args else 0) + + monkeypatch.setattr(subprocess, "run", run) + foundation = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "foundation") + + with pytest.raises(OciTerraformBootstrapError, match="found Terraform drift"): + foundation.verify_no_drift(**_FOUNDATION) + + +@pytest.mark.parametrize( + ("update", "message"), + [ + ({"tenancy_id": "not-an-ocid"}, "tenancy"), + ({"state_key": "/absolute.tfstate"}, "state key"), + ({"repository_name": "dander//runtime"}, "repository"), + ({"config_file_profile": "profile with spaces"}, "SecurityToken profile"), + ], +) +def test_admin_rejects_unsafe_inputs_before_terraform( + tmp_path: Path, + terraform_calls: list[tuple[str, ...]], + update: dict[str, str], + message: str, +) -> None: + bootstrap = OciAdministrativeBootstrap( + _ROOT / "infra/oci/bootstrap-admin", + tmp_path / "operator", + ) + + with pytest.raises(OciTerraformBootstrapError, match=message): + bootstrap.execute(**{**_ADMIN, **update}) + + assert terraform_calls == [] + + +def test_oci_bootstrap_rejects_repository_owned_operator_artifacts() -> None: + with pytest.raises(OciTerraformBootstrapError, match="outside the repository"): + OciTerraformBootstrap(_ROOT / "infra/oci", _ROOT / "tmp/oci-operator") + + +def test_foundation_apply_requires_reviewed_plan( + tmp_path: Path, + terraform_calls: list[tuple[str, ...]], +) -> None: + bootstrap = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "operator") + + with pytest.raises(OciTerraformBootstrapError, match="Saved OCI plan is missing"): + bootstrap.apply_saved_plan(**_FOUNDATION) + + assert terraform_calls == [] diff --git a/tests/cli/test_oci_cli.py b/tests/cli/test_oci_cli.py new file mode 100644 index 00000000..93bc8753 --- /dev/null +++ b/tests/cli/test_oci_cli.py @@ -0,0 +1,137 @@ +"""OCI plan-first CLI command boundaries.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from typer.testing import CliRunner + +from dander.cli.main import app + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + + +def test_oci_admin_plan_is_non_mutating_and_prints_review_command( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + + def fake_execute(self: object, **kwargs: object) -> Path: + del self + captured.update(kwargs) + return tmp_path / "operator/dander-oci-admin-bootstrap.tfplan" + + monkeypatch.setattr( + "dander.cli.oci_command.OciAdministrativeBootstrap.execute", + fake_execute, + ) + operator = tmp_path / "operator" + result = CliRunner().invoke( + app, + [ + "init-oci-admin-plan", + "--tenancy-id", + "ocid1.tenancy.oc1.." + "a" * 32, + "--compartment-id", + "ocid1.compartment.oc1.." + "b" * 32, + "--state-bucket", + "dander-phase7-state", + "--oci-profile", + "DANDER", + "--operator-artifact-dir", + str(operator), + ], + ) + + assert result.exit_code == 0, result.output + assert captured["repository_name"] == "dander/runtime" + assert captured["config_file_profile"] == "DANDER" + assert "OCI administrative bootstrap planned" in result.output + assert "terraform-workspace" in result.output + assert "No OCI resources were created" in result.output + + +def test_oci_foundation_apply_requires_confirmation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + called = False + + def fake_apply(self: object, **kwargs: object) -> Path: + del self, kwargs + nonlocal called + called = True + return tmp_path / "operator/dander-oci-foundation.tfplan" + + monkeypatch.setattr( + "dander.cli.oci_command.OciTerraformBootstrap.apply_saved_plan", + fake_apply, + ) + result = CliRunner().invoke( + app, + [ + "init-oci-apply", + "--tenancy-id", + "ocid1.tenancy.oc1.." + "a" * 32, + "--compartment-id", + "ocid1.compartment.oc1.." + "b" * 32, + "--object-storage-namespace", + "unitnamespace", + "--state-bucket", + "dander-phase7-state", + "--operator-artifact-dir", + str(tmp_path / "operator"), + ], + input="n\n", + ) + + assert result.exit_code == 1 + assert called is False + + +def test_oci_deployment_verifier_checks_stage_zero_and_foundation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + verified: list[dict[str, object]] = [] + + def fake_verify(self: object, **kwargs: object) -> None: + del self + verified.append(kwargs) + + monkeypatch.setattr( + "dander.cli.oci_command.OciAdministrativeBootstrap.verify_no_drift", + fake_verify, + ) + monkeypatch.setattr( + "dander.cli.oci_command.OciTerraformBootstrap.verify_no_drift", + fake_verify, + ) + result = CliRunner().invoke( + app, + [ + "verify-oci-deployment", + "--tenancy-id", + "ocid1.tenancy.oc1.." + "a" * 32, + "--compartment-id", + "ocid1.compartment.oc1.." + "b" * 32, + "--object-storage-namespace", + "unitnamespace", + "--state-bucket", + "dander-phase7-state", + "--admin-operator-artifact-dir", + str(tmp_path / "admin"), + "--foundation-operator-artifact-dir", + str(tmp_path / "foundation"), + ], + ) + + assert result.exit_code == 0, result.output + assert len(verified) == 2 + assert verified[0]["state_key"] == "dander/oci/bootstrap-admin/terraform.tfstate" + assert verified[1]["state_key"] == "dander/oci/foundation/terraform.tfstate" + assert '"status": "no_drift"' in result.output From 33154a023c8e6704d71dee43cc0245160f75496e Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 02:36:52 -0400 Subject: [PATCH 149/163] feat(oci): add lifecycle controller and operations --- .dockerignore | 2 + .github/workflows/ci.yml | 24 + HANDOFF.md | 40 +- docs/cloud-portability-plan.md | 16 +- docs/decisions.md | 18 + docs/known-limitations.md | 12 +- infra/oci/README.md | 21 +- infra/oci/controller/Dockerfile | 23 + infra/oci/controller/README.md | 24 + infra/oci/controller/func.py | 5 + infra/oci/controller/requirements.txt | 3 + infra/oci/main.tf | 213 ++++++++ infra/oci/outputs.tf | 16 + infra/oci/terraform.tfvars.example | 17 +- infra/oci/tests/oci.tftest.hcl | 102 +++- infra/oci/variables.tf | 99 ++++ pyproject.toml | 4 + src/dander/bootstrap/__init__.py | 2 + src/dander/bootstrap/oci_terraform.py | 138 ++++++ src/dander/cli/oci_command.py | 376 +++++++++++++- src/dander/cli/runtime_command.py | 9 +- src/dander/deployment/projection.py | 4 +- src/dander/providers/__init__.py | 17 +- .../oci_container_instances/__init__.py | 49 +- .../oci_container_instances/controller.py | 466 ++++++++++++++++++ .../function_handler.py | 157 ++++++ .../oci_container_instances/oci_adapter.py | 414 ++++++++++++++++ .../oci_container_instances/operations.py | 310 ++++++++++++ src/dander/runtime_secrets.py | 99 ++++ src/dander/state/failure.py | 6 +- tests/bootstrap/test_oci_controller_image.py | 54 ++ tests/bootstrap/test_oci_terraform.py | 10 +- tests/cli/test_oci_cli.py | 71 ++- tests/cli/test_oci_operations_cli.py | 164 ++++++ .../test_oci_container_instances_adapter.py | 191 +++++++ ...test_oci_container_instances_controller.py | 301 +++++++++++ ...test_oci_container_instances_operations.py | 207 ++++++++ tests/providers/test_oci_function_handler.py | 134 +++++ tests/state/test_failure.py | 14 + tests/test_runtime_secrets.py | 122 +++++ .../DANDER-115-oci-lifecycle-controller.md | 53 ++ 41 files changed, 3952 insertions(+), 55 deletions(-) create mode 100644 infra/oci/controller/Dockerfile create mode 100644 infra/oci/controller/README.md create mode 100644 infra/oci/controller/func.py create mode 100644 infra/oci/controller/requirements.txt create mode 100644 src/dander/providers/oci_container_instances/controller.py create mode 100644 src/dander/providers/oci_container_instances/function_handler.py create mode 100644 src/dander/providers/oci_container_instances/oci_adapter.py create mode 100644 src/dander/providers/oci_container_instances/operations.py create mode 100644 src/dander/runtime_secrets.py create mode 100644 tests/bootstrap/test_oci_controller_image.py create mode 100644 tests/cli/test_oci_operations_cli.py create mode 100644 tests/providers/test_oci_container_instances_adapter.py create mode 100644 tests/providers/test_oci_container_instances_controller.py create mode 100644 tests/providers/test_oci_container_instances_operations.py create mode 100644 tests/providers/test_oci_function_handler.py create mode 100644 tests/test_runtime_secrets.py create mode 100644 tickets/DANDER-115-oci-lifecycle-controller.md diff --git a/.dockerignore b/.dockerignore index a943d4d8..45e97bea 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,6 +18,8 @@ !examples/** !infra/ !infra/** +!dist/ +!dist/*.whl infra/.DS_Store infra/.terraform/ infra/**/.terraform/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 684d039a..9e485cec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,6 +177,7 @@ jobs: needs: [python-quality, terraform-quality] steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2 + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - name: Build immutable-check image run: docker build --tag dander-ci:${{ github.sha }} . - name: Verify CLI starts @@ -196,6 +197,29 @@ jobs: exit-code: 1 ignore-unfixed: true severity: CRITICAL,HIGH + - name: Build OCI lifecycle controller + run: | + uv build --wheel + VERSION="$(uv version --short)" + docker build --platform linux/amd64 \ + --file infra/oci/controller/Dockerfile \ + --build-arg DANDER_WHEEL="dist/dander_platform-${VERSION}-py3-none-any.whl" \ + --tag dander-oci-controller:${{ github.sha }} . + - name: Verify OCI lifecycle controller contract + run: | + test "$(docker image inspect dander-oci-controller:${{ github.sha }} --format '{{.Architecture}}')" = "amd64" + test "$(docker run --rm --platform linux/amd64 --entrypoint /usr/bin/id dander-oci-controller:${{ github.sha }} -u)" = "1000" + docker run --rm --platform linux/amd64 --read-only --tmpfs /tmp \ + --entrypoint /usr/bin/python3.12 dander-oci-controller:${{ github.sha }} \ + -c 'import dander, func, oci; assert callable(func.handler)' + - name: Scan OCI lifecycle controller + uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0 + with: + image-ref: dander-oci-controller:${{ github.sha }} + format: table + exit-code: 1 + ignore-unfixed: true + severity: CRITICAL,HIGH secrets: name: Secret scan diff --git a/HANDOFF.md b/HANDOFF.md index 26c792c1..9d847318 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,39 @@ ## Finished -- Merged the OCI SDK/Vault provider slice through protected PR #212. -- Merged the typed OCI Container Instances launcher through protected PR #213. -- Added native Object Storage state and immutable private OCIR stage zero. -- Added the private VCN, Vault/key, resource-principal policy, Logging, and Notifications foundation. -- Added SecurityToken-only plan/apply CLI and read-only no-drift verification for both remote states. +- Implemented an idempotent OCI Function controller with one fresh Container Instance per attempt. +- Added Resource Scheduler, pipeline-scoped lifecycle events, exact resource principals, logs, and alarms. +- Added crash-recoverable Object Storage run records plus bounded logs, interruption, replay, and retry. +- Added manifest-bound OCI run/status/logs/cancel/replay and exact launcher planning/no-drift inputs. +- Added runtime resolution of versionless OCI Vault references and a source-free controller image. ## Try It -Run `uv run pytest tests/bootstrap/test_oci_terraform.py tests/cli/test_oci_cli.py`. +Run `uv run pytest tests/providers/test_oci_container_instances_controller.py`. ## Checks -- Both OCI Terraform roots validate and their native tests pass. -- The complete pytest suite, Ruff, and repository-wide strict Mypy passed. -- Wheel and source distribution built and passed the distribution contract check. -- Trivy reported no HIGH or CRITICAL infrastructure misconfigurations. -- Protected-main CI for merged PR #213 passed all five jobs. +- Complete pytest, Ruff lint/format, and repository-wide Mypy pass. +- Both OCI Terraform roots validate; their native tests pass (2 foundation/controller, 1 stage zero). +- Wheel/sdist build and the distribution contract pass. +- The `linux/amd64` Function image builds and imports Dander `0.9.0rc1` with OCI SDK `2.184.1`. +- No state, plan, key, or credential file is present in the worktree diff. ## Decisions -- Stage zero and foundation are separate reviewed saved plans; secret values never enter Terraform. -- Terraform's native OCI backend requires Terraform 1.12+ and uses only SecurityToken auth here. -- Live OCI writes remain at $0 pending preflight and a numeric per-attempt ceiling. +- The Function owns parallelism one, exit-75 whole-task retry, a 3,300-second deadline, and cleanup. +- Scheduler delivery is UTC/hourly; event rules and controller dynamic groups are pipeline/exact-OCID scoped. +- Runtime task images remain source-free; the Function image contains only the exact wheel and dependencies. ## Remaining -- Merge this Terraform foundation through a protected PR. -- Implement scheduling/lifecycle operations in a separate PR. -- Build one new post-merge candidate and run the complete OCI live gate after account preflight. -- Complete Phase 8 only after Phase 7 passes. +- Merge this implementation through a protected PR and verify protected-main CI. +- Obtain explicit approval for public candidate publication and a numeric OCI per-attempt ceiling. +- Run read-only OCI preflight, reviewed plans, live profile/rotation/rollback/cleanup, and no drift. +- Merge sanitized live evidence, then make the binary Phase 7 exit-gate recommendation. ## Review First -- `src/dander/bootstrap/oci_terraform.py` +- `src/dander/providers/oci_container_instances/controller.py` +- `src/dander/providers/oci_container_instances/oci_adapter.py` - `infra/oci/main.tf` -- `infra/oci/bootstrap-admin/main.tf` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index c28f8d3b..8099714d 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1202,17 +1202,23 @@ Tickets: 7. schedule, retry, parallelism, interruption, rollback, and reconciliation proof. Progress on 2026-08-12: the OCI SDK/Vault provider, typed Container Instances launcher projection, -and plan-first Terraform foundations are implemented behind provider-specific boundaries. The OCI +plan-first Terraform foundations, and lifecycle-controller implementation are behind +provider-specific boundaries. The OCI native remote-state bootstrap creates only a private versioned Object Storage bucket and private immutable OCIR repository; the second root creates a private VCN/subnet, default Vault with an auto-rotating key, compartment-scoped Container Instance resource-principal policy, Logging, and Notifications. Both use short-lived SecurityToken auth, saved plans outside the repository, and locally passing Terraform tests; a read-only verifier refreshes both remote states and fails on any planned drift. The launcher still accepts only the named PostgreSQL/PostgreSQL/ -no-catalog/OCI-Vault profile and exact digest-qualified OCIR images. Promotion, scheduling Function, -operations, reconciliation, credential-refresh, live-profile, cleanup, and no-drift evidence remain -open. This progress is not an exit-gate or support claim, and paid OCI mutation remains disabled -pending credential preflight and an explicit per-attempt ceiling. +no-catalog/OCI-Vault profile and exact digest-qualified OCIR images. The narrow Python 3.12 OCI +Function owns deterministic idempotency, overlap exclusion, whole-task retries, deadlines, +interruption, replay, bounded logs, terminal history, per-pipeline lifecycle-event reconciliation, +and UTC Resource Scheduler delivery; operator commands use only expiring SecurityToken sessions. +Runtime processes resolve versionless Vault references through resource principals without putting +values in Terraform or Function configuration. Controller/runtime image publication, +credential-refresh, live-profile, rollback, cleanup, and no-drift evidence remain open. This +progress is not an exit-gate or support claim, and paid OCI mutation remains disabled pending +credential preflight and an explicit per-attempt ceiling. Exit gate: diff --git a/docs/decisions.md b/docs/decisions.md index 31d01184..d4b4838b 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1345,3 +1345,21 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Sequencing:** Secret values are seeded outside Terraform after the foundation apply, and the output subnet/Vault OCIDs become typed launcher inputs. Saved plans, bootstrap state, backend metadata, and Terraform caches stay in a mode-restricted operator directory outside Git. + +## 2026-08-12 — One narrow Function owns OCI lifecycle semantics + +- **Controller:** One Python 3.12 OCI Function per pipeline uses Object Storage conditional writes + for deterministic idempotency and maximum parallelism one. It creates a fresh Container Instance + for each whole-task attempt, retries only exit code 75, captures bounded logs before deletion, + and retains sanitized terminal history. A 3,300-second task limit reserves five minutes inside + OCI Functions' one-hour detached limit for interruption and cleanup. +- **Delivery and identity:** UTC Resource Scheduler starts the Function no more often than hourly; + a pipeline-tagged Events rule assists reconciliation. Exact Function OCIDs form the controller + dynamic group. Functions and Container Instances use resource principals, while local operations + accept only expiring SecurityToken profiles. Static OCI API keys and cross-cloud keys are not + fallbacks. +- **Secrets and artifacts:** Launch projections contain only validated Vault references. Each + runtime process resolves the `CURRENT` version and removes injected values at exit. The + controller is a digest-pinned `GENERIC_X86` Function image assembled from the exact reviewed + wheel; the task runtime remains the separately accepted source-free release image. Live OCI + publication and provider proof remain separate gates. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index abc558c0..dac88b5f 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -76,8 +76,16 @@ remain Phase 8 work. whole-GiB memory, and one container attempt with provider restart policy `NEVER`; unsupported profiles and inputs fail before OCI access. OCI OCPUs are provider billing/resource units rather than a cross-provider performance claim, and the fixed 15 GB ephemeral-storage allocation is not - configurable through this projection. Terraform, the launch Function, lifecycle reconciliation, - CLI operations, digest promotion, and the live OCI profile remain incomplete. No OCI launcher + configurable through this projection. A resource-principal OCI Function now owns deterministic + run identity, maximum parallelism one, whole-task retry only for exit code 75, bounded logs, + stop/delete cleanup, immutable terminal history, Resource Scheduler delivery, and lifecycle-event + reconciliation. The Function's one-hour detached limit leaves a five-minute cleanup reserve, so + OCI tasks are capped at 3,300 seconds. Resource Scheduler is UTC-only and cannot schedule an + interval shorter than one hour. OCI Vault references resolve the `CURRENT` version at the start + of every runtime process and are removed from the process environment afterward; in-process + rotation is not claimed. The controller image is a Python 3.12 `GENERIC_X86` image built from an + exact wheel, not the source-free task image. Digest promotion, provider deployment verification, + credential-rotation proof, and the complete live OCI profile remain unqualified. No OCI launcher support claim exists until those separate gates pass. ## Ingestion and schemas diff --git a/infra/oci/README.md b/infra/oci/README.md index 167ce438..c421a199 100644 --- a/infra/oci/README.md +++ b/infra/oci/README.md @@ -6,7 +6,8 @@ This root creates the provider-owned foundation for Dander's named OCI profile: - NAT egress for PostgreSQL plus an OCI Services gateway for OCIR, Vault, and control APIs; - a default OCI Vault and auto-rotating software-protected key; - a compartment-scoped Container Instance dynamic group and least-privilege runtime policy; and -- a log group and Notifications topic for the lifecycle controller. +- a private versioned run-record bucket, per-pipeline OCI Function, Resource Scheduler schedule, + lifecycle event rule, invocation log, error alarm, and Notifications topic. It deliberately does **not** create long-lived Container Instances. Every task attempt is a new run-scoped instance created by the Phase 7 launch Function and stopped by the reconciler. It also @@ -16,3 +17,21 @@ apply and before enabling schedules. Use Terraform's native `oci` backend with the private stage-zero bucket and a short-lived `SecurityToken` profile. Review a saved plan before apply. The output subnet and Vault OCIDs are then copied into the typed `oci_container_instances` launcher configuration. + +## Plan sequence + +1. Run `dander init-oci-admin-plan`, review the saved stage-zero plan, and apply it only after + explicit cost approval with `dander init-oci-admin-apply`. +2. Run `dander init-oci-plan` and apply the reviewed foundation plan with `dander init-oci-apply`. +3. Seed only the manifest-declared secret names in Vault. Secret values never enter Terraform. +4. Build and push the [controller image](controller/README.md) from a clean protected-main wheel. +5. Run `dander init-oci-launcher-plan` with the exact runtime digest, controller tag and controller + digest. Review and apply that saved plan through the same `init-oci-apply` boundary. +6. Use `dander verify-oci-deployment` with all four controller inputs to require exact no drift. + +The Function owns maximum parallelism one, whole-task retry only for runtime exit code 75, a +3,300-second maximum runtime deadline, stop/delete cleanup, bounded logs, and immutable history. +Resource Scheduler is UTC-only and does not admit a recurrence interval shorter than one hour. +`dander oci run`, `cancel`, and `replay` require confirmation; `status` and `logs` are read-only. +All operator API calls use an expiring `SecurityToken` profile, while the Function and Container +Instances use resource principals. Static OCI API keys and static cloud keys are not fallbacks. diff --git a/infra/oci/controller/Dockerfile b/infra/oci/controller/Dockerfile new file mode 100644 index 00000000..3b5d2597 --- /dev/null +++ b/infra/oci/controller/Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1 + +# OCI Functions defaults to GENERIC_X86. Pin both Python 3.12 FDK images to their +# linux/amd64 manifests so the controller build cannot silently change architecture. +FROM fnproject/python:3.12-dev@sha256:12fcb8be166981acdcf52bbad73ea4833c73634a31b7c63a9c13912e499ba201 AS build + +ARG DANDER_WHEEL +WORKDIR /function +COPY infra/oci/controller/requirements.txt /function/requirements.txt +RUN pip3 install --target /python --no-cache-dir -r /function/requirements.txt +COPY ${DANDER_WHEEL} /tmp/dander_platform-0-py3-none-any.whl +RUN pip3 install --target /python --no-cache-dir --no-deps /tmp/dander_platform-0-py3-none-any.whl \ + && rm -rf /root/.cache /tmp/* + +FROM fnproject/python:3.12@sha256:fa298b94e95a54e6d10c54aa5691fefdb01c76c5c38b3da352a2f9dc0fbdf739 + +WORKDIR /function +COPY --from=build /python /python +COPY infra/oci/controller/func.py /function/func.py +RUN chmod -R a+rX /function /python +ENV PYTHONPATH=/function:/python +USER fn +ENTRYPOINT ["/python/bin/fdk", "/function/func.py", "handler"] diff --git a/infra/oci/controller/README.md b/infra/oci/controller/README.md new file mode 100644 index 00000000..6317bab5 --- /dev/null +++ b/infra/oci/controller/README.md @@ -0,0 +1,24 @@ +# OCI lifecycle controller image + +This image contains only the OCI Functions shim, the exact Dander wheel under review, the OCI SDK, +and the Python FDK. It does not contain repository source, project data, credentials, or Terraform +artifacts. The Dockerfile is pinned to the Python 3.12 `linux/amd64` FDK manifests because the +Terraform application explicitly selects `GENERIC_X86`. + +From a clean protected-main checkout, build the wheel and controller image with a unique candidate +tag: + +```console +uv build --wheel +docker buildx build --platform linux/amd64 \ + --file infra/oci/controller/Dockerfile \ + --build-arg DANDER_WHEEL=dist/dander_platform--py3-none-any.whl \ + --tag ocir..oci.oraclecloud.com//: \ + --push . +``` + +Inspect the pushed image, record its `sha256:` digest outside Git, and pass the tag and digest +together to `dander init-oci-launcher-plan`. The controller uses the same private immutable OCIR +repository selected by the manifest, but a separate unique tag from the task image. Never reuse a +tag, rebuild an accepted candidate, or +commit the wheel, registry credentials, saved plan, or Terraform state. diff --git a/infra/oci/controller/func.py b/infra/oci/controller/func.py new file mode 100644 index 00000000..a0b48be0 --- /dev/null +++ b/infra/oci/controller/func.py @@ -0,0 +1,5 @@ +"""OCI Functions shim for Dander's packaged lifecycle controller.""" + +from dander.providers.oci_container_instances.function_handler import handler + +__all__ = ["handler"] diff --git a/infra/oci/controller/requirements.txt b/infra/oci/controller/requirements.txt new file mode 100644 index 00000000..ab0995e9 --- /dev/null +++ b/infra/oci/controller/requirements.txt @@ -0,0 +1,3 @@ +fdk==0.1.117 +oci==2.184.1 +pydantic==2.13.4 diff --git a/infra/oci/main.tf b/infra/oci/main.tf index f952c998..2a5f85fa 100644 --- a/infra/oci/main.tf +++ b/infra/oci/main.tf @@ -3,6 +3,20 @@ locals { component = "oci-container-instances" managed-by = "dander" }) + controller_enabled = var.controller_image != null && var.controller_image_digest != null && length(var.execution_projections) > 0 + resource_schedule_ids = [ + for schedule in oci_resource_scheduler_schedule.pipeline : schedule.id + ] +} + +check "controller_inputs_are_atomic" { + assert { + condition = ( + (var.controller_image == null && var.controller_image_digest == null && length(var.execution_projections) == 0) || + local.controller_enabled + ) + error_message = "controller_image, controller_image_digest, and execution_projections must be supplied together." + } } data "oci_core_services" "all_services" { @@ -134,6 +148,167 @@ resource "oci_identity_policy" "runtime" { ] } +resource "oci_objectstorage_bucket" "run_records" { + compartment_id = var.compartment_id + namespace = var.object_storage_namespace + name = "${var.name}-oci-runs-${substr(sha256(var.compartment_id), 0, 8)}" + access_type = "NoPublicAccess" + object_events_enabled = false + storage_tier = "Standard" + versioning = "Enabled" + auto_tiering = "Disabled" + freeform_tags = local.tags +} + +resource "oci_objectstorage_object" "projection" { + for_each = local.controller_enabled ? var.execution_projections : {} + + namespace = var.object_storage_namespace + bucket = oci_objectstorage_bucket.run_records.name + object = "projections/${each.key}.json" + content = jsonencode(each.value) + content_type = "application/json" +} + +resource "oci_functions_application" "controller" { + count = local.controller_enabled ? 1 : 0 + + compartment_id = var.compartment_id + display_name = "${var.name}-lifecycle" + subnet_ids = [oci_core_subnet.runtime.id] + shape = "GENERIC_X86" + freeform_tags = local.tags + + logging { + line_format = "JSON" + } +} + +resource "oci_functions_function" "pipeline" { + for_each = local.controller_enabled ? var.execution_projections : {} + + application_id = one(oci_functions_application.controller).id + display_name = "${var.name}-${each.key}" + image = var.controller_image + image_digest = var.controller_image_digest + memory_in_mbs = var.controller_memory_mib + timeout_in_seconds = 300 + detached_mode_timeout_in_seconds = 3600 + freeform_tags = merge(local.tags, { pipeline = each.key }) + config = { + DANDER_OCI_NAMESPACE = var.object_storage_namespace + DANDER_OCI_RUN_BUCKET = oci_objectstorage_bucket.run_records.name + DANDER_OCI_PIPELINE = each.key + DANDER_OCI_PROJECTION_KEY = oci_objectstorage_object.projection[each.key].object + } + + failure_destination { + kind = "NOTIFICATION" + topic_id = oci_ons_notification_topic.runtime.id + } +} + +resource "oci_identity_dynamic_group" "controller" { + count = local.controller_enabled ? 1 : 0 + + compartment_id = var.tenancy_id + name = var.controller_dynamic_group_name + description = "Dander OCI lifecycle Function resource principals" + matching_rule = format( + "ANY {%s}", + join(", ", [for function in oci_functions_function.pipeline : "resource.id='${function.id}'"]), + ) + freeform_tags = local.tags +} + +resource "oci_identity_policy" "controller" { + count = local.controller_enabled ? 1 : 0 + + compartment_id = var.tenancy_id + name = "${var.name}-lifecycle-controller" + description = "Least-privilege OCI lifecycle Function mutations" + freeform_tags = local.tags + statements = [ + "Allow dynamic-group ${one(oci_identity_dynamic_group.controller).name} to manage compute-container-family in compartment id ${var.compartment_id}", + "Allow dynamic-group ${one(oci_identity_dynamic_group.controller).name} to use virtual-network-family in compartment id ${var.compartment_id}", + "Allow dynamic-group ${one(oci_identity_dynamic_group.controller).name} to manage objects in compartment id ${var.compartment_id} where target.bucket.name='${oci_objectstorage_bucket.run_records.name}'", + ] +} + +resource "oci_resource_scheduler_schedule" "pipeline" { + for_each = local.controller_enabled ? var.execution_projections : {} + + action = "START_RESOURCE" + compartment_id = var.compartment_id + display_name = "${var.name}-${each.key}" + description = "Invoke the Dander OCI lifecycle controller for ${each.key}" + recurrence_type = "CRON" + recurrence_details = each.value.schedule.expression + state = each.value.schedule.paused ? "INACTIVE" : "ACTIVE" + freeform_tags = merge(local.tags, { pipeline = each.key }) + + resources { + id = oci_functions_function.pipeline[each.key].id + + parameters { + parameter_type = "BODY" + value = [jsonencode({ action = "start", source = "resource_scheduler" })] + } + } +} + +resource "oci_identity_dynamic_group" "scheduler" { + count = local.controller_enabled ? 1 : 0 + + compartment_id = var.tenancy_id + name = var.scheduler_dynamic_group_name + description = "Dander OCI Resource Scheduler invocations" + matching_rule = format( + "ANY {%s}", + join(", ", [for id in local.resource_schedule_ids : "resource.id='${id}'"]), + ) + freeform_tags = local.tags +} + +resource "oci_identity_policy" "scheduler" { + count = local.controller_enabled ? 1 : 0 + + compartment_id = var.tenancy_id + name = "${var.name}-lifecycle-scheduler" + description = "Permit only Dander Resource Schedules to invoke lifecycle Functions" + freeform_tags = local.tags + statements = [ + "Allow dynamic-group ${one(oci_identity_dynamic_group.scheduler).name} to use fn-invocation in compartment id ${var.compartment_id}", + ] +} + +resource "oci_events_rule" "container_lifecycle" { + for_each = local.controller_enabled ? var.execution_projections : {} + + compartment_id = var.compartment_id + display_name = "${var.name}-${each.key}-container-lifecycle" + description = "Reconcile ${each.key} after its Container Instance lifecycle events" + is_enabled = true + freeform_tags = local.tags + condition = jsonencode({ + data = { + freeFormTags = { + managed-by = "dander" + dander-pipeline = each.key + } + } + }) + + actions { + action { + action_type = "FAAS" + is_enabled = true + function_id = oci_functions_function.pipeline[each.key].id + description = "Reconcile ${each.key} after an OCI lifecycle event" + } + } +} + resource "oci_logging_log_group" "runtime" { compartment_id = var.compartment_id display_name = "${var.name}-runtime" @@ -147,3 +322,41 @@ resource "oci_ons_notification_topic" "runtime" { description = "Dander OCI execution and reconciliation alerts" freeform_tags = local.tags } + +resource "oci_logging_log" "function_invocations" { + count = local.controller_enabled ? 1 : 0 + + display_name = "${var.name}-lifecycle-invoke" + log_group_id = oci_logging_log_group.runtime.id + log_type = "SERVICE" + is_enabled = true + retention_duration = 30 + freeform_tags = local.tags + + configuration { + compartment_id = var.compartment_id + + source { + category = "invoke" + resource = one(oci_functions_application.controller).id + service = "functions" + source_type = "OCISERVICE" + } + } +} + +resource "oci_monitoring_alarm" "function_errors" { + count = local.controller_enabled ? 1 : 0 + + compartment_id = var.compartment_id + metric_compartment_id = var.compartment_id + namespace = "oci_faas" + display_name = "${var.name}-lifecycle-errors" + body = "Dander lifecycle Function returned an error. Inspect the bounded invocation log and active run record." + destinations = [oci_ons_notification_topic.runtime.id] + is_enabled = true + pending_duration = "PT1M" + query = "FunctionResponseCount[1m]{applicationId = \"${one(oci_functions_application.controller).id}\",responseType = \"Error\"}.sum() > 0" + severity = "ERROR" + freeform_tags = local.tags +} diff --git a/infra/oci/outputs.tf b/infra/oci/outputs.tf index 4898833c..feac8be5 100644 --- a/infra/oci/outputs.tf +++ b/infra/oci/outputs.tf @@ -28,6 +28,22 @@ output "notification_topic_id" { value = oci_ons_notification_topic.runtime.id } +output "run_records_bucket" { + description = "Private versioned Object Storage bucket for non-secret projections and runs." + value = oci_objectstorage_bucket.run_records.name +} + +output "controller" { + description = "Lifecycle Function and Resource Scheduler identifiers by pipeline." + value = { + for id, function in oci_functions_function.pipeline : id => { + function_id = function.id + schedule_id = oci_resource_scheduler_schedule.pipeline[id].id + schedule_state = oci_resource_scheduler_schedule.pipeline[id].state + } + } +} + output "network" { description = "Private runtime VCN resources used by run-scoped Container Instances." value = { diff --git a/infra/oci/terraform.tfvars.example b/infra/oci/terraform.tfvars.example index b29de61c..484d834e 100644 --- a/infra/oci/terraform.tfvars.example +++ b/infra/oci/terraform.tfvars.example @@ -1,7 +1,12 @@ -tenancy_id = "ocid1.tenancy.oc1..replace_me" -compartment_id = "ocid1.compartment.oc1..replace_me" -region = "us-ashburn-1" -config_file_profile = "DANDER" -dynamic_group_name = "dander_phase7_runtime" +tenancy_id = "ocid1.tenancy.oc1..replace_me" +compartment_id = "ocid1.compartment.oc1..replace_me" +region = "us-ashburn-1" +object_storage_namespace = "replace_me" +config_file_profile = "DANDER" +dynamic_group_name = "dander_phase7_runtime" +controller_dynamic_group_name = "dander_phase7_controller" +scheduler_dynamic_group_name = "dander_phase7_scheduler" -# Keep all values non-secret. Authenticate through a short-lived OCI SecurityToken profile. +# The bootstrap CLI passes the reviewed remote-state bucket/key and optional controller image, +# digest, and execution projections. Keep every value non-secret and authenticate only through a +# short-lived OCI SecurityToken profile. diff --git a/infra/oci/tests/oci.tftest.hcl b/infra/oci/tests/oci.tftest.hcl index 78fd5f00..77a1897a 100644 --- a/infra/oci/tests/oci.tftest.hcl +++ b/infra/oci/tests/oci.tftest.hcl @@ -1,4 +1,11 @@ mock_provider "oci" { + mock_resource "oci_functions_function" { + override_during = plan + defaults = { + id = "ocid1.fnfunc.oc1.iad.ffffffffffffffffffffffffffffffff" + } + } + mock_data "oci_core_services" { defaults = { services = [{ @@ -11,10 +18,11 @@ mock_provider "oci" { } variables { - tenancy_id = "ocid1.tenancy.oc1..aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - compartment_id = "ocid1.compartment.oc1..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - region = "us-ashburn-1" - dynamic_group_name = "dander_phase7_runtime" + tenancy_id = "ocid1.tenancy.oc1..aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + compartment_id = "ocid1.compartment.oc1..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + region = "us-ashburn-1" + object_storage_namespace = "unitnamespace" + dynamic_group_name = "dander_phase7_runtime" } run "projects_private_keyless_foundation" { @@ -47,3 +55,89 @@ run "projects_private_keyless_foundation" { error_message = "Only Container Instances in the selected compartment may receive runtime OCI permissions." } } + +run "projects_idempotent_lifecycle_controller" { + command = plan + + variables { + controller_image = "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime:phase7-controller-unit" + controller_image_digest = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + execution_projections = { + jobs = { + schema = "io.dander.execution/v1" + contract = "io.dander.runtime/v1" + pipeline_id = "jobs" + profile_id = "oci_postgresql" + launcher = "oci_container_instances" + image = "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command = ["runtime", "execute"] + configuration_reference = "/app/dander.yaml" + environment = {} + secret_bindings = {} + workload_identity = "oci-resource-principal://dynamic-group/dander_phase7_runtime" + resources = { + cpu_millis = 1000 + memory_mib = 2048 + ephemeral_storage_mib = null + deadline_seconds = 900 + runtime_retry_count = 0 + launcher_retry_count = 1 + } + schedule = { + task_count = 1 + maximum_parallelism = 1 + expression = "0 * * * *" + time_zone = "UTC" + paused = true + } + network = { + placement = "ocid1.subnet.oc1.iad.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + extensions = { + oci_assign_public_ip = "false" + oci_availability_domain = "unit:US-ASHBURN-AD-1" + } + } + labels = {} + observability = { + log_destination = "oci_logging" + metric_namespace = "oci_computecontainerinstance" + alert_target = "oci_notifications" + retention_days = 30 + } + extensions = { + oci_compartment_id = "ocid1.compartment.oc1..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + oci_graceful_shutdown_seconds = "120" + oci_registry_endpoint = "ocir.us-ashburn-1.oci.oraclecloud.com" + oci_restart_policy = "NEVER" + oci_shape = "CI.Standard.E4.Flex" + oci_tenancy_id = "ocid1.tenancy.oc1..aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + oci_vault_id = "ocid1.vault.oc1.iad.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + } + } + } + + assert { + condition = ( + oci_objectstorage_bucket.run_records.access_type == "NoPublicAccess" && + oci_objectstorage_bucket.run_records.versioning == "Enabled" && + oci_functions_application.controller[0].shape == "GENERIC_X86" && + oci_functions_function.pipeline["jobs"].image_digest == "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" && + oci_functions_function.pipeline["jobs"].detached_mode_timeout_in_seconds == 3600 && + oci_resource_scheduler_schedule.pipeline["jobs"].state == "INACTIVE" + ) + error_message = "The lifecycle controller must use private versioned records, an immutable Function, and paused-aware scheduling." + } + + assert { + condition = ( + oci_identity_dynamic_group.controller[0].matching_rule == format("ANY {resource.id='%s'}", oci_functions_function.pipeline["jobs"].id) && + length(oci_identity_policy.controller[0].statements) == 3 && + oci_events_rule.container_lifecycle["jobs"].is_enabled && + strcontains(oci_events_rule.container_lifecycle["jobs"].condition, "dander-pipeline") && + oci_logging_log.function_invocations[0].configuration[0].source[0].category == "invoke" && + oci_monitoring_alarm.function_errors[0].namespace == "oci_faas" + ) + error_message = "The controller must remain keyless, event-assisted, logged, and alarmed." + } +} diff --git a/infra/oci/variables.tf b/infra/oci/variables.tf index 6f1f77b9..9efe6fae 100644 --- a/infra/oci/variables.tf +++ b/infra/oci/variables.tf @@ -28,6 +28,16 @@ variable "region" { } } +variable "object_storage_namespace" { + description = "OCI Object Storage namespace used for non-secret projections and run records." + type = string + + validation { + condition = can(regex("^[a-z0-9][a-z0-9_-]{0,99}$", var.object_storage_namespace)) + error_message = "object_storage_namespace must be a normalized OCI namespace." + } +} + variable "config_file_profile" { description = "Short-lived OCI SecurityToken profile used by Terraform." type = string @@ -55,6 +65,95 @@ variable "dynamic_group_name" { } } +variable "controller_dynamic_group_name" { + description = "Tenancy-unique dynamic group for lifecycle Function resource principals." + type = string + default = "dander_phase7_controller" + + validation { + condition = can(regex("^[A-Za-z][A-Za-z0-9_-]{0,99}$", var.controller_dynamic_group_name)) + error_message = "controller_dynamic_group_name must be a valid OCI dynamic-group name." + } +} + +variable "scheduler_dynamic_group_name" { + description = "Tenancy-unique dynamic group for Dander Resource Schedules." + type = string + default = "dander_phase7_scheduler" + + validation { + condition = can(regex("^[A-Za-z][A-Za-z0-9_-]{0,99}$", var.scheduler_dynamic_group_name)) + error_message = "scheduler_dynamic_group_name must be a valid OCI dynamic-group name." + } +} + +variable "controller_image" { + description = "Immutable-tagged OCI Functions controller image in same-region OCIR." + type = string + default = null + nullable = true + + validation { + condition = var.controller_image == null || can(regex( + "^[a-z0-9.-]+/[a-z0-9_-]+/[a-z0-9._/-]+:[A-Za-z0-9._-]+$", + var.controller_image, + )) + error_message = "controller_image must be a tagged OCIR image." + } +} + +variable "controller_image_digest" { + description = "Exact sha256 digest bound to the OCI Functions controller image tag." + type = string + default = null + nullable = true + + validation { + condition = var.controller_image_digest == null || can(regex("^sha256:[0-9a-f]{64}$", var.controller_image_digest)) + error_message = "controller_image_digest must be an immutable sha256 digest." + } +} + +variable "controller_memory_mib" { + description = "Memory assigned to each lifecycle Function." + type = number + default = 512 + + validation { + condition = contains([128, 256, 512, 1024, 2048], var.controller_memory_mib) + error_message = "controller_memory_mib must be a supported bounded Functions allocation." + } +} + +variable "execution_projections" { + description = "Validated OCI Container Instances templates keyed by pipeline id." + type = map(any) + default = {} + + validation { + condition = alltrue([ + for id, projection in var.execution_projections : ( + can(regex("^[A-Za-z][A-Za-z0-9_-]{0,63}$", id)) && + projection.schema == "io.dander.execution/v1" && + projection.launcher == "oci_container_instances" && + projection.pipeline_id == id && + can(regex("@sha256:[0-9a-f]{64}$", projection.image)) && + projection.resources.runtime_retry_count == 0 && + projection.resources.deadline_seconds >= 1 && + projection.resources.deadline_seconds <= 3300 && + projection.resources.launcher_retry_count >= 0 && + projection.resources.launcher_retry_count <= 10 && + projection.schedule.task_count == 1 && + projection.schedule.maximum_parallelism == 1 && + projection.schedule.time_zone == "UTC" && + projection.network.extensions.oci_assign_public_ip == "false" && + projection.extensions.oci_restart_policy == "NEVER" + ) + ]) + error_message = "execution_projections must satisfy the bounded OCI lifecycle contract." + } +} + variable "vcn_cidr" { description = "Private VCN CIDR for the OCI launcher." type = string diff --git a/pyproject.toml b/pyproject.toml index 4e82d850..072c0cbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,6 +187,10 @@ packages = ["src/dander"] "infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl" = "dander/templates/project/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl" "infra/oci/bootstrap-admin/variables.tf" = "dander/templates/project/infra/oci/bootstrap-admin/variables.tf" "infra/oci/bootstrap-admin/versions.tf" = "dander/templates/project/infra/oci/bootstrap-admin/versions.tf" +"infra/oci/controller/Dockerfile" = "dander/templates/project/infra/oci/controller/Dockerfile" +"infra/oci/controller/README.md" = "dander/templates/project/infra/oci/controller/README.md" +"infra/oci/controller/func.py" = "dander/templates/project/infra/oci/controller/func.py" +"infra/oci/controller/requirements.txt" = "dander/templates/project/infra/oci/controller/requirements.txt" "infra/oci/main.tf" = "dander/templates/project/infra/oci/main.tf" "infra/oci/outputs.tf" = "dander/templates/project/infra/oci/outputs.tf" "infra/oci/terraform.tfvars.example" = "dander/templates/project/infra/oci/terraform.tfvars.example" diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index 4df0d0a9..e6f3c241 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -16,6 +16,7 @@ OciAdministrativeBootstrap, OciTerraformBootstrap, OciTerraformBootstrapError, + build_oci_execution_projections, ) from dander.bootstrap.permissions import require_stage_zero_permissions from dander.bootstrap.project import ( @@ -51,6 +52,7 @@ "OciAdministrativeBootstrap", "OciTerraformBootstrap", "OciTerraformBootstrapError", + "build_oci_execution_projections", "DeploymentSummary", "DeploymentVerificationError", "DeploymentVerifier", diff --git a/src/dander/bootstrap/oci_terraform.py b/src/dander/bootstrap/oci_terraform.py index a8c1a9c2..c56081b7 100644 --- a/src/dander/bootstrap/oci_terraform.py +++ b/src/dander/bootstrap/oci_terraform.py @@ -9,6 +9,15 @@ import subprocess from typing import TYPE_CHECKING +from dander.bootstrap.terraform import ( + TerraformBootstrapError, + build_launcher_runtime, + validate_runtime_settings, + validate_terraform_pipelines, +) +from dander.deployment import ExecutionProjectionError, ResolvedTemplateRequest +from dander.providers import ProviderFactoryError + if TYPE_CHECKING: from collections.abc import Mapping from pathlib import Path @@ -23,6 +32,8 @@ _DYNAMIC_GROUP = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,99}$") _NAMESPACE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,99}$") _STATE_KEY = re.compile(r"^(?!/)(?!.*//)[A-Za-z0-9][A-Za-z0-9._/-]{0,1023}$") +_CONTROLLER_IMAGE = re.compile(r"^[a-z0-9.-]+/[a-z0-9_-]+/[a-z0-9._/-]+:[A-Za-z0-9._-]+$") +_IMAGE_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") _BACKEND_SCHEMA = "io.dander.oci-bootstrap-backend/v1" @@ -30,6 +41,57 @@ class OciTerraformBootstrapError(RuntimeError): """Raised when an OCI saved-plan lifecycle cannot complete safely.""" +def build_oci_execution_projections( + *, + container_image: str, + launcher_config: Mapping[str, object], + profile_id: str, + runtime_cpu: int, + runtime_memory: str, + runtime_timeout_seconds: int, + runtime_max_retries: int, + runtime_batch_rows: int, + require_guarded_free_tier: bool, + pipelines: Mapping[str, Mapping[str, object]], +) -> dict[str, dict[str, object]]: + """Build validated OCI templates without contacting OCI.""" + expanded_pipelines = dict(pipelines) + if not expanded_pipelines: + raise OciTerraformBootstrapError("OCI launcher planning requires at least one pipeline") + try: + validate_runtime_settings( + cpu=runtime_cpu, + memory=runtime_memory, + timeout_seconds=runtime_timeout_seconds, + max_retries=runtime_max_retries, + batch_rows=runtime_batch_rows, + require_guarded_free_tier=require_guarded_free_tier, + ) + validate_terraform_pipelines(expanded_pipelines) + launcher = build_launcher_runtime( + launcher_config=dict(launcher_config), + gcp_context=None, + ) + return { + pipeline_id: template.as_dict() + for pipeline_id, template in launcher.templates.build( + ResolvedTemplateRequest( + pipelines=expanded_pipelines, + image=container_image, + profile_id=profile_id, + cpu=runtime_cpu, + memory=runtime_memory, + deadline_seconds=runtime_timeout_seconds, + launcher_retry_count=runtime_max_retries, + batch_rows=runtime_batch_rows, + alert_target="oci_notifications", + ) + ).items() + } + except (ExecutionProjectionError, ProviderFactoryError, TerraformBootstrapError) as error: + raise OciTerraformBootstrapError(str(error)) from error + + class OciAdministrativeBootstrap: """Create OCI state and registry prerequisites, then migrate to native remote state.""" @@ -318,6 +380,11 @@ def execute( dynamic_group_name: str, config_file_profile: str = "DEFAULT", name: str = "dander", + controller_image: str | None = None, + controller_image_digest: str | None = None, + execution_projections: Mapping[str, object] | None = None, + controller_dynamic_group_name: str = "dander_phase7_controller", + scheduler_dynamic_group_name: str = "dander_phase7_scheduler", ) -> Path: """Produce one immutable remote-state-backed OCI foundation plan.""" self._validate( @@ -331,6 +398,13 @@ def execute( config_file_profile=config_file_profile, name=name, ) + controller_args = self._controller_args( + controller_image=controller_image, + controller_image_digest=controller_image_digest, + execution_projections=execution_projections, + controller_dynamic_group_name=controller_dynamic_group_name, + scheduler_dynamic_group_name=scheduler_dynamic_group_name, + ) self._workspace.prepare() self._init( tenancy_id=tenancy_id, @@ -348,8 +422,10 @@ def execute( f"-var=compartment_id={compartment_id}", f"-var=region={region}", f"-var=config_file_profile={config_file_profile}", + f"-var=object_storage_namespace={namespace}", f"-var=dynamic_group_name={dynamic_group_name}", f"-var=name={name}", + *controller_args, f"-out={self._workspace.plan_path}", ) self._workspace.secure_plan() @@ -405,6 +481,11 @@ def verify_no_drift( dynamic_group_name: str, config_file_profile: str = "DEFAULT", name: str = "dander", + controller_image: str | None = None, + controller_image_digest: str | None = None, + execution_projections: Mapping[str, object] | None = None, + controller_dynamic_group_name: str = "dander_phase7_controller", + scheduler_dynamic_group_name: str = "dander_phase7_scheduler", ) -> None: """Refresh the OCI foundation and fail if its reviewed configuration would change.""" self._validate( @@ -418,6 +499,13 @@ def verify_no_drift( config_file_profile=config_file_profile, name=name, ) + controller_args = self._controller_args( + controller_image=controller_image, + controller_image_digest=controller_image_digest, + execution_projections=execution_projections, + controller_dynamic_group_name=controller_dynamic_group_name, + scheduler_dynamic_group_name=scheduler_dynamic_group_name, + ) self._workspace.prepare(preserve_plan=True) self._init( tenancy_id=tenancy_id, @@ -436,8 +524,10 @@ def verify_no_drift( f"-var=compartment_id={compartment_id}", f"-var=region={region}", f"-var=config_file_profile={config_file_profile}", + f"-var=object_storage_namespace={namespace}", f"-var=dynamic_group_name={dynamic_group_name}", f"-var=name={name}", + *controller_args, ) def _init( @@ -464,6 +554,53 @@ def _init( f"-backend-config=config_file_profile={config_file_profile}", ) + @staticmethod + def _controller_args( + *, + controller_image: str | None, + controller_image_digest: str | None, + execution_projections: Mapping[str, object] | None, + controller_dynamic_group_name: str, + scheduler_dynamic_group_name: str, + ) -> tuple[str, ...]: + projections = dict(execution_projections or {}) + supplied = ( + controller_image is not None, + controller_image_digest is not None, + bool(projections), + ) + if any(supplied) and not all(supplied): + raise OciTerraformBootstrapError( + "OCI controller image, digest, and execution projections must be supplied together" + ) + for value, label in ( + (controller_dynamic_group_name, "OCI controller dynamic group"), + (scheduler_dynamic_group_name, "OCI scheduler dynamic group"), + ): + _require(_DYNAMIC_GROUP, value, label) + if not all(supplied): + return () + assert controller_image is not None and controller_image_digest is not None + _require(_CONTROLLER_IMAGE, controller_image, "OCI controller image") + _require(_IMAGE_DIGEST, controller_image_digest, "OCI controller image digest") + try: + projections_json = json.dumps( + projections, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError) as error: + raise OciTerraformBootstrapError( + "OCI execution projections are not JSON serializable" + ) from error + return ( + f"-var=controller_image={controller_image}", + f"-var=controller_image_digest={controller_image_digest}", + f"-var=controller_dynamic_group_name={controller_dynamic_group_name}", + f"-var=scheduler_dynamic_group_name={scheduler_dynamic_group_name}", + f"-var=execution_projections={projections_json}", + ) + @staticmethod def _validate( *, @@ -636,4 +773,5 @@ def _write_json(path: Path, document: Mapping[str, object]) -> None: "OciAdministrativeBootstrap", "OciTerraformBootstrap", "OciTerraformBootstrapError", + "build_oci_execution_projections", ] diff --git a/src/dander/cli/oci_command.py b/src/dander/cli/oci_command.py index c1735fe4..05896663 100644 --- a/src/dander/cli/oci_command.py +++ b/src/dander/cli/oci_command.py @@ -4,6 +4,7 @@ import shlex from pathlib import Path +from typing import Any, cast import typer from click import ClickException @@ -13,11 +14,196 @@ OciAdministrativeBootstrap, OciTerraformBootstrap, OciTerraformBootstrapError, + build_oci_execution_projections, +) +from dander.project import ProjectConfigError, load_project_config +from dander.providers.oci_container_instances import ( + OciContainerInstanceOperations, + OciLifecycleError, + OciOperationBinding, + OciOperationError, ) _DEFAULT_OCI_INFRA_DIR = Path("infra/oci") _DEFAULT_OCI_ADMIN_DIR = Path("infra/oci/bootstrap-admin") console = Console() +oci_app = typer.Typer(help="Operate manifest-bound OCI Container Instances pipelines.") + + +def _oci_operations( + *, + config: Path, + deployment: str, + pipeline: str, + function_id: str, + oci_profile: str, + name: str, +) -> OciContainerInstanceOperations: + binding = OciOperationBinding.from_project( + config=config, + deployment=deployment, + pipeline_id=pipeline, + function_id=function_id, + name=name, + ) + return OciContainerInstanceOperations.from_security_token_profile( + binding, + profile=oci_profile, + ) + + +def _operation( + *, + config: Path, + deployment: str, + pipeline: str, + function_id: str, + oci_profile: str, + name: str, +) -> OciContainerInstanceOperations: + try: + return _oci_operations( + config=config, + deployment=deployment, + pipeline=pipeline, + function_id=function_id, + oci_profile=oci_profile, + name=name, + ) + except (OciOperationError, OciLifecycleError) as error: + raise ClickException(str(error)) from error + + +@oci_app.command("run") +def oci_run( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + function_id: str = typer.Option(..., "--function-id"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Start one potentially billable Container Instance after confirmation.""" + if not typer.confirm(f"Start OCI Container Instances pipeline {pipeline!r}?", default=False): + raise typer.Abort() + try: + invocation = _operation( + config=config, + deployment=deployment, + pipeline=pipeline, + function_id=function_id, + oci_profile=oci_profile, + name=name, + ).start() + except (OciOperationError, OciLifecycleError) as error: + raise ClickException(str(error)) from error + console.print_json(data=invocation.as_dict()) + + +@oci_app.command("status") +def oci_status( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + function_id: str = typer.Option(..., "--function-id"), + run_id: str | None = typer.Option(None, "--run-id"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Describe one exact execution, or the currently active execution.""" + try: + operations = _operation( + config=config, + deployment=deployment, + pipeline=pipeline, + function_id=function_id, + oci_profile=oci_profile, + name=name, + ) + execution = operations.describe(run_id) if run_id is not None else operations.latest() + except (OciOperationError, OciLifecycleError) as error: + raise ClickException(str(error)) from error + console.print_json(data=None if execution is None else execution.as_dict()) + + +@oci_app.command("logs") +def oci_logs( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + function_id: str = typer.Option(..., "--function-id"), + run_id: str = typer.Option(..., "--run-id"), + attempt: int | None = typer.Option(None, "--attempt", min=1, max=11), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Read the bounded output retained for one execution attempt.""" + try: + content = _operation( + config=config, + deployment=deployment, + pipeline=pipeline, + function_id=function_id, + oci_profile=oci_profile, + name=name, + ).logs(run_id, attempt=attempt) + except (OciOperationError, OciLifecycleError) as error: + raise ClickException(str(error)) from error + console.print(content.decode("utf-8", errors="replace"), markup=False, end="") + + +@oci_app.command("cancel") +def oci_cancel( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + function_id: str = typer.Option(..., "--function-id"), + run_id: str = typer.Option(..., "--run-id"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Stop and delete one active run after confirmation.""" + if not typer.confirm(f"Cancel OCI execution {run_id!r}?", default=False): + raise typer.Abort() + try: + execution = _operation( + config=config, + deployment=deployment, + pipeline=pipeline, + function_id=function_id, + oci_profile=oci_profile, + name=name, + ).cancel(run_id) + except (OciOperationError, OciLifecycleError) as error: + raise ClickException(str(error)) from error + console.print_json(data=execution.as_dict()) + + +@oci_app.command("replay") +def oci_replay( + deployment: str = typer.Option(..., "--deployment"), + pipeline: str = typer.Option(..., "--pipeline"), + function_id: str = typer.Option(..., "--function-id"), + run_id: str = typer.Option(..., "--run-id"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), +) -> None: + """Replay one terminal run with a fresh caller-known identity.""" + if not typer.confirm(f"Replay OCI execution {run_id!r}?", default=False): + raise typer.Abort() + try: + invocation = _operation( + config=config, + deployment=deployment, + pipeline=pipeline, + function_id=function_id, + oci_profile=oci_profile, + name=name, + ).replay(run_id) + except (OciOperationError, OciLifecycleError) as error: + raise ClickException(str(error)) from error + console.print_json(data=invocation.as_dict()) def _admin_arguments( @@ -138,13 +324,85 @@ def init_oci_plan( name=name, ) try: - plan_path = OciTerraformBootstrap(infra_dir, operator_artifact_dir).execute(**arguments) + plan_path = OciTerraformBootstrap(infra_dir, operator_artifact_dir).execute( + **cast("Any", arguments) + ) except OciTerraformBootstrapError as error: raise ClickException(str(error)) from error _print_plan("OCI foundation", operator_artifact_dir, plan_path) console.print("No OCI resources were changed. Apply requires explicit cost approval.") +def init_oci_launcher_plan( + tenancy_id: str = typer.Option(..., "--tenancy-id"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--object-storage-namespace"), + state_bucket_name: str = typer.Option(..., "--state-bucket"), + state_key: str = typer.Option("dander/oci/foundation/terraform.tfstate", "--state-key"), + container_image: str = typer.Option(..., "--container-image"), + controller_image: str = typer.Option(..., "--controller-image"), + controller_image_digest: str = typer.Option(..., "--controller-image-digest"), + deployment: str = typer.Option(..., "--deployment"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + dynamic_group_name: str = typer.Option("dander_phase7_runtime", "--dynamic-group"), + controller_dynamic_group_name: str = typer.Option( + "dander_phase7_controller", "--controller-dynamic-group" + ), + scheduler_dynamic_group_name: str = typer.Option( + "dander_phase7_scheduler", "--scheduler-dynamic-group" + ), + config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), + name: str = typer.Option("dander", "--name"), + operator_artifact_dir: Path = typer.Option(..., "--operator-artifact-dir"), # noqa: B008 + infra_dir: Path = typer.Option(_DEFAULT_OCI_INFRA_DIR, hidden=True), # noqa: B008 +) -> None: + """Plan one manifest-selected OCI launcher and its keyless lifecycle controller.""" + try: + projections = _deployment_projections( + config=config, + platforms_config=platforms_config, + deployment=deployment, + container_image=container_image, + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + controller_image=controller_image, + ) + arguments = cast( + "dict[str, object]", + _foundation_arguments( + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + state_bucket_name=state_bucket_name, + state_key=state_key, + dynamic_group_name=dynamic_group_name, + config_file_profile=config_file_profile, + name=name, + ), + ) + arguments.update( + { + "controller_image": controller_image, + "controller_image_digest": controller_image_digest, + "execution_projections": projections, + "controller_dynamic_group_name": controller_dynamic_group_name, + "scheduler_dynamic_group_name": scheduler_dynamic_group_name, + } + ) + plan_path = OciTerraformBootstrap(infra_dir, operator_artifact_dir).execute( + **cast("Any", arguments) + ) + except (OciTerraformBootstrapError, ProjectConfigError) as error: + raise ClickException(str(error)) from error + _print_plan("OCI launcher", operator_artifact_dir, plan_path) + console.print("No OCI resources were changed. Apply requires explicit cost approval.") + + def init_oci_apply( tenancy_id: str = typer.Option(..., "--tenancy-id"), compartment_id: str = typer.Option(..., "--compartment-id"), @@ -199,6 +457,18 @@ def verify_oci_deployment( dynamic_group_name: str = typer.Option("dander_phase7_runtime", "--dynamic-group"), config_file_profile: str = typer.Option("DEFAULT", "--oci-profile"), name: str = typer.Option("dander", "--name"), + container_image: str | None = typer.Option(None, "--container-image"), + controller_image: str | None = typer.Option(None, "--controller-image"), + controller_image_digest: str | None = typer.Option(None, "--controller-image-digest"), + deployment: str | None = typer.Option(None, "--deployment"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 + platforms_config: Path | None = typer.Option(None, "--platforms-config"), # noqa: B008 + controller_dynamic_group_name: str = typer.Option( + "dander_phase7_controller", "--controller-dynamic-group" + ), + scheduler_dynamic_group_name: str = typer.Option( + "dander_phase7_scheduler", "--scheduler-dynamic-group" + ), admin_operator_artifact_dir: Path = typer.Option( # noqa: B008 ..., "--admin-operator-artifact-dir" ), @@ -210,6 +480,36 @@ def verify_oci_deployment( ) -> None: """Read OCI APIs through Terraform and fail if stage zero or the foundation drifted.""" try: + controller_values = (container_image, controller_image, controller_image_digest, deployment) + if any(value is not None for value in controller_values) and not all( + value is not None for value in controller_values + ): + raise OciTerraformBootstrapError( + "OCI launcher verification requires deployment, runtime image, controller image, " + "and controller digest together" + ) + controller_arguments: dict[str, object] = {} + if deployment is not None: + assert container_image is not None + assert controller_image is not None + assert controller_image_digest is not None + controller_arguments = { + "controller_image": controller_image, + "controller_image_digest": controller_image_digest, + "execution_projections": _deployment_projections( + config=config, + platforms_config=platforms_config, + deployment=deployment, + container_image=container_image, + tenancy_id=tenancy_id, + compartment_id=compartment_id, + region=region, + namespace=namespace, + controller_image=controller_image, + ), + "controller_dynamic_group_name": controller_dynamic_group_name, + "scheduler_dynamic_group_name": scheduler_dynamic_group_name, + } OciAdministrativeBootstrap(admin_infra_dir, admin_operator_artifact_dir).verify_no_drift( namespace=namespace, **_admin_arguments( @@ -222,8 +522,9 @@ def verify_oci_deployment( config_file_profile=config_file_profile, ), ) - OciTerraformBootstrap(infra_dir, foundation_operator_artifact_dir).verify_no_drift( - **_foundation_arguments( + verification_arguments = cast( + "dict[str, object]", + _foundation_arguments( tenancy_id=tenancy_id, compartment_id=compartment_id, region=region, @@ -233,9 +534,13 @@ def verify_oci_deployment( dynamic_group_name=dynamic_group_name, config_file_profile=config_file_profile, name=name, - ) + ), ) - except OciTerraformBootstrapError as error: + verification_arguments.update(controller_arguments) + OciTerraformBootstrap(infra_dir, foundation_operator_artifact_dir).verify_no_drift( + **cast("Any", verification_arguments) + ) + except (OciTerraformBootstrapError, ProjectConfigError) as error: raise ClickException(str(error)) from error console.print_json( data={ @@ -254,6 +559,65 @@ def _foundation_arguments(**values: str) -> dict[str, str]: return dict(values) +def _deployment_projections( + *, + config: Path, + platforms_config: Path | None, + deployment: str, + container_image: str, + tenancy_id: str, + compartment_id: str, + region: str, + namespace: str, + controller_image: str, +) -> dict[str, dict[str, object]]: + manifest = load_project_config( + config, + platforms_path=platforms_config, + deployment=deployment, + ) + if manifest.launcher_provider != "oci_container_instances": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select launcher.provider='oci_container_instances'" + ) + manifest.validate_references(config.resolve().parent) + launcher = manifest.resolved_launcher_config() + expected = { + "tenancy_id": tenancy_id, + "compartment_id": compartment_id, + "region": region, + "registry_namespace": namespace, + } + if any(launcher.get(key) != value for key, value in expected.items()): + raise ProjectConfigError( + "OCI manifest tenancy, compartment, region, and registry namespace must match " + "the selected Terraform foundation" + ) + repository_name = launcher.get("repository_name") + expected_controller_repository = ( + f"ocir.{region}.oci.oraclecloud.com/{namespace}/{repository_name}:" + ) + if not isinstance(repository_name, str) or not controller_image.startswith( + expected_controller_repository + ): + raise ProjectConfigError( + "OCI controller image must use a unique tag in the manifest-selected OCIR repository" + ) + runtime = manifest.platform.runtime + return build_oci_execution_projections( + container_image=container_image, + launcher_config=launcher, + profile_id=deployment, + runtime_cpu=runtime.cpu, + runtime_memory=runtime.memory, + runtime_timeout_seconds=runtime.timeout_seconds, + runtime_max_retries=runtime.max_retries, + runtime_batch_rows=runtime.batch_rows, + require_guarded_free_tier=manifest.platform.safety.require_guarded_free_tier, + pipelines=manifest.terraform_pipelines(), + ) + + def _print_plan(label: str, operator_artifact_dir: Path, plan_path: Path) -> None: workspace = operator_artifact_dir.expanduser().resolve() / "terraform-workspace" console.print(f"[green]{label} planned.[/green] Saved plan: {plan_path}") @@ -266,9 +630,11 @@ def _print_plan(label: str, operator_artifact_dir: Path, plan_path: Path) -> Non def register_oci_commands(app: typer.Typer) -> None: """Register OCI plan-first administrative commands.""" + app.add_typer(oci_app, name="oci") app.command("init-oci-admin-plan")(init_oci_admin_plan) app.command("init-oci-admin-apply")(init_oci_admin_apply) app.command("init-oci-plan")(init_oci_plan) + app.command("init-oci-launcher-plan")(init_oci_launcher_plan) app.command("init-oci-apply")(init_oci_apply) app.command("verify-oci-deployment")(verify_oci_deployment) diff --git a/src/dander/cli/runtime_command.py b/src/dander/cli/runtime_command.py index 5bef2316..09e93187 100644 --- a/src/dander/cli/runtime_command.py +++ b/src/dander/cli/runtime_command.py @@ -37,6 +37,7 @@ validate_runtime_identifier, ) from dander.runtime_inspection import inspect_runtime, run_local_conformance +from dander.runtime_secrets import RuntimeSecretBindingError, projected_secret_environment from dander.state import RunStage, classify_failure runtime_app = typer.Typer( @@ -202,7 +203,11 @@ def execute_runtime( dataplex_location="us", ) try: - with graceful_signal_handlers(), launcher_identity(context): + with ( + graceful_signal_handlers(), + launcher_identity(context), + projected_secret_environment(), + ): result = execute_run( options, console=_CONSOLE, @@ -222,7 +227,7 @@ def execute_runtime( ).to_json() ) raise typer.Exit(code=RuntimeExitCode.CANCELLED) from None - except ClickException as error: + except (ClickException, RuntimeSecretBindingError) as error: failure = classify_failure(error, stage=RunStage.INGEST, run_id=context.run_id) if failure.code != "unexpected_error": retryable = is_retryable_failure(failure.code) diff --git a/src/dander/deployment/projection.py b/src/dander/deployment/projection.py index 2a6d23b4..420dd4c0 100644 --- a/src/dander/deployment/projection.py +++ b/src/dander/deployment/projection.py @@ -408,7 +408,9 @@ class LauncherCapabilities: cpu_millis=frozenset({1_000, 2_000, 4_000, 6_000, 8_000}), minimum_memory_mib=1_024, maximum_memory_mib=1_540_096, - maximum_deadline_seconds=86_400, + # The detached lifecycle Function owns interruption and whole-task retries. OCI permits + # detached Functions to run for at most one hour; reserve five minutes for cleanup. + maximum_deadline_seconds=3_300, maximum_launcher_retries=10, maximum_task_count=1, maximum_parallelism=1, diff --git a/src/dander/providers/__init__.py b/src/dander/providers/__init__.py index 714c8a8d..a9e44fe0 100644 --- a/src/dander/providers/__init__.py +++ b/src/dander/providers/__init__.py @@ -1,6 +1,10 @@ """Cloud-provider registration and construction contracts.""" -from dander.providers.defaults import default_provider_registry +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING + from dander.providers.dependencies import ( FULL_RUNTIME_DISTRIBUTIONS, PROVIDER_DEPENDENCY_SETS, @@ -18,6 +22,17 @@ lazy_provider_factory, ) +if TYPE_CHECKING: + from dander.providers.defaults import default_provider_registry + + +def __getattr__(name: str) -> object: + """Avoid importing every provider config when one provider submodule is selected.""" + if name == "default_provider_registry": + return getattr(import_module("dander.providers.defaults"), name) + raise AttributeError(name) + + __all__ = [ "FULL_RUNTIME_DISTRIBUTIONS", "PROVIDER_API_VERSION", diff --git a/src/dander/providers/oci_container_instances/__init__.py b/src/dander/providers/oci_container_instances/__init__.py index a0d5638a..be310c56 100644 --- a/src/dander/providers/oci_container_instances/__init__.py +++ b/src/dander/providers/oci_container_instances/__init__.py @@ -1,7 +1,54 @@ """OCI Container Instances launcher configuration.""" +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING + from dander.providers.oci_container_instances.config import ( OciContainerInstancesLauncherConfig, ) +from dander.providers.oci_container_instances.controller import ( + OCI_EXECUTION_SCHEMA, + OciExecution, + OciLifecycleController, + OciLifecycleError, +) + +if TYPE_CHECKING: + from dander.providers.oci_container_instances.operations import ( + OciContainerInstanceOperations, + OciInvocation, + OciOperationBinding, + OciOperationError, + ) + +_OPERATION_EXPORTS = frozenset( + { + "OciContainerInstanceOperations", + "OciInvocation", + "OciOperationBinding", + "OciOperationError", + } +) + + +def __getattr__(name: str) -> object: + """Keep manifest/project operation dependencies out of OCI Function startup.""" + if name in _OPERATION_EXPORTS: + module = import_module("dander.providers.oci_container_instances.operations") + return getattr(module, name) + raise AttributeError(name) + -__all__ = ["OciContainerInstancesLauncherConfig"] +__all__ = [ + "OCI_EXECUTION_SCHEMA", + "OciContainerInstancesLauncherConfig", + "OciContainerInstanceOperations", + "OciExecution", + "OciLifecycleController", + "OciLifecycleError", + "OciInvocation", + "OciOperationBinding", + "OciOperationError", +] diff --git a/src/dander/providers/oci_container_instances/controller.py b/src/dander/providers/oci_container_instances/controller.py new file mode 100644 index 00000000..71ccafed --- /dev/null +++ b/src/dander/providers/oci_container_instances/controller.py @@ -0,0 +1,466 @@ +"""Idempotent OCI Container Instances lifecycle controller.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import asdict, dataclass, replace +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Literal, Protocol, cast + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + +OCI_EXECUTION_SCHEMA = "io.dander.oci-execution/v1" +_TERMINAL_STATES = frozenset({"succeeded", "failed", "cancelled"}) +_RUN_ID = re.compile(r"^oci-[0-9a-f]{24}$") +_IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$") +_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9_.:+-]{1,256}$") +_IMMUTABLE_IMAGE = re.compile(r"^[a-z0-9.-]+/[A-Za-z0-9._/-]+@sha256:[0-9a-f]{64}$") +_ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]{0,126}$") +_FAILURE_CODE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") + + +class OciLifecycleError(RuntimeError): + """The OCI controller received invalid state or could not complete an operation.""" + + +@dataclass(frozen=True, slots=True) +class OciInstanceStatus: + """Small provider-neutral view of a Container Instance attempt.""" + + state: Literal["pending", "running", "succeeded", "failed"] + container_id: str | None = None + exit_code: int | None = None + failure_code: str | None = None + + +@dataclass(frozen=True, slots=True) +class OciExecution: + """Sanitized durable execution record; it never contains row or secret data.""" + + schema: str + run_id: str + pipeline_id: str + idempotency_key: str + image: str + state: Literal["pending", "running", "succeeded", "failed", "cancelled"] + attempt: int + max_attempts: int + created_at: str + updated_at: str + deadline_at: str + instance_id: str | None = None + container_id: str | None = None + exit_code: int | None = None + failure_code: str | None = None + replay_of: str | None = None + + @property + def terminal(self) -> bool: + return self.state in _TERMINAL_STATES + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class StoredExecution: + """One execution plus the repository compare-and-swap token.""" + + execution: OciExecution + version: str + + +class OciRunRepository(Protocol): + """Atomic active-run lock and immutable terminal-history boundary.""" + + def claim(self, execution: OciExecution) -> tuple[StoredExecution, bool]: ... + + def get(self, pipeline_id: str, run_id: str | None = None) -> StoredExecution | None: ... + + def save(self, stored: StoredExecution, execution: OciExecution) -> StoredExecution: ... + + def finish(self, stored: StoredExecution, execution: OciExecution) -> StoredExecution: ... + + def save_logs(self, execution: OciExecution, content: bytes) -> None: ... + + +class OciContainerGateway(Protocol): + """Provider mutations and observations owned by the lifecycle controller.""" + + def create(self, projection: Mapping[str, object], execution: OciExecution) -> str: ... + + def status(self, instance_id: str) -> OciInstanceStatus: ... + + def stop(self, instance_id: str) -> None: ... + + def delete(self, instance_id: str) -> None: ... + + def logs(self, container_id: str, *, limit_bytes: int) -> bytes: ... + + +class OciLifecycleController: + """Launch, reconcile, retry, interrupt, and replay one OCI pipeline.""" + + def __init__( + self, + *, + projection: Mapping[str, object], + repository: OciRunRepository, + gateway: OciContainerGateway, + clock: Callable[[], datetime] | None = None, + ) -> None: + self._projection = _validated_projection(projection) + self._pipeline_id = str(self._projection["pipeline_id"]) + self._repository = repository + self._gateway = gateway + self._clock = clock or (lambda: datetime.now(UTC)) + + def start( + self, + *, + idempotency_key: str, + replay_of: str | None = None, + ) -> OciExecution: + """Claim the pipeline atomically and create at most one first attempt.""" + if _IDEMPOTENCY_KEY.fullmatch(idempotency_key) is None: + raise OciLifecycleError("OCI idempotency key contains unsupported characters") + now = self._now() + run_id = _run_id(self._pipeline_id, idempotency_key) + resources = _mapping(self._projection, "resources") + record = OciExecution( + schema=OCI_EXECUTION_SCHEMA, + run_id=run_id, + pipeline_id=self._pipeline_id, + idempotency_key=idempotency_key, + image=str(self._projection["image"]), + state="pending", + attempt=1, + max_attempts=cast("int", resources["launcher_retry_count"]) + 1, + created_at=_timestamp(now), + updated_at=_timestamp(now), + deadline_at=_timestamp( + now + timedelta(seconds=cast("int", resources["deadline_seconds"])) + ), + replay_of=replay_of, + ) + stored, claimed = self._repository.claim(record) + if not claimed: + existing = stored.execution + if existing.idempotency_key != idempotency_key: + raise OciLifecycleError( + f"OCI pipeline {self._pipeline_id!r} already has active run {existing.run_id!r}" + ) + return existing + return self._launch(stored).execution + + def reconcile(self, run_id: str | None = None) -> OciExecution | None: + """Advance one run exactly one provider observation.""" + stored = self._repository.get(self._pipeline_id, run_id) + if stored is None: + return None + execution = stored.execution + if execution.terminal: + # A prior invocation may have persisted history but failed while releasing the active + # Object Storage lock. Re-running finish is idempotent and repairs that partial step. + return self._repository.finish(stored, execution).execution + if execution.state == "pending" or execution.instance_id is None: + return self._launch(stored).execution + now = self._now() + if now >= _parse_timestamp(execution.deadline_at): + self._stop_capture_delete(execution) + return self._finish( + stored, + state="failed", + failure_code="launcher_deadline_exceeded", + ).execution + status = self._gateway.status(execution.instance_id) + if status.state in {"pending", "running"}: + updated = replace( + execution, + state="running", + container_id=status.container_id or execution.container_id, + updated_at=_timestamp(now), + ) + return self._repository.save(stored, updated).execution + updated = replace( + execution, + container_id=status.container_id or execution.container_id, + exit_code=status.exit_code, + updated_at=_timestamp(now), + ) + self._capture_logs(updated) + self._gateway.delete(execution.instance_id) + if status.state == "succeeded" and status.exit_code == 0: + return self._finish(stored, execution=updated, state="succeeded").execution + if status.exit_code == 75 and execution.attempt < execution.max_attempts: + retry = replace( + updated, + state="pending", + attempt=execution.attempt + 1, + instance_id=None, + container_id=None, + exit_code=None, + failure_code=None, + updated_at=_timestamp(now), + ) + return self._launch(self._repository.save(stored, retry)).execution + failure = status.failure_code or ( + "launcher_retry_exhausted" if status.exit_code == 75 else "runtime_failed" + ) + return self._finish( + stored, + execution=updated, + state="failed", + failure_code=failure, + ).execution + + def cancel(self, run_id: str) -> OciExecution: + """Stop and delete one owned active instance, preserving a terminal record.""" + stored = self._repository.get(self._pipeline_id, run_id) + if stored is None: + raise OciLifecycleError("OCI execution was not found") + if stored.execution.terminal: + return stored.execution + if stored.execution.instance_id is not None: + self._stop_capture_delete(stored.execution) + return self._finish( + stored, + state="cancelled", + failure_code="interrupted_run", + ).execution + + def replay(self, run_id: str, *, idempotency_key: str) -> OciExecution: + """Start a fresh attempt chain only after the selected run is terminal.""" + previous = self._repository.get(self._pipeline_id, run_id) + if previous is None: + raise OciLifecycleError("OCI execution was not found") + if not previous.execution.terminal: + raise OciLifecycleError("Only a terminal OCI execution can be replayed") + return self.start(idempotency_key=idempotency_key, replay_of=run_id) + + def _launch(self, stored: StoredExecution) -> StoredExecution: + execution = stored.execution + try: + instance_id = self._gateway.create(self._projection, execution) + except Exception as error: # noqa: BLE001 - SDK errors vary by version. + self._finish( + stored, + state="failed", + failure_code="launcher_create_failed", + ) + raise OciLifecycleError("OCI Container Instance creation failed") from error + updated = replace( + execution, + state="running", + instance_id=instance_id, + updated_at=_timestamp(self._now()), + ) + return self._repository.save(stored, updated) + + def _finish( + self, + stored: StoredExecution, + *, + state: Literal["succeeded", "failed", "cancelled"], + failure_code: str | None = None, + execution: OciExecution | None = None, + ) -> StoredExecution: + updated = replace( + execution or stored.execution, + state=state, + failure_code=failure_code, + updated_at=_timestamp(self._now()), + ) + return self._repository.finish(stored, updated) + + def _capture_logs(self, execution: OciExecution) -> None: + if execution.container_id is None: + return + try: + content = self._gateway.logs(execution.container_id, limit_bytes=262_144) + except Exception: # noqa: BLE001 - log preservation must not mask task outcome. + return + self._repository.save_logs(execution, content) + + def _stop_capture_delete(self, execution: OciExecution) -> None: + if execution.instance_id is None: + return + try: + self._gateway.stop(execution.instance_id) + finally: + self._capture_logs(execution) + self._gateway.delete(execution.instance_id) + + def _now(self) -> datetime: + current = self._clock() + if current.tzinfo is None: + raise OciLifecycleError("OCI lifecycle clock must be timezone-aware") + return current.astimezone(UTC) + + +def _validated_projection(projection: Mapping[str, object]) -> dict[str, object]: + required = { + "schema", + "contract", + "pipeline_id", + "profile_id", + "launcher", + "image", + "command", + "configuration_reference", + "environment", + "secret_bindings", + "workload_identity", + "resources", + "schedule", + "network", + "labels", + "observability", + "extensions", + } + if set(projection) != required: + raise OciLifecycleError("OCI execution projection is incomplete") + if ( + projection.get("schema") != "io.dander.execution/v1" + or projection.get("launcher") != "oci_container_instances" + ): + raise OciLifecycleError("OCI execution projection has an unsupported contract") + pipeline_id = projection.get("pipeline_id") + image = projection.get("image") + command = projection.get("command") + if ( + not isinstance(pipeline_id, str) + or _IDENTIFIER.fullmatch(pipeline_id) is None + or not isinstance(image, str) + or _IMMUTABLE_IMAGE.fullmatch(image) is None + or not isinstance(command, list) + or not 1 <= len(command) <= 64 + or any(not isinstance(part, str) or not part or len(part) > 256 for part in command) + ): + raise OciLifecycleError("OCI execution projection identity or command is invalid") + if command[:2] != ["runtime", "execute"]: + raise OciLifecycleError("OCI execution projection command is unsupported") + environment = _mapping(projection, "environment") + secret_bindings = _mapping(projection, "secret_bindings") + if len(environment) > 128 or any( + not isinstance(name, str) + or _ENVIRONMENT_NAME.fullmatch(name) is None + or not isinstance(value, str) + or len(value) > 2_048 + for name, value in environment.items() + ): + raise OciLifecycleError("OCI execution projection environment is invalid") + if len(secret_bindings) > 128 or set(environment) & set(secret_bindings): + raise OciLifecycleError("OCI execution projection secret bindings are invalid") + for name, binding in secret_bindings.items(): + if ( + not isinstance(name, str) + or _ENVIRONMENT_NAME.fullmatch(name) is None + or not isinstance(binding, dict) + or set(binding) != {"provider", "reference"} + or binding.get("provider") != "oci_vault" + or not isinstance(binding.get("reference"), str) + or not str(binding["reference"]).startswith("oci-vault://") + ): + raise OciLifecycleError("OCI execution projection secret bindings are invalid") + resources = _mapping(projection, "resources") + deadline = resources.get("deadline_seconds") + retries = resources.get("launcher_retry_count") + if ( + isinstance(deadline, bool) + or not isinstance(deadline, int) + or not 1 <= deadline <= 3_300 + or isinstance(retries, bool) + or not isinstance(retries, int) + or not 0 <= retries <= 10 + or resources.get("runtime_retry_count") != 0 + ): + raise OciLifecycleError( + "OCI controller requires a 1-3300 second deadline and 0-10 launcher retries" + ) + return dict(projection) + + +def _mapping(document: Mapping[str, object], key: str) -> Mapping[str, object]: + value = document.get(key) + if not isinstance(value, dict): + raise OciLifecycleError(f"OCI execution projection {key} is invalid") + return value + + +def _run_id(pipeline_id: str, idempotency_key: str) -> str: + digest = hashlib.sha256(f"{pipeline_id}\0{idempotency_key}".encode()).hexdigest()[:24] + return f"oci-{digest}" + + +def execution_run_id(pipeline_id: str, idempotency_key: str) -> str: + """Return the controller identity a detached caller can know before execution starts.""" + return _run_id(pipeline_id, idempotency_key) + + +def _timestamp(value: datetime) -> str: + return value.astimezone(UTC).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _parse_timestamp(value: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise OciLifecycleError("OCI execution timestamp is invalid") from error + if parsed.tzinfo is None: + raise OciLifecycleError("OCI execution timestamp must include a timezone") + return parsed.astimezone(UTC) + + +def execution_from_json(content: bytes) -> OciExecution: + """Parse one repository record without accepting unknown or secret-bearing fields.""" + try: + document = json.loads(content) + if not isinstance(document, dict): + raise TypeError + execution = OciExecution(**document) + except (TypeError, ValueError, json.JSONDecodeError) as error: + raise OciLifecycleError("OCI execution record is invalid") from error + integer_values = (execution.attempt, execution.max_attempts) + if ( + execution.schema != OCI_EXECUTION_SCHEMA + or _RUN_ID.fullmatch(execution.run_id) is None + or _IDENTIFIER.fullmatch(execution.pipeline_id) is None + or _IDEMPOTENCY_KEY.fullmatch(execution.idempotency_key) is None + or _IMMUTABLE_IMAGE.fullmatch(execution.image) is None + or execution.state not in _TERMINAL_STATES | {"pending", "running"} + or any(isinstance(value, bool) or not isinstance(value, int) for value in integer_values) + or not 1 <= execution.attempt <= execution.max_attempts <= 11 + or ( + execution.exit_code is not None + and (isinstance(execution.exit_code, bool) or not isinstance(execution.exit_code, int)) + ) + or ( + execution.failure_code is not None + and _FAILURE_CODE.fullmatch(execution.failure_code) is None + ) + or (execution.replay_of is not None and _RUN_ID.fullmatch(execution.replay_of) is None) + ): + raise OciLifecycleError("OCI execution record has an unsupported contract") + created = _parse_timestamp(execution.created_at) + _parse_timestamp(execution.updated_at) + if _parse_timestamp(execution.deadline_at) <= created: + raise OciLifecycleError("OCI execution record has an invalid deadline") + return execution + + +__all__ = [ + "OCI_EXECUTION_SCHEMA", + "OciContainerGateway", + "OciExecution", + "OciInstanceStatus", + "OciLifecycleController", + "OciLifecycleError", + "OciRunRepository", + "StoredExecution", + "execution_from_json", + "execution_run_id", +] diff --git a/src/dander/providers/oci_container_instances/function_handler.py b/src/dander/providers/oci_container_instances/function_handler.py new file mode 100644 index 00000000..fff58cde --- /dev/null +++ b/src/dander/providers/oci_container_instances/function_handler.py @@ -0,0 +1,157 @@ +"""OCI Functions entrypoint for scheduled and event-assisted lifecycle reconciliation.""" + +from __future__ import annotations + +import json +import os +import time +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from dander.providers.oci_container_instances.controller import ( + OciExecution, + OciLifecycleController, + OciLifecycleError, +) +from dander.providers.oci_container_instances.oci_adapter import ( + build_resource_principal_adapters, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from typing import BinaryIO + + +def handler(ctx: object, data: BinaryIO | None = None) -> object: + """Handle detached schedules/manual starts and short event/cancel operations.""" + del ctx + try: + payload = _payload(data) + namespace = _required_environment("DANDER_OCI_NAMESPACE") + bucket = _required_environment("DANDER_OCI_RUN_BUCKET") + pipeline_id = _required_environment("DANDER_OCI_PIPELINE") + projection_key = _required_environment("DANDER_OCI_PROJECTION_KEY") + repository, gateway = build_resource_principal_adapters( + namespace=namespace, + bucket=bucket, + ) + projection = repository.get_projection(projection_key) + if projection.get("pipeline_id") != pipeline_id: + raise OciLifecycleError("OCI Function projection does not match its pipeline") + controller = OciLifecycleController( + projection=projection, + repository=repository, + gateway=gateway, + ) + action = _action(payload) + execution: OciExecution | None + if action == "start": + key = payload.get("idempotency_key") + if not isinstance(key, str): + key = _scheduled_key(pipeline_id) + execution = controller.start(idempotency_key=key) + execution = _monitor(controller, execution) + elif action == "reconcile": + run_id = payload.get("run_id") + execution = controller.reconcile(run_id if isinstance(run_id, str) else None) + elif action == "cancel": + execution = controller.cancel(_required_string(payload, "run_id")) + elif action == "replay": + execution = controller.replay( + _required_string(payload, "run_id"), + idempotency_key=_required_string(payload, "idempotency_key"), + ) + execution = _monitor(controller, execution) + else: # pragma: no cover - _action is exhaustive. + raise OciLifecycleError("Unsupported OCI controller action") + return _response(200, None if execution is None else execution.as_dict()) + except OciLifecycleError as error: + return _response( + 409, + { + "schema": "io.dander.oci-controller-error/v1", + "failure_code": "controller_rejected", + "message": str(error), + }, + ) + + +def _monitor( + controller: OciLifecycleController, + execution: OciExecution, + *, + sleep: Callable[[float], None] = time.sleep, +) -> OciExecution: + current = execution + while not current.terminal: + sleep(5) + observed = controller.reconcile(current.run_id) + if observed is None: + raise OciLifecycleError("OCI execution disappeared during reconciliation") + current = observed + return current + + +def _payload(data: BinaryIO | None) -> dict[str, object]: + if data is None: + return {} + content = data.read(65_537) + if len(content) > 65_536: + raise OciLifecycleError("OCI controller request is too large") + if not content: + return {} + try: + document = json.loads(content) + except json.JSONDecodeError as error: + raise OciLifecycleError("OCI controller request is invalid JSON") from error + if not isinstance(document, dict): + raise OciLifecycleError("OCI controller request must be a JSON object") + return document + + +def _action(payload: Mapping[str, object]) -> str: + action = payload.get("action") + if action is None and ("eventType" in payload or payload.get("cloudEventsVersion") is not None): + return "reconcile" + if action is None: + return "start" + if action not in {"start", "reconcile", "cancel", "replay"}: + raise OciLifecycleError("OCI controller action is unsupported") + return str(action) + + +def _scheduled_key(pipeline_id: str) -> str: + # Resource Scheduler invokes functions no more often than hourly. The UTC-hour bucket makes + # delivery retries idempotent while the active-run lock enforces maximum parallelism one. + hour = datetime.now(UTC).strftime("%Y-%m-%dT%H") + return f"schedule:{pipeline_id}:{hour}" + + +def _required_environment(name: str) -> str: + value = os.environ.get(name) + if not value: + raise OciLifecycleError(f"OCI Function configuration {name} is missing") + return value + + +def _required_string(payload: Mapping[str, object], name: str) -> str: + value = payload.get(name) + if not isinstance(value, str) or not value: + raise OciLifecycleError(f"OCI controller request {name} is missing") + return value + + +def _response(status: int, document: object) -> object: + body = json.dumps(document, sort_keys=True, separators=(",", ":")) + try: + from fdk import response # type: ignore[import-not-found] + except ImportError: + return {"status": status, "body": body} + return response.Response( + status_code=status, + response_data=body, + headers={"Content-Type": "application/json"}, + ) + + +__all__ = ["handler"] diff --git a/src/dander/providers/oci_container_instances/oci_adapter.py b/src/dander/providers/oci_container_instances/oci_adapter.py new file mode 100644 index 00000000..b618a453 --- /dev/null +++ b/src/dander/providers/oci_container_instances/oci_adapter.py @@ -0,0 +1,414 @@ +"""Lazy OCI SDK adapters for lifecycle records and Container Instances.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from importlib import import_module +from typing import Any, cast + +from dander.providers.oci_container_instances.controller import ( + OciExecution, + OciInstanceStatus, + OciLifecycleError, + StoredExecution, + execution_from_json, +) + +_ACTIVE_PREFIX = "active" +_HISTORY_PREFIX = "history" +_LOG_PREFIX = "logs" + + +@dataclass(frozen=True, slots=True) +class _Object: + content: bytes + etag: str + + +class OciObjectRunRepository: + """Object Storage repository using conditional writes as the pipeline lock.""" + + def __init__( + self, + *, + client: object, + namespace: str, + bucket: str, + ) -> None: + self._client = cast("Any", client) + self._namespace = namespace + self._bucket = bucket + + def claim(self, execution: OciExecution) -> tuple[StoredExecution, bool]: + historical = self._read(self._history_key(execution.pipeline_id, execution.run_id)) + if historical is not None: + return StoredExecution(execution_from_json(historical.content), historical.etag), False + key = self._active_key(execution.pipeline_id) + try: + response = self._put(key, execution, if_none_match="*") + except Exception as error: # noqa: BLE001 - SDK exception classes remain optional. + if _status(error) != 412: + raise OciLifecycleError("Could not claim OCI pipeline execution") from error + active = self._read(key) + if active is None: + raise OciLifecycleError("OCI pipeline lock changed during claim") from error + return StoredExecution(execution_from_json(active.content), active.etag), False + return StoredExecution(execution, _etag(response)), True + + def get(self, pipeline_id: str, run_id: str | None = None) -> StoredExecution | None: + active = self._read(self._active_key(pipeline_id)) + if active is not None: + execution = execution_from_json(active.content) + if run_id is None or execution.run_id == run_id: + return StoredExecution(execution, active.etag) + if run_id is None: + return None + historical = self._read(self._history_key(pipeline_id, run_id)) + if historical is None: + return None + return StoredExecution(execution_from_json(historical.content), historical.etag) + + def save(self, stored: StoredExecution, execution: OciExecution) -> StoredExecution: + _require_same_run(stored.execution, execution) + response = self._put( + self._active_key(execution.pipeline_id), + execution, + if_match=stored.version, + ) + return StoredExecution(execution, _etag(response)) + + def finish(self, stored: StoredExecution, execution: OciExecution) -> StoredExecution: + if not execution.terminal: + raise OciLifecycleError("OCI repository can finish only terminal executions") + active_key = self._active_key(execution.pipeline_id) + active = self._read(active_key) + history_key = self._history_key(execution.pipeline_id, execution.run_id) + if active is None: + historical = self._read(history_key) + if historical is not None and execution_from_json(historical.content) == execution: + return StoredExecution(execution, historical.etag) + raise OciLifecycleError("OCI active execution disappeared before it was preserved") + active_execution = execution_from_json(active.content) + _require_same_run(active_execution, execution) + current = self.save(StoredExecution(active_execution, active.etag), execution) + try: + response = self._put(history_key, execution, if_none_match="*") + history_version = _etag(response) + except Exception as error: # noqa: BLE001 - an idempotent repeat may already exist. + if _status(error) != 412: + raise OciLifecycleError("Could not preserve OCI terminal execution") from error + existing = self._read(history_key) + if existing is None or execution_from_json(existing.content) != execution: + raise OciLifecycleError("OCI terminal execution history conflicts") from error + history_version = existing.etag + try: + self._client.delete_object( + self._namespace, + self._bucket, + active_key, + if_match=current.version, + ) + except Exception as error: # noqa: BLE001 + if _status(error) != 404: + raise OciLifecycleError("Could not release OCI pipeline execution") from error + return StoredExecution(execution, history_version) + + def save_logs(self, execution: OciExecution, content: bytes) -> None: + if len(content) > 262_144: + content = content[-262_144:] + key = f"{_LOG_PREFIX}/{execution.pipeline_id}/{execution.run_id}/{execution.attempt}.log" + try: + self._client.put_object( + self._namespace, + self._bucket, + key, + content, + content_type="text/plain; charset=utf-8", + if_none_match="*", + ) + except Exception as error: # noqa: BLE001 + if _status(error) != 412: + raise OciLifecycleError("Could not preserve bounded OCI runtime logs") from error + + def get_logs(self, execution: OciExecution, *, attempt: int | None = None) -> bytes: + selected = attempt or execution.attempt + result = self._read( + f"{_LOG_PREFIX}/{execution.pipeline_id}/{execution.run_id}/{selected}.log" + ) + return b"" if result is None else result.content + + def get_projection(self, key: str) -> dict[str, object]: + """Read one Terraform-owned non-secret projection document.""" + if not key.startswith("projections/") or not key.endswith(".json"): + raise OciLifecycleError("OCI projection object key is invalid") + result = self._read(key) + if result is None or len(result.content) > 131_072: + raise OciLifecycleError("OCI execution projection was not found") + try: + document = json.loads(result.content) + except json.JSONDecodeError as error: + raise OciLifecycleError("OCI execution projection is invalid") from error + if not isinstance(document, dict): + raise OciLifecycleError("OCI execution projection is invalid") + return cast("dict[str, object]", document) + + def _put( + self, + key: str, + execution: OciExecution, + **conditions: str, + ) -> object: + content = json.dumps(execution.as_dict(), sort_keys=True, separators=(",", ":")).encode() + return self._client.put_object( + self._namespace, + self._bucket, + key, + content, + content_type="application/json", + **conditions, + ) + + def _read(self, key: str) -> _Object | None: + try: + response = self._client.get_object(self._namespace, self._bucket, key) + except Exception as error: # noqa: BLE001 + if _status(error) == 404: + return None + raise OciLifecycleError("Could not read OCI execution record") from error + data = getattr(response, "data", None) + content = getattr(data, "content", None) + if not isinstance(content, bytes): + raw = getattr(data, "raw", None) + content = raw.read() if raw is not None else None + if not isinstance(content, bytes): + raise OciLifecycleError("OCI execution record body is invalid") + return _Object(content=content, etag=_etag(response)) + + @staticmethod + def _active_key(pipeline_id: str) -> str: + return f"{_ACTIVE_PREFIX}/{pipeline_id}.json" + + @staticmethod + def _history_key(pipeline_id: str, run_id: str) -> str: + return f"{_HISTORY_PREFIX}/{pipeline_id}/{run_id}.json" + + +class OciSdkContainerGateway: + """Create run-scoped instances and normalize lifecycle state through the OCI SDK.""" + + def __init__(self, *, client: object) -> None: + self._client = cast("Any", client) + + def create(self, projection: Mapping[str, object], execution: OciExecution) -> str: + try: + oci = cast("Any", import_module("oci")) + except ImportError as error: # pragma: no cover - packaged OCI extra covers this path. + raise OciLifecycleError("OCI SDK is unavailable") from error + resources = _mapping(projection, "resources") + network = _mapping(projection, "network") + extensions = _mapping(projection, "extensions") + environment = {str(k): str(v) for k, v in _mapping(projection, "environment").items()} + environment.update( + { + "DANDER_RUN_ID": execution.run_id, + "DANDER_LAUNCHER_EXECUTION_ID": execution.run_id, + "DANDER_ATTEMPT": str(execution.attempt), + "DANDER_SHARD_INDEX": "0", + "DANDER_SHARD_COUNT": "1", + "DANDER_DEADLINE_AT": execution.deadline_at, + "DANDER_SECRET_BINDINGS_JSON": json.dumps( + _mapping(projection, "secret_bindings"), + sort_keys=True, + separators=(",", ":"), + ), + } + ) + cpu = cast("int", resources["cpu_millis"]) / 1_000 + memory = cast("int", resources["memory_mib"]) / 1_024 + details = oci.container_instances.models.CreateContainerInstanceDetails( + compartment_id=str(extensions["oci_compartment_id"]), + availability_domain=str(_mapping(network, "extensions")["oci_availability_domain"]), + display_name=f"dander-{execution.pipeline_id}-{execution.run_id[-8:]}-a{execution.attempt}", + shape=str(extensions["oci_shape"]), + shape_config=oci.container_instances.models.CreateContainerInstanceShapeConfigDetails( + ocpus=cpu, + memory_in_gbs=memory, + ), + container_restart_policy="NEVER", + graceful_shutdown_timeout_in_seconds=int( + cast("str", extensions["oci_graceful_shutdown_seconds"]) + ), + vnics=[ + oci.container_instances.models.CreateContainerVnicDetails( + subnet_id=str(network["placement"]), + is_public_ip_assigned=False, + ) + ], + volumes=[ + oci.container_instances.models.CreateContainerEmptyDirVolumeDetails( + name="dander-tmp", + volume_type="EMPTYDIR", + backing_store="EPHEMERAL_STORAGE", + ) + ], + containers=[ + oci.container_instances.models.CreateContainerDetails( + display_name="runtime", + image_url=str(projection["image"]), + arguments=[str(item) for item in _sequence(projection, "command")], + environment_variables=environment, + is_resource_principal_disabled=False, + resource_config=oci.container_instances.models.CreateContainerResourceConfigDetails( + vcpus_limit=cpu, + memory_limit_in_gbs=memory, + ), + volume_mounts=[ + oci.container_instances.models.CreateVolumeMountDetails( + mount_path="/tmp", + volume_name="dander-tmp", + is_read_only=False, + ) + ], + security_context=oci.container_instances.models.CreateLinuxSecurityContextDetails( + run_as_user=65532, + run_as_group=65532, + is_non_root_user_check_enabled=True, + is_root_file_system_readonly=True, + ), + freeform_tags={ + "dander-run-id": execution.run_id, + "dander-pipeline": execution.pipeline_id, + "dander-attempt": str(execution.attempt), + }, + ) + ], + freeform_tags={ + "managed-by": "dander", + "dander-run-id": execution.run_id, + "dander-pipeline": execution.pipeline_id, + "dander-attempt": str(execution.attempt), + }, + ) + response = self._client.create_container_instance( + details, + opc_retry_token=f"{execution.run_id}-a{execution.attempt}", + ) + instance_id = getattr(getattr(response, "data", None), "id", None) + if not isinstance(instance_id, str) or not instance_id.startswith( + "ocid1.computecontainerinstance" + ): + raise OciLifecycleError("OCI did not return a Container Instance OCID") + return instance_id + + def status(self, instance_id: str) -> OciInstanceStatus: + response = self._client.get_container_instance(instance_id) + instance = getattr(response, "data", None) + lifecycle = str(getattr(instance, "lifecycle_state", "")).upper() + containers = getattr(instance, "containers", None) or [] + if containers: + container = containers[0] + container_id = getattr(container, "id", None) + container_state = str(getattr(container, "lifecycle_state", "")).upper() + exit_code = getattr(container, "exit_code", None) + if container_state in {"TERMINATED", "FAILED"}: + state = "succeeded" if exit_code == 0 else "failed" + return OciInstanceStatus( + state=cast("Any", state), + container_id=container_id if isinstance(container_id, str) else None, + exit_code=exit_code if isinstance(exit_code, int) else None, + failure_code=(None if exit_code in {0, 75} else "runtime_failed"), + ) + return OciInstanceStatus( + state="running" if container_state == "ACTIVE" else "pending", + container_id=container_id if isinstance(container_id, str) else None, + ) + if lifecycle == "FAILED": + return OciInstanceStatus(state="failed", failure_code="container_instance_failed") + return OciInstanceStatus(state="running" if lifecycle == "ACTIVE" else "pending") + + def stop(self, instance_id: str) -> None: + try: + self._client.stop_container_instance(instance_id) + except Exception as error: # noqa: BLE001 + if _status(error) not in {404, 409}: + raise + + def delete(self, instance_id: str) -> None: + try: + self._client.delete_container_instance(instance_id) + except Exception as error: # noqa: BLE001 + if _status(error) != 404: + raise + + def logs(self, container_id: str, *, limit_bytes: int) -> bytes: + response = self._client.retrieve_logs(container_id) + data = getattr(response, "data", None) + content = getattr(data, "content", None) + if not isinstance(content, bytes): + raw = getattr(data, "raw", None) + content = raw.read() if raw is not None else b"" + return bytes(content)[-limit_bytes:] + + +def build_resource_principal_adapters( + *, + namespace: str, + bucket: str, +) -> tuple[OciObjectRunRepository, OciSdkContainerGateway]: + """Build controller adapters without credential files or static cloud keys.""" + try: + oci = cast("Any", import_module("oci")) + except ImportError as error: # pragma: no cover + raise OciLifecycleError("OCI SDK is unavailable") from error + signer = oci.auth.signers.get_resource_principals_signer() + return ( + OciObjectRunRepository( + client=oci.object_storage.ObjectStorageClient(config={}, signer=signer), + namespace=namespace, + bucket=bucket, + ), + OciSdkContainerGateway( + client=oci.container_instances.ContainerInstanceClient(config={}, signer=signer) + ), + ) + + +def _etag(response: object) -> str: + headers = getattr(response, "headers", None) + value = headers.get("etag") if isinstance(headers, Mapping) else None + if not isinstance(value, str) or not value: + raise OciLifecycleError("OCI Object Storage response omitted an ETag") + return value + + +def _status(error: Exception) -> int | None: + value = getattr(error, "status", None) + return value if isinstance(value, int) else None + + +def _require_same_run(existing: OciExecution, updated: OciExecution) -> None: + if (existing.pipeline_id, existing.run_id) != (updated.pipeline_id, updated.run_id): + raise OciLifecycleError("OCI execution update changed its identity") + + +def _mapping(document: Mapping[str, object], key: str) -> Mapping[str, object]: + value = document.get(key) + if not isinstance(value, dict): + raise OciLifecycleError(f"OCI execution projection {key} is invalid") + return value + + +def _sequence(document: Mapping[str, object], key: str) -> list[object]: + value = document.get(key) + if not isinstance(value, list): + raise OciLifecycleError(f"OCI execution projection {key} is invalid") + return value + + +__all__ = [ + "OciObjectRunRepository", + "OciSdkContainerGateway", + "build_resource_principal_adapters", +] diff --git a/src/dander/providers/oci_container_instances/operations.py b/src/dander/providers/oci_container_instances/operations.py new file mode 100644 index 00000000..c5c774cf --- /dev/null +++ b/src/dander/providers/oci_container_instances/operations.py @@ -0,0 +1,310 @@ +"""Manifest-bound OCI Container Instances lifecycle operations.""" + +from __future__ import annotations + +import hashlib +import io +import json +import re +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from importlib import import_module +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast +from uuid import uuid4 + +from dander.project import ProjectConfigError, load_project_config +from dander.providers.oci_container_instances.controller import ( + OciExecution, + OciLifecycleError, + execution_from_json, + execution_run_id, +) +from dander.providers.oci_container_instances.oci_adapter import OciObjectRunRepository + +if TYPE_CHECKING: + from collections.abc import Callable + +_FUNCTION_OCID = re.compile(r"^ocid1\.fnfunc\.oc[0-9]+\.[a-z0-9-]+\.[A-Za-z0-9]+$") +_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") +_PIPELINE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$") + + +class OciOperationError(RuntimeError): + """An OCI operation was unbound, rejected, or returned invalid data.""" + + +@dataclass(frozen=True, slots=True) +class OciOperationBinding: + """Exact project pipeline to OCI Function and run-record binding.""" + + region: str + namespace: str + bucket: str + pipeline_id: str + function_id: str + project_dir: Path + + @classmethod + def from_project( + cls, + *, + config: Path, + deployment: str, + pipeline_id: str, + function_id: str, + name: str = "dander", + ) -> OciOperationBinding: + if _PIPELINE.fullmatch(pipeline_id) is None: + raise OciOperationError("Invalid OCI pipeline identifier") + if _FUNCTION_OCID.fullmatch(function_id) is None: + raise OciOperationError("Invalid OCI Function OCID") + resolved = config.expanduser().resolve() + try: + manifest = load_project_config(resolved, deployment=deployment) + if manifest.launcher_provider != "oci_container_instances": + raise ProjectConfigError( + f"Deployment {deployment!r} does not select " + "launcher.provider='oci_container_instances'" + ) + manifest.validate_references(resolved.parent) + manifest.pipelines[pipeline_id] + launcher = manifest.resolved_launcher_config() + except KeyError as error: + raise OciOperationError( + f"Pipeline {pipeline_id!r} is not declared in the project manifest" + ) from error + except ProjectConfigError as error: + raise OciOperationError(str(error)) from error + namespace = launcher.get("registry_namespace") + compartment_id = launcher.get("compartment_id") + region = launcher.get("region") + if not all(isinstance(value, str) for value in (namespace, compartment_id, region)): + raise OciOperationError("OCI launcher binding is incomplete") + compartment = cast("str", compartment_id) + suffix = hashlib.sha256(compartment.encode()).hexdigest()[:8] + return cls( + region=cast("str", region), + namespace=cast("str", namespace), + bucket=f"{name}-oci-runs-{suffix}", + pipeline_id=pipeline_id, + function_id=function_id, + project_dir=resolved.parent, + ) + + +@dataclass(frozen=True, slots=True) +class OciInvocation: + """Sanitized acknowledgement for a detached lifecycle Function invocation.""" + + action: str + run_id: str + accepted: bool = True + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +class _FunctionInvoker: + def __init__( + self, + *, + management_client: object, + signer: object, + config: dict[str, Any], + expected_pipeline: str, + ) -> None: + self._management = cast("Any", management_client) + self._signer = signer + self._config = config + self._expected_pipeline = expected_pipeline + + def invoke(self, function_id: str, payload: dict[str, object], *, detached: bool) -> bytes: + try: + oci = cast("Any", import_module("oci")) + except ImportError as error: # pragma: no cover + raise OciOperationError("OCI SDK is unavailable") from error + try: + function = getattr(self._management.get_function(function_id), "data", None) + except Exception as error: # noqa: BLE001 - OCI SDK errors vary by release. + raise OciOperationError("OCI Function metadata could not be read") from error + function_config = getattr(function, "config", None) + if ( + not isinstance(function_config, dict) + or function_config.get("DANDER_OCI_PIPELINE") != self._expected_pipeline + ): + raise OciOperationError("OCI Function does not match the selected pipeline") + endpoint = getattr(function, "invoke_endpoint", None) + if not isinstance(endpoint, str) or not endpoint.startswith("https://"): + raise OciOperationError("OCI Function invoke endpoint is unavailable") + client = oci.functions.FunctionsInvokeClient( + self._config, + signer=self._signer, + service_endpoint=endpoint, + ) + body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + try: + response = client.invoke_function( + function_id, + invoke_function_body=io.BytesIO(body), + fn_invoke_type="detached" if detached else "sync", + ) + except Exception as error: # noqa: BLE001 - OCI SDK errors vary by release. + raise OciOperationError("OCI Function invocation failed") from error + if detached: + return b"" + data = getattr(response, "data", None) + content = getattr(data, "content", None) + if isinstance(content, bytes): + return content + raw = getattr(data, "raw", None) + result = raw.read() if raw is not None else b"" + return result if isinstance(result, bytes) else b"" + + +class OciContainerInstanceOperations: + """Start, observe, cancel, replay, and read bounded logs for one OCI pipeline.""" + + def __init__( + self, + binding: OciOperationBinding, + *, + repository: OciObjectRunRepository, + invoker: object, + clock: Callable[[], datetime] | None = None, + nonce: Callable[[], str] | None = None, + ) -> None: + self.binding = binding + self._repository = repository + self._invoker = cast("Any", invoker) + self._clock = clock or (lambda: datetime.now(UTC)) + self._nonce = nonce or (lambda: uuid4().hex[:12]) + + @classmethod + def from_security_token_profile( + cls, + binding: OciOperationBinding, + *, + profile: str = "DEFAULT", + ) -> OciContainerInstanceOperations: + """Build SDK clients from one expiring OCI session profile, never an API key signer.""" + if _PROFILE.fullmatch(profile) is None: + raise OciOperationError("Invalid OCI SecurityToken profile") + try: + oci = cast("Any", import_module("oci")) + except ImportError as error: # pragma: no cover + raise OciOperationError("OCI SDK is unavailable") from error + try: + config = oci.config.from_file(profile_name=profile) + token_path = config.get("security_token_file") + key_path = config.get("key_file") + if not isinstance(token_path, str) or not isinstance(key_path, str): + raise OciOperationError( + "OCI profile must be an authenticated SecurityToken session" + ) + token = Path(token_path).expanduser().read_text(encoding="utf-8").strip() + private_key = oci.signer.load_private_key_from_file( + Path(key_path).expanduser(), + pass_phrase=config.get("pass_phrase"), + ) + signer = oci.auth.signers.SecurityTokenSigner(token, private_key) + except (OSError, ValueError) as error: + raise OciOperationError("OCI SecurityToken session could not be loaded") from error + config["region"] = binding.region + repository = OciObjectRunRepository( + client=oci.object_storage.ObjectStorageClient(config, signer=signer), + namespace=binding.namespace, + bucket=binding.bucket, + ) + return cls( + binding, + repository=repository, + invoker=_FunctionInvoker( + management_client=oci.functions.FunctionsManagementClient(config, signer=signer), + signer=signer, + config=config, + expected_pipeline=binding.pipeline_id, + ), + ) + + def start(self) -> OciInvocation: + """Queue one detached manual execution with a caller-known stable run id.""" + key = self._new_key("manual") + self._invoker.invoke( + self.binding.function_id, + {"action": "start", "idempotency_key": key}, + detached=True, + ) + return OciInvocation( + action="start", + run_id=execution_run_id(self.binding.pipeline_id, key), + ) + + def latest(self) -> OciExecution | None: + """Return the active execution, if any.""" + stored = self._repository.get(self.binding.pipeline_id) + return None if stored is None else stored.execution + + def describe(self, run_id: str) -> OciExecution: + """Return one exact active or terminal execution.""" + stored = self._repository.get(self.binding.pipeline_id, run_id) + if stored is None: + raise OciOperationError("OCI execution was not found") + return stored.execution + + def logs(self, run_id: str, *, attempt: int | None = None) -> bytes: + """Return at most 256 KiB of runtime output retained outside Git.""" + execution = self.describe(run_id) + return self._repository.get_logs(execution, attempt=attempt) + + def cancel(self, run_id: str) -> OciExecution: + """Synchronously request interruption and return the controller record.""" + content = self._invoker.invoke( + self.binding.function_id, + {"action": "cancel", "run_id": run_id}, + detached=False, + ) + return _execution_response(content) + + def replay(self, run_id: str) -> OciInvocation: + """Queue a fresh attempt chain for one terminal execution.""" + previous = self.describe(run_id) + if not previous.terminal: + raise OciOperationError("Only a terminal OCI execution can be replayed") + key = self._new_key("replay") + self._invoker.invoke( + self.binding.function_id, + {"action": "replay", "run_id": run_id, "idempotency_key": key}, + detached=True, + ) + return OciInvocation( + action="replay", + run_id=execution_run_id(self.binding.pipeline_id, key), + ) + + def _new_key(self, prefix: str) -> str: + now = self._clock().astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + return f"{prefix}:{self.binding.pipeline_id}:{now}:{self._nonce()}" + + +def _execution_response(content: bytes) -> OciExecution: + try: + document = json.loads(content) + except json.JSONDecodeError as error: + raise OciOperationError("OCI Function returned an invalid response") from error + if not isinstance(document, dict): + raise OciOperationError("OCI Function returned an invalid response") + try: + return execution_from_json( + json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + ) + except OciLifecycleError as error: + raise OciOperationError(str(error)) from error + + +__all__ = [ + "OciContainerInstanceOperations", + "OciInvocation", + "OciOperationBinding", + "OciOperationError", +] diff --git a/src/dander/runtime_secrets.py b/src/dander/runtime_secrets.py new file mode 100644 index 00000000..cfce9e76 --- /dev/null +++ b/src/dander/runtime_secrets.py @@ -0,0 +1,99 @@ +"""Resolve launcher-projected secret references only for one runtime execution.""" + +from __future__ import annotations + +import json +import os +import re +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from dander.providers import ProviderFactoryError, ProviderKind, default_provider_registry +from dander.security import SecretRuntime + +if TYPE_CHECKING: + from collections.abc import Iterator, MutableMapping + + from dander.core.interfaces import SecretStoreProvider + +_BINDINGS_ENV = "DANDER_SECRET_BINDINGS_JSON" +_ENVIRONMENT_NAME = re.compile(r"^[A-Z][A-Z0-9_]{0,126}$") +_SUPPORTED_PROVIDERS = frozenset({"oci_vault"}) + + +class RuntimeSecretBindingError(RuntimeError): + """A launcher supplied malformed, ambiguous, or unresolvable secret bindings.""" + + +@contextmanager +def projected_secret_environment( + *, + environ: MutableMapping[str, str] = os.environ, +) -> Iterator[None]: + """Temporarily resolve validated launcher bindings, then remove every secret value.""" + raw = environ.get(_BINDINGS_ENV) + if raw is None: + yield + return + bindings = _parse_bindings(raw) + overlap = sorted(set(bindings) & set(environ)) + if overlap: + raise RuntimeSecretBindingError( + "Launcher secret bindings overlap preconfigured environment values: " + + ", ".join(overlap) + ) + resolved: dict[str, str] = {} + try: + registry = default_provider_registry() + stores: dict[str, SecretStoreProvider] = {} + for name, (provider, reference) in bindings.items(): + store = stores.get(provider) + if store is None: + config = registry.parse(ProviderKind.SECRETS, {"provider": provider}) + runtime = registry.build(ProviderKind.SECRETS, config) + if not isinstance(runtime, SecretRuntime): + raise RuntimeSecretBindingError( + "Selected launcher secret provider returned an invalid runtime" + ) + store = runtime.store + stores[provider] = store + value = store.get_secret(reference) + if not isinstance(value, str) or not value: + raise RuntimeSecretBindingError( + f"Launcher secret binding {name} resolved to an empty value" + ) + resolved[name] = value + environ.update(resolved) + yield + except ProviderFactoryError as error: + raise RuntimeSecretBindingError(str(error)) from error + finally: + for name in resolved: + environ.pop(name, None) + + +def _parse_bindings(raw: str) -> dict[str, tuple[str, str]]: + try: + document = json.loads(raw) + except json.JSONDecodeError as error: + raise RuntimeSecretBindingError("Launcher secret bindings are not valid JSON") from error + if not isinstance(document, dict) or not document: + raise RuntimeSecretBindingError("Launcher secret bindings must be a non-empty object") + parsed: dict[str, tuple[str, str]] = {} + for name, binding in document.items(): + if not isinstance(name, str) or _ENVIRONMENT_NAME.fullmatch(name) is None: + raise RuntimeSecretBindingError("Launcher secret binding names are invalid") + if not isinstance(binding, dict) or set(binding) != {"provider", "reference"}: + raise RuntimeSecretBindingError("Launcher secret binding entries are invalid") + provider = binding.get("provider") + reference = binding.get("reference") + if provider not in _SUPPORTED_PROVIDERS or not isinstance(reference, str): + raise RuntimeSecretBindingError("Launcher secret binding provider is unsupported") + expected_prefix = "oci-vault://" + if not reference.startswith(expected_prefix) or len(reference) > 1_024: + raise RuntimeSecretBindingError("Launcher secret binding reference is invalid") + parsed[name] = (provider, reference) + return parsed + + +__all__ = ["RuntimeSecretBindingError", "projected_secret_environment"] diff --git a/src/dander/state/failure.py b/src/dander/state/failure.py index a82342c4..3088c7eb 100644 --- a/src/dander/state/failure.py +++ b/src/dander/state/failure.py @@ -129,7 +129,11 @@ def _exception_chain(error: BaseException) -> tuple[BaseException, ...]: def _status_code(error: BaseException) -> int | None: response = getattr(error, "response", None) value = getattr(response, "status_code", None) - return value if isinstance(value, int) else None + if isinstance(value, int): + return value + # OCI SDK ServiceError and several provider SDKs expose the status directly. + direct = getattr(error, "status", None) + return direct if isinstance(direct, int) else None def _has_known_phrase(chain: tuple[BaseException, ...], phrase: str) -> bool: diff --git a/tests/bootstrap/test_oci_controller_image.py b/tests/bootstrap/test_oci_controller_image.py new file mode 100644 index 00000000..1a03cb09 --- /dev/null +++ b/tests/bootstrap/test_oci_controller_image.py @@ -0,0 +1,54 @@ +"""Source-free OCI Functions controller image contract.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +_ROOT = Path(__file__).parents[2] + + +def test_controller_image_is_python_312_x86_digest_pinned_and_wheel_only() -> None: + dockerfile = (_ROOT / "infra/oci/controller/Dockerfile").read_text(encoding="utf-8") + + assert "fnproject/python:3.12-dev@sha256:" in dockerfile + assert "fnproject/python:3.12@sha256:" in dockerfile + assert "ARG DANDER_WHEEL" in dockerfile + assert "pip3 install --target /python --no-cache-dir --no-deps" in dockerfile + assert "COPY src/" not in dockerfile + assert "COPY . " not in dockerfile + assert 'ENTRYPOINT ["/python/bin/fdk", "/function/func.py", "handler"]' in dockerfile + + +def test_controller_dependencies_and_shim_are_exact_and_secret_free() -> None: + requirements = (_ROOT / "infra/oci/controller/requirements.txt").read_text(encoding="utf-8") + shim = (_ROOT / "infra/oci/controller/func.py").read_text(encoding="utf-8") + + assert requirements.splitlines() == [ + "fdk==0.1.117", + "oci==2.184.1", + "pydantic==2.13.4", + ] + assert "function_handler import handler" in shim + assert "token" not in requirements.lower() + assert "password" not in shim.lower() + + +def test_oci_function_import_does_not_load_other_provider_operations() -> None: + script = """ +import sys +from dander.providers.oci_container_instances.function_handler import handler +assert callable(handler) +assert 'dander.providers.azure_container_apps.operations' not in sys.modules +assert 'dander.providers.fargate.operations' not in sys.modules +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=_ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/bootstrap/test_oci_terraform.py b/tests/bootstrap/test_oci_terraform.py index 7488e7c5..e63f6076 100644 --- a/tests/bootstrap/test_oci_terraform.py +++ b/tests/bootstrap/test_oci_terraform.py @@ -6,6 +6,7 @@ import stat import subprocess from pathlib import Path +from typing import TYPE_CHECKING, cast import pytest @@ -15,6 +16,9 @@ OciTerraformBootstrapError, ) +if TYPE_CHECKING: + from typing import Any + _ROOT = Path(__file__).parents[2] _TENANCY = "ocid1.tenancy.oc1.." + "a" * 32 _COMPARTMENT = "ocid1.compartment.oc1.." + "b" * 32 @@ -105,7 +109,7 @@ def test_foundation_bootstrap_uses_native_backend_and_saved_plan( ) -> None: bootstrap = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "foundation") - plan = bootstrap.execute(**_FOUNDATION) + plan = bootstrap.execute(**cast("Any", _FOUNDATION)) bootstrap.apply_saved_plan(**_FOUNDATION) init = terraform_calls[0] @@ -127,7 +131,7 @@ def test_deployment_verification_reads_both_remote_states_and_requires_no_drift( foundation = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "foundation") admin.verify_no_drift(namespace="unitnamespace", **_ADMIN) - foundation.verify_no_drift(**_FOUNDATION) + foundation.verify_no_drift(**cast("Any", _FOUNDATION)) verification_plans = [call for call in terraform_calls if "-detailed-exitcode" in call] assert len(verification_plans) == 2 @@ -146,7 +150,7 @@ def run(args: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[ foundation = OciTerraformBootstrap(_ROOT / "infra/oci", tmp_path / "foundation") with pytest.raises(OciTerraformBootstrapError, match="found Terraform drift"): - foundation.verify_no_drift(**_FOUNDATION) + foundation.verify_no_drift(**cast("Any", _FOUNDATION)) @pytest.mark.parametrize( diff --git a/tests/cli/test_oci_cli.py b/tests/cli/test_oci_cli.py index 93bc8753..5f0c8a61 100644 --- a/tests/cli/test_oci_cli.py +++ b/tests/cli/test_oci_cli.py @@ -2,16 +2,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from types import SimpleNamespace +from typing import TYPE_CHECKING, cast +import pytest from typer.testing import CliRunner +import dander.cli.oci_command as oci_command from dander.cli.main import app +from dander.project import ProjectConfigError if TYPE_CHECKING: from pathlib import Path - - import pytest + from typing import Any def test_oci_admin_plan_is_non_mutating_and_prints_review_command( @@ -135,3 +138,65 @@ def fake_verify(self: object, **kwargs: object) -> None: assert verified[0]["state_key"] == "dander/oci/bootstrap-admin/terraform.tfstate" assert verified[1]["state_key"] == "dander/oci/foundation/terraform.tfstate" assert '"status": "no_drift"' in result.output + + +def test_launcher_projection_requires_controller_tag_in_selected_immutable_repository( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + launcher = { + "tenancy_id": "ocid1.tenancy.oc1.." + "a" * 32, + "compartment_id": "ocid1.compartment.oc1.." + "b" * 32, + "region": "us-ashburn-1", + "registry_namespace": "unitnamespace", + "repository_name": "dander/runtime", + } + manifest = SimpleNamespace( + launcher_provider="oci_container_instances", + validate_references=lambda _path: None, + resolved_launcher_config=lambda: launcher, + platform=SimpleNamespace( + runtime=SimpleNamespace( + cpu=1, + memory="2Gi", + timeout_seconds=900, + max_retries=1, + batch_rows=1_000, + ), + safety=SimpleNamespace(require_guarded_free_tier=True), + ), + terraform_pipelines=lambda: {}, + ) + monkeypatch.setattr(oci_command, "load_project_config", lambda *_args, **_kwargs: manifest) + monkeypatch.setattr(oci_command, "build_oci_execution_projections", lambda **_kwargs: {}) + common = { + "config": tmp_path / "dander.yaml", + "platforms_config": None, + "deployment": "oci_postgresql", + "container_image": ( + "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime@sha256:" + "a" * 64 + ), + "tenancy_id": launcher["tenancy_id"], + "compartment_id": launcher["compartment_id"], + "region": launcher["region"], + "namespace": launcher["registry_namespace"], + } + + assert ( + oci_command._deployment_projections( # noqa: SLF001 + **cast("Any", common), + controller_image=( + "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/" + "dander/runtime:phase7-controller" + ), + ) + == {} + ) + with pytest.raises(ProjectConfigError, match="manifest-selected OCIR repository"): + oci_command._deployment_projections( # noqa: SLF001 + **cast("Any", common), + controller_image=( + "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/" + "dander/controller:phase7-controller" + ), + ) diff --git a/tests/cli/test_oci_operations_cli.py b/tests/cli/test_oci_operations_cli.py new file mode 100644 index 00000000..0c8bacde --- /dev/null +++ b/tests/cli/test_oci_operations_cli.py @@ -0,0 +1,164 @@ +"""Thin CLI coverage for manifest-bound OCI Container Instances operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from typer.testing import CliRunner + +import dander.cli.oci_command as oci_command +from dander.cli.main import app +from dander.providers.oci_container_instances import ( + OCI_EXECUTION_SCHEMA, + OciExecution, + OciInvocation, +) + +if TYPE_CHECKING: + from pathlib import Path + + from pytest import MonkeyPatch + +_FUNCTION = "ocid1.fnfunc.oc1.iad." + "f" * 32 +_RUN_ID = "oci-" + "a" * 24 + + +def _execution() -> OciExecution: + return OciExecution( + schema=OCI_EXECUTION_SCHEMA, + run_id=_RUN_ID, + pipeline_id="warehouse_fixture", + idempotency_key="unit", + image="iad.ocir.io/unit/dander/runtime@sha256:" + "a" * 64, + state="succeeded", + attempt=1, + max_attempts=2, + created_at="2026-08-12T12:00:00Z", + updated_at="2026-08-12T12:01:00Z", + deadline_at="2026-08-12T12:15:00Z", + exit_code=0, + ) + + +class _Operations: + def __init__(self) -> None: + self.calls: list[tuple[str, object]] = [] + self.execution = _execution() + + def start(self) -> OciInvocation: + self.calls.append(("start", None)) + return OciInvocation("start", _RUN_ID) + + def latest(self) -> OciExecution: + self.calls.append(("latest", None)) + return self.execution + + def describe(self, run_id: str) -> OciExecution: + self.calls.append(("describe", run_id)) + return self.execution + + def logs(self, run_id: str, *, attempt: int | None) -> bytes: + self.calls.append(("logs", (run_id, attempt))) + return b"runtime complete\n" + + def cancel(self, run_id: str) -> OciExecution: + self.calls.append(("cancel", run_id)) + return self.execution + + def replay(self, run_id: str) -> OciInvocation: + self.calls.append(("replay", run_id)) + return OciInvocation("replay", _RUN_ID) + + +def _install_fake(monkeypatch: MonkeyPatch) -> tuple[_Operations, dict[str, object]]: + operations = _Operations() + binding: dict[str, object] = {} + + def factory(**kwargs: object) -> _Operations: + binding.update(kwargs) + return operations + + monkeypatch.setattr(oci_command, "_oci_operations", factory) + return operations, binding + + +def _base_args(tmp_path: Path) -> list[str]: + return [ + "--deployment", + "oci_postgres", + "--pipeline", + "warehouse_fixture", + "--function-id", + _FUNCTION, + "--config", + str(tmp_path / "dander.yaml"), + "--oci-profile", + "DANDER", + ] + + +def test_status_and_logs_bind_to_the_selected_manifest_pipeline( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + operations, binding = _install_fake(monkeypatch) + + status = CliRunner().invoke( + app, + ["oci", "status", *_base_args(tmp_path), "--run-id", _RUN_ID], + ) + logs = CliRunner().invoke( + app, + ["oci", "logs", *_base_args(tmp_path), "--run-id", _RUN_ID, "--attempt", "1"], + ) + + assert status.exit_code == 0, status.output + assert logs.exit_code == 0, logs.output + assert '"state": "succeeded"' in status.output + assert logs.output == "runtime complete\n" + assert operations.calls == [ + ("describe", _RUN_ID), + ("logs", (_RUN_ID, 1)), + ] + assert binding == { + "config": tmp_path / "dander.yaml", + "deployment": "oci_postgres", + "pipeline": "warehouse_fixture", + "function_id": _FUNCTION, + "oci_profile": "DANDER", + "name": "dander", + } + + +@pytest.mark.parametrize( + ("command", "extra", "expected"), + [ + ("run", [], ("start", None)), + ("cancel", ["--run-id", _RUN_ID], ("cancel", _RUN_ID)), + ("replay", ["--run-id", _RUN_ID], ("replay", _RUN_ID)), + ], +) +def test_mutating_operations_require_confirmation_and_invoke_one_action( + monkeypatch: MonkeyPatch, + tmp_path: Path, + command: str, + extra: list[str], + expected: tuple[str, object], +) -> None: + operations, _binding = _install_fake(monkeypatch) + + refused = CliRunner().invoke( + app, + ["oci", command, *_base_args(tmp_path), *extra], + input="n\n", + ) + accepted = CliRunner().invoke( + app, + ["oci", command, *_base_args(tmp_path), *extra], + input="y\n", + ) + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert operations.calls == [expected] diff --git a/tests/providers/test_oci_container_instances_adapter.py b/tests/providers/test_oci_container_instances_adapter.py new file mode 100644 index 00000000..4df1e4b4 --- /dev/null +++ b/tests/providers/test_oci_container_instances_adapter.py @@ -0,0 +1,191 @@ +"""OCI Object Storage lifecycle repository correctness without provider access.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from dander.providers.oci_container_instances.controller import ( + OCI_EXECUTION_SCHEMA, + OciExecution, + OciLifecycleError, + execution_run_id, +) +from dander.providers.oci_container_instances.oci_adapter import OciObjectRunRepository + + +class _ServiceError(RuntimeError): + def __init__(self, status: int) -> None: + super().__init__(f"provider-{status}") + self.status = status + + +class _ObjectClient: + def __init__(self) -> None: + self.objects: dict[str, tuple[bytes, str]] = {} + self.version = 0 + self.delete_failure: int | None = None + + def put_object( + self, + namespace: str, + bucket: str, + key: str, + content: bytes, + **kwargs: object, + ) -> object: + assert namespace == "unitnamespace" + assert bucket == "unit-runs" + existing = self.objects.get(key) + if kwargs.get("if_none_match") == "*" and existing is not None: + raise _ServiceError(412) + if_match = kwargs.get("if_match") + if if_match is not None and (existing is None or existing[1] != if_match): + raise _ServiceError(412) + self.version += 1 + etag = f"etag-{self.version}" + self.objects[key] = (bytes(content), etag) + return SimpleNamespace(headers={"etag": etag}) + + def get_object(self, namespace: str, bucket: str, key: str) -> object: + assert namespace == "unitnamespace" + assert bucket == "unit-runs" + try: + content, etag = self.objects[key] + except KeyError as error: + raise _ServiceError(404) from error + return SimpleNamespace(data=SimpleNamespace(content=content), headers={"etag": etag}) + + def delete_object( + self, + namespace: str, + bucket: str, + key: str, + **kwargs: object, + ) -> None: + assert namespace == "unitnamespace" + assert bucket == "unit-runs" + if self.delete_failure is not None: + status = self.delete_failure + self.delete_failure = None + raise _ServiceError(status) + existing = self.objects.get(key) + if existing is None: + raise _ServiceError(404) + if kwargs.get("if_match") != existing[1]: + raise _ServiceError(412) + del self.objects[key] + + +def _execution(key: str = "manual:unit") -> OciExecution: + return OciExecution( + schema=OCI_EXECUTION_SCHEMA, + run_id=execution_run_id("jobs", key), + pipeline_id="jobs", + idempotency_key=key, + image="ocir.us-ashburn-1.oci.oraclecloud.com/unit/dander@sha256:" + "a" * 64, + state="pending", + attempt=1, + max_attempts=2, + created_at="2026-08-12T12:00:00Z", + updated_at="2026-08-12T12:00:00Z", + deadline_at="2026-08-12T12:15:00Z", + ) + + +def _repository(client: _ObjectClient) -> OciObjectRunRepository: + return OciObjectRunRepository( + client=client, + namespace="unitnamespace", + bucket="unit-runs", + ) + + +def test_claim_compare_and_swap_finish_and_history_are_idempotent() -> None: + client = _ObjectClient() + repository = _repository(client) + execution = _execution() + + stored, claimed = repository.claim(execution) + repeated, repeated_claim = repository.claim(execution) + running = replace(execution, state="running", updated_at="2026-08-12T12:00:01Z") + stored = repository.save(stored, running) + terminal = replace( + running, + state="succeeded", + exit_code=0, + updated_at="2026-08-12T12:01:00Z", + ) + finished = repository.finish(stored, terminal) + + assert claimed is True + assert repeated_claim is False + assert repeated.execution == execution + assert repository.get("jobs") is None + assert repository.get("jobs", execution.run_id) == finished + assert repository.finish(finished, terminal) == finished + + +def test_finish_recovers_after_history_was_written_but_active_release_failed() -> None: + client = _ObjectClient() + repository = _repository(client) + execution = _execution("manual:recovery") + stored, _ = repository.claim(execution) + terminal = replace( + execution, + state="failed", + failure_code="runtime_failed", + exit_code=1, + updated_at="2026-08-12T12:01:00Z", + ) + client.delete_failure = 500 + + with pytest.raises(OciLifecycleError, match="release"): + repository.finish(stored, terminal) + + partial = repository.get("jobs", terminal.run_id) + assert partial is not None and partial.execution == terminal + recovered = repository.finish(partial, terminal) + assert recovered.execution == terminal + assert repository.get("jobs") is None + + +def test_logs_are_tail_bounded_and_projection_reads_are_size_and_shape_checked() -> None: + client = _ObjectClient() + repository = _repository(client) + execution = _execution() + repository.save_logs(execution, b"a" * 300_000) + + assert repository.get_logs(execution) == b"a" * 262_144 + + projection = {"schema": "io.dander.execution/v1", "pipeline_id": "jobs"} + client.put_object( + "unitnamespace", + "unit-runs", + "projections/jobs.json", + json.dumps(projection).encode(), + ) + assert repository.get_projection("projections/jobs.json") == projection + with pytest.raises(OciLifecycleError, match="key"): + repository.get_projection("history/jobs.json") + + +def test_corrupt_execution_records_fail_closed_without_echoing_content() -> None: + client = _ObjectClient() + repository = _repository(client) + document = _execution().as_dict() + document["state"] = "customer-secret-content" + client.put_object( + "unitnamespace", + "unit-runs", + "active/jobs.json", + json.dumps(document).encode(), + ) + + with pytest.raises(OciLifecycleError, match="unsupported contract") as raised: + repository.get("jobs") + + assert "customer-secret-content" not in str(raised.value) diff --git a/tests/providers/test_oci_container_instances_controller.py b/tests/providers/test_oci_container_instances_controller.py new file mode 100644 index 00000000..a8192a70 --- /dev/null +++ b/tests/providers/test_oci_container_instances_controller.py @@ -0,0 +1,301 @@ +"""OCI lifecycle controller correctness without provider access.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +import pytest + +from dander.providers.oci_container_instances.controller import ( + OciExecution, + OciInstanceStatus, + OciLifecycleController, + OciLifecycleError, + StoredExecution, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + +_IMAGE = "ocir.us-ashburn-1.oci.oraclecloud.com/unit/dander/runtime@sha256:" + "a" * 64 + + +class _Repository: + def __init__(self) -> None: + self.active: OciExecution | None = None + self.history: dict[str, OciExecution] = {} + self.logs: list[tuple[str, bytes]] = [] + self.version = 0 + + def claim(self, execution: OciExecution) -> tuple[StoredExecution, bool]: + historical = self.history.get(execution.run_id) + if historical is not None: + return self._stored(historical), False + if self.active is not None: + return self._stored(self.active), False + self.active = execution + return self._stored(execution), True + + def get(self, pipeline_id: str, run_id: str | None = None) -> StoredExecution | None: + assert pipeline_id == "jobs" + if self.active is not None and (run_id is None or self.active.run_id == run_id): + return self._stored(self.active) + if run_id is not None and run_id in self.history: + return self._stored(self.history[run_id]) + return None + + def save(self, stored: StoredExecution, execution: OciExecution) -> StoredExecution: + assert self.active is not None + assert stored.execution.run_id == execution.run_id == self.active.run_id + self.active = execution + return self._stored(execution) + + def finish(self, stored: StoredExecution, execution: OciExecution) -> StoredExecution: + if self.active is None: + historical = self.history.get(execution.run_id) + assert historical == execution + return self._stored(execution) + result = self.save(stored, execution) + self.history[execution.run_id] = execution + self.active = None + return result + + def save_logs(self, execution: OciExecution, content: bytes) -> None: + self.logs.append((execution.run_id, content)) + + def _stored(self, execution: OciExecution) -> StoredExecution: + self.version += 1 + return StoredExecution(execution, f"v{self.version}") + + +class _Gateway: + def __init__(self) -> None: + self.created: list[tuple[str, int]] = [] + self.statuses: list[OciInstanceStatus] = [] + self.stopped: list[str] = [] + self.deleted: list[str] = [] + self.fail_create = False + + def create(self, projection: Mapping[str, object], execution: OciExecution) -> str: + assert projection["pipeline_id"] == execution.pipeline_id + if self.fail_create: + raise RuntimeError("provider failure") + instance_id = f"instance-{execution.attempt}" + self.created.append((instance_id, execution.attempt)) + return instance_id + + def status(self, instance_id: str) -> OciInstanceStatus: + assert instance_id.startswith("instance-") + return self.statuses.pop(0) + + def stop(self, instance_id: str) -> None: + self.stopped.append(instance_id) + + def delete(self, instance_id: str) -> None: + self.deleted.append(instance_id) + + def logs(self, container_id: str, *, limit_bytes: int) -> bytes: + assert limit_bytes == 262_144 + return f"bounded:{container_id}".encode() + + +def _projection(*, retries: int = 1, deadline: int = 900) -> dict[str, object]: + return { + "schema": "io.dander.execution/v1", + "contract": "io.dander.runtime/v1", + "pipeline_id": "jobs", + "profile_id": "oci_postgresql", + "launcher": "oci_container_instances", + "image": _IMAGE, + "command": ["runtime", "execute"], + "configuration_reference": "/app/dander.yaml", + "environment": {}, + "secret_bindings": {}, + "workload_identity": "oci-resource-principal://dynamic-group/unit", + "resources": { + "cpu_millis": 1000, + "memory_mib": 2048, + "deadline_seconds": deadline, + "runtime_retry_count": 0, + "launcher_retry_count": retries, + }, + "network": {"placement": "subnet", "extensions": {}}, + "schedule": { + "task_count": 1, + "maximum_parallelism": 1, + "expression": "0 * * * *", + "time_zone": "UTC", + "paused": True, + }, + "labels": {}, + "observability": { + "log_destination": "oci_logging", + "metric_namespace": "oci_computecontainerinstance", + "alert_target": "oci_notifications", + "retention_days": 30, + }, + "extensions": {}, + } + + +def _controller( + *, + repository: _Repository, + gateway: _Gateway, + clock: object | None = None, + retries: int = 1, + deadline: int = 900, +) -> OciLifecycleController: + return OciLifecycleController( + projection=_projection(retries=retries, deadline=deadline), + repository=repository, + gateway=gateway, + clock=clock, # type: ignore[arg-type] + ) + + +def test_start_is_idempotent_and_parallelism_is_one() -> None: + repository = _Repository() + gateway = _Gateway() + controller = _controller(repository=repository, gateway=gateway) + + first = controller.start(idempotency_key="manual:one") + repeated = controller.start(idempotency_key="manual:one") + + assert repeated == first + assert gateway.created == [("instance-1", 1)] + with pytest.raises(OciLifecycleError, match="already has active run"): + controller.start(idempotency_key="manual:two") + + with pytest.raises(OciLifecycleError, match="unsupported characters"): + controller.start(idempotency_key="private value with spaces") + + +def test_success_preserves_logs_deletes_instance_and_releases_pipeline() -> None: + repository = _Repository() + gateway = _Gateway() + gateway.statuses = [OciInstanceStatus("succeeded", "container-one", 0)] + controller = _controller(repository=repository, gateway=gateway) + started = controller.start(idempotency_key="manual:success") + + finished = controller.reconcile(started.run_id) + + assert finished is not None and finished.state == "succeeded" + assert gateway.deleted == ["instance-1"] + assert repository.logs == [(started.run_id, b"bounded:container-one")] + assert repository.active is None + + +def test_retryable_exit_creates_fresh_attempt_then_succeeds() -> None: + repository = _Repository() + gateway = _Gateway() + gateway.statuses = [ + OciInstanceStatus("failed", "container-one", 75), + OciInstanceStatus("succeeded", "container-two", 0), + ] + controller = _controller(repository=repository, gateway=gateway, retries=1) + started = controller.start(idempotency_key="manual:retry") + + retried = controller.reconcile(started.run_id) + finished = controller.reconcile(started.run_id) + + assert retried is not None and retried.attempt == 2 and retried.state == "running" + assert finished is not None and finished.state == "succeeded" + assert gateway.created == [("instance-1", 1), ("instance-2", 2)] + assert gateway.deleted == ["instance-1", "instance-2"] + + +def test_retry_exhaustion_has_stable_failure_code() -> None: + repository = _Repository() + gateway = _Gateway() + gateway.statuses = [OciInstanceStatus("failed", "container-one", 75)] + controller = _controller(repository=repository, gateway=gateway, retries=0) + started = controller.start(idempotency_key="manual:exhaust") + + finished = controller.reconcile(started.run_id) + + assert finished is not None and finished.state == "failed" + assert finished.failure_code == "launcher_retry_exhausted" + + +def test_deadline_stops_and_deletes_before_failing() -> None: + now = datetime(2026, 8, 12, 12, tzinfo=UTC) + moments = iter((now, now, now + timedelta(seconds=11), now + timedelta(seconds=11))) + repository = _Repository() + gateway = _Gateway() + controller = _controller( + repository=repository, + gateway=gateway, + clock=lambda: next(moments), + deadline=10, + ) + started = controller.start(idempotency_key="manual:deadline") + + finished = controller.reconcile(started.run_id) + + assert finished is not None and finished.failure_code == "launcher_deadline_exceeded" + assert gateway.stopped == ["instance-1"] + assert gateway.deleted == ["instance-1"] + + +def test_cancel_is_terminal_and_replay_creates_a_new_run() -> None: + repository = _Repository() + gateway = _Gateway() + controller = _controller(repository=repository, gateway=gateway) + started = controller.start(idempotency_key="manual:cancel") + + cancelled = controller.cancel(started.run_id) + replayed = controller.replay(started.run_id, idempotency_key="replay:one") + + assert cancelled.state == "cancelled" + assert cancelled.failure_code == "interrupted_run" + assert replayed.replay_of == started.run_id + assert replayed.run_id != started.run_id + assert gateway.stopped == ["instance-1"] + assert gateway.deleted == ["instance-1"] + + +def test_create_failure_is_recorded_and_raised() -> None: + repository = _Repository() + gateway = _Gateway() + gateway.fail_create = True + controller = _controller(repository=repository, gateway=gateway) + + with pytest.raises(OciLifecycleError, match="creation failed"): + controller.start(idempotency_key="manual:create-failure") + + assert len(repository.history) == 1 + assert next(iter(repository.history.values())).failure_code == "launcher_create_failed" + + +@pytest.mark.parametrize("deadline", [0, 3301]) +def test_controller_rejects_deadlines_it_cannot_own(deadline: int) -> None: + with pytest.raises(OciLifecycleError, match="1-3300"): + _controller(repository=_Repository(), gateway=_Gateway(), deadline=deadline) + + +def test_controller_rejects_unknown_projection_fields_and_secret_value_overlap() -> None: + projection = _projection() + projection["unknown"] = "must-not-be-retained" + with pytest.raises(OciLifecycleError, match="incomplete"): + OciLifecycleController( + projection=projection, + repository=_Repository(), + gateway=_Gateway(), + ) + + projection = _projection() + projection["environment"] = {"TOKEN": "plain"} + projection["secret_bindings"] = { + "TOKEN": { + "provider": "oci_vault", + "reference": "oci-vault://ocid1.vault.oc1.iad.unit/secrets/token", + } + } + with pytest.raises(OciLifecycleError, match="secret bindings"): + OciLifecycleController( + projection=projection, + repository=_Repository(), + gateway=_Gateway(), + ) diff --git a/tests/providers/test_oci_container_instances_operations.py b/tests/providers/test_oci_container_instances_operations.py new file mode 100644 index 00000000..066fef66 --- /dev/null +++ b/tests/providers/test_oci_container_instances_operations.py @@ -0,0 +1,207 @@ +"""Manifest-bound OCI lifecycle operation contract tests.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import TYPE_CHECKING, Literal, cast + +import pytest + +import dander.providers.oci_container_instances.operations as operations_module +from dander.providers.oci_container_instances import ( + OCI_EXECUTION_SCHEMA, + OciContainerInstanceOperations, + OciExecution, + OciInvocation, + OciOperationBinding, + OciOperationError, +) +from dander.providers.oci_container_instances.controller import StoredExecution + +if TYPE_CHECKING: + from pytest import MonkeyPatch + + from dander.providers.oci_container_instances.oci_adapter import OciObjectRunRepository + +_COMPARTMENT = "ocid1.compartment.oc1.." + "b" * 32 +_FUNCTION = "ocid1.fnfunc.oc1.iad." + "f" * 32 +_RUN_ID = "oci-" + "a" * 24 + + +def _execution( + *, + state: Literal["pending", "running", "succeeded", "failed", "cancelled"] = "succeeded", + run_id: str = _RUN_ID, +) -> OciExecution: + return OciExecution( + schema=OCI_EXECUTION_SCHEMA, + run_id=run_id, + pipeline_id="warehouse_fixture", + idempotency_key="unit", + image="iad.ocir.io/unit/dander/runtime@sha256:" + "a" * 64, + state=state, + attempt=1, + max_attempts=2, + created_at="2026-08-12T12:00:00Z", + updated_at="2026-08-12T12:01:00Z", + deadline_at="2026-08-12T12:15:00Z", + instance_id="ocid1.computecontainerinstance.oc1.iad." + "c" * 32, + container_id="ocid1.computecontainer.oc1.iad." + "d" * 32, + exit_code=0, + ) + + +class _Repository: + def __init__(self, execution: OciExecution | None = None) -> None: + self.execution = execution + + def get( + self, + pipeline_id: str, + run_id: str | None = None, + ) -> StoredExecution | None: + assert pipeline_id == "warehouse_fixture" + if self.execution is None or (run_id is not None and run_id != self.execution.run_id): + return None + return StoredExecution(self.execution, "etag-unit") + + def get_logs(self, execution: OciExecution, *, attempt: int | None = None) -> bytes: + assert execution == self.execution + assert attempt in {None, 1} + return b"runtime complete\n" + + +class _Invoker: + def __init__(self, response: bytes = b"") -> None: + self.response = response + self.calls: list[tuple[str, dict[str, object], bool]] = [] + + def invoke(self, function_id: str, payload: dict[str, object], *, detached: bool) -> bytes: + self.calls.append((function_id, payload, detached)) + return self.response + + +def _binding() -> OciOperationBinding: + return OciOperationBinding( + region="us-ashburn-1", + namespace="unitnamespace", + bucket="dander-oci-runs-unit", + pipeline_id="warehouse_fixture", + function_id=_FUNCTION, + project_dir=Path("/tmp/dander-oci-operations-test"), + ) + + +def _operations( + repository: _Repository, + invoker: _Invoker, +) -> OciContainerInstanceOperations: + return OciContainerInstanceOperations( + _binding(), + repository=cast("OciObjectRunRepository", repository), + invoker=invoker, + clock=lambda: datetime(2026, 8, 12, 12, 0, tzinfo=UTC), + nonce=lambda: "nonceunit123", + ) + + +def test_binding_uses_manifest_launcher_and_exact_terraform_bucket_formula( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + manifest = SimpleNamespace( + launcher_provider="oci_container_instances", + pipelines={"warehouse_fixture": object()}, + validate_references=lambda _path: None, + resolved_launcher_config=lambda: { + "registry_namespace": "unitnamespace", + "compartment_id": _COMPARTMENT, + "region": "us-ashburn-1", + }, + ) + monkeypatch.setattr( + operations_module, + "load_project_config", + lambda *_args, **_kwargs: manifest, + ) + + binding = OciOperationBinding.from_project( + config=tmp_path / "dander.yaml", + deployment="oci_postgres", + pipeline_id="warehouse_fixture", + function_id=_FUNCTION, + ) + + suffix = hashlib.sha256(_COMPARTMENT.encode()).hexdigest()[:8] + assert binding.bucket == f"dander-oci-runs-{suffix}" + assert binding.function_id == _FUNCTION + assert binding.project_dir == tmp_path + + +def test_detached_start_returns_the_same_deterministic_run_identity_as_controller() -> None: + invoker = _Invoker() + operation = _operations(_Repository(), invoker) + + result = operation.start() + + assert isinstance(result, OciInvocation) + assert result.action == "start" + assert result.run_id.startswith("oci-") + assert invoker.calls == [ + ( + _FUNCTION, + { + "action": "start", + "idempotency_key": ("manual:warehouse_fixture:20260812T120000Z:nonceunit123"), + }, + True, + ) + ] + + +def test_status_and_logs_are_read_from_the_exact_manifest_bound_repository() -> None: + execution = _execution() + operation = _operations(_Repository(execution), _Invoker()) + + assert operation.latest() == execution + assert operation.describe(_RUN_ID) == execution + assert operation.logs(_RUN_ID, attempt=1) == b"runtime complete\n" + + +def test_cancel_parses_only_the_sanitized_controller_execution_contract() -> None: + execution = _execution(state="cancelled") + invoker = _Invoker(json.dumps(execution.as_dict()).encode()) + + cancelled = _operations(_Repository(execution), invoker).cancel(_RUN_ID) + + assert cancelled == execution + assert invoker.calls == [(_FUNCTION, {"action": "cancel", "run_id": _RUN_ID}, False)] + + +def test_replay_requires_terminal_history_and_returns_a_new_caller_known_run_id() -> None: + invoker = _Invoker() + operation = _operations(_Repository(_execution()), invoker) + + replay = operation.replay(_RUN_ID) + + assert replay.action == "replay" + assert replay.run_id != _RUN_ID + assert invoker.calls[0][1] == { + "action": "replay", + "run_id": _RUN_ID, + "idempotency_key": "replay:warehouse_fixture:20260812T120000Z:nonceunit123", + } + assert invoker.calls[0][2] is True + + +def test_operation_failures_do_not_echo_invalid_provider_responses() -> None: + operation = _operations(_Repository(_execution()), _Invoker(b"not-json-secret")) + + with pytest.raises(OciOperationError, match="invalid response") as raised: + operation.cancel(_RUN_ID) + + assert "not-json-secret" not in str(raised.value) diff --git a/tests/providers/test_oci_function_handler.py b/tests/providers/test_oci_function_handler.py new file mode 100644 index 00000000..ba4e2e48 --- /dev/null +++ b/tests/providers/test_oci_function_handler.py @@ -0,0 +1,134 @@ +"""OCI Functions lifecycle dispatch contract without provider access.""" + +from __future__ import annotations + +import io +import json +from dataclasses import replace +from typing import TYPE_CHECKING + +import pytest + +import dander.providers.oci_container_instances.function_handler as function_handler +from dander.providers.oci_container_instances.controller import ( + OCI_EXECUTION_SCHEMA, + OciExecution, + OciLifecycleError, +) + +if TYPE_CHECKING: + from pytest import MonkeyPatch + + +def _execution(*, state: str = "running") -> OciExecution: + return OciExecution( + schema=OCI_EXECUTION_SCHEMA, + run_id="oci-" + "a" * 24, + pipeline_id="jobs", + idempotency_key="manual:unit", + image="ocir.us-ashburn-1.oci.oraclecloud.com/unit/dander@sha256:" + "a" * 64, + state=state, # type: ignore[arg-type] + attempt=1, + max_attempts=2, + created_at="2026-08-12T12:00:00Z", + updated_at="2026-08-12T12:00:01Z", + deadline_at="2026-08-12T12:15:00Z", + ) + + +class _Repository: + def get_projection(self, key: str) -> dict[str, object]: + assert key == "projections/jobs.json" + return {"pipeline_id": "jobs"} + + +class _Controller: + calls: list[tuple[str, object]] = [] + + def __init__(self, **kwargs: object) -> None: + assert kwargs["projection"] == {"pipeline_id": "jobs"} + + def start(self, *, idempotency_key: str) -> OciExecution: + self.calls.append(("start", idempotency_key)) + return _execution() + + def reconcile(self, run_id: str | None = None) -> OciExecution | None: + self.calls.append(("reconcile", run_id)) + if run_id is None: + return None + return replace(_execution(), state="succeeded", exit_code=0) + + def cancel(self, run_id: str) -> OciExecution: + self.calls.append(("cancel", run_id)) + return replace(_execution(), state="cancelled", failure_code="interrupted_run") + + def replay(self, run_id: str, *, idempotency_key: str) -> OciExecution: + self.calls.append(("replay", (run_id, idempotency_key))) + return _execution() + + +@pytest.fixture(autouse=True) +def _install_function_boundary(monkeypatch: MonkeyPatch) -> None: + _Controller.calls = [] + monkeypatch.setenv("DANDER_OCI_NAMESPACE", "unitnamespace") + monkeypatch.setenv("DANDER_OCI_RUN_BUCKET", "unit-runs") + monkeypatch.setenv("DANDER_OCI_PIPELINE", "jobs") + monkeypatch.setenv("DANDER_OCI_PROJECTION_KEY", "projections/jobs.json") + monkeypatch.setattr( + function_handler, + "build_resource_principal_adapters", + lambda **_kwargs: (_Repository(), object()), + ) + monkeypatch.setattr(function_handler, "OciLifecycleController", _Controller) + monkeypatch.setattr( + "dander.providers.oci_container_instances.function_handler.time.sleep", + lambda _seconds: None, + ) + + +def _document(result: object) -> tuple[int, object]: + assert isinstance(result, dict) + return int(result["status"]), json.loads(str(result["body"])) + + +def test_manual_start_monitors_to_terminal_with_exact_idempotency_key() -> None: + result = function_handler.handler( + object(), + io.BytesIO(b'{"action":"start","idempotency_key":"manual:unit"}'), + ) + + status, document = _document(result) + assert status == 200 + assert isinstance(document, dict) and document["state"] == "succeeded" + assert _Controller.calls == [ + ("start", "manual:unit"), + ("reconcile", "oci-" + "a" * 24), + ] + + +def test_lifecycle_event_reconciles_once_without_starting_or_polling() -> None: + result = function_handler.handler( + object(), + io.BytesIO(b'{"cloudEventsVersion":"0.1","eventType":"unit"}'), + ) + + status, document = _document(result) + assert status == 200 + assert document is None + assert _Controller.calls == [("reconcile", None)] + + +def test_rejected_payload_is_bounded_and_does_not_echo_input() -> None: + result = function_handler.handler( + object(), + io.BytesIO(b'{"action":"customer-secret-value"}'), + ) + + status, document = _document(result) + assert status == 409 + assert isinstance(document, dict) + assert document["failure_code"] == "controller_rejected" + assert "customer-secret-value" not in json.dumps(document) + + with pytest.raises(OciLifecycleError, match="too large"): + function_handler._payload(io.BytesIO(b"x" * 65_537)) diff --git a/tests/state/test_failure.py b/tests/state/test_failure.py index d04e4585..bb9fd41a 100644 --- a/tests/state/test_failure.py +++ b/tests/state/test_failure.py @@ -31,6 +31,20 @@ def test_failure_classifier_uses_http_status_without_persisting_exception_text() assert len(failure.summary) <= 512 +def test_failure_classifier_normalizes_direct_provider_status() -> None: + class ServiceError(RuntimeError): + status = 429 + + failure = classify_failure( + ServiceError("OCI request detail must not persist"), + stage=RunStage.INGEST, + run_id="safe-run", + ) + + assert failure.code == "rate_limited" + assert "OCI request detail" not in failure.summary + + def test_unknown_failure_is_stage_specific_and_points_to_run_logs() -> None: failure = classify_failure( RuntimeError("customer payload must never persist"), diff --git a/tests/test_runtime_secrets.py b/tests/test_runtime_secrets.py new file mode 100644 index 00000000..f0956e1c --- /dev/null +++ b/tests/test_runtime_secrets.py @@ -0,0 +1,122 @@ +"""Run-scoped launcher secret-reference resolution.""" + +from __future__ import annotations + +import json + +import pytest + +from dander.runtime_secrets import ( + RuntimeSecretBindingError, + projected_secret_environment, +) + + +class _Store: + def __init__(self) -> None: + self.references: list[str] = [] + + def get_secret(self, reference: str) -> str: + self.references.append(reference) + return "resolved-value" + + +class _Runtime: + def __init__(self, store: _Store) -> None: + self.store = store + + +class _Registry: + def __init__(self, store: _Store) -> None: + self.store = store + + def parse(self, kind: object, raw: object) -> object: + del kind + assert raw == {"provider": "oci_vault"} + return object() + + def build(self, kind: object, config: object) -> object: + del kind, config + from dander.security import SecretCapabilities, SecretRuntime + + return SecretRuntime( + provider_id="oci_vault", + store=self.store, + capabilities=SecretCapabilities( + provider_id="oci_vault", + reference_forms=frozenset({"oci_vault_secret_name"}), + environment_indirection=False, + audited_access=True, + ), + ) + + +def test_projected_secret_environment_is_scoped_and_resolves_only_references( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _Store() + monkeypatch.setattr( + "dander.runtime_secrets.default_provider_registry", lambda: _Registry(store) + ) + environment = { + "DANDER_SECRET_BINDINGS_JSON": json.dumps( + { + "DANDER_POSTGRES_DSN": { + "provider": "oci_vault", + "reference": "oci-vault://ocid1.vault.oc1.iad.unit/secrets/postgres-dsn", + } + } + ) + } + + with projected_secret_environment(environ=environment): + assert environment["DANDER_POSTGRES_DSN"] == "resolved-value" + + assert "DANDER_POSTGRES_DSN" not in environment + assert store.references == ["oci-vault://ocid1.vault.oc1.iad.unit/secrets/postgres-dsn"] + + +@pytest.mark.parametrize( + ("document", "message"), + [ + ({}, "non-empty"), + ({"bad-name": {}}, "names"), + ( + {"TOKEN": {"provider": "environment", "reference": "env://TOKEN"}}, + "provider", + ), + ( + {"TOKEN": {"provider": "oci_vault", "reference": "plain-text"}}, + "reference", + ), + ], +) +def test_projected_secret_environment_fails_closed( + document: object, + message: str, +) -> None: + environment = {"DANDER_SECRET_BINDINGS_JSON": json.dumps(document)} + with ( + pytest.raises(RuntimeSecretBindingError, match=message), + projected_secret_environment(environ=environment), + ): + pass + + +def test_projected_secret_environment_rejects_ambiguous_existing_values() -> None: + environment = { + "DANDER_SECRET_BINDINGS_JSON": json.dumps( + { + "TOKEN": { + "provider": "oci_vault", + "reference": "oci-vault://ocid1.vault.oc1.iad.unit/secrets/token", + } + } + ), + "TOKEN": "already-set", + } + with ( + pytest.raises(RuntimeSecretBindingError, match="preconfigured"), + projected_secret_environment(environ=environment), + ): + pass diff --git a/tickets/DANDER-115-oci-lifecycle-controller.md b/tickets/DANDER-115-oci-lifecycle-controller.md new file mode 100644 index 00000000..f4c7c3ff --- /dev/null +++ b/tickets/DANDER-115-oci-lifecycle-controller.md @@ -0,0 +1,53 @@ +--- +id: DANDER-115 +title: Control OCI Container Instances lifecycle +status: in-review +component: python +epic: cloud-portability-phase-7 +depends_on: [DANDER-84] +created: 2026-08-12 +--- + +## Context + +OCI Container Instances has no native batch-job controller. Phase 7 therefore requires one narrow +Function that projects the existing provider-neutral runtime contract without adding warehouse, +state, SQL, ingestion, planner, or orchestration semantics. + +## Acceptance Criteria + +- [x] Object Storage conditional writes provide deterministic delivery idempotency, one active run + per pipeline, sanitized terminal history, and bounded per-attempt logs. +- [x] Each attempt creates a new private, resource-principal Container Instance with restart policy + `NEVER`; only runtime exit code 75 receives a bounded whole-task retry. +- [x] The controller owns the absolute deadline, interruption, replay, log capture, and deletion. +- [x] Per-pipeline UTC Resource Scheduler and lifecycle-event rules invoke only the matching + Function; Function dynamic-group membership uses exact Function OCIDs. +- [x] Runtime processes resolve only validated OCI Vault references, never put values in Terraform + or Function configuration, and remove injected values after execution. +- [x] Manifest-bound `run`, `status`, `logs`, `cancel`, and `replay` operations use only an expiring + SecurityToken profile and require confirmation for mutations. +- [x] The source-free Python 3.12 controller recipe installs one exact Dander wheel and uses pinned + `linux/amd64` Function base images matching Terraform's `GENERIC_X86` application. +- [x] Focused Python and Terraform tests pass without OCI access. +- [ ] Protected CI passes and this implementation merges before any live OCI deployment. +- [ ] A separately approved live proof covers schedule, retry, overlap, interruption, replay, + secret rotation, rollback, cleanup, and no drift. + +## Design + +Keep lifecycle state provider-local: immutable projection objects and compare-and-swap active run +records in Object Storage, one short Function shim, and one run-scoped Container Instance per +attempt. The existing runtime command remains the task boundary. Provider output is normalized at +the adapter edge and repository records reject unknown fields. + +## Implementation Notes + +The controller reserves five minutes of the one-hour detached Function ceiling, caps tasks at +3,300 seconds, and retains at most 256 KiB of logs per attempt. Resource Scheduler's provider limit +is represented honestly as UTC-only and no more frequent than hourly. Cross-cloud identity is not +implemented and static keys remain rejected. + +## Review Log + +Protected review and live acceptance remain pending. From be9d797d325dffe714977b06c149ab99896f6b13 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 02:58:29 -0400 Subject: [PATCH 150/163] feat(oci): add digest-preserving OCIR promotion --- HANDOFF.md | 35 +- docs/cloud-portability-plan.md | 7 +- docs/decisions.md | 13 + docs/known-limitations.md | 5 +- infra/oci/README.md | 29 +- src/dander/bootstrap/__init__.py | 2 + src/dander/bootstrap/oci_image.py | 412 ++++++++++++++++++ src/dander/cli/oci_command.py | 40 ++ tests/bootstrap/test_oci_image_promotion.py | 290 ++++++++++++ tests/cli/test_oci_cli.py | 50 +++ .../DANDER-115-oci-lifecycle-controller.md | 5 +- tickets/DANDER-116-ocir-runtime-promotion.md | 41 ++ 12 files changed, 899 insertions(+), 30 deletions(-) create mode 100644 src/dander/bootstrap/oci_image.py create mode 100644 tests/bootstrap/test_oci_image_promotion.py create mode 100644 tickets/DANDER-116-ocir-runtime-promotion.md diff --git a/HANDOFF.md b/HANDOFF.md index 9d847318..97ceb3f5 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,39 +2,38 @@ ## Finished -- Implemented an idempotent OCI Function controller with one fresh Container Instance per attempt. -- Added Resource Scheduler, pipeline-scoped lifecycle events, exact resource principals, logs, and alarms. -- Added crash-recoverable Object Storage run records plus bounded logs, interruption, replay, and retry. -- Added manifest-bound OCI run/status/logs/cancel/replay and exact launcher planning/no-drift inputs. -- Added runtime resolution of versionless OCI Vault references and a source-free controller image. +- Merged protected PR #215 for the OCI lifecycle controller; protected-main CI is green. +- Added digest-preserving OCIR runtime promotion with no provider-specific rebuild. +- Derived a repository-scoped registry token from the expiring OCI SecurityToken session. +- Kept the token in a mode-0600 temporary Docker config while preserving source-registry helpers. +- Added fail-closed repository, index, platform-map, idempotency, cleanup, and CLI contracts. ## Try It -Run `uv run pytest tests/providers/test_oci_container_instances_controller.py`. +Run `uv run pytest tests/bootstrap/test_oci_image_promotion.py`. ## Checks -- Complete pytest, Ruff lint/format, and repository-wide Mypy pass. -- Both OCI Terraform roots validate; their native tests pass (2 foundation/controller, 1 stage zero). -- Wheel/sdist build and the distribution contract pass. -- The `linux/amd64` Function image builds and imports Dander `0.9.0rc1` with OCI SDK `2.184.1`. -- No state, plan, key, or credential file is present in the worktree diff. +- Focused OCIR promotion and OCI CLI tests pass. +- Focused Ruff lint/format and Mypy pass. +- Protected-main CI at `e79e30be67cc9abd1df14dfb941a0046c8bacc50` passes. ## Decisions -- The Function owns parallelism one, exit-75 whole-task retry, a 3,300-second deadline, and cleanup. -- Scheduler delivery is UTC/hourly; event rules and controller dynamic groups are pipeline/exact-OCID scoped. -- Runtime task images remain source-free; the Function image contains only the exact wheel and dependencies. +- Runtime indexes are copied, never rebuilt, and must retain index plus platform digests. +- OCIR user auth tokens and registry passwords remain prohibited; use scoped ephemeral access tokens. +- The reviewed stage-zero repository must already be private, immutable, and available. ## Remaining -- Merge this implementation through a protected PR and verify protected-main CI. +- Merge the focused OCIR promotion implementation and verify protected-main CI. +- Add protected controller-image publication from an exact reviewed wheel. - Obtain explicit approval for public candidate publication and a numeric OCI per-attempt ceiling. - Run read-only OCI preflight, reviewed plans, live profile/rotation/rollback/cleanup, and no drift. - Merge sanitized live evidence, then make the binary Phase 7 exit-gate recommendation. ## Review First -- `src/dander/providers/oci_container_instances/controller.py` -- `src/dander/providers/oci_container_instances/oci_adapter.py` -- `infra/oci/main.tf` +- `src/dander/bootstrap/oci_image.py` +- `tests/bootstrap/test_oci_image_promotion.py` +- `src/dander/cli/oci_command.py` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 8099714d..bf43901f 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1215,8 +1215,11 @@ Function owns deterministic idempotency, overlap exclusion, whole-task retries, interruption, replay, bounded logs, terminal history, per-pipeline lifecycle-event reconciliation, and UTC Resource Scheduler delivery; operator commands use only expiring SecurityToken sessions. Runtime processes resolve versionless Vault references through resource principals without putting -values in Terraform or Function configuration. Controller/runtime image publication, -credential-refresh, live-profile, rollback, cleanup, and no-drift evidence remain open. This +values in Terraform or Function configuration. Accepted runtime indexes can be copied into the +exact private immutable OCIR repository without rebuild using a short-lived repository-scoped token +derived from the operator's SecurityToken session; source, destination, and per-platform digests +must remain equal. Controller/runtime image publication, credential-refresh, live-profile, +rollback, cleanup, and no-drift evidence remain open. This progress is not an exit-gate or support claim, and paid OCI mutation remains disabled pending credential preflight and an explicit per-attempt ceiling. diff --git a/docs/decisions.md b/docs/decisions.md index d4b4838b..550e9d38 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1363,3 +1363,16 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc controller is a digest-pinned `GENERIC_X86` Function image assembled from the exact reviewed wheel; the task runtime remains the separately accepted source-free release image. Live OCI publication and provider proof remain separate gates. + +## 2026-08-12 — OCIR promotion uses a scoped token derived from SecurityToken auth + +- **Decision:** Copy the already accepted OCI runtime index with `docker buildx imagetools`; do not + rebuild it for OCI. Verify the source artifact record, source platform map, private immutable + destination repository, destination index digest, and destination platform map. +- **Authentication:** Request a short-lived token limited to `pull,push` on the exact OCIR + repository through the OCI CLI's signed `SecurityToken` session. Put it only in a mode-`0600` + temporary Docker configuration that preserves existing source-registry helpers and is deleted at + command exit. Do not create or accept a user auth token, registry password, or static API key. +- **Failure boundary:** Existing tags are accepted only when their digest is already identical. + Any repository-policy mismatch, short token lifetime, digest rewrite, or platform drift fails the + promotion before an OCI launcher may consume the image. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index dac88b5f..33b2a63f 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -84,8 +84,9 @@ remain Phase 8 work. interval shorter than one hour. OCI Vault references resolve the `CURRENT` version at the start of every runtime process and are removed from the process environment afterward; in-process rotation is not claimed. The controller image is a Python 3.12 `GENERIC_X86` image built from an - exact wheel, not the source-free task image. Digest promotion, provider deployment verification, - credential-rotation proof, and the complete live OCI profile remain unqualified. No OCI launcher + exact wheel, not the source-free task image. Digest-preserving OCIR promotion is implemented with + an ephemeral repository-scoped token, but live publication, provider deployment verification, + credential-rotation proof, and the complete OCI profile remain unqualified. No OCI launcher support claim exists until those separate gates pass. ## Ingestion and schemas diff --git a/infra/oci/README.md b/infra/oci/README.md index c421a199..34084da1 100644 --- a/infra/oci/README.md +++ b/infra/oci/README.md @@ -22,16 +22,33 @@ then copied into the typed `oci_container_instances` launcher configuration. 1. Run `dander init-oci-admin-plan`, review the saved stage-zero plan, and apply it only after explicit cost approval with `dander init-oci-admin-apply`. -2. Run `dander init-oci-plan` and apply the reviewed foundation plan with `dander init-oci-apply`. -3. Seed only the manifest-declared secret names in Vault. Secret values never enter Terraform. -4. Build and push the [controller image](controller/README.md) from a clean protected-main wheel. -5. Run `dander init-oci-launcher-plan` with the exact runtime digest, controller tag and controller +2. Promote the accepted source-free runtime index without rebuilding it: + + ```console + dander image-promote-oci \ + --source-image SOURCE@sha256:DIGEST \ + --compartment-id ocid1.compartment... \ + --registry-namespace NAMESPACE \ + --repository dander/runtime \ + --oci-profile DANDER + ``` + + The command verifies the accepted local artifact record and source platform map, requires the + exact private immutable repository, requests a repository-scoped access token from the expiring + OCI `SecurityToken` session, and uses only a temporary Docker configuration. It rejects any + index or platform-digest rewrite and records the immutable result locally. +3. Run `dander init-oci-plan` and apply the reviewed foundation plan with `dander init-oci-apply`. +4. Seed only the manifest-declared secret names in Vault. Secret values never enter Terraform. +5. Build and push the [controller image](controller/README.md) from a clean protected-main wheel. +6. Run `dander init-oci-launcher-plan` with the exact runtime digest, controller tag and controller digest. Review and apply that saved plan through the same `init-oci-apply` boundary. -6. Use `dander verify-oci-deployment` with all four controller inputs to require exact no drift. +7. Use `dander verify-oci-deployment` with all four controller inputs to require exact no drift. The Function owns maximum parallelism one, whole-task retry only for runtime exit code 75, a 3,300-second maximum runtime deadline, stop/delete cleanup, bounded logs, and immutable history. Resource Scheduler is UTC-only and does not admit a recurrence interval shorter than one hour. `dander oci run`, `cancel`, and `replay` require confirmation; `status` and `logs` are read-only. All operator API calls use an expiring `SecurityToken` profile, while the Function and Container -Instances use resource principals. Static OCI API keys and static cloud keys are not fallbacks. +Instances use resource principals. Runtime promotion derives a short-lived, repository-scoped +registry token from that session and never changes the operator's persistent Docker configuration. +Static OCI API keys, user auth tokens, registry passwords, and static cloud keys are not fallbacks. diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index e6f3c241..7bbab18b 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -12,6 +12,7 @@ ) from dander.bootstrap.azure_image import AzureRuntimeImagePromoter from dander.bootstrap.azure_terraform import AzureTerraformBootstrap, AzureTerraformBootstrapError +from dander.bootstrap.oci_image import OciRuntimeImagePromoter from dander.bootstrap.oci_terraform import ( OciAdministrativeBootstrap, OciTerraformBootstrap, @@ -50,6 +51,7 @@ "AzureTerraformBootstrap", "AzureTerraformBootstrapError", "OciAdministrativeBootstrap", + "OciRuntimeImagePromoter", "OciTerraformBootstrap", "OciTerraformBootstrapError", "build_oci_execution_projections", diff --git a/src/dander/bootstrap/oci_image.py b/src/dander/bootstrap/oci_image.py new file mode 100644 index 00000000..315b59ab --- /dev/null +++ b/src/dander/bootstrap/oci_image.py @@ -0,0 +1,412 @@ +"""Digest-preserving promotion of accepted runtime images into OCI OCIR.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING, Protocol + +from dander.bootstrap.project import ( + _ARTIFACT_SCHEMA, + _DIGEST, + _OCI_IMAGE, + ProjectBootstrapError, + _platform_manifests, + _write_artifact_record, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + +_REGION = re.compile(r"^[a-z]{2}(?:-[a-z0-9]+)+-[1-9][0-9]*$") +_COMPARTMENT_OCID = re.compile(r"^ocid1\.compartment\.oc[0-9]+\.\.[A-Za-z0-9]+$") +_NAMESPACE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,99}$") +_REPOSITORY = re.compile(r"^[a-z0-9]+(?:[._/-][a-z0-9]+)*$") +_PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") +_TAG_PREFIX = re.compile(r"^[a-z][a-z0-9-]{0,31}$") + + +class _Runner(Protocol): + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + ) -> subprocess.CompletedProcess[str]: ... + + +def _subprocess_runner( + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + check=check, + capture_output=capture_output, + text=text, + ) + + +class OciRuntimeImagePromoter: + """Copy one accepted OCI index into OCIR without rebuilding or static credentials.""" + + def __init__(self, project_dir: Path, *, runner: _Runner | None = None) -> None: + self._project_dir = project_dir.resolve() + self._runner = runner or _subprocess_runner + self._artifact_record_path: Path | None = None + + @property + def artifact_record_path(self) -> Path | None: + """Return the OCI artifact record written by the latest successful promotion.""" + return self._artifact_record_path + + def promote( + self, + *, + source_image: str, + compartment_id: str, + region: str, + namespace: str, + repository_name: str, + oci_profile: str = "DEFAULT", + tag_prefix: str = "promoted", + ) -> str: + """Promote an accepted image with a scoped token and verify identical OCI content.""" + self._artifact_record_path = None + source_match = _OCI_IMAGE.fullmatch(source_image) + if source_match is None: + raise ProjectBootstrapError("Source image must be an immutable OCI digest reference") + if _COMPARTMENT_OCID.fullmatch(compartment_id) is None: + raise ProjectBootstrapError("Invalid OCI image-promotion compartment") + if _REGION.fullmatch(region) is None: + raise ProjectBootstrapError("Invalid OCI image-promotion region") + if _NAMESPACE.fullmatch(namespace) is None: + raise ProjectBootstrapError("Invalid OCI registry namespace") + if _REPOSITORY.fullmatch(repository_name) is None: + raise ProjectBootstrapError("Invalid OCI registry repository name") + if _PROFILE.fullmatch(oci_profile) is None: + raise ProjectBootstrapError("Invalid OCI SecurityToken profile") + if _TAG_PREFIX.fullmatch(tag_prefix) is None: + raise ProjectBootstrapError("Invalid OCI image-promotion tag prefix") + + source_record = self._source_record(source_image=source_image) + expected_digest = source_match.group("digest") + expected_platforms = source_record["platform_manifests"] + assert isinstance(expected_platforms, dict) + source_raw = self._inspect(source_image).stdout + if _platform_manifests(source_raw, default_digest=expected_digest) != expected_platforms: + raise ProjectBootstrapError( + "Source registry content no longer matches the accepted artifact record" + ) + + host = f"ocir.{region}.oci.oraclecloud.com" + destination_repository = f"{host}/{namespace}/{repository_name}" + tag = f"{tag_prefix}-{expected_digest.removeprefix('sha256:')[:12]}" + tagged_destination = f"{destination_repository}:{tag}" + image_uri = f"{namespace}/{repository_name}:{tag}" + oci_prefix = ( + "oci", + "--profile", + oci_profile, + "--auth", + "security_token", + "--region", + region, + ) + lookup = ( + *oci_prefix, + "artifacts", + "container", + "image", + "lookup", + "--image-uri", + image_uri, + "--query", + "data.digest", + "--raw-output", + ) + + try: + self._verify_repository( + oci_prefix=oci_prefix, + compartment_id=compartment_id, + repository_name=repository_name, + ) + existing = self._runner( + lookup, + cwd=self._project_dir, + check=False, + capture_output=True, + text=True, + ) + created = existing.returncode != 0 + token = self._registry_token( + oci_prefix=oci_prefix, + host=host, + namespace=namespace, + repository_name=repository_name, + ) + with self._temporary_docker_config(host=host, identity_token=token) as config_dir: + if created: + self._runner( + ( + "docker", + "--config", + str(config_dir), + "buildx", + "imagetools", + "create", + "--prefer-index=false", + "--tag", + tagged_destination, + source_image, + ), + cwd=self._project_dir, + check=True, + ) + destination_digest = self._runner( + lookup, + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + else: + destination_digest = existing.stdout.strip() + + if destination_digest != expected_digest: + if created: + raise ProjectBootstrapError( + "OCIR rewrote the OCI index; the promoted artifact is not " + "byte-identical" + ) + raise ProjectBootstrapError( + "Existing immutable OCIR tag does not match the accepted OCI index" + ) + immutable_destination = f"{destination_repository}@{destination_digest}" + destination_raw = self._inspect( + immutable_destination, + docker_config=config_dir, + ).stdout + if ( + _platform_manifests( + destination_raw, + default_digest=destination_digest, + ) + != expected_platforms + ): + raise ProjectBootstrapError( + "OCIR platform manifests differ from the accepted source-free artifact" + ) + except (FileNotFoundError, subprocess.CalledProcessError) as error: + raise ProjectBootstrapError("Could not promote the runtime image into OCIR") from error + + record_path = self._project_dir / ".dander" / "runtime-artifact-oci.json" + try: + _write_artifact_record( + record_path, + { + **source_record, + "image": immutable_destination, + "tagged_image": tagged_destination, + "source_image": source_image, + "promotion": "registry-copy", + "authentication": "SecurityToken-scoped-access-token", + }, + ) + except OSError as error: + raise ProjectBootstrapError( + "Runtime image was promoted but its OCI artifact record could not be written" + ) from error + self._artifact_record_path = record_path + return immutable_destination + + def _source_record(self, *, source_image: str) -> dict[str, object]: + source_record_path = self._project_dir / ".dander" / "runtime-artifact.json" + try: + source_record = json.loads(source_record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ProjectBootstrapError( + "A valid source-free runtime artifact record is required before OCI promotion" + ) from error + source_match = _OCI_IMAGE.fullmatch(source_image) + assert source_match is not None + expected_digest = source_match.group("digest") + if not isinstance(source_record, dict): + raise ProjectBootstrapError( + "Source image does not match its accepted runtime artifact record" + ) + expected_platforms = source_record.get("platform_manifests") + if ( + source_record.get("schema") != _ARTIFACT_SCHEMA + or source_record.get("image") != source_image + or source_record.get("index_digest") != expected_digest + or not isinstance(expected_platforms, dict) + or not expected_platforms + or any( + not isinstance(key, str) + or not isinstance(value, str) + or _DIGEST.fullmatch(value) is None + for key, value in expected_platforms.items() + ) + ): + raise ProjectBootstrapError( + "Source image does not match its accepted runtime artifact record" + ) + return source_record + + def _verify_repository( + self, + *, + oci_prefix: tuple[str, ...], + compartment_id: str, + repository_name: str, + ) -> None: + response = self._runner( + ( + *oci_prefix, + "artifacts", + "container", + "repository", + "list", + "--compartment-id", + compartment_id, + "--all", + "--output", + "json", + ), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ) + try: + document = json.loads(response.stdout) + items = document["data"]["items"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise ProjectBootstrapError("OCI returned invalid repository metadata") from error + if not isinstance(items, list): + raise ProjectBootstrapError("OCI returned invalid repository metadata") + matches = [ + item + for item in items + if isinstance(item, dict) and item.get("display-name") == repository_name + ] + if len(matches) != 1: + raise ProjectBootstrapError("The exact OCI runtime repository does not exist") + repository = matches[0] + if ( + repository.get("is-immutable") is not True + or repository.get("is-public") is not False + or repository.get("lifecycle-state") != "AVAILABLE" + ): + raise ProjectBootstrapError( + "OCI runtime repository must be private, immutable, and available" + ) + + def _registry_token( + self, + *, + oci_prefix: tuple[str, ...], + host: str, + namespace: str, + repository_name: str, + ) -> str: + scope = f"repository:{namespace}/{repository_name}:pull,push" + response = self._runner( + ( + *oci_prefix, + "container-registry", + "access-token", + "get", + "--service", + host, + "--scope", + scope, + "--output", + "json", + ), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ) + try: + document = json.loads(response.stdout) + data = document["data"] + token = data["token"] + expires_in = data["expires-in"] + actual_scope = data["scope"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise ProjectBootstrapError("OCI returned an invalid registry access token") from error + if ( + not isinstance(token, str) + or not token + or not isinstance(expires_in, int) + or expires_in < 300 + or actual_scope != scope + ): + raise ProjectBootstrapError("OCI returned an invalid registry access token") + return token + + @contextmanager + def _temporary_docker_config(self, *, host: str, identity_token: str) -> Iterator[Path]: + try: + source_dir = Path(os.environ.get("DOCKER_CONFIG", Path.home() / ".docker")) + source_path = source_dir / "config.json" + if source_path.is_file(): + document = json.loads(source_path.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise ProjectBootstrapError("Docker configuration is not a JSON object") + else: + document = {} + auths = document.get("auths", {}) + if not isinstance(auths, dict): + raise ProjectBootstrapError("Docker configuration has invalid registry entries") + document["auths"] = {**auths, host: {"identitytoken": identity_token}} + with TemporaryDirectory(prefix="dander-ocir-") as directory: + config_dir = Path(directory) + config_path = config_dir / "config.json" + config_path.write_text( + json.dumps(document, separators=(",", ":")), + encoding="utf-8", + ) + config_path.chmod(0o600) + yield config_dir + except (OSError, json.JSONDecodeError) as error: + raise ProjectBootstrapError( + "Could not create an isolated Docker configuration for OCIR" + ) from error + + def _inspect( + self, + image: str, + *, + docker_config: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + prefix: tuple[str, ...] = ("docker",) + if docker_config is not None: + prefix = ("docker", "--config", str(docker_config)) + return self._runner( + (*prefix, "buildx", "imagetools", "inspect", "--raw", image), + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ) + + +__all__ = ["OciRuntimeImagePromoter"] diff --git a/src/dander/cli/oci_command.py b/src/dander/cli/oci_command.py index 05896663..e32fd87c 100644 --- a/src/dander/cli/oci_command.py +++ b/src/dander/cli/oci_command.py @@ -12,8 +12,10 @@ from dander.bootstrap import ( OciAdministrativeBootstrap, + OciRuntimeImagePromoter, OciTerraformBootstrap, OciTerraformBootstrapError, + ProjectBootstrapError, build_oci_execution_projections, ) from dander.project import ProjectConfigError, load_project_config @@ -30,6 +32,43 @@ oci_app = typer.Typer(help="Operate manifest-bound OCI Container Instances pipelines.") +def image_promote_oci( + source_image: str = typer.Option( + ..., "--source-image", help="Accepted source-free OCI image ending in @sha256 digest." + ), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--registry-namespace"), + repository_name: str = typer.Option("dander/runtime", "--repository"), + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + tag_prefix: str = typer.Option("promoted", "--tag-prefix"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 +) -> None: + """Copy an accepted OCI index into OCIR using one short-lived scoped token.""" + if not typer.confirm( + f"Copy the accepted runtime image into OCI repository {repository_name!r}?", + default=False, + ): + raise typer.Abort() + try: + promoter = OciRuntimeImagePromoter(config.resolve().parent) + image = promoter.promote( + source_image=source_image, + compartment_id=compartment_id, + region=region, + namespace=namespace, + repository_name=repository_name, + oci_profile=oci_profile, + tag_prefix=tag_prefix, + ) + except ProjectBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]Promoted byte-identical runtime image:[/green] {image}") + if promoter.artifact_record_path is not None: + console.print(f"OCI artifact record: {promoter.artifact_record_path}") + console.print("Next: use this immutable digest with dander init-oci-launcher-plan.") + + def _oci_operations( *, config: Path, @@ -631,6 +670,7 @@ def _print_plan(label: str, operator_artifact_dir: Path, plan_path: Path) -> Non def register_oci_commands(app: typer.Typer) -> None: """Register OCI plan-first administrative commands.""" app.add_typer(oci_app, name="oci") + app.command("image-promote-oci")(image_promote_oci) app.command("init-oci-admin-plan")(init_oci_admin_plan) app.command("init-oci-admin-apply")(init_oci_admin_apply) app.command("init-oci-plan")(init_oci_plan) diff --git a/tests/bootstrap/test_oci_image_promotion.py b/tests/bootstrap/test_oci_image_promotion.py new file mode 100644 index 00000000..9999dccd --- /dev/null +++ b/tests/bootstrap/test_oci_image_promotion.py @@ -0,0 +1,290 @@ +"""Scoped-token registry-copy coverage for accepted OCI runtime images.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +import pytest + +from dander.bootstrap import OciRuntimeImagePromoter, ProjectBootstrapError + +_AMD64_DIGEST = "sha256:" + "b" * 64 +_ARM64_DIGEST = "sha256:" + "c" * 64 +_COMPARTMENT_ID = "ocid1.compartment.oc1.." + "a" * 32 +_TOKEN = "unit-scoped-registry-token" + + +def _index(*, arm64_digest: str = _ARM64_DIGEST) -> str: + return json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "digest": _AMD64_DIGEST, + "platform": {"os": "linux", "architecture": "amd64"}, + }, + { + "digest": arm64_digest, + "platform": {"os": "linux", "architecture": "arm64"}, + }, + ], + }, + separators=(",", ":"), + ) + + +_INDEX = _index() +_INDEX_DIGEST = "sha256:" + hashlib.sha256(_INDEX.encode()).hexdigest() +_SOURCE_IMAGE = f"us-central1-docker.pkg.dev/unit-project/dander/dander@{_INDEX_DIGEST}" + + +def _write_source_record(project_dir: Path, *, image: str = _SOURCE_IMAGE) -> None: + artifact_dir = project_dir / ".dander" + artifact_dir.mkdir() + (artifact_dir / "runtime-artifact.json").write_text( + json.dumps( + { + "schema": "io.dander.runtime.artifact/v1", + "image": image, + "tagged_image": "source:accepted", + "index_digest": _INDEX_DIGEST, + "platform_manifests": { + "linux/amd64": _AMD64_DIGEST, + "linux/arm64": _ARM64_DIGEST, + }, + "source": "https://github.com/harrisonoconnorhover/dander", + "revision": "unit", + "created": "2026-08-12T00:00:00Z", + "sbom_attached": True, + "provenance_attached": True, + } + ), + encoding="utf-8", + ) + + +class _Runner: + def __init__( + self, + *, + existing: bool = False, + destination_digest: str = _INDEX_DIGEST, + destination_raw: str = _INDEX, + expires_in: int = 900, + repository_private: bool = True, + ) -> None: + self.existing = existing + self.destination_digest = destination_digest + self.destination_raw = destination_raw + self.expires_in = expires_in + self.repository_private = repository_private + self.created = False + self.commands: list[tuple[str, ...]] = [] + self.config_paths: list[Path] = [] + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + ) -> subprocess.CompletedProcess[str]: + del capture_output, text + assert cwd.is_absolute() + self.commands.append(args) + if args[:4] == ("docker", "buildx", "imagetools", "inspect"): + assert args[-1] == _SOURCE_IMAGE + return subprocess.CompletedProcess(args, 0, stdout=_INDEX) + if args[0] == "oci" and "repository" in args and "list" in args: + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "data": { + "items": [ + { + "display-name": "dander/runtime", + "is-immutable": True, + "is-public": not self.repository_private, + "lifecycle-state": "AVAILABLE", + } + ] + } + } + ), + ) + if args[0] == "oci" and "lookup" in args: + if not self.existing and not self.created: + assert not check + return subprocess.CompletedProcess(args, 1, stdout="") + return subprocess.CompletedProcess(args, 0, stdout=self.destination_digest + "\n") + if args[0] == "oci" and "access-token" in args: + scope = "repository:unitnamespace/dander/runtime:pull,push" + assert args[args.index("--scope") + 1] == scope + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "data": { + "token": _TOKEN, + "expires-in": self.expires_in, + "scope": scope, + } + } + ), + ) + if args[:2] == ("docker", "--config"): + config_path = Path(args[2]) / "config.json" + self.config_paths.append(config_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + assert config["auths"]["ocir.us-ashburn-1.oci.oraclecloud.com"] == { + "identitytoken": _TOKEN + } + assert config["credHelpers"] == {"us-central1-docker.pkg.dev": "gcloud"} + if "create" in args: + self.created = True + return subprocess.CompletedProcess(args, 0, stdout="") + if "inspect" in args: + return subprocess.CompletedProcess(args, 0, stdout=self.destination_raw) + raise AssertionError(f"Unexpected command: {args}") + + +def _docker_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + directory = tmp_path / "docker" + directory.mkdir() + (directory / "config.json").write_text( + json.dumps({"credHelpers": {"us-central1-docker.pkg.dev": "gcloud"}}), + encoding="utf-8", + ) + monkeypatch.setenv("DOCKER_CONFIG", str(directory)) + return directory + + +def _promote(project_dir: Path, runner: _Runner) -> tuple[OciRuntimeImagePromoter, str]: + promoter = OciRuntimeImagePromoter(project_dir, runner=runner) + image = promoter.promote( + source_image=_SOURCE_IMAGE, + compartment_id=_COMPARTMENT_ID, + region="us-ashburn-1", + namespace="unitnamespace", + repository_name="dander/runtime", + oci_profile="DANDER", + ) + return promoter, image + + +def test_oci_promoter_copies_and_verifies_with_only_a_scoped_temporary_token( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + runner = _Runner() + + promoter, image = _promote(tmp_path, runner) + + assert image == ( + f"ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime@{_INDEX_DIGEST}" + ) + create = next(command for command in runner.commands if "create" in command) + assert "--prefer-index=false" in create + assert create[-1] == _SOURCE_IMAGE + assert all("build" not in command for command in runner.commands) + assert all(_TOKEN not in command for command in runner.commands) + assert any( + "--auth" in command and command[command.index("--auth") + 1] == "security_token" + for command in runner.commands + if command[0] == "oci" + ) + assert runner.config_paths + assert all(not path.exists() for path in runner.config_paths) + assert promoter.artifact_record_path == tmp_path / ".dander" / "runtime-artifact-oci.json" + record = json.loads(promoter.artifact_record_path.read_text(encoding="utf-8")) + assert record["source_image"] == _SOURCE_IMAGE + assert record["image"] == image + assert record["promotion"] == "registry-copy" + assert record["authentication"] == "SecurityToken-scoped-access-token" + + +def test_oci_promoter_is_idempotent_for_an_existing_identical_tag( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + runner = _Runner(existing=True) + + _, image = _promote(tmp_path, runner) + + assert image.endswith("@" + _INDEX_DIGEST) + assert not any("create" in command for command in runner.commands) + + +def test_oci_promoter_rejects_registry_digest_rewrites( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + + with pytest.raises(ProjectBootstrapError, match="rewrote the OCI index"): + _promote( + tmp_path, + _Runner(destination_digest="sha256:" + "d" * 64), + ) + + +def test_oci_promoter_rejects_platform_manifest_drift( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + + with pytest.raises(ProjectBootstrapError, match="platform manifests differ"): + _promote( + tmp_path, + _Runner(destination_raw=_index(arm64_digest="sha256:" + "e" * 64)), + ) + + +def test_oci_promoter_rejects_short_lived_or_malformed_token( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + + with pytest.raises(ProjectBootstrapError, match="invalid registry access token"): + _promote(tmp_path, _Runner(expires_in=60)) + + +def test_oci_promoter_requires_the_reviewed_private_immutable_repository( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + + with pytest.raises(ProjectBootstrapError, match="private, immutable, and available"): + _promote(tmp_path, _Runner(repository_private=False)) + + +def test_oci_promoter_rejects_source_record_mismatch_before_provider_access( + tmp_path: Path, +) -> None: + _write_source_record(tmp_path, image="registry.example.invalid/dander@" + _INDEX_DIGEST) + runner = _Runner() + + with pytest.raises(ProjectBootstrapError, match="does not match"): + _promote(tmp_path, runner) + + assert runner.commands == [] diff --git a/tests/cli/test_oci_cli.py b/tests/cli/test_oci_cli.py index 5f0c8a61..667ce28f 100644 --- a/tests/cli/test_oci_cli.py +++ b/tests/cli/test_oci_cli.py @@ -58,6 +58,56 @@ def fake_execute(self: object, **kwargs: object) -> Path: assert "No OCI resources were created" in result.output +def test_oci_image_promotion_requires_confirmation_and_passes_typed_inputs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[dict[str, object]] = [] + digest = "sha256:" + "a" * 64 + + class _Promoter: + artifact_record_path = tmp_path / ".dander" / "runtime-artifact-oci.json" + + def __init__(self, project_dir: Path) -> None: + assert project_dir == tmp_path + + def promote(self, **kwargs: object) -> str: + calls.append(kwargs) + return f"ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime@{digest}" + + monkeypatch.setattr(oci_command, "OciRuntimeImagePromoter", _Promoter) + args = [ + "image-promote-oci", + "--source-image", + f"registry.example/source@{digest}", + "--compartment-id", + "ocid1.compartment.oc1.." + "b" * 32, + "--registry-namespace", + "unitnamespace", + "--oci-profile", + "DANDER", + "--config", + str(tmp_path / "dander.yaml"), + ] + + refused = CliRunner().invoke(app, args, input="n\n") + accepted = CliRunner().invoke(app, args, input="y\n") + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert calls == [ + { + "source_image": f"registry.example/source@{digest}", + "compartment_id": "ocid1.compartment.oc1.." + "b" * 32, + "region": "us-ashburn-1", + "namespace": "unitnamespace", + "repository_name": "dander/runtime", + "oci_profile": "DANDER", + "tag_prefix": "promoted", + } + ] + + def test_oci_foundation_apply_requires_confirmation( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tickets/DANDER-115-oci-lifecycle-controller.md b/tickets/DANDER-115-oci-lifecycle-controller.md index f4c7c3ff..612557e0 100644 --- a/tickets/DANDER-115-oci-lifecycle-controller.md +++ b/tickets/DANDER-115-oci-lifecycle-controller.md @@ -30,7 +30,7 @@ state, SQL, ingestion, planner, or orchestration semantics. - [x] The source-free Python 3.12 controller recipe installs one exact Dander wheel and uses pinned `linux/amd64` Function base images matching Terraform's `GENERIC_X86` application. - [x] Focused Python and Terraform tests pass without OCI access. -- [ ] Protected CI passes and this implementation merges before any live OCI deployment. +- [x] Protected CI passes and this implementation merges before any live OCI deployment. - [ ] A separately approved live proof covers schedule, retry, overlap, interruption, replay, secret rotation, rollback, cleanup, and no drift. @@ -50,4 +50,5 @@ implemented and static keys remain rejected. ## Review Log -Protected review and live acceptance remain pending. +Protected implementation PR #215 merged at `e79e30be67cc9abd1df14dfb941a0046c8bacc50`, and the +protected-main CI run passed. Live acceptance remains pending. diff --git a/tickets/DANDER-116-ocir-runtime-promotion.md b/tickets/DANDER-116-ocir-runtime-promotion.md new file mode 100644 index 00000000..116f8cb3 --- /dev/null +++ b/tickets/DANDER-116-ocir-runtime-promotion.md @@ -0,0 +1,41 @@ +--- +id: DANDER-116 +title: Promote the accepted runtime index into OCIR +status: in-review +component: python +epic: cloud-portability-phase-7 +depends_on: [DANDER-115] +created: 2026-08-12 +--- + +## Context + +Phase 7 must run the same accepted source-free release digest on OCI. OCI-specific rebuilding would +invalidate that artifact identity, while long-lived OCIR user auth tokens would violate the +keyless operator boundary. + +## Acceptance Criteria + +- [x] Require the exact accepted runtime artifact record and re-verify its source platform map. +- [x] Require the selected destination repository to exist, be private, immutable, and available. +- [x] Derive a short-lived `pull,push` token scoped to only that repository from an authenticated + OCI `SecurityToken` profile. +- [x] Preserve existing source-registry helpers in a mode-`0600` temporary Docker configuration; + never put the token in arguments or persistent configuration. +- [x] Copy without rebuilding and require equal index and per-platform digests at the destination. +- [x] Treat an existing equal immutable tag as idempotent and reject all mismatches. +- [x] Require explicit CLI confirmation and write only a local sanitized artifact record. +- [x] Focused contracts cover copy, idempotency, repository policy, token lifetime, digest rewrite, + platform drift, source-record mismatch, cleanup, and CLI confirmation. +- [ ] Protected CI passes and this implementation merges before live OCIR publication. +- [ ] Separately approved live publication proves the real registry preserves the accepted index. + +## Design + +Use OCI's Container Registry access-token endpoint with signature-header authentication from the +existing SecurityToken session. The returned identity token remains memory/temporary-file only and +is removed even when copy or verification fails. + +## Review Log + +Protected review and live publication remain pending. From cffa9d0fdb10a805213df965a7826213c6cf559c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 03:19:25 -0400 Subject: [PATCH 151/163] feat(oci): publish controller from reviewed wheel --- HANDOFF.md | 32 +- docs/cloud-portability-plan.md | 7 +- docs/decisions.md | 11 + src/dander/bootstrap/__init__.py | 3 +- src/dander/bootstrap/oci_image.py | 577 +++++++++++++----- src/dander/cli/oci_command.py | 39 ++ .../test_oci_controller_publication.py | 306 ++++++++++ tests/cli/test_oci_cli.py | 54 ++ tickets/DANDER-116-ocir-runtime-promotion.md | 5 +- .../DANDER-117-oci-controller-publication.md | 42 ++ 10 files changed, 916 insertions(+), 160 deletions(-) create mode 100644 tests/bootstrap/test_oci_controller_publication.py create mode 100644 tickets/DANDER-117-oci-controller-publication.md diff --git a/HANDOFF.md b/HANDOFF.md index 97ceb3f5..3458a327 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Merged protected PR #215 for the OCI lifecycle controller; protected-main CI is green. -- Added digest-preserving OCIR runtime promotion with no provider-specific rebuild. -- Derived a repository-scoped registry token from the expiring OCI SecurityToken session. -- Kept the token in a mode-0600 temporary Docker config while preserving source-registry helpers. -- Added fail-closed repository, index, platform-map, idempotency, cleanup, and CLI contracts. +- Merged protected PR #216 for digest-preserving OCIR runtime promotion. +- Added OCI controller publication bound to one exact reviewed wheel SHA-256. +- Build inputs now come only from the wheel through an ephemeral source-free context. +- Added deterministic immutable tagging, amd64 verification, and sanitized artifact recording. +- Added fail-closed rerun, scoped-token, confirmation, and cleanup contracts. ## Try It -Run `uv run pytest tests/bootstrap/test_oci_image_promotion.py`. +Run `uv run pytest tests/bootstrap/test_oci_controller_publication.py tests/cli/test_oci_cli.py`. ## Checks -- Focused OCIR promotion and OCI CLI tests pass. +- Focused controller, OCIR promotion, and OCI CLI tests pass. - Focused Ruff lint/format and Mypy pass. -- Protected-main CI at `e79e30be67cc9abd1df14dfb941a0046c8bacc50` passes. +- Protected-main CI at `fae47a3cf860ba74a7c40b63c84ca21b9db7c6a2` passes. ## Decisions -- Runtime indexes are copied, never rebuilt, and must retain index plus platform digests. -- OCIR user auth tokens and registry passwords remain prohibited; use scoped ephemeral access tokens. -- The reviewed stage-zero repository must already be private, immutable, and available. +- The controller is a wheel-built amd64 Function image, separate from the copied task runtime. +- A dirty checkout cannot contribute files to the controller build context. +- Existing controller tags require an exact local wheel/tag/digest binding. ## Remaining -- Merge the focused OCIR promotion implementation and verify protected-main CI. -- Add protected controller-image publication from an exact reviewed wheel. +- Merge the protected controller-publication PR and verify protected-main CI. +- Prepare the Phase 7 release candidate without publishing it. - Obtain explicit approval for public candidate publication and a numeric OCI per-attempt ceiling. -- Run read-only OCI preflight, reviewed plans, live profile/rotation/rollback/cleanup, and no drift. -- Merge sanitized live evidence, then make the binary Phase 7 exit-gate recommendation. +- Run approved live OCI publication, profile, rotation, rollback, cleanup, and no-drift proof. +- Merge sanitized evidence and make the binary Phase 7 exit-gate recommendation. ## Review First - `src/dander/bootstrap/oci_image.py` -- `tests/bootstrap/test_oci_image_promotion.py` +- `tests/bootstrap/test_oci_controller_publication.py` - `src/dander/cli/oci_command.py` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index bf43901f..10e4d5cd 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1218,8 +1218,11 @@ Runtime processes resolve versionless Vault references through resource principa values in Terraform or Function configuration. Accepted runtime indexes can be copied into the exact private immutable OCIR repository without rebuild using a short-lived repository-scoped token derived from the operator's SecurityToken session; source, destination, and per-platform digests -must remain equal. Controller/runtime image publication, credential-refresh, live-profile, -rollback, cleanup, and no-drift evidence remain open. This +must remain equal. The separately built Function controller is bound to the exact reviewed wheel +SHA-256: its Dockerfile, shim, and dependency pins are extracted from that wheel into an ephemeral +source-free build context, published only for `linux/amd64`, and recorded by immutable digest. +Live runtime/controller publication, credential-refresh, live-profile, rollback, cleanup, and +no-drift evidence remain open. This progress is not an exit-gate or support claim, and paid OCI mutation remains disabled pending credential preflight and an explicit per-attempt ceiling. diff --git a/docs/decisions.md b/docs/decisions.md index 550e9d38..8c676ac1 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1376,3 +1376,14 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Failure boundary:** Existing tags are accepted only when their digest is already identical. Any repository-policy mismatch, short token lifetime, digest rewrite, or platform drift fails the promotion before an OCI launcher may consume the image. + +## 2026-08-12 — OCI controller publication is bound to the reviewed wheel + +- **Build boundary:** The Function controller is not the portable task runtime. Build it for + `linux/amd64` from one exact SHA-256-qualified Dander wheel, extracting the Dockerfile, shim, and + dependency pins from that wheel into an ephemeral context; never build from the working tree. +- **Registry boundary:** Publish a deterministic wheel-bound tag to the same reviewed private, + immutable OCIR repository using the repository-scoped SecurityToken-derived access token. Record + the resulting digest locally without credentials or local paths. +- **Idempotency:** An existing immutable controller tag is reusable only when its digest has an + exact local artifact binding to the same wheel hash. Missing or mismatched bindings fail closed. diff --git a/src/dander/bootstrap/__init__.py b/src/dander/bootstrap/__init__.py index 7bbab18b..e79ebe7f 100644 --- a/src/dander/bootstrap/__init__.py +++ b/src/dander/bootstrap/__init__.py @@ -12,7 +12,7 @@ ) from dander.bootstrap.azure_image import AzureRuntimeImagePromoter from dander.bootstrap.azure_terraform import AzureTerraformBootstrap, AzureTerraformBootstrapError -from dander.bootstrap.oci_image import OciRuntimeImagePromoter +from dander.bootstrap.oci_image import OciControllerImagePublisher, OciRuntimeImagePromoter from dander.bootstrap.oci_terraform import ( OciAdministrativeBootstrap, OciTerraformBootstrap, @@ -51,6 +51,7 @@ "AzureTerraformBootstrap", "AzureTerraformBootstrapError", "OciAdministrativeBootstrap", + "OciControllerImagePublisher", "OciRuntimeImagePromoter", "OciTerraformBootstrap", "OciTerraformBootstrapError", diff --git a/src/dander/bootstrap/oci_image.py b/src/dander/bootstrap/oci_image.py index 315b59ab..0c522870 100644 --- a/src/dander/bootstrap/oci_image.py +++ b/src/dander/bootstrap/oci_image.py @@ -1,12 +1,16 @@ -"""Digest-preserving promotion of accepted runtime images into OCI OCIR.""" +"""Reviewed runtime and lifecycle-controller publication into OCI OCIR.""" from __future__ import annotations +import hashlib import json import os import re +import shutil import subprocess +import zipfile from contextlib import contextmanager +from email.parser import Parser from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Protocol @@ -29,6 +33,13 @@ _REPOSITORY = re.compile(r"^[a-z0-9]+(?:[._/-][a-z0-9]+)*$") _PROFILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") _TAG_PREFIX = re.compile(r"^[a-z][a-z0-9-]{0,31}$") +_WHEEL_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_CONTROLLER_ARTIFACT_SCHEMA = "io.dander.oci.controller.artifact/v1" +_CONTROLLER_BUILD_FILES = ( + "dander/templates/project/infra/oci/controller/Dockerfile", + "dander/templates/project/infra/oci/controller/func.py", + "dander/templates/project/infra/oci/controller/requirements.txt", +) class _Runner(Protocol): @@ -60,6 +71,165 @@ def _subprocess_runner( ) +def _oci_prefix(*, profile: str, region: str) -> tuple[str, ...]: + return ( + "oci", + "--profile", + profile, + "--auth", + "security_token", + "--region", + region, + ) + + +def _verify_repository( + *, + runner: _Runner, + cwd: Path, + oci_prefix: tuple[str, ...], + compartment_id: str, + repository_name: str, +) -> None: + response = runner( + ( + *oci_prefix, + "artifacts", + "container", + "repository", + "list", + "--compartment-id", + compartment_id, + "--all", + "--output", + "json", + ), + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + try: + document = json.loads(response.stdout) + items = document["data"]["items"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise ProjectBootstrapError("OCI returned invalid repository metadata") from error + if not isinstance(items, list): + raise ProjectBootstrapError("OCI returned invalid repository metadata") + matches = [ + item + for item in items + if isinstance(item, dict) and item.get("display-name") == repository_name + ] + if len(matches) != 1: + raise ProjectBootstrapError("The exact OCI runtime repository does not exist") + repository = matches[0] + if ( + repository.get("is-immutable") is not True + or repository.get("is-public") is not False + or repository.get("lifecycle-state") != "AVAILABLE" + ): + raise ProjectBootstrapError( + "OCI runtime repository must be private, immutable, and available" + ) + + +def _registry_token( + *, + runner: _Runner, + cwd: Path, + oci_prefix: tuple[str, ...], + host: str, + namespace: str, + repository_name: str, +) -> str: + scope = f"repository:{namespace}/{repository_name}:pull,push" + response = runner( + ( + *oci_prefix, + "container-registry", + "access-token", + "get", + "--service", + host, + "--scope", + scope, + "--output", + "json", + ), + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + try: + document = json.loads(response.stdout) + data = document["data"] + token = data["token"] + expires_in = data["expires-in"] + actual_scope = data["scope"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise ProjectBootstrapError("OCI returned an invalid registry access token") from error + if ( + not isinstance(token, str) + or not token + or not isinstance(expires_in, int) + or expires_in < 300 + or actual_scope != scope + ): + raise ProjectBootstrapError("OCI returned an invalid registry access token") + return token + + +@contextmanager +def _temporary_docker_config(*, host: str, identity_token: str) -> Iterator[Path]: + try: + source_dir = Path(os.environ.get("DOCKER_CONFIG", Path.home() / ".docker")) + source_path = source_dir / "config.json" + if source_path.is_file(): + document = json.loads(source_path.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise ProjectBootstrapError("Docker configuration is not a JSON object") + else: + document = {} + auths = document.get("auths", {}) + if not isinstance(auths, dict): + raise ProjectBootstrapError("Docker configuration has invalid registry entries") + document["auths"] = {**auths, host: {"identitytoken": identity_token}} + with TemporaryDirectory(prefix="dander-ocir-") as directory: + config_dir = Path(directory) + config_path = config_dir / "config.json" + config_path.write_text( + json.dumps(document, separators=(",", ":")), + encoding="utf-8", + ) + config_path.chmod(0o600) + yield config_dir + except (OSError, json.JSONDecodeError) as error: + raise ProjectBootstrapError( + "Could not create an isolated Docker configuration for OCIR" + ) from error + + +def _inspect( + *, + runner: _Runner, + cwd: Path, + image: str, + docker_config: Path | None = None, +) -> subprocess.CompletedProcess[str]: + prefix: tuple[str, ...] = ("docker",) + if docker_config is not None: + prefix = ("docker", "--config", str(docker_config)) + return runner( + (*prefix, "buildx", "imagetools", "inspect", "--raw", image), + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + class OciRuntimeImagePromoter: """Copy one accepted OCI index into OCIR without rebuilding or static credentials.""" @@ -106,7 +276,11 @@ def promote( expected_digest = source_match.group("digest") expected_platforms = source_record["platform_manifests"] assert isinstance(expected_platforms, dict) - source_raw = self._inspect(source_image).stdout + source_raw = _inspect( + runner=self._runner, + cwd=self._project_dir, + image=source_image, + ).stdout if _platform_manifests(source_raw, default_digest=expected_digest) != expected_platforms: raise ProjectBootstrapError( "Source registry content no longer matches the accepted artifact record" @@ -117,15 +291,7 @@ def promote( tag = f"{tag_prefix}-{expected_digest.removeprefix('sha256:')[:12]}" tagged_destination = f"{destination_repository}:{tag}" image_uri = f"{namespace}/{repository_name}:{tag}" - oci_prefix = ( - "oci", - "--profile", - oci_profile, - "--auth", - "security_token", - "--region", - region, - ) + oci_prefix = _oci_prefix(profile=oci_profile, region=region) lookup = ( *oci_prefix, "artifacts", @@ -140,7 +306,9 @@ def promote( ) try: - self._verify_repository( + _verify_repository( + runner=self._runner, + cwd=self._project_dir, oci_prefix=oci_prefix, compartment_id=compartment_id, repository_name=repository_name, @@ -153,13 +321,15 @@ def promote( text=True, ) created = existing.returncode != 0 - token = self._registry_token( + token = _registry_token( + runner=self._runner, + cwd=self._project_dir, oci_prefix=oci_prefix, host=host, namespace=namespace, repository_name=repository_name, ) - with self._temporary_docker_config(host=host, identity_token=token) as config_dir: + with _temporary_docker_config(host=host, identity_token=token) as config_dir: if created: self._runner( ( @@ -197,8 +367,10 @@ def promote( "Existing immutable OCIR tag does not match the accepted OCI index" ) immutable_destination = f"{destination_repository}@{destination_digest}" - destination_raw = self._inspect( - immutable_destination, + destination_raw = _inspect( + runner=self._runner, + cwd=self._project_dir, + image=immutable_destination, docker_config=config_dir, ).stdout if ( @@ -268,145 +440,272 @@ def _source_record(self, *, source_image: str) -> dict[str, object]: ) return source_record - def _verify_repository( + +class OciControllerImagePublisher: + """Build one OCI Functions controller solely from an exact reviewed wheel.""" + + def __init__(self, project_dir: Path, *, runner: _Runner | None = None) -> None: + self._project_dir = project_dir.resolve() + self._runner = runner or _subprocess_runner + self._artifact_record_path: Path | None = None + + @property + def artifact_record_path(self) -> Path | None: + """Return the controller artifact record written by the latest publication.""" + return self._artifact_record_path + + def publish( self, *, - oci_prefix: tuple[str, ...], + wheel: Path, + wheel_sha256: str, compartment_id: str, + region: str, + namespace: str, repository_name: str, - ) -> None: - response = self._runner( - ( - *oci_prefix, - "artifacts", - "container", - "repository", - "list", - "--compartment-id", - compartment_id, - "--all", - "--output", - "json", - ), - cwd=self._project_dir, - check=True, - capture_output=True, - text=True, + oci_profile: str = "DEFAULT", + ) -> str: + """Publish an amd64 controller with a wheel-bound tag and scoped registry token.""" + self._artifact_record_path = None + wheel_path = wheel.expanduser().resolve() + if _WHEEL_SHA256.fullmatch(wheel_sha256) is None: + raise ProjectBootstrapError("Invalid reviewed wheel SHA-256") + if _COMPARTMENT_OCID.fullmatch(compartment_id) is None: + raise ProjectBootstrapError("Invalid OCI controller-publication compartment") + if _REGION.fullmatch(region) is None: + raise ProjectBootstrapError("Invalid OCI controller-publication region") + if _NAMESPACE.fullmatch(namespace) is None: + raise ProjectBootstrapError("Invalid OCI registry namespace") + if _REPOSITORY.fullmatch(repository_name) is None: + raise ProjectBootstrapError("Invalid OCI registry repository name") + if _PROFILE.fullmatch(oci_profile) is None: + raise ProjectBootstrapError("Invalid OCI SecurityToken profile") + + version = self._verify_wheel(wheel_path=wheel_path, wheel_sha256=wheel_sha256) + host = f"ocir.{region}.oci.oraclecloud.com" + destination_repository = f"{host}/{namespace}/{repository_name}" + tag = f"controller-{wheel_sha256[:12]}" + tagged_destination = f"{destination_repository}:{tag}" + image_uri = f"{namespace}/{repository_name}:{tag}" + oci_prefix = _oci_prefix(profile=oci_profile, region=region) + lookup = ( + *oci_prefix, + "artifacts", + "container", + "image", + "lookup", + "--image-uri", + image_uri, + "--query", + "data.digest", + "--raw-output", ) + try: - document = json.loads(response.stdout) - items = document["data"]["items"] - except (json.JSONDecodeError, KeyError, TypeError) as error: - raise ProjectBootstrapError("OCI returned invalid repository metadata") from error - if not isinstance(items, list): - raise ProjectBootstrapError("OCI returned invalid repository metadata") - matches = [ - item - for item in items - if isinstance(item, dict) and item.get("display-name") == repository_name - ] - if len(matches) != 1: - raise ProjectBootstrapError("The exact OCI runtime repository does not exist") - repository = matches[0] - if ( - repository.get("is-immutable") is not True - or repository.get("is-public") is not False - or repository.get("lifecycle-state") != "AVAILABLE" - ): - raise ProjectBootstrapError( - "OCI runtime repository must be private, immutable, and available" + _verify_repository( + runner=self._runner, + cwd=self._project_dir, + oci_prefix=oci_prefix, + compartment_id=compartment_id, + repository_name=repository_name, ) + existing = self._runner( + lookup, + cwd=self._project_dir, + check=False, + capture_output=True, + text=True, + ) + created = existing.returncode != 0 + if created: + destination_digest = "" + else: + destination_digest = existing.stdout.strip() + self._require_matching_record( + wheel_sha256=wheel_sha256, + tagged_image=tagged_destination, + digest=destination_digest, + ) - def _registry_token( - self, - *, - oci_prefix: tuple[str, ...], - host: str, - namespace: str, - repository_name: str, - ) -> str: - scope = f"repository:{namespace}/{repository_name}:pull,push" - response = self._runner( - ( - *oci_prefix, - "container-registry", - "access-token", - "get", - "--service", - host, - "--scope", - scope, - "--output", - "json", - ), - cwd=self._project_dir, - check=True, - capture_output=True, - text=True, - ) + token = _registry_token( + runner=self._runner, + cwd=self._project_dir, + oci_prefix=oci_prefix, + host=host, + namespace=namespace, + repository_name=repository_name, + ) + with _temporary_docker_config(host=host, identity_token=token) as config_dir: + if created: + with TemporaryDirectory(prefix="dander-oci-controller-") as directory: + context_dir = Path(directory) + self._extract_controller_context( + wheel_path=wheel_path, + context_dir=context_dir, + ) + self._runner( + ( + "docker", + "--config", + str(config_dir), + "buildx", + "build", + "--platform", + "linux/amd64", + "--file", + "infra/oci/controller/Dockerfile", + "--build-arg", + "DANDER_WHEEL=dander-controller.whl", + "--provenance=false", + "--sbom=false", + "--push", + "--tag", + tagged_destination, + ".", + ), + cwd=context_dir, + check=True, + ) + destination_digest = self._runner( + lookup, + cwd=self._project_dir, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if _DIGEST.fullmatch(destination_digest) is None: + raise ProjectBootstrapError("OCIR returned an invalid controller image digest") + immutable_destination = f"{destination_repository}@{destination_digest}" + raw = _inspect( + runner=self._runner, + cwd=self._project_dir, + image=immutable_destination, + docker_config=config_dir, + ).stdout + platforms = _platform_manifests(raw, default_digest=destination_digest) + if set(platforms) != {"linux/amd64"}: + raise ProjectBootstrapError( + "OCI controller image must contain only the linux/amd64 runtime platform" + ) + except (FileNotFoundError, subprocess.CalledProcessError) as error: + raise ProjectBootstrapError("Could not publish the OCI lifecycle controller") from error + + record_path = self._project_dir / ".dander" / "oci-controller-artifact.json" try: - document = json.loads(response.stdout) - data = document["data"] - token = data["token"] - expires_in = data["expires-in"] - actual_scope = data["scope"] - except (json.JSONDecodeError, KeyError, TypeError) as error: - raise ProjectBootstrapError("OCI returned an invalid registry access token") from error - if ( - not isinstance(token, str) - or not token - or not isinstance(expires_in, int) - or expires_in < 300 - or actual_scope != scope - ): - raise ProjectBootstrapError("OCI returned an invalid registry access token") - return token + _write_artifact_record( + record_path, + { + "schema": _CONTROLLER_ARTIFACT_SCHEMA, + "image": immutable_destination, + "tagged_image": tagged_destination, + "image_digest": destination_digest, + "platform_manifests": platforms, + "wheel": wheel_path.name, + "wheel_sha256": wheel_sha256, + "package": "dander-platform", + "version": version, + "build": "exact-reviewed-wheel", + "authentication": "SecurityToken-scoped-access-token", + }, + ) + except OSError as error: + raise ProjectBootstrapError( + "OCI controller was published but its artifact record could not be written" + ) from error + self._artifact_record_path = record_path + return immutable_destination - @contextmanager - def _temporary_docker_config(self, *, host: str, identity_token: str) -> Iterator[Path]: + def _verify_wheel(self, *, wheel_path: Path, wheel_sha256: str) -> str: + if not wheel_path.is_file() or not wheel_path.name.startswith("dander_platform-"): + raise ProjectBootstrapError("The reviewed Dander wheel does not exist") + hasher = hashlib.sha256() try: - source_dir = Path(os.environ.get("DOCKER_CONFIG", Path.home() / ".docker")) - source_path = source_dir / "config.json" - if source_path.is_file(): - document = json.loads(source_path.read_text(encoding="utf-8")) - if not isinstance(document, dict): - raise ProjectBootstrapError("Docker configuration is not a JSON object") - else: - document = {} - auths = document.get("auths", {}) - if not isinstance(auths, dict): - raise ProjectBootstrapError("Docker configuration has invalid registry entries") - document["auths"] = {**auths, host: {"identitytoken": identity_token}} - with TemporaryDirectory(prefix="dander-ocir-") as directory: - config_dir = Path(directory) - config_path = config_dir / "config.json" - config_path.write_text( - json.dumps(document, separators=(",", ":")), - encoding="utf-8", + with wheel_path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(chunk) + except OSError as error: + raise ProjectBootstrapError("Could not read the reviewed Dander wheel") from error + if hasher.hexdigest() != wheel_sha256: + raise ProjectBootstrapError("The Dander wheel does not match the reviewed SHA-256") + + try: + with zipfile.ZipFile(wheel_path) as archive: + infos = archive.infolist() + names = [info.filename for info in infos] + if len(names) != len(set(names)) or any( + name.startswith("/") or "\\" in name or ".." in Path(name).parts + for name in names + ): + raise ProjectBootstrapError("The reviewed Dander wheel has unsafe paths") + for required in _CONTROLLER_BUILD_FILES: + try: + info = archive.getinfo(required) + except KeyError as error: + raise ProjectBootstrapError( + "The reviewed Dander wheel is missing OCI controller build files" + ) from error + if info.is_dir(): + raise ProjectBootstrapError( + "The reviewed Dander wheel has invalid OCI controller build files" + ) + metadata_names = [ + name + for name in names + if re.fullmatch(r"dander_platform-[^/]+\.dist-info/METADATA", name) + ] + if len(metadata_names) != 1: + raise ProjectBootstrapError("The reviewed Dander wheel has invalid metadata") + metadata = Parser().parsestr( + archive.read(metadata_names[0]).decode("utf-8", errors="strict") ) - config_path.chmod(0o600) - yield config_dir - except (OSError, json.JSONDecodeError) as error: + except (OSError, UnicodeDecodeError, zipfile.BadZipFile) as error: + raise ProjectBootstrapError("The reviewed Dander wheel is invalid") from error + version = metadata.get("Version", "") + if metadata.get("Name") != "dander-platform" or not re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9.!+-]*", version + ): + raise ProjectBootstrapError("The reviewed Dander wheel has invalid metadata") + return version + + def _extract_controller_context(self, *, wheel_path: Path, context_dir: Path) -> None: + try: + with zipfile.ZipFile(wheel_path) as archive: + for source in _CONTROLLER_BUILD_FILES: + relative = source.removeprefix("dander/templates/project/") + destination = context_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(archive.read(source)) + shutil.copyfile(wheel_path, context_dir / "dander-controller.whl") + except (KeyError, OSError, zipfile.BadZipFile) as error: raise ProjectBootstrapError( - "Could not create an isolated Docker configuration for OCIR" + "Could not create the reviewed OCI controller build context" ) from error - def _inspect( + def _require_matching_record( self, - image: str, *, - docker_config: Path | None = None, - ) -> subprocess.CompletedProcess[str]: - prefix: tuple[str, ...] = ("docker",) - if docker_config is not None: - prefix = ("docker", "--config", str(docker_config)) - return self._runner( - (*prefix, "buildx", "imagetools", "inspect", "--raw", image), - cwd=self._project_dir, - check=True, - capture_output=True, - text=True, - ) + wheel_sha256: str, + tagged_image: str, + digest: str, + ) -> None: + record_path = self._project_dir / ".dander" / "oci-controller-artifact.json" + try: + record = json.loads(record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ProjectBootstrapError( + "Existing immutable OCI controller tag has no matching local artifact record" + ) from error + if ( + not isinstance(record, dict) + or record.get("schema") != _CONTROLLER_ARTIFACT_SCHEMA + or record.get("wheel_sha256") != wheel_sha256 + or record.get("tagged_image") != tagged_image + or record.get("image_digest") != digest + or record.get("image") != tagged_image.rsplit(":", 1)[0] + f"@{digest}" + ): + raise ProjectBootstrapError( + "Existing immutable OCI controller tag does not match the reviewed wheel" + ) -__all__ = ["OciRuntimeImagePromoter"] +__all__ = ["OciControllerImagePublisher", "OciRuntimeImagePromoter"] diff --git a/src/dander/cli/oci_command.py b/src/dander/cli/oci_command.py index e32fd87c..9699d434 100644 --- a/src/dander/cli/oci_command.py +++ b/src/dander/cli/oci_command.py @@ -12,6 +12,7 @@ from dander.bootstrap import ( OciAdministrativeBootstrap, + OciControllerImagePublisher, OciRuntimeImagePromoter, OciTerraformBootstrap, OciTerraformBootstrapError, @@ -69,6 +70,43 @@ def image_promote_oci( console.print("Next: use this immutable digest with dander init-oci-launcher-plan.") +def image_publish_oci_controller( + wheel: Path = typer.Option( # noqa: B008 + ..., "--wheel", help="Exact reviewed Dander wheel." + ), + wheel_sha256: str = typer.Option(..., "--wheel-sha256"), + compartment_id: str = typer.Option(..., "--compartment-id"), + region: str = typer.Option("us-ashburn-1", "--region"), + namespace: str = typer.Option(..., "--registry-namespace"), + repository_name: str = typer.Option("dander/runtime", "--repository"), + oci_profile: str = typer.Option("DEFAULT", "--oci-profile"), + config: Path = typer.Option(Path("dander.yaml"), "--config"), # noqa: B008 +) -> None: + """Build the OCI lifecycle controller solely from an exact reviewed wheel.""" + if not typer.confirm( + f"Publish the wheel-bound OCI controller into repository {repository_name!r}?", + default=False, + ): + raise typer.Abort() + try: + publisher = OciControllerImagePublisher(config.resolve().parent) + image = publisher.publish( + wheel=wheel, + wheel_sha256=wheel_sha256, + compartment_id=compartment_id, + region=region, + namespace=namespace, + repository_name=repository_name, + oci_profile=oci_profile, + ) + except ProjectBootstrapError as error: + raise ClickException(str(error)) from error + console.print(f"[green]Published immutable OCI controller image:[/green] {image}") + if publisher.artifact_record_path is not None: + console.print(f"OCI controller artifact record: {publisher.artifact_record_path}") + console.print("Next: pass this digest-qualified image to dander init-oci-launcher-plan.") + + def _oci_operations( *, config: Path, @@ -671,6 +709,7 @@ def register_oci_commands(app: typer.Typer) -> None: """Register OCI plan-first administrative commands.""" app.add_typer(oci_app, name="oci") app.command("image-promote-oci")(image_promote_oci) + app.command("image-publish-oci-controller")(image_publish_oci_controller) app.command("init-oci-admin-plan")(init_oci_admin_plan) app.command("init-oci-admin-apply")(init_oci_admin_apply) app.command("init-oci-plan")(init_oci_plan) diff --git a/tests/bootstrap/test_oci_controller_publication.py b/tests/bootstrap/test_oci_controller_publication.py new file mode 100644 index 00000000..9d284a06 --- /dev/null +++ b/tests/bootstrap/test_oci_controller_publication.py @@ -0,0 +1,306 @@ +"""Exact-wheel OCI lifecycle-controller publication contracts.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import zipfile +from pathlib import Path + +import pytest + +from dander.bootstrap import OciControllerImagePublisher, ProjectBootstrapError + +_COMPARTMENT_ID = "ocid1.compartment.oc1.." + "a" * 32 +_DESTINATION_DIGEST = "sha256:" + "d" * 64 +_TOKEN = "unit-controller-registry-token" +_MANIFEST = json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"digest": "sha256:" + "e" * 64}, + "layers": [], + } +) +_ASSETS = { + "dander/templates/project/infra/oci/controller/Dockerfile": ( + "ARG DANDER_WHEEL\nCOPY ${DANDER_WHEEL} /tmp/dander.whl\n" + ), + "dander/templates/project/infra/oci/controller/func.py": "from dander import __version__\n", + "dander/templates/project/infra/oci/controller/requirements.txt": "oci==2.184.1\n", +} + + +def _write_wheel(project_dir: Path, *, missing: str = "") -> tuple[Path, str]: + wheel = project_dir / "dander_platform-0.9.0rc2-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for name, contents in _ASSETS.items(): + if name != missing: + archive.writestr(name, contents) + archive.writestr( + "dander_platform-0.9.0rc2.dist-info/METADATA", + "Metadata-Version: 2.4\nName: dander-platform\nVersion: 0.9.0rc2\n", + ) + return wheel, hashlib.sha256(wheel.read_bytes()).hexdigest() + + +def _docker_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + directory = tmp_path / "docker" + directory.mkdir() + (directory / "config.json").write_text( + json.dumps({"credHelpers": {"us-central1-docker.pkg.dev": "gcloud"}}), + encoding="utf-8", + ) + monkeypatch.setenv("DOCKER_CONFIG", str(directory)) + + +class _Runner: + def __init__( + self, + project_dir: Path, + *, + existing: bool = False, + manifest: str = _MANIFEST, + fail_build: bool = False, + ) -> None: + self.project_dir = project_dir + self.existing = existing + self.manifest = manifest + self.fail_build = fail_build + self.created = False + self.commands: list[tuple[str, ...]] = [] + self.config_paths: list[Path] = [] + self.context_paths: list[Path] = [] + self.wheel_sha256 = "" + + def __call__( + self, + args: tuple[str, ...], + *, + cwd: Path, + check: bool, + capture_output: bool = False, + text: bool = False, + ) -> subprocess.CompletedProcess[str]: + del capture_output, text + self.commands.append(args) + if args[0] == "oci" and "repository" in args and "list" in args: + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "data": { + "items": [ + { + "display-name": "dander/runtime", + "is-immutable": True, + "is-public": False, + "lifecycle-state": "AVAILABLE", + } + ] + } + } + ), + ) + if args[0] == "oci" and "lookup" in args: + if not self.existing and not self.created: + assert check is False + return subprocess.CompletedProcess(args, 1, stdout="") + return subprocess.CompletedProcess(args, 0, stdout=_DESTINATION_DIGEST + "\n") + if args[0] == "oci" and "access-token" in args: + scope = "repository:unitnamespace/dander/runtime:pull,push" + assert args[args.index("--scope") + 1] == scope + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "data": { + "token": _TOKEN, + "expires-in": 900, + "scope": scope, + } + } + ), + ) + if args[:2] == ("docker", "--config"): + config_path = Path(args[2]) / "config.json" + self.config_paths.append(config_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + assert config["auths"]["ocir.us-ashburn-1.oci.oraclecloud.com"] == { + "identitytoken": _TOKEN + } + assert config["credHelpers"] == {"us-central1-docker.pkg.dev": "gcloud"} + if "build" in args: + assert cwd != self.project_dir + self.context_paths.append(cwd) + assert not (cwd / "src").exists() + assert (cwd / "infra/oci/controller/Dockerfile").read_text() == _ASSETS[ + "dander/templates/project/infra/oci/controller/Dockerfile" + ] + assert (cwd / "infra/oci/controller/func.py").read_text() == _ASSETS[ + "dander/templates/project/infra/oci/controller/func.py" + ] + self.wheel_sha256 = hashlib.sha256( + (cwd / "dander-controller.whl").read_bytes() + ).hexdigest() + if self.fail_build: + raise subprocess.CalledProcessError(1, args) + self.created = True + return subprocess.CompletedProcess(args, 0, stdout="") + if "inspect" in args: + return subprocess.CompletedProcess(args, 0, stdout=self.manifest) + raise AssertionError(f"Unexpected command: {args}") + + +def _publish( + project_dir: Path, + runner: _Runner, + wheel: Path, + wheel_sha256: str, +) -> tuple[OciControllerImagePublisher, str]: + publisher = OciControllerImagePublisher(project_dir, runner=runner) + image = publisher.publish( + wheel=wheel, + wheel_sha256=wheel_sha256, + compartment_id=_COMPARTMENT_ID, + region="us-ashburn-1", + namespace="unitnamespace", + repository_name="dander/runtime", + oci_profile="DANDER", + ) + return publisher, image + + +def test_controller_is_built_only_from_the_exact_reviewed_wheel( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + wheel, wheel_sha256 = _write_wheel(tmp_path) + _docker_config(monkeypatch, tmp_path) + runner = _Runner(tmp_path) + + publisher, image = _publish(tmp_path, runner, wheel, wheel_sha256) + + repository = "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime" + assert image == f"{repository}@{_DESTINATION_DIGEST}" + build = next(command for command in runner.commands if "build" in command) + assert build[build.index("--platform") + 1] == "linux/amd64" + assert "--provenance=false" in build + assert "--sbom=false" in build + assert build[build.index("--tag") + 1] == f"{repository}:controller-{wheel_sha256[:12]}" + assert runner.wheel_sha256 == wheel_sha256 + assert all(_TOKEN not in command for command in runner.commands) + assert runner.context_paths and all(not path.exists() for path in runner.context_paths) + assert runner.config_paths and all(not path.exists() for path in runner.config_paths) + assert publisher.artifact_record_path == tmp_path / ".dander/oci-controller-artifact.json" + record = json.loads(publisher.artifact_record_path.read_text(encoding="utf-8")) + assert record["wheel_sha256"] == wheel_sha256 + assert record["version"] == "0.9.0rc2" + assert record["platform_manifests"] == {"linux/amd64": _DESTINATION_DIGEST} + assert _TOKEN not in json.dumps(record) + + +def test_existing_controller_tag_requires_an_exact_local_binding( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + wheel, wheel_sha256 = _write_wheel(tmp_path) + _docker_config(monkeypatch, tmp_path) + runner = _Runner(tmp_path, existing=True) + repository = "ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime" + artifact_dir = tmp_path / ".dander" + artifact_dir.mkdir() + (artifact_dir / "oci-controller-artifact.json").write_text( + json.dumps( + { + "schema": "io.dander.oci.controller.artifact/v1", + "image": f"{repository}@{_DESTINATION_DIGEST}", + "tagged_image": f"{repository}:controller-{wheel_sha256[:12]}", + "image_digest": _DESTINATION_DIGEST, + "wheel_sha256": wheel_sha256, + } + ), + encoding="utf-8", + ) + + _, image = _publish(tmp_path, runner, wheel, wheel_sha256) + + assert image == f"{repository}@{_DESTINATION_DIGEST}" + assert not any("build" in command for command in runner.commands) + + +def test_existing_controller_tag_without_binding_fails_closed( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + wheel, wheel_sha256 = _write_wheel(tmp_path) + _docker_config(monkeypatch, tmp_path) + runner = _Runner(tmp_path, existing=True) + + with pytest.raises(ProjectBootstrapError, match="no matching local artifact record"): + _publish(tmp_path, runner, wheel, wheel_sha256) + + assert not any("access-token" in command for command in runner.commands) + + +def test_controller_rejects_wrong_wheel_hash_before_provider_access(tmp_path: Path) -> None: + wheel, _ = _write_wheel(tmp_path) + runner = _Runner(tmp_path) + + with pytest.raises(ProjectBootstrapError, match="does not match the reviewed SHA-256"): + _publish(tmp_path, runner, wheel, "0" * 64) + + assert runner.commands == [] + + +def test_controller_rejects_incomplete_wheel_before_provider_access(tmp_path: Path) -> None: + wheel, wheel_sha256 = _write_wheel( + tmp_path, + missing="dander/templates/project/infra/oci/controller/func.py", + ) + runner = _Runner(tmp_path) + + with pytest.raises(ProjectBootstrapError, match="missing OCI controller build files"): + _publish(tmp_path, runner, wheel, wheel_sha256) + + assert runner.commands == [] + + +def test_controller_rejects_non_amd64_registry_content( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + wheel, wheel_sha256 = _write_wheel(tmp_path) + _docker_config(monkeypatch, tmp_path) + manifest = json.dumps( + { + "schemaVersion": 2, + "manifests": [ + { + "digest": "sha256:" + "f" * 64, + "platform": {"os": "linux", "architecture": "arm64"}, + } + ], + } + ) + + with pytest.raises(ProjectBootstrapError, match="only the linux/amd64"): + _publish(tmp_path, _Runner(tmp_path, manifest=manifest), wheel, wheel_sha256) + + +def test_temporary_wheel_context_and_token_are_removed_after_build_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + wheel, wheel_sha256 = _write_wheel(tmp_path) + _docker_config(monkeypatch, tmp_path) + runner = _Runner(tmp_path, fail_build=True) + + with pytest.raises(ProjectBootstrapError, match="Could not publish"): + _publish(tmp_path, runner, wheel, wheel_sha256) + + assert runner.context_paths and all(not path.exists() for path in runner.context_paths) + assert runner.config_paths and all(not path.exists() for path in runner.config_paths) diff --git a/tests/cli/test_oci_cli.py b/tests/cli/test_oci_cli.py index 667ce28f..231909ce 100644 --- a/tests/cli/test_oci_cli.py +++ b/tests/cli/test_oci_cli.py @@ -108,6 +108,60 @@ def promote(self, **kwargs: object) -> str: ] +def test_oci_controller_publication_requires_confirmation_and_exact_wheel_hash( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: list[dict[str, object]] = [] + digest = "sha256:" + "c" * 64 + wheel = tmp_path / "dander_platform-0.9.0rc2-py3-none-any.whl" + wheel.write_bytes(b"reviewed-wheel") + + class _Publisher: + artifact_record_path = tmp_path / ".dander" / "oci-controller-artifact.json" + + def __init__(self, project_dir: Path) -> None: + assert project_dir == tmp_path + + def publish(self, **kwargs: object) -> str: + calls.append(kwargs) + return f"ocir.us-ashburn-1.oci.oraclecloud.com/unitnamespace/dander/runtime@{digest}" + + monkeypatch.setattr(oci_command, "OciControllerImagePublisher", _Publisher) + args = [ + "image-publish-oci-controller", + "--wheel", + str(wheel), + "--wheel-sha256", + "a" * 64, + "--compartment-id", + "ocid1.compartment.oc1.." + "b" * 32, + "--registry-namespace", + "unitnamespace", + "--oci-profile", + "DANDER", + "--config", + str(tmp_path / "dander.yaml"), + ] + + refused = CliRunner().invoke(app, args, input="n\n") + accepted = CliRunner().invoke(app, args, input="y\n") + + assert refused.exit_code == 1 + assert accepted.exit_code == 0, accepted.output + assert calls == [ + { + "wheel": wheel, + "wheel_sha256": "a" * 64, + "compartment_id": "ocid1.compartment.oc1.." + "b" * 32, + "region": "us-ashburn-1", + "namespace": "unitnamespace", + "repository_name": "dander/runtime", + "oci_profile": "DANDER", + } + ] + + def test_oci_foundation_apply_requires_confirmation( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tickets/DANDER-116-ocir-runtime-promotion.md b/tickets/DANDER-116-ocir-runtime-promotion.md index 116f8cb3..3f12bfb8 100644 --- a/tickets/DANDER-116-ocir-runtime-promotion.md +++ b/tickets/DANDER-116-ocir-runtime-promotion.md @@ -27,7 +27,7 @@ keyless operator boundary. - [x] Require explicit CLI confirmation and write only a local sanitized artifact record. - [x] Focused contracts cover copy, idempotency, repository policy, token lifetime, digest rewrite, platform drift, source-record mismatch, cleanup, and CLI confirmation. -- [ ] Protected CI passes and this implementation merges before live OCIR publication. +- [x] Protected CI passes and this implementation merges before live OCIR publication. - [ ] Separately approved live publication proves the real registry preserves the accepted index. ## Design @@ -38,4 +38,5 @@ is removed even when copy or verification fails. ## Review Log -Protected review and live publication remain pending. +Protected implementation PR #216 merged at `fae47a3cf860ba74a7c40b63c84ca21b9db7c6a2`, and the +protected-main CI run passed. Live publication remains pending. diff --git a/tickets/DANDER-117-oci-controller-publication.md b/tickets/DANDER-117-oci-controller-publication.md new file mode 100644 index 00000000..e93846aa --- /dev/null +++ b/tickets/DANDER-117-oci-controller-publication.md @@ -0,0 +1,42 @@ +--- +id: DANDER-117 +title: Publish the OCI lifecycle controller from the reviewed wheel +status: in-review +component: python +epic: cloud-portability-phase-7 +depends_on: [DANDER-115, DANDER-116] +created: 2026-08-12 +--- + +## Context + +The OCI Function controller is intentionally distinct from the portable task runtime. Its image +must contain Phase 7 controller code without allowing uncommitted checkout files or a different +wheel to enter the live deployment. + +## Acceptance Criteria + +- [x] Require an exact lowercase SHA-256 for one valid `dander-platform` wheel before OCI access. +- [x] Require the wheel to contain its own OCI controller Dockerfile, shim, dependency pins, and + package metadata. +- [x] Extract only those reviewed files and the wheel into an ephemeral source-free build context. +- [x] Build and publish only `linux/amd64` with a deterministic wheel-bound tag in the reviewed + private immutable OCIR repository. +- [x] Use the repository-scoped SecurityToken-derived access token only through the temporary + mode-`0600` Docker configuration. +- [x] Verify the digest and runnable platform, then write a sanitized local artifact record. +- [x] Fail closed on a pre-existing tag unless a local artifact record binds the same wheel hash, + tag, and digest. +- [x] Focused contracts cover wheel integrity, source-free context, idempotency, platform checks, + CLI confirmation, and cleanup after failure. +- [ ] Protected CI passes and this implementation merges before live controller publication. +- [ ] Separately approved live publication proves the real OCIR and OCI Functions path. + +## Design + +Treat the wheel as the reviewed controller source boundary. The deterministic tag supports safe +reruns, while the immutable digest remains the only deployment identity. + +## Review Log + +Protected review and live publication remain pending. From 49bab65594a52566ca3201b590395028a2c3c092 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 03:35:08 -0400 Subject: [PATCH 152/163] Prepare 0.9.0rc2 release candidate --- CHANGELOG.md | 19 ++++++++++ HANDOFF.md | 35 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 46 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd8ebe1c..efe7ebc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,25 @@ capabilities enter through the next minor release. ## Unreleased +## 0.9.0rc2 — 2026-08-12 (beta) + +### Added + +- Add the first-class experimental OCI Container Instances profile for PostgreSQL state and + warehouse, no catalog, and OCI Vault, with a typed provider-neutral launcher projection. +- Add plan-first native OCI stage-zero and foundation Terraform with Object Storage remote state, + a private immutable OCIR repository, private networking, Vault, resource principals, logging, + notifications, schedules, and deployment no-drift verification. +- Add the narrow OCI Functions lifecycle controller for idempotent launch, overlap exclusion, + bounded retry, interruption, replay, deadline cleanup, log capture, and event reconciliation. +- Add digest-preserving runtime promotion into OCIR and exact-wheel controller publication using + short-lived repository-scoped access tokens derived from OCI SecurityToken sessions. + +### Fixed + +- Require the OCI controller build context to come only from an exact SHA-256-qualified reviewed + wheel and fail closed on immutable tag, repository policy, platform, or artifact-record drift. + ## 0.9.0rc1 — 2026-08-11 (beta) ### Added diff --git a/HANDOFF.md b/HANDOFF.md index 3458a327..600f6279 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,35 @@ ## Finished -- Merged protected PR #216 for digest-preserving OCIR runtime promotion. -- Added OCI controller publication bound to one exact reviewed wheel SHA-256. -- Build inputs now come only from the wheel through an ephemeral source-free context. -- Added deterministic immutable tagging, amd64 verification, and sanitized artifact recording. -- Added fail-closed rerun, scoped-token, confirmation, and cleanup contracts. +- Merged protected PR #217 for exact-wheel OCI controller publication. +- Prepared Dander `0.9.0rc2` from protected main without publishing it. +- Reconciled exact release-version references and added Phase 7 candidate notes. ## Try It -Run `uv run pytest tests/bootstrap/test_oci_controller_publication.py tests/cli/test_oci_cli.py`. +Run `python3 scripts/check_release_metadata.py`, then build and inspect the distribution. ## Checks -- Focused controller, OCIR promotion, and OCI CLI tests pass. -- Focused Ruff lint/format and Mypy pass. -- Protected-main CI at `fae47a3cf860ba74a7c40b63c84ca21b9db7c6a2` passes. +- Protected-main CI at `653e93f498b3b702daefc8872d48f9c589a31e46` passes. +- Release metadata, lock, wheel/sdist, exact-wheel, Ruff, Mypy, and focused tests pass locally. +- Full protected CI remains required before merge and again on protected main before tagging. ## Decisions -- The controller is a wheel-built amd64 Function image, separate from the copied task runtime. -- A dirty checkout cannot contribute files to the controller build context. -- Existing controller tags require an exact local wheel/tag/digest binding. +- Publish only from an exact `v0.9.0rc2` tag on the green protected-main merge commit. +- Keep OCI experimental until Phase 7 live acceptance and Phase 8 support qualification pass. +- Do not tag, publish, or create paid OCI resources without explicit approvals. ## Remaining -- Merge the protected controller-publication PR and verify protected-main CI. -- Prepare the Phase 7 release candidate without publishing it. -- Obtain explicit approval for public candidate publication and a numeric OCI per-attempt ceiling. -- Run approved live OCI publication, profile, rotation, rollback, cleanup, and no-drift proof. +- Merge this version-only PR through protected CI after publication approval. +- Tag and publish `0.9.0rc2` through the reviewed `pypi` environment. +- Run the approved live OCI profile, rotation, rollback, cleanup, and no-drift proof. - Merge sanitized evidence and make the binary Phase 7 exit-gate recommendation. ## Review First -- `src/dander/bootstrap/oci_image.py` -- `tests/bootstrap/test_oci_controller_publication.py` -- `src/dander/cli/oci_command.py` +- `pyproject.toml` +- `CHANGELOG.md` +- `scripts/check_release_metadata.py` diff --git a/README.md b/README.md index 6c886861..acfdac28 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.9.0rc1 +uv tool install dander-platform==0.9.0rc2 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.9.0rc1` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.9.0rc2` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 79067051..a6cb14f2 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.9.0rc1`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.9.0rc2`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 9e438adb..5f2b7bb1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.9.0rc1 +uv tool install dander-platform==0.9.0rc2 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index d61e188f..31d2f640 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -24,7 +24,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.9.0rc1`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.9.0rc2`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index daef73b7..edb1fe79 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.9.0rc1` is the current public beta candidate. +- Dander `0.9.0rc2` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index ae9a775b..456417e3 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.9.0rc1" +export DANDER_TARGET_VERSION="0.9.0rc2" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 072c0cbb..4248d21b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.9.0rc1" +version = "0.9.0rc2" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 5a033d9a..453b06c9 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.9.0rc1", + "dander-platform==0.9.0rc2", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.9.0rc1" + "README install command uses 0.1.0; package metadata uses 0.9.0rc2" ] diff --git a/uv.lock b/uv.lock index 649ee5d5..f2bb8f5c 100644 --- a/uv.lock +++ b/uv.lock @@ -683,7 +683,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.9.0rc1" +version = "0.9.0rc2" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From f902e3f6e6f12f72fab725ae245e8bdf27da8c3d Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 07:04:10 -0400 Subject: [PATCH 153/163] fix(oci): preserve digest identity without tag immutability --- HANDOFF.md | 34 +++++++++---------- docs/cloud-portability-plan.md | 11 +++--- docs/decisions.md | 22 +++++++++--- docs/known-limitations.md | 6 ++++ infra/README.md | 2 +- infra/oci/bootstrap-admin/README.md | 8 ++++- infra/oci/bootstrap-admin/main.tf | 1 - infra/oci/bootstrap-admin/outputs.tf | 2 +- infra/oci/bootstrap-admin/variables.tf | 2 +- infra/oci/controller/README.md | 2 +- src/dander/bootstrap/oci_image.py | 30 +++++++++------- src/dander/cli/oci_command.py | 2 +- .../test_oci_controller_publication.py | 4 ++- tests/bootstrap/test_oci_image_promotion.py | 24 +++++++++++-- tests/bootstrap/test_oci_terraform.py | 7 ++++ 15 files changed, 108 insertions(+), 49 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 600f6279..0f80f4fd 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,35 @@ ## Finished -- Merged protected PR #217 for exact-wheel OCI controller publication. -- Prepared Dander `0.9.0rc2` from protected main without publishing it. -- Reconciled exact release-version references and added Phase 7 candidate notes. +- Confirmed live OCI account activation and isolated the Phase 7 compartment. +- Captured Oracle's live rejection of repository-level tag immutability on create and update. +- Adapted OCIR bootstrap/publication to private, verified digest addressing without false parity. ## Try It -Run `python3 scripts/check_release_metadata.py`, then build and inspect the distribution. +Run `uv run pytest -q` against the OCI Terraform, image-promotion, and controller-publication tests. ## Checks -- Protected-main CI at `653e93f498b3b702daefc8872d48f9c589a31e46` passes. -- Release metadata, lock, wheel/sdist, exact-wheel, Ruff, Mypy, and focused tests pass locally. -- Full protected CI remains required before merge and again on protected main before tagging. +- Live OCI SecurityToken, Object Storage, and Container Instances preflight pass. +- Stage zero created only its private versioned bucket before OCIR rejected `isImmutable`. +- All 106 focused OCI/distribution/runtime tests, Ruff, Mypy, Terraform validation, and packaged + bootstrap inspection pass; full protected CI remains to run. ## Decisions -- Publish only from an exact `v0.9.0rc2` tag on the green protected-main merge commit. -- Keep OCI experimental until Phase 7 live acceptance and Phase 8 support qualification pass. -- Do not tag, publish, or create paid OCI resources without explicit approvals. +- Treat OCIR tag immutability as an explicit unavailable capability in this tenancy. +- Preserve artifact immutability through verified digests and matching Function tag/digest inputs. +- Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Merge this version-only PR through protected CI after publication approval. -- Tag and publish `0.9.0rc2` through the reviewed `pypi` environment. -- Run the approved live OCI profile, rotation, rollback, cleanup, and no-drift proof. -- Merge sanitized evidence and make the binary Phase 7 exit-gate recommendation. +- Merge the correction through protected CI and publish a new exact release candidate. +- Resume the approved live OCI profile from the preserved partial stage-zero state. +- Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `pyproject.toml` -- `CHANGELOG.md` -- `scripts/check_release_metadata.py` +- `infra/oci/bootstrap-admin/main.tf` +- `src/dander/bootstrap/oci_image.py` +- `docs/decisions.md` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 10e4d5cd..89ce3059 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1205,7 +1205,7 @@ Progress on 2026-08-12: the OCI SDK/Vault provider, typed Container Instances la plan-first Terraform foundations, and lifecycle-controller implementation are behind provider-specific boundaries. The OCI native remote-state bootstrap creates only a private versioned Object Storage bucket and private -immutable OCIR repository; the second root creates a private VCN/subnet, default Vault with an +digest-addressed OCIR repository; the second root creates a private VCN/subnet, default Vault with an auto-rotating key, compartment-scoped Container Instance resource-principal policy, Logging, and Notifications. Both use short-lived SecurityToken auth, saved plans outside the repository, and locally passing Terraform tests; a read-only verifier refreshes both remote states and fails on any @@ -1216,9 +1216,12 @@ interruption, replay, bounded logs, terminal history, per-pipeline lifecycle-eve and UTC Resource Scheduler delivery; operator commands use only expiring SecurityToken sessions. Runtime processes resolve versionless Vault references through resource principals without putting values in Terraform or Function configuration. Accepted runtime indexes can be copied into the -exact private immutable OCIR repository without rebuild using a short-lived repository-scoped token -derived from the operator's SecurityToken session; source, destination, and per-platform digests -must remain equal. The separately built Function controller is bound to the exact reviewed wheel +exact private OCIR repository without rebuild using a short-lived repository-scoped token derived +from the operator's SecurityToken session; source, destination, and per-platform digests must +remain equal. OCI currently rejects its advertised repository-level tag-immutability setting in +the live Ashburn tenancy; Dander records that limitation, rejects existing-tag mismatches, and +deploys only the verified digest (or OCI Functions' required matching tag-and-digest pair). The +separately built Function controller is bound to the exact reviewed wheel SHA-256: its Dockerfile, shim, and dependency pins are extracted from that wheel into an ephemeral source-free build context, published only for `linux/amd64`, and recorded by immutable digest. Live runtime/controller publication, credential-refresh, live-profile, rollback, cleanup, and diff --git a/docs/decisions.md b/docs/decisions.md index 8c676ac1..97e8cc21 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1336,8 +1336,8 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc ## 2026-08-12 — OCI foundation uses native state and two reviewed applies - **State boundary:** Stage zero creates only a private versioned Object Storage bucket and a - private immutable OCIR repository, then migrates its temporary local state to Terraform's native - OCI backend. Terraform 1.12 or newer and a short-lived `SecurityToken` profile are required; + private digest-addressed OCIR repository, then migrates its temporary local state to Terraform's + native OCI backend. Terraform 1.12 or newer and a short-lived `SecurityToken` profile are required; API-key and registry-password credentials are not accepted by this workflow. - **Foundation:** A second remote-state-backed plan creates the private egress-only VCN/subnet, default Vault and auto-rotating software key, compartment-scoped Container Instance dynamic @@ -1383,7 +1383,19 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc `linux/amd64` from one exact SHA-256-qualified Dander wheel, extracting the Dockerfile, shim, and dependency pins from that wheel into an ephemeral context; never build from the working tree. - **Registry boundary:** Publish a deterministic wheel-bound tag to the same reviewed private, - immutable OCIR repository using the repository-scoped SecurityToken-derived access token. Record - the resulting digest locally without credentials or local paths. -- **Idempotency:** An existing immutable controller tag is reusable only when its digest has an + OCIR repository using the repository-scoped SecurityToken-derived access token. Record the + resulting digest locally without credentials or local paths. +- **Idempotency:** An existing controller tag is reusable only when its digest has an exact local artifact binding to the same wheel hash. Missing or mismatched bindings fail closed. + +## 2026-08-12 — OCIR capability is explicit while artifact identity remains digest-bound + +- **Provider evidence:** A live Ashburn tenancy returned `400 BAD_REQUEST` with “Setting + isImmutable is not currently supported” for both repository create and update, despite the + property appearing in Oracle's API, CLI, and Terraform surfaces. +- **Decision:** Create a private OCIR repository without requesting that unsupported property. + Record whether OCI reports repository tag immutability, reject pre-existing tag mismatches, and + verify the destination index plus every runnable platform digest after publication. +- **Safety boundary:** Container Instances consume only `@sha256` references. OCI Functions require + a tag but also receive its exact `image_digest`; existing controller tags require a local binding + to the same reviewed wheel. This is an explicit provider limitation, not a claim of false parity. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 33b2a63f..31f3052d 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -88,6 +88,12 @@ remain Phase 8 work. an ephemeral repository-scoped token, but live publication, provider deployment verification, credential-rotation proof, and the complete OCI profile remain unqualified. No OCI launcher support claim exists until those separate gates pass. +- OCI Container Registry advertises repository tag immutability, but the live Ashburn tenancy + rejected `isImmutable` on both create and update with `400 BAD_REQUEST`. Dander represents this + limitation honestly: the repository remains private, publication rejects an existing tag whose + digest differs, promoted indexes and platform manifests are re-read and compared, and every task + deployment uses an exact digest while OCI Functions receive the matching tag-and-digest pair. + Repository-level tag immutability is not claimed where OCI reports it unavailable. ## Ingestion and schemas diff --git a/infra/README.md b/infra/README.md index 68b88c98..a297a551 100644 --- a/infra/README.md +++ b/infra/README.md @@ -16,7 +16,7 @@ call sites (mirrors the `SecretStoreProvider` / `ComputeProvider` abstractions i | `kubernetes/chart/dander` | Versioned Helm chart for an existing conforming cluster. **Locally validated; not live-qualified.** | | `azure/bootstrap-admin` | Azure Storage state, ACR, and user-assigned runtime identity. **Live-proven for the named experimental Phase 6 profiles.** | | `azure/modules/container-apps-jobs` | Container Apps Jobs, Key Vault references, Log Analytics, alerts, and optional network placement. **Live-proven for the named experimental Phase 6 profiles.** | -| `oci/bootstrap-admin` | Native OCI Object Storage state and a private immutable OCIR repository. **Implemented and locally validated; live proof pending.** | +| `oci/bootstrap-admin` | Native OCI Object Storage state and a private digest-addressed OCIR repository. **Implemented and locally validated; live proof pending.** | | `oci` | Private VCN/subnet, default Vault and rotating key, Container Instance resource-principal policy, Logging, and Notifications foundation. **Implemented and locally validated; live proof pending.** | The main root always calls `modules/bigquery` and can opt into the remaining workload modules. The diff --git a/infra/oci/bootstrap-admin/README.md b/infra/oci/bootstrap-admin/README.md index 83bfae80..8e3bfe19 100644 --- a/infra/oci/bootstrap-admin/README.md +++ b/infra/oci/bootstrap-admin/README.md @@ -3,7 +3,13 @@ This root creates the two prerequisites that must exist before Dander can use remote state: - a private, versioned Object Storage bucket; and -- a private, immutable OCIR repository. +- a private OCIR repository for digest-addressed runtime and controller images. + +OCIR currently rejects its advertised repository-level `is_immutable` setting in some tenancies. +Dander therefore does not ask Terraform to set that unsupported property. Publication still fails +closed on an existing tag mismatch, records the observed repository capability, verifies the exact +index and platform digests after every push, and supplies the immutable digest alongside the tag to +OCI Functions and Container Instances. Use an OCI CLI `SecurityToken` profile. Do not use API-key, auth-token, or registry-password credentials. The administrative bootstrap copies this root to a private operator directory, diff --git a/infra/oci/bootstrap-admin/main.tf b/infra/oci/bootstrap-admin/main.tf index f1979656..1251c1a8 100644 --- a/infra/oci/bootstrap-admin/main.tf +++ b/infra/oci/bootstrap-admin/main.tf @@ -26,7 +26,6 @@ resource "oci_objectstorage_bucket" "terraform_state" { resource "oci_artifacts_container_repository" "runtime" { compartment_id = var.compartment_id display_name = var.repository_name - is_immutable = true is_public = false freeform_tags = local.tags } diff --git a/infra/oci/bootstrap-admin/outputs.tf b/infra/oci/bootstrap-admin/outputs.tf index 94da7912..b6e0c589 100644 --- a/infra/oci/bootstrap-admin/outputs.tf +++ b/infra/oci/bootstrap-admin/outputs.tf @@ -9,7 +9,7 @@ output "state_bucket_name" { } output "repository_id" { - description = "Immutable private OCIR repository OCID." + description = "Private OCIR repository OCID for digest-addressed artifacts." value = oci_artifacts_container_repository.runtime.id } diff --git a/infra/oci/bootstrap-admin/variables.tf b/infra/oci/bootstrap-admin/variables.tf index 5f081b6b..f1d9505d 100644 --- a/infra/oci/bootstrap-admin/variables.tf +++ b/infra/oci/bootstrap-admin/variables.tf @@ -50,7 +50,7 @@ variable "state_bucket_name" { } variable "repository_name" { - description = "Private immutable OCIR repository receiving verified Dander artifacts." + description = "Private OCIR repository receiving verified digest-addressed Dander artifacts." type = string default = "dander/runtime" diff --git a/infra/oci/controller/README.md b/infra/oci/controller/README.md index 6317bab5..fe4bb574 100644 --- a/infra/oci/controller/README.md +++ b/infra/oci/controller/README.md @@ -18,7 +18,7 @@ docker buildx build --platform linux/amd64 \ ``` Inspect the pushed image, record its `sha256:` digest outside Git, and pass the tag and digest -together to `dander init-oci-launcher-plan`. The controller uses the same private immutable OCIR +together to `dander init-oci-launcher-plan`. The controller uses the same private OCIR repository selected by the manifest, but a separate unique tag from the task image. Never reuse a tag, rebuild an accepted candidate, or commit the wheel, registry credentials, saved plan, or Terraform state. diff --git a/src/dander/bootstrap/oci_image.py b/src/dander/bootstrap/oci_image.py index 0c522870..9d584ee7 100644 --- a/src/dander/bootstrap/oci_image.py +++ b/src/dander/bootstrap/oci_image.py @@ -90,7 +90,7 @@ def _verify_repository( oci_prefix: tuple[str, ...], compartment_id: str, repository_name: str, -) -> None: +) -> bool: response = runner( ( *oci_prefix, @@ -124,14 +124,12 @@ def _verify_repository( if len(matches) != 1: raise ProjectBootstrapError("The exact OCI runtime repository does not exist") repository = matches[0] - if ( - repository.get("is-immutable") is not True - or repository.get("is-public") is not False - or repository.get("lifecycle-state") != "AVAILABLE" - ): - raise ProjectBootstrapError( - "OCI runtime repository must be private, immutable, and available" - ) + repository_immutable = repository.get("is-immutable") + if not isinstance(repository_immutable, bool): + raise ProjectBootstrapError("OCI returned invalid repository metadata") + if repository.get("is-public") is not False or repository.get("lifecycle-state") != "AVAILABLE": + raise ProjectBootstrapError("OCI runtime repository must be private and available") + return repository_immutable def _registry_token( @@ -306,7 +304,7 @@ def promote( ) try: - _verify_repository( + repository_immutable = _verify_repository( runner=self._runner, cwd=self._project_dir, oci_prefix=oci_prefix, @@ -364,7 +362,7 @@ def promote( "byte-identical" ) raise ProjectBootstrapError( - "Existing immutable OCIR tag does not match the accepted OCI index" + "Existing OCIR tag does not match the accepted OCI index" ) immutable_destination = f"{destination_repository}@{destination_digest}" destination_raw = _inspect( @@ -397,6 +395,10 @@ def promote( "source_image": source_image, "promotion": "registry-copy", "authentication": "SecurityToken-scoped-access-token", + "repository_tag_immutability": ( + "enabled" if repository_immutable else "provider-unavailable" + ), + "deployment_reference": "digest", }, ) except OSError as error: @@ -502,7 +504,7 @@ def publish( ) try: - _verify_repository( + repository_immutable = _verify_repository( runner=self._runner, cwd=self._project_dir, oci_prefix=oci_prefix, @@ -606,6 +608,10 @@ def publish( "version": version, "build": "exact-reviewed-wheel", "authentication": "SecurityToken-scoped-access-token", + "repository_tag_immutability": ( + "enabled" if repository_immutable else "provider-unavailable" + ), + "deployment_reference": "tag-and-digest", }, ) except OSError as error: diff --git a/src/dander/cli/oci_command.py b/src/dander/cli/oci_command.py index 9699d434..e4ec5002 100644 --- a/src/dander/cli/oci_command.py +++ b/src/dander/cli/oci_command.py @@ -319,7 +319,7 @@ def init_oci_admin_plan( ), infra_dir: Path = typer.Option(_DEFAULT_OCI_ADMIN_DIR, hidden=True), # noqa: B008 ) -> None: - """Plan the OCI state bucket and immutable OCIR repository without applying.""" + """Plan the OCI state bucket and private digest-addressed OCIR repository.""" arguments = _admin_arguments( tenancy_id=tenancy_id, compartment_id=compartment_id, diff --git a/tests/bootstrap/test_oci_controller_publication.py b/tests/bootstrap/test_oci_controller_publication.py index 9d284a06..966361fe 100644 --- a/tests/bootstrap/test_oci_controller_publication.py +++ b/tests/bootstrap/test_oci_controller_publication.py @@ -95,7 +95,7 @@ def __call__( "items": [ { "display-name": "dander/runtime", - "is-immutable": True, + "is-immutable": False, "is-public": False, "lifecycle-state": "AVAILABLE", } @@ -200,6 +200,8 @@ def test_controller_is_built_only_from_the_exact_reviewed_wheel( assert record["wheel_sha256"] == wheel_sha256 assert record["version"] == "0.9.0rc2" assert record["platform_manifests"] == {"linux/amd64": _DESTINATION_DIGEST} + assert record["repository_tag_immutability"] == "provider-unavailable" + assert record["deployment_reference"] == "tag-and-digest" assert _TOKEN not in json.dumps(record) diff --git a/tests/bootstrap/test_oci_image_promotion.py b/tests/bootstrap/test_oci_image_promotion.py index 9999dccd..9ac2823e 100644 --- a/tests/bootstrap/test_oci_image_promotion.py +++ b/tests/bootstrap/test_oci_image_promotion.py @@ -76,12 +76,14 @@ def __init__( destination_raw: str = _INDEX, expires_in: int = 900, repository_private: bool = True, + repository_immutable: bool = False, ) -> None: self.existing = existing self.destination_digest = destination_digest self.destination_raw = destination_raw self.expires_in = expires_in self.repository_private = repository_private + self.repository_immutable = repository_immutable self.created = False self.commands: list[tuple[str, ...]] = [] self.config_paths: list[Path] = [] @@ -111,7 +113,7 @@ def __call__( "items": [ { "display-name": "dander/runtime", - "is-immutable": True, + "is-immutable": self.repository_immutable, "is-public": not self.repository_private, "lifecycle-state": "AVAILABLE", } @@ -212,6 +214,8 @@ def test_oci_promoter_copies_and_verifies_with_only_a_scoped_temporary_token( assert record["image"] == image assert record["promotion"] == "registry-copy" assert record["authentication"] == "SecurityToken-scoped-access-token" + assert record["repository_tag_immutability"] == "provider-unavailable" + assert record["deployment_reference"] == "digest" def test_oci_promoter_is_idempotent_for_an_existing_identical_tag( @@ -267,17 +271,31 @@ def test_oci_promoter_rejects_short_lived_or_malformed_token( _promote(tmp_path, _Runner(expires_in=60)) -def test_oci_promoter_requires_the_reviewed_private_immutable_repository( +def test_oci_promoter_requires_the_reviewed_private_available_repository( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: _write_source_record(tmp_path) _docker_config(monkeypatch, tmp_path) - with pytest.raises(ProjectBootstrapError, match="private, immutable, and available"): + with pytest.raises(ProjectBootstrapError, match="private and available"): _promote(tmp_path, _Runner(repository_private=False)) +def test_oci_promoter_records_repository_immutability_when_provider_supports_it( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + _docker_config(monkeypatch, tmp_path) + + promoter, _ = _promote(tmp_path, _Runner(repository_immutable=True)) + + assert promoter.artifact_record_path is not None + record = json.loads(promoter.artifact_record_path.read_text(encoding="utf-8")) + assert record["repository_tag_immutability"] == "enabled" + + def test_oci_promoter_rejects_source_record_mismatch_before_provider_access( tmp_path: Path, ) -> None: diff --git a/tests/bootstrap/test_oci_terraform.py b/tests/bootstrap/test_oci_terraform.py index e63f6076..69fd891e 100644 --- a/tests/bootstrap/test_oci_terraform.py +++ b/tests/bootstrap/test_oci_terraform.py @@ -103,6 +103,13 @@ def test_admin_bootstrap_applies_exact_plan_then_migrates_native_state( assert "user_ocid" not in backend +def test_admin_repository_is_private_and_does_not_request_unsupported_immutability() -> None: + configuration = (_ROOT / "infra/oci/bootstrap-admin/main.tf").read_text(encoding="utf-8") + + assert "is_public = false" in configuration + assert "is_immutable" not in configuration + + def test_foundation_bootstrap_uses_native_backend_and_saved_plan( tmp_path: Path, terraform_calls: list[tuple[str, ...]], From cf8d80857a8be8362fabb82297964dbca28e7738 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 07:07:52 -0400 Subject: [PATCH 154/163] test(oci): assert supported registry contract --- infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl b/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl index 68df35a2..7ef3fbc4 100644 --- a/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl +++ b/infra/oci/bootstrap-admin/tests/bootstrap.tftest.hcl @@ -14,7 +14,7 @@ variables { repository_name = "dander/runtime" } -run "hardened_stage_zero" { +run "private_digest_addressed_stage_zero" { command = plan assert { @@ -28,9 +28,9 @@ run "hardened_stage_zero" { assert { condition = ( - oci_artifacts_container_repository.runtime.is_immutable && - !oci_artifacts_container_repository.runtime.is_public + !oci_artifacts_container_repository.runtime.is_public && + oci_artifacts_container_repository.runtime.display_name == "dander/runtime" ) - error_message = "OCIR must reject mutable tags and unauthenticated reads." + error_message = "OCIR must remain private and use the selected digest-addressed repository." } } From 817fee7f1896e09502fce67c9e9c7429a0eed4c6 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 07:23:50 -0400 Subject: [PATCH 155/163] Prepare 0.9.0rc3 release candidate --- CHANGELOG.md | 10 +++++++ HANDOFF.md | 29 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 35 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efe7ebc4..ca0b60df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ capabilities enter through the next minor release. ## Unreleased +## 0.9.0rc3 — 2026-08-12 (beta) + +### Fixed + +- Preserve OCI artifact identity when OCIR rejects its advertised repository tag-immutability + setting: keep the repository private, reject existing-tag mismatches, verify copied index and + platform digests, and deploy only digest-bound runtime/controller references. +- Record the observed OCIR tag-immutability capability explicitly instead of claiming unsupported + provider parity. + ## 0.9.0rc2 — 2026-08-12 (beta) ### Added diff --git a/HANDOFF.md b/HANDOFF.md index 0f80f4fd..222fc65f 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,34 @@ ## Finished -- Confirmed live OCI account activation and isolated the Phase 7 compartment. -- Captured Oracle's live rejection of repository-level tag immutability on create and update. -- Adapted OCIR bootstrap/publication to private, verified digest addressing without false parity. +- Merged protected PR #219 for OCI digest identity without false tag-immutability parity. +- Verified full protected-main CI at `1cc7b1e47aaf1550ec299944c040343d3491643f`. +- Prepared Dander `0.9.0rc3` release metadata from that protected baseline. ## Try It -Run `uv run pytest -q` against the OCI Terraform, image-promotion, and controller-publication tests. +Run `python3 scripts/check_release_metadata.py`, then build and inspect the distribution. ## Checks -- Live OCI SecurityToken, Object Storage, and Container Instances preflight pass. -- Stage zero created only its private versioned bucket before OCIR rejected `isImmutable`. -- All 106 focused OCI/distribution/runtime tests, Ruff, Mypy, Terraform validation, and packaged - bootstrap inspection pass; full protected CI remains to run. +- Protected-main Python, Terraform, secret, distribution, container, and vulnerability checks pass. +- Release metadata, lock, focused tests, Ruff, Mypy, wheel/sdist inspection, and packaged OCI + bootstrap checks pass for `0.9.0rc3`; protected PR CI remains required. ## Decisions -- Treat OCIR tag immutability as an explicit unavailable capability in this tenancy. -- Preserve artifact immutability through verified digests and matching Function tag/digest inputs. +- Publish only from exact tag `v0.9.0rc3` on the next green protected-main merge commit. +- Keep the accepted public wheel and its digest outside the repository for live OCI proof. - Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Merge the correction through protected CI and publish a new exact release candidate. -- Resume the approved live OCI profile from the preserved partial stage-zero state. +- Merge the version-only PR through protected CI, then tag and publish `0.9.0rc3`. +- Resume the approved live OCI profile from the preserved partial stage-zero state with that wheel. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `infra/oci/bootstrap-admin/main.tf` -- `src/dander/bootstrap/oci_image.py` -- `docs/decisions.md` +- `pyproject.toml` +- `CHANGELOG.md` +- `scripts/check_release_metadata.py` diff --git a/README.md b/README.md index acfdac28..c0e7311f 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.9.0rc2 +uv tool install dander-platform==0.9.0rc3 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.9.0rc2` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.9.0rc3` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index a6cb14f2..537f2bbb 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.9.0rc2`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.9.0rc3`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index 5f2b7bb1..fb5b3d8e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.9.0rc2 +uv tool install dander-platform==0.9.0rc3 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 31d2f640..b310e6f8 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -24,7 +24,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.9.0rc2`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.9.0rc3`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index edb1fe79..98c10152 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.9.0rc2` is the current public beta candidate. +- Dander `0.9.0rc3` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 456417e3..715f10f1 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.9.0rc2" +export DANDER_TARGET_VERSION="0.9.0rc3" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index 4248d21b..bf30c1d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.9.0rc2" +version = "0.9.0rc3" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 453b06c9..41715c29 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.9.0rc2", + "dander-platform==0.9.0rc3", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.9.0rc2" + "README install command uses 0.1.0; package metadata uses 0.9.0rc3" ] diff --git a/uv.lock b/uv.lock index f2bb8f5c..99425bbf 100644 --- a/uv.lock +++ b/uv.lock @@ -683,7 +683,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.9.0rc2" +version = "0.9.0rc3" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 52fcbeb13f5e21c1c6c5485ef42910e2165021f3 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 07:56:55 -0400 Subject: [PATCH 156/163] fix(oci): schedule automatic key rotation --- HANDOFF.md | 29 +++++++++++++++-------------- docs/decisions.md | 10 ++++++++++ infra/oci/README.md | 2 +- infra/oci/main.tf | 4 ++++ infra/oci/tests/oci.tftest.hcl | 5 +++-- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 222fc65f..8936dce8 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,34 +2,35 @@ ## Finished -- Merged protected PR #219 for OCI digest identity without false tag-immutability parity. -- Verified full protected-main CI at `1cc7b1e47aaf1550ec299944c040343d3491643f`. -- Prepared Dander `0.9.0rc3` release metadata from that protected baseline. +- Published exact protected-main candidate `dander-platform==0.9.0rc3` and verified its public wheel. +- Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. +- Published the source-free RC3 runtime index in the disposable GCP proof registry. +- Captured the live OCI KMS requirement for an explicit automatic-rotation schedule. ## Try It -Run `python3 scripts/check_release_metadata.py`, then build and inspect the distribution. +Run `terraform -chdir=infra/oci test` to verify the scheduled key-rotation contract. ## Checks -- Protected-main Python, Terraform, secret, distribution, container, and vulnerability checks pass. -- Release metadata, lock, focused tests, Ruff, Mypy, wheel/sdist inspection, and packaged OCI - bootstrap checks pass for `0.9.0rc3`; protected PR CI remains required. +- Protected-main CI passed for the exact `v0.9.0rc3` candidate. +- Live OCI stage-zero retry reached zero drift after one private repository create. +- Focused Terraform test and full protected CI remain required for this correction. ## Decisions -- Publish only from exact tag `v0.9.0rc3` on the next green protected-main merge commit. -- Keep the accepted public wheel and its digest outside the repository for live OCI proof. +- Use an explicit 365-day OCI KMS automatic-rotation interval. +- Continue live proof only from the public RC3 wheel and its source-free image digest. - Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Merge the version-only PR through protected CI, then tag and publish `0.9.0rc3`. -- Resume the approved live OCI profile from the preserved partial stage-zero state with that wheel. +- Merge the key-rotation correction through protected CI, then retry the one-resource foundation plan. +- Promote the accepted GCP runtime index to OCIR without rebuilding it. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `pyproject.toml` -- `CHANGELOG.md` -- `scripts/check_release_metadata.py` +- `infra/oci/main.tf` +- `infra/oci/tests/oci.tftest.hcl` +- `docs/decisions.md` diff --git a/docs/decisions.md b/docs/decisions.md index 97e8cc21..1a3054e0 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1399,3 +1399,13 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Safety boundary:** Container Instances consume only `@sha256` references. OCI Functions require a tag but also receive its exact `image_digest`; existing controller tags require a local binding to the same reviewed wheel. This is an explicit provider limitation, not a claim of false parity. + +## 2026-08-12 — OCI key rotation uses an explicit annual schedule + +- **Provider evidence:** The live Ashburn Key Management API rejected a key create with automatic + rotation enabled but no `autoKeyRotationDetails`, even though Terraform accepted that plan. +- **Decision:** Supply a 365-day rotation interval explicitly. This stays inside OCI's documented + 60-to-365-day range, minimizes unnecessary key-version churn, and preserves automatic rotation. +- **Boundary:** The schedule is part of the reviewed Terraform projection and its focused contract + test; it contains no secret material and does not weaken the versionless application-secret + rotation proof required by Phase 7. diff --git a/infra/oci/README.md b/infra/oci/README.md index 34084da1..02b456f7 100644 --- a/infra/oci/README.md +++ b/infra/oci/README.md @@ -4,7 +4,7 @@ This root creates the provider-owned foundation for Dander's named OCI profile: - a private VCN/subnet with no public IPs or inbound rules; - NAT egress for PostgreSQL plus an OCI Services gateway for OCIR, Vault, and control APIs; -- a default OCI Vault and auto-rotating software-protected key; +- a default OCI Vault and software-protected key with an explicit 365-day auto-rotation schedule; - a compartment-scoped Container Instance dynamic group and least-privilege runtime policy; and - a private versioned run-record bucket, per-pipeline OCI Function, Resource Scheduler schedule, lifecycle event rule, invocation log, error alarm, and Notifications topic. diff --git a/infra/oci/main.tf b/infra/oci/main.tf index 2a5f85fa..def7b597 100644 --- a/infra/oci/main.tf +++ b/infra/oci/main.tf @@ -121,6 +121,10 @@ resource "oci_kms_key" "runtime" { is_auto_rotation_enabled = true freeform_tags = local.tags + auto_key_rotation_details { + rotation_interval_in_days = 365 + } + key_shape { algorithm = "AES" length = 32 diff --git a/infra/oci/tests/oci.tftest.hcl b/infra/oci/tests/oci.tftest.hcl index 77a1897a..a090bff3 100644 --- a/infra/oci/tests/oci.tftest.hcl +++ b/infra/oci/tests/oci.tftest.hcl @@ -42,9 +42,10 @@ run "projects_private_keyless_foundation" { condition = ( oci_kms_vault.runtime.vault_type == "DEFAULT" && oci_kms_key.runtime.protection_mode == "SOFTWARE" && - oci_kms_key.runtime.is_auto_rotation_enabled + oci_kms_key.runtime.is_auto_rotation_enabled && + oci_kms_key.runtime.auto_key_rotation_details[0].rotation_interval_in_days == 365 ) - error_message = "The runtime Vault must use the bounded-cost default tier and an auto-rotating key." + error_message = "The runtime Vault must use the bounded-cost default tier and an explicitly scheduled auto-rotating key." } assert { From 39881c0d3e4bc616d644633a6fe712a156ebc264 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 07:59:41 -0400 Subject: [PATCH 157/163] fix(oci): verify exact repository metadata --- HANDOFF.md | 20 ++++++----- docs/decisions.md | 9 +++++ src/dander/bootstrap/oci_image.py | 36 +++++++++++++++++-- .../test_oci_controller_publication.py | 18 +++++++++- tests/bootstrap/test_oci_image_promotion.py | 18 +++++++++- 5 files changed, 89 insertions(+), 12 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 8936dce8..09b450f0 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -6,31 +6,35 @@ - Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. - Published the source-free RC3 runtime index in the disposable GCP proof registry. - Captured the live OCI KMS requirement for an explicit automatic-rotation schedule. +- Captured OCI's live list-summary versus exact-repository metadata behavior. ## Try It -Run `terraform -chdir=infra/oci test` to verify the scheduled key-rotation contract. +Run the focused OCI image-promotion and controller-publication tests. ## Checks - Protected-main CI passed for the exact `v0.9.0rc3` candidate. - Live OCI stage-zero retry reached zero drift after one private repository create. -- Focused Terraform test and full protected CI remain required for this correction. +- The rotation-schedule correction passed PR CI; protected-main CI is running. +- Live promotion failed closed before copying because OCI's list summary omitted immutability. +- Focused tests and full protected CI remain required for the metadata correction. ## Decisions - Use an explicit 365-day OCI KMS automatic-rotation interval. -- Continue live proof only from the public RC3 wheel and its source-free image digest. +- Verify exact OCIR metadata with a repository get after resolving one named list summary. - Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Merge the key-rotation correction through protected CI, then retry the one-resource foundation plan. -- Promote the accepted GCP runtime index to OCIR without rebuilding it. +- Merge the exact-repository verification correction through protected CI. +- Retry the one-resource foundation plan after the combined protected-main CI passes. +- Resume digest-preserving OCIR promotion from the accepted GCP runtime index. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `infra/oci/main.tf` -- `infra/oci/tests/oci.tftest.hcl` -- `docs/decisions.md` +- `src/dander/bootstrap/oci_image.py` +- `tests/bootstrap/test_oci_image_promotion.py` +- `tests/bootstrap/test_oci_controller_publication.py` diff --git a/docs/decisions.md b/docs/decisions.md index 1a3054e0..a93911b8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1409,3 +1409,12 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc - **Boundary:** The schedule is part of the reviewed Terraform projection and its focused contract test; it contains no secret material and does not weaken the versionless application-secret rotation proof required by Phase 7. + +## 2026-08-12 — OCIR repository verification follows summary with exact get + +- **Provider evidence:** The live repository-list summary omitted `is-immutable` when false, while + an exact repository get returned the required boolean together with private and available state. +- **Decision:** Select exactly one named repository from the compartment list, then verify its ID, + name, visibility, lifecycle, and immutability capability from the exact get response. +- **Boundary:** Missing or inconsistent exact metadata still fails closed before registry-token + creation or artifact copy; an omitted optional field in list summaries is not treated as false. diff --git a/src/dander/bootstrap/oci_image.py b/src/dander/bootstrap/oci_image.py index 9d584ee7..df11c3c3 100644 --- a/src/dander/bootstrap/oci_image.py +++ b/src/dander/bootstrap/oci_image.py @@ -123,11 +123,43 @@ def _verify_repository( ] if len(matches) != 1: raise ProjectBootstrapError("The exact OCI runtime repository does not exist") - repository = matches[0] + repository_summary = matches[0] + repository_id = repository_summary.get("id") + if not isinstance(repository_id, str) or not repository_id: + raise ProjectBootstrapError("OCI returned invalid repository metadata") + response = runner( + ( + *oci_prefix, + "artifacts", + "container", + "repository", + "get", + "--repository-id", + repository_id, + "--output", + "json", + ), + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + try: + document = json.loads(response.stdout) + repository = document["data"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise ProjectBootstrapError("OCI returned invalid repository metadata") from error + if not isinstance(repository, dict): + raise ProjectBootstrapError("OCI returned invalid repository metadata") repository_immutable = repository.get("is-immutable") if not isinstance(repository_immutable, bool): raise ProjectBootstrapError("OCI returned invalid repository metadata") - if repository.get("is-public") is not False or repository.get("lifecycle-state") != "AVAILABLE": + if ( + repository.get("id") != repository_id + or repository.get("display-name") != repository_name + or repository.get("is-public") is not False + or repository.get("lifecycle-state") != "AVAILABLE" + ): raise ProjectBootstrapError("OCI runtime repository must be private and available") return repository_immutable diff --git a/tests/bootstrap/test_oci_controller_publication.py b/tests/bootstrap/test_oci_controller_publication.py index 966361fe..81967f0d 100644 --- a/tests/bootstrap/test_oci_controller_publication.py +++ b/tests/bootstrap/test_oci_controller_publication.py @@ -95,7 +95,7 @@ def __call__( "items": [ { "display-name": "dander/runtime", - "is-immutable": False, + "id": "ocid1.containerrepo.oc1.iad.0.unitnamespace.runtime", "is-public": False, "lifecycle-state": "AVAILABLE", } @@ -104,6 +104,22 @@ def __call__( } ), ) + if args[0] == "oci" and "repository" in args and "get" in args: + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "data": { + "display-name": "dander/runtime", + "id": "ocid1.containerrepo.oc1.iad.0.unitnamespace.runtime", + "is-immutable": False, + "is-public": False, + "lifecycle-state": "AVAILABLE", + } + } + ), + ) if args[0] == "oci" and "lookup" in args: if not self.existing and not self.created: assert check is False diff --git a/tests/bootstrap/test_oci_image_promotion.py b/tests/bootstrap/test_oci_image_promotion.py index 9ac2823e..d43dd8f0 100644 --- a/tests/bootstrap/test_oci_image_promotion.py +++ b/tests/bootstrap/test_oci_image_promotion.py @@ -113,7 +113,7 @@ def __call__( "items": [ { "display-name": "dander/runtime", - "is-immutable": self.repository_immutable, + "id": "ocid1.containerrepo.oc1.iad.0.unitnamespace.runtime", "is-public": not self.repository_private, "lifecycle-state": "AVAILABLE", } @@ -122,6 +122,22 @@ def __call__( } ), ) + if args[0] == "oci" and "repository" in args and "get" in args: + return subprocess.CompletedProcess( + args, + 0, + stdout=json.dumps( + { + "data": { + "display-name": "dander/runtime", + "id": "ocid1.containerrepo.oc1.iad.0.unitnamespace.runtime", + "is-immutable": self.repository_immutable, + "is-public": not self.repository_private, + "lifecycle-state": "AVAILABLE", + } + } + ), + ) if args[0] == "oci" and "lookup" in args: if not self.existing and not self.created: assert not check From e6fc14a91665707941a83c42ea061e612ee19223 Mon Sep 17 00:00:00 2001 From: harrisonoconnorhover <130776285+harrisonoconnorhover@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:30:24 -0400 Subject: [PATCH 158/163] Prepare 0.9.0rc4 release candidate (#223) Co-authored-by: Harrison O'Connor-Hoover --- CHANGELOG.md | 7 ++++++ HANDOFF.md | 25 +++++++++---------- README.md | 4 +-- .../cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 +-- uv.lock | 2 +- 11 files changed, 30 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca0b60df..8cac3832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ capabilities enter through the next minor release. ## Unreleased +## 0.9.0rc4 — 2026-08-12 (beta) + +### Fixed + +- Give OCI KMS automatic key rotation an explicit 365-day schedule required by the live API. +- Verify OCIR capability from exact repository metadata after list summaries omit optional fields. + ## 0.9.0rc3 — 2026-08-12 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 09b450f0..c9af22ea 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,23 +2,22 @@ ## Finished -- Published exact protected-main candidate `dander-platform==0.9.0rc3` and verified its public wheel. +- Merged the OCI KMS rotation correction through protected CI. +- Merged exact OCIR repository-metadata verification through protected CI. - Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. - Published the source-free RC3 runtime index in the disposable GCP proof registry. -- Captured the live OCI KMS requirement for an explicit automatic-rotation schedule. -- Captured OCI's live list-summary versus exact-repository metadata behavior. +- Prepared `dander-platform==0.9.0rc4` release metadata for the combined live corrections. ## Try It -Run the focused OCI image-promotion and controller-publication tests. +Run `uv run python scripts/check_release_metadata.py`. ## Checks -- Protected-main CI passed for the exact `v0.9.0rc3` candidate. +- Protected-main CI passed after both OCI live corrections merged. - Live OCI stage-zero retry reached zero drift after one private repository create. -- The rotation-schedule correction passed PR CI; protected-main CI is running. -- Live promotion failed closed before copying because OCI's list summary omitted immutability. -- Focused tests and full protected CI remain required for the metadata correction. +- Focused OCI tests, Ruff, Mypy, Terraform validation, and protected CI passed for the corrections. +- RC4 distribution inspection and protected release CI remain required. ## Decisions @@ -28,13 +27,13 @@ Run the focused OCI image-promotion and controller-publication tests. ## Remaining -- Merge the exact-repository verification correction through protected CI. -- Retry the one-resource foundation plan after the combined protected-main CI passes. +- Publish and verify the exact protected-main `v0.9.0rc4` candidate. +- Retry the one-resource foundation plan with the public RC4 operator. - Resume digest-preserving OCIR promotion from the accepted GCP runtime index. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `src/dander/bootstrap/oci_image.py` -- `tests/bootstrap/test_oci_image_promotion.py` -- `tests/bootstrap/test_oci_controller_publication.py` +- `CHANGELOG.md` +- `pyproject.toml` +- `HANDOFF.md` diff --git a/README.md b/README.md index c0e7311f..f9b79a14 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.9.0rc3 +uv tool install dander-platform==0.9.0rc4 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.9.0rc3` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.9.0rc4` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 537f2bbb..4777e780 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.9.0rc3`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.9.0rc4`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index fb5b3d8e..b297ae80 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.9.0rc3 +uv tool install dander-platform==0.9.0rc4 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index b310e6f8..d5293d73 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -24,7 +24,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.9.0rc3`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.9.0rc4`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 98c10152..c62b9f4e 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.9.0rc3` is the current public beta candidate. +- Dander `0.9.0rc4` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 715f10f1..16aab2f8 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.9.0rc3" +export DANDER_TARGET_VERSION="0.9.0rc4" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index bf30c1d4..d50eee1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.9.0rc3" +version = "0.9.0rc4" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 41715c29..925c4a66 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.9.0rc3", + "dander-platform==0.9.0rc4", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.9.0rc3" + "README install command uses 0.1.0; package metadata uses 0.9.0rc4" ] diff --git a/uv.lock b/uv.lock index 99425bbf..ae9fff69 100644 --- a/uv.lock +++ b/uv.lock @@ -683,7 +683,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.9.0rc3" +version = "0.9.0rc4" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 75f7456feb0c465342a0fff49a75ab81618a6e76 Mon Sep 17 00:00:00 2001 From: harrisonoconnorhover <130776285+harrisonoconnorhover@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:53:22 -0400 Subject: [PATCH 159/163] Respect OCI default Vault rotation limits (#224) Co-authored-by: Harrison O'Connor-Hoover --- CHANGELOG.md | 5 +++++ HANDOFF.md | 26 +++++++++++++------------- docs/cloud-portability-plan.md | 9 ++++++--- docs/decisions.md | 13 +++++++++++++ docs/known-limitations.md | 6 ++++++ infra/oci/README.md | 7 ++++++- infra/oci/main.tf | 6 +----- infra/oci/outputs.tf | 2 +- infra/oci/tests/oci.tftest.hcl | 5 ++--- 9 files changed, 53 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cac3832..2fb815ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ capabilities enter through the next minor release. ## Unreleased +### Fixed + +- Keep the OCI foundation on its bounded-cost default Vault and disable unsupported automatic + master-key rotation instead of requiring the separately billed virtual private Vault tier. + ## 0.9.0rc4 — 2026-08-12 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index c9af22ea..6ed68c31 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,38 @@ ## Finished -- Merged the OCI KMS rotation correction through protected CI. -- Merged exact OCIR repository-metadata verification through protected CI. +- Published and verified the exact protected-main `dander-platform==0.9.0rc4` candidate. - Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. - Published the source-free RC3 runtime index in the disposable GCP proof registry. -- Prepared `dander-platform==0.9.0rc4` release metadata for the combined live corrections. +- Confirmed the default OCI Vault rejects automatic master-key rotation before creating the key. +- Corrected the foundation to retain the bounded-cost default Vault with manual key rotation. ## Try It -Run `uv run python scripts/check_release_metadata.py`. +Run `terraform -chdir=infra/oci test`. ## Checks -- Protected-main CI passed after both OCI live corrections merged. +- Protected-main CI and exact-tag publication passed for `v0.9.0rc4`. - Live OCI stage-zero retry reached zero drift after one private repository create. -- Focused OCI tests, Ruff, Mypy, Terraform validation, and protected CI passed for the corrections. -- RC4 distribution inspection and protected release CI remain required. +- The live RC4 foundation plan was exactly one key create, zero changes, zero destroys. +- Its apply failed closed with OCI's explicit default-Vault automatic-rotation limitation. ## Decisions -- Use an explicit 365-day OCI KMS automatic-rotation interval. +- Keep the bounded-cost default Vault and represent its manual master-key rotation honestly. - Verify exact OCIR metadata with a repository get after resolving one named list summary. - Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Publish and verify the exact protected-main `v0.9.0rc4` candidate. -- Retry the one-resource foundation plan with the public RC4 operator. +- Merge and publish the default-Vault correction through protected CI. +- Retry the one-resource foundation plan with the corrected public operator. - Resume digest-preserving OCIR promotion from the accepted GCP runtime index. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `CHANGELOG.md` -- `pyproject.toml` -- `HANDOFF.md` +- `infra/oci/main.tf` +- `infra/oci/tests/oci.tftest.hcl` +- `docs/known-limitations.md` diff --git a/docs/cloud-portability-plan.md b/docs/cloud-portability-plan.md index 89ce3059..cda3c9b0 100644 --- a/docs/cloud-portability-plan.md +++ b/docs/cloud-portability-plan.md @@ -1205,8 +1205,8 @@ Progress on 2026-08-12: the OCI SDK/Vault provider, typed Container Instances la plan-first Terraform foundations, and lifecycle-controller implementation are behind provider-specific boundaries. The OCI native remote-state bootstrap creates only a private versioned Object Storage bucket and private -digest-addressed OCIR repository; the second root creates a private VCN/subnet, default Vault with an -auto-rotating key, compartment-scoped Container Instance resource-principal policy, Logging, and +digest-addressed OCIR repository; the second root creates a private VCN/subnet, default Vault with a +software-protected key, compartment-scoped Container Instance resource-principal policy, Logging, and Notifications. Both use short-lived SecurityToken auth, saved plans outside the repository, and locally passing Terraform tests; a read-only verifier refreshes both remote states and fails on any planned drift. The launcher still accepts only the named PostgreSQL/PostgreSQL/ @@ -1221,7 +1221,10 @@ from the operator's SecurityToken session; source, destination, and per-platform remain equal. OCI currently rejects its advertised repository-level tag-immutability setting in the live Ashburn tenancy; Dander records that limitation, rejects existing-tag mismatches, and deploys only the verified digest (or OCI Functions' required matching tag-and-digest pair). The -separately built Function controller is bound to the exact reviewed wheel +default Vault also rejects automatic master-key rotation, which Oracle reserves for the separately +billed virtual private Vault tier; Dander records that limitation and retains the distinct required +proof that a later run observes a new versionless application-secret value without an image rebuild. +The separately built Function controller is bound to the exact reviewed wheel SHA-256: its Dockerfile, shim, and dependency pins are extracted from that wheel into an ephemeral source-free build context, published only for `linux/amd64`, and recorded by immutable digest. Live runtime/controller publication, credential-refresh, live-profile, rollback, cleanup, and diff --git a/docs/decisions.md b/docs/decisions.md index a93911b8..3a12cb64 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1410,6 +1410,19 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc test; it contains no secret material and does not weaken the versionless application-secret rotation proof required by Phase 7. +## 2026-08-12 — The bounded-cost default OCI Vault uses manual master-key rotation + +- **Provider evidence:** After the explicit schedule reached the live Ashburn API, OCI rejected the + create with `400 InvalidParameter: Automatic key rotation is not supported with VIRTUAL vaults`. + Oracle documents automatic key rotation only for the separately billed virtual private Vault; + the selected Terraform `DEFAULT` tier is the shared virtual Vault. +- **Decision:** Preserve the bounded-cost default Vault and software-protected key, disable its + unsupported automatic rotation setting, and represent manual key-version rotation honestly. Do + not silently select a roughly USD 3.72/hour private Vault for an experimental profile. +- **Boundary:** This supersedes the preceding annual-schedule decision for the default Vault only. + Phase 7 still requires a live versionless application-secret rotation between runs without image + rebuild or secret logging; master-key scheduling and application-secret refresh are distinct. + ## 2026-08-12 — OCIR repository verification follows summary with exact get - **Provider evidence:** The live repository-list summary omitted `is-immutable` when false, while diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 31f3052d..aa87e434 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -94,6 +94,12 @@ remain Phase 8 work. digest differs, promoted indexes and platform manifests are re-read and compared, and every task deployment uses an exact digest while OCI Functions receive the matching tag-and-digest pair. Repository-level tag immutability is not claimed where OCI reports it unavailable. +- OCI automatic master-key rotation is not available on the bounded-cost default Vault. A live + Ashburn create rejected it with `400 InvalidParameter`; Oracle documents the feature for the + separately billed virtual private Vault tier. Dander therefore provisions a software-protected + key on the default Vault with manual key-version rotation and does not claim automatic key + rotation. The named profile must still prove versionless application-secret rotation between + runs without rebuilding the image or logging secret values. ## Ingestion and schemas diff --git a/infra/oci/README.md b/infra/oci/README.md index 02b456f7..2da19463 100644 --- a/infra/oci/README.md +++ b/infra/oci/README.md @@ -4,7 +4,7 @@ This root creates the provider-owned foundation for Dander's named OCI profile: - a private VCN/subnet with no public IPs or inbound rules; - NAT egress for PostgreSQL plus an OCI Services gateway for OCIR, Vault, and control APIs; -- a default OCI Vault and software-protected key with an explicit 365-day auto-rotation schedule; +- a default OCI Vault and software-protected key with manual key-version rotation; - a compartment-scoped Container Instance dynamic group and least-privilege runtime policy; and - a private versioned run-record bucket, per-pipeline OCI Function, Resource Scheduler schedule, lifecycle event rule, invocation log, error alarm, and Notifications topic. @@ -52,3 +52,8 @@ All operator API calls use an expiring `SecurityToken` profile, while the Functi Instances use resource principals. Runtime promotion derives a short-lived, repository-scoped registry token from that session and never changes the operator's persistent Docker configuration. Static OCI API keys, user auth tokens, registry passwords, and static cloud keys are not fallbacks. + +OCI automatic master-key rotation is available only on the separately billed virtual private +Vault tier. The bounded-cost default Vault therefore uses a software key with manual key-version +rotation. This provider limitation does not weaken the Phase 7 application-secret proof: changing +the `CURRENT` secret version must still be observed by a later run without rebuilding its image. diff --git a/infra/oci/main.tf b/infra/oci/main.tf index def7b597..eaedaf1a 100644 --- a/infra/oci/main.tf +++ b/infra/oci/main.tf @@ -118,13 +118,9 @@ resource "oci_kms_key" "runtime" { display_name = "${var.name}-runtime" management_endpoint = oci_kms_vault.runtime.management_endpoint protection_mode = "SOFTWARE" - is_auto_rotation_enabled = true + is_auto_rotation_enabled = false freeform_tags = local.tags - auto_key_rotation_details { - rotation_interval_in_days = 365 - } - key_shape { algorithm = "AES" length = 32 diff --git a/infra/oci/outputs.tf b/infra/oci/outputs.tf index feac8be5..7d9d4672 100644 --- a/infra/oci/outputs.tf +++ b/infra/oci/outputs.tf @@ -9,7 +9,7 @@ output "vault_id" { } output "vault_key_id" { - description = "Auto-rotating software-protected key for runtime secrets." + description = "Software-protected key for runtime secrets; the default Vault supports manual key-version rotation." value = oci_kms_key.runtime.id } diff --git a/infra/oci/tests/oci.tftest.hcl b/infra/oci/tests/oci.tftest.hcl index a090bff3..8b02385f 100644 --- a/infra/oci/tests/oci.tftest.hcl +++ b/infra/oci/tests/oci.tftest.hcl @@ -42,10 +42,9 @@ run "projects_private_keyless_foundation" { condition = ( oci_kms_vault.runtime.vault_type == "DEFAULT" && oci_kms_key.runtime.protection_mode == "SOFTWARE" && - oci_kms_key.runtime.is_auto_rotation_enabled && - oci_kms_key.runtime.auto_key_rotation_details[0].rotation_interval_in_days == 365 + !oci_kms_key.runtime.is_auto_rotation_enabled ) - error_message = "The runtime Vault must use the bounded-cost default tier and an explicitly scheduled auto-rotating key." + error_message = "The runtime Vault must use the bounded-cost default tier and must not request its unsupported automatic key rotation." } assert { From 9b4448d0cc3393bf629363a00206dc17cc2368fd Mon Sep 17 00:00:00 2001 From: harrisonoconnorhover <130776285+harrisonoconnorhover@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:07:58 -0400 Subject: [PATCH 160/163] release: prepare 0.9.0rc5 (#225) Co-authored-by: Harrison O'Connor-Hoover --- CHANGELOG.md | 2 ++ HANDOFF.md | 6 +++--- README.md | 4 ++-- acceptance/cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 ++-- uv.lock | 2 +- 11 files changed, 16 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb815ac..880f105d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ capabilities enter through the next minor release. ## Unreleased +## 0.9.0rc5 — 2026-08-12 (beta) + ### Fixed - Keep the OCI foundation on its bounded-cost default Vault and disable unsupported automatic diff --git a/HANDOFF.md b/HANDOFF.md index 6ed68c31..40439949 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,7 +2,7 @@ ## Finished -- Published and verified the exact protected-main `dander-platform==0.9.0rc4` candidate. +- Prepared the exact protected-main `dander-platform==0.9.0rc5` candidate. - Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. - Published the source-free RC3 runtime index in the disposable GCP proof registry. - Confirmed the default OCI Vault rejects automatic master-key rotation before creating the key. @@ -14,7 +14,7 @@ Run `terraform -chdir=infra/oci test`. ## Checks -- Protected-main CI and exact-tag publication passed for `v0.9.0rc4`. +- Protected-main CI passed for the default-Vault correction before the RC5 cut. - Live OCI stage-zero retry reached zero drift after one private repository create. - The live RC4 foundation plan was exactly one key create, zero changes, zero destroys. - Its apply failed closed with OCI's explicit default-Vault automatic-rotation limitation. @@ -27,7 +27,7 @@ Run `terraform -chdir=infra/oci test`. ## Remaining -- Merge and publish the default-Vault correction through protected CI. +- Merge and publish `v0.9.0rc5` through protected CI. - Retry the one-resource foundation plan with the corrected public operator. - Resume digest-preserving OCIR promotion from the accepted GCP runtime index. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. diff --git a/README.md b/README.md index f9b79a14..1ea1de57 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.9.0rc4 +uv tool install dander-platform==0.9.0rc5 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.9.0rc4` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.9.0rc5` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 4777e780..247536fa 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.9.0rc4`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.9.0rc5`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index b297ae80..ff5ae19f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.9.0rc4 +uv tool install dander-platform==0.9.0rc5 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index d5293d73..50d2953a 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -24,7 +24,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.9.0rc4`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.9.0rc5`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index c62b9f4e..06f61b5e 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.9.0rc4` is the current public beta candidate. +- Dander `0.9.0rc5` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 16aab2f8..75a2f013 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.9.0rc4" +export DANDER_TARGET_VERSION="0.9.0rc5" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index d50eee1f..e013f912 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.9.0rc4" +version = "0.9.0rc5" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 925c4a66..0e78e101 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.9.0rc4", + "dander-platform==0.9.0rc5", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.9.0rc4" + "README install command uses 0.1.0; package metadata uses 0.9.0rc5" ] diff --git a/uv.lock b/uv.lock index ae9fff69..a587b70a 100644 --- a/uv.lock +++ b/uv.lock @@ -683,7 +683,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.9.0rc4" +version = "0.9.0rc5" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From b2f528840bb82437db2770408bdc7da3caecac16 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 09:46:10 -0400 Subject: [PATCH 161/163] fix(oci): preserve Buildx in scoped Docker config --- CHANGELOG.md | 5 +++ HANDOFF.md | 34 +++++++++++---------- docs/decisions.md | 10 ++++++ src/dander/bootstrap/oci_image.py | 14 +++++++++ tests/bootstrap/test_oci_image_promotion.py | 27 +++++++++++++++- 5 files changed, 73 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 880f105d..ef59faf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ capabilities enter through the next minor release. ## Unreleased +### Fixed + +- Keep OCI's temporary scoped-token Docker configuration independent of Docker Desktop's named + builder metadata while preserving Buildx plugin discovery from the operator configuration. + ## 0.9.0rc5 — 2026-08-12 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 40439949..322e066d 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,38 +2,40 @@ ## Finished -- Prepared the exact protected-main `dander-platform==0.9.0rc5` candidate. +- Published the exact protected-main `dander-platform==0.9.0rc5` candidate. - Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. -- Published the source-free RC3 runtime index in the disposable GCP proof registry. -- Confirmed the default OCI Vault rejects automatic master-key rotation before creating the key. -- Corrected the foundation to retain the bounded-cost default Vault with manual key rotation. +- Applied the RC5 OCI foundation and verified both stage-zero and foundation no-drift. +- Published the source-free RC5 runtime index in the disposable GCP proof registry. +- Corrected isolated OCIR promotion so Docker Desktop Buildx remains discoverable without copying + named-builder metadata into the temporary scoped-token configuration. ## Try It -Run `terraform -chdir=infra/oci test`. +Run `pytest -q tests/bootstrap/test_oci_image_promotion.py`. ## Checks -- Protected-main CI passed for the default-Vault correction before the RC5 cut. -- Live OCI stage-zero retry reached zero drift after one private repository create. -- The live RC4 foundation plan was exactly one key create, zero changes, zero destroys. -- Its apply failed closed with OCI's explicit default-Vault automatic-rotation limitation. +- Protected-main CI passed for the default-Vault correction and RC5 release. +- Live OCI stage zero and foundation both report no drift after RC5 apply. +- RC5 source-free GAR publication preserved the reviewed multi-platform digest and attestations. +- Isolated Docker-config regression coverage includes Docker Desktop context and plugin discovery. ## Decisions -- Keep the bounded-cost default Vault and represent its manual master-key rotation honestly. -- Verify exact OCIR metadata with a repository get after resolving one named list summary. +- Keep Buildx available through only its non-secret plugin directory; do not copy named-builder + state into the temporary credential directory. - Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Merge and publish `v0.9.0rc5` through protected CI. -- Retry the one-resource foundation plan with the corrected public operator. +- Merge the isolated Docker-config fix and publish its release candidate through protected CI. - Resume digest-preserving OCIR promotion from the accepted GCP runtime index. +- Publish and deploy the exact-wheel OCI lifecycle controller. +- Complete the bounded live lifecycle acceptance matrix. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. ## Review First -- `infra/oci/main.tf` -- `infra/oci/tests/oci.tftest.hcl` -- `docs/known-limitations.md` +- `src/dander/bootstrap/oci_image.py` +- `tests/bootstrap/test_oci_image_promotion.py` +- `HANDOFF.md` diff --git a/docs/decisions.md b/docs/decisions.md index 3a12cb64..c28ff571 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1,5 +1,15 @@ # Engineering Decisions +## 2026-08-12 — Scoped OCIR credentials do not inherit named-builder state + +- **Isolation:** OCIR promotion keeps its repository token in a one-use Docker configuration and + carries forward only ordinary registry helpers plus non-secret Buildx plugin search paths. +- **Context:** The temporary configuration removes Docker Desktop's `currentContext`; Buildx + registry operations use the context-independent default because named-builder metadata is not + copied into the credential directory. +- **Boundary:** This changes operator compatibility only. Artifact identity checks, repository + scope, short-lived authentication, and runtime image bytes remain unchanged. + ## 2026-08-10 — PostgreSQL graphs reuse canonical plans and database-local fencing - **Planning:** PostgreSQL consumes the existing `GraphExecutionPlan` and renders its relational diff --git a/src/dander/bootstrap/oci_image.py b/src/dander/bootstrap/oci_image.py index df11c3c3..2a374849 100644 --- a/src/dander/bootstrap/oci_image.py +++ b/src/dander/bootstrap/oci_image.py @@ -222,6 +222,20 @@ def _temporary_docker_config(*, host: str, identity_token: str) -> Iterator[Path raise ProjectBootstrapError("Docker configuration is not a JSON object") else: document = {} + # Buildx keeps named-builder metadata below DOCKER_CONFIG. Copying a Docker + # Desktop currentContext into this one-use directory selects metadata that + # is intentionally absent, so registry-only imagetools operations must use + # the context-independent default instead. + document.pop("currentContext", None) + source_plugin_dir = source_dir / "cli-plugins" + if source_plugin_dir.is_dir(): + plugin_dirs = document.get("cliPluginsExtraDirs", []) + if not isinstance(plugin_dirs, list) or not all( + isinstance(path, str) for path in plugin_dirs + ): + raise ProjectBootstrapError("Docker configuration has invalid plugin paths") + plugin_path = str(source_plugin_dir) + document["cliPluginsExtraDirs"] = list(dict.fromkeys((*plugin_dirs, plugin_path))) auths = document.get("auths", {}) if not isinstance(auths, dict): raise ProjectBootstrapError("Docker configuration has invalid registry entries") diff --git a/tests/bootstrap/test_oci_image_promotion.py b/tests/bootstrap/test_oci_image_promotion.py index d43dd8f0..a8471613 100644 --- a/tests/bootstrap/test_oci_image_promotion.py +++ b/tests/bootstrap/test_oci_image_promotion.py @@ -167,6 +167,9 @@ def __call__( "identitytoken": _TOKEN } assert config["credHelpers"] == {"us-central1-docker.pkg.dev": "gcloud"} + assert "currentContext" not in config + assert config["cliPluginsExtraDirs"][0] == str(Path(__file__).parent) + assert config["cliPluginsExtraDirs"][1].endswith("/docker/cli-plugins") if "create" in args: self.created = True return subprocess.CompletedProcess(args, 0, stdout="") @@ -178,8 +181,15 @@ def __call__( def _docker_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: directory = tmp_path / "docker" directory.mkdir() + (directory / "cli-plugins").mkdir() (directory / "config.json").write_text( - json.dumps({"credHelpers": {"us-central1-docker.pkg.dev": "gcloud"}}), + json.dumps( + { + "credHelpers": {"us-central1-docker.pkg.dev": "gcloud"}, + "currentContext": "desktop-linux", + "cliPluginsExtraDirs": [str(Path(__file__).parent)], + } + ), encoding="utf-8", ) monkeypatch.setenv("DOCKER_CONFIG", str(directory)) @@ -287,6 +297,21 @@ def test_oci_promoter_rejects_short_lived_or_malformed_token( _promote(tmp_path, _Runner(expires_in=60)) +def test_oci_promoter_rejects_malformed_docker_plugin_paths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _write_source_record(tmp_path) + directory = _docker_config(monkeypatch, tmp_path) + (directory / "config.json").write_text( + json.dumps({"cliPluginsExtraDirs": "not-a-list"}), + encoding="utf-8", + ) + + with pytest.raises(ProjectBootstrapError, match="invalid plugin paths"): + _promote(tmp_path, _Runner()) + + def test_oci_promoter_requires_the_reviewed_private_available_repository( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From ee477ae591207b11452d8e303cbef4e01f06464c Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 10:03:22 -0400 Subject: [PATCH 162/163] release: prepare 0.9.0rc6 --- CHANGELOG.md | 2 ++ HANDOFF.md | 5 +++-- README.md | 4 ++-- acceptance/cloud-portability/phase1b/README.md | 2 +- docs/getting-started.md | 2 +- docs/release-audit.md | 2 +- docs/session-resume.md | 2 +- docs/upgrading.md | 2 +- pyproject.toml | 2 +- tests/test_release_metadata.py | 4 ++-- uv.lock | 2 +- 11 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef59faf8..8b9305bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ capabilities enter through the next minor release. ## Unreleased +## 0.9.0rc6 — 2026-08-12 (beta) + ### Fixed - Keep OCI's temporary scoped-token Docker configuration independent of Docker Desktop's named diff --git a/HANDOFF.md b/HANDOFF.md index 322e066d..59068059 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,7 +2,7 @@ ## Finished -- Published the exact protected-main `dander-platform==0.9.0rc5` candidate. +- Prepared the exact protected-main `dander-platform==0.9.0rc6` candidate. - Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. - Applied the RC5 OCI foundation and verified both stage-zero and foundation no-drift. - Published the source-free RC5 runtime index in the disposable GCP proof registry. @@ -16,6 +16,7 @@ Run `pytest -q tests/bootstrap/test_oci_image_promotion.py`. ## Checks - Protected-main CI passed for the default-Vault correction and RC5 release. +- Protected-main CI passed for the isolated Docker-config correction before the RC6 cut. - Live OCI stage zero and foundation both report no drift after RC5 apply. - RC5 source-free GAR publication preserved the reviewed multi-platform digest and attestations. - Isolated Docker-config regression coverage includes Docker Desktop context and plugin discovery. @@ -28,7 +29,7 @@ Run `pytest -q tests/bootstrap/test_oci_image_promotion.py`. ## Remaining -- Merge the isolated Docker-config fix and publish its release candidate through protected CI. +- Merge and publish `v0.9.0rc6` through protected CI. - Resume digest-preserving OCIR promotion from the accepted GCP runtime index. - Publish and deploy the exact-wheel OCI lifecycle controller. - Complete the bounded live lifecycle acceptance matrix. diff --git a/README.md b/README.md index 1ea1de57..ac1261f8 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The Python distribution is named `dander-platform` because the `dander` name on different project. The import package and command remain `dander`: ```bash -uv tool install dander-platform==0.9.0rc5 +uv tool install dander-platform==0.9.0rc6 dander --version dander new my-data-platform cd my-data-platform @@ -627,7 +627,7 @@ shows each run's agents with their role, ticket, and live PASS/FAIL verdicts: ## Status -Dander `0.9.0rc5` is the current public beta: its source-free GCP platform, connector plugins, +Dander `0.9.0rc6` is the current public beta: its source-free GCP platform, connector plugins, transform/test engine, run ledger, metadata spine, and Druff operator interface have completed bounded live proofs. It remains pre-1.0 software; review the documented limitations before using business-critical data. diff --git a/acceptance/cloud-portability/phase1b/README.md b/acceptance/cloud-portability/phase1b/README.md index 247536fa..20bca7e1 100644 --- a/acceptance/cloud-portability/phase1b/README.md +++ b/acceptance/cloud-portability/phase1b/README.md @@ -12,7 +12,7 @@ Terraform state, task descriptions, or logs. ## Prerequisites -- Dander `0.9.0rc5`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with +- Dander `0.9.0rc6`, Docker Buildx, Terraform 1.9+, `gcloud`, and AWS CLI authenticated with short-lived browser/SSO credentials. - `crane` pinned to `v0.20.6` (`go install github.com/google/go-containerregistry/cmd/crane@v0.20.6`). diff --git a/docs/getting-started.md b/docs/getting-started.md index ff5ae19f..5a81e26a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ gcloud auth application-default set-quota-project "$DANDER_PROJECT" ## Install and scaffold ```bash -uv tool install dander-platform==0.9.0rc5 +uv tool install dander-platform==0.9.0rc6 dander --version dander new my-dander-project cd my-dander-project diff --git a/docs/release-audit.md b/docs/release-audit.md index 50d2953a..a5977c22 100644 --- a/docs/release-audit.md +++ b/docs/release-audit.md @@ -24,7 +24,7 @@ cloud path remains outside the live proof. ## Current release and deployment record -- Public Dander beta: `0.9.0rc5`; public Salesforce connector: `0.3.1`; public ServiceNow connector: +- Public Dander beta: `0.9.0rc6`; public Salesforce connector: `0.3.1`; public ServiceNow connector: `0.2.2`. - Isolated portability acceptance used Dander `0.7.0rc2` and the public plugin candidates in a source-free image. The same accepted OCI content passed local and Cloud Run conformance. diff --git a/docs/session-resume.md b/docs/session-resume.md index 06f61b5e..1fd41cbb 100644 --- a/docs/session-resume.md +++ b/docs/session-resume.md @@ -5,7 +5,7 @@ Read `HANDOFF.md`, `docs/decisions.md`, `docs/spec-alignment.md`, and ## Public releases -- Dander `0.9.0rc5` is the current public beta candidate. +- Dander `0.9.0rc6` is the current public beta candidate. - Salesforce `0.3.1` and ServiceNow `0.2.2` are the current stable connector releases for Dander `0.7.x`; their accepted release candidates remain recorded in the Phase 1 evidence. - Druff's fork contains Josh's reconciled graph-client ancestry and the later persistence, diff --git a/docs/upgrading.md b/docs/upgrading.md index 75a2f013..434291f0 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -11,7 +11,7 @@ the [hosted quickstart](getting-started.md). A guarded installation must also re ## Upgrade the CLI and project image ```bash -export DANDER_TARGET_VERSION="0.9.0rc5" +export DANDER_TARGET_VERSION="0.9.0rc6" uv tool install --force "dander-platform==$DANDER_TARGET_VERSION" dander --version ``` diff --git a/pyproject.toml b/pyproject.toml index e013f912..e1dce7c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dander-platform" -version = "0.9.0rc5" +version = "0.9.0rc6" description = "Opinionated, GCP-native, self-hosted data platform: ingest + transform + catalog behind one CLI." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 0e78e101..6f81804e 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -37,7 +37,7 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: readme = tmp_path / "README.md" readme.write_text( readme.read_text(encoding="utf-8").replace( - "dander-platform==0.9.0rc5", + "dander-platform==0.9.0rc6", "dander-platform==0.1.0", 1, ), @@ -45,5 +45,5 @@ def test_release_metadata_check_reports_stale_readme(tmp_path: Path) -> None: ) assert release_metadata_errors(tmp_path) == [ - "README install command uses 0.1.0; package metadata uses 0.9.0rc5" + "README install command uses 0.1.0; package metadata uses 0.9.0rc6" ] diff --git a/uv.lock b/uv.lock index a587b70a..2917a0b9 100644 --- a/uv.lock +++ b/uv.lock @@ -683,7 +683,7 @@ wheels = [ [[package]] name = "dander-platform" -version = "0.9.0rc5" +version = "0.9.0rc6" source = { editable = "." } dependencies = [ { name = "dlt", extra = ["bigquery"] }, From 55fa677e340625b7bd7264ed5f72fcf034162330 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Wed, 12 Aug 2026 10:31:01 -0400 Subject: [PATCH 163/163] Fix OCIR scoped token authentication --- CHANGELOG.md | 5 ++++ HANDOFF.md | 29 +++++++++---------- docs/decisions.md | 13 +++++++++ src/dander/bootstrap/oci_image.py | 14 ++++++--- .../test_oci_controller_publication.py | 9 ++++-- tests/bootstrap/test_oci_image_promotion.py | 9 ++++-- 6 files changed, 53 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b9305bd..de5ee529 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ capabilities enter through the next minor release. ## Unreleased +### Fixed + +- Present short-lived repository-scoped OCI access tokens to Docker in OCIR's supported + `BEARER_TOKEN` login form while retaining isolated, ephemeral credential storage. + ## 0.9.0rc6 — 2026-08-12 (beta) ### Fixed diff --git a/HANDOFF.md b/HANDOFF.md index 59068059..0f1bde04 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,35 +2,32 @@ ## Finished -- Prepared the exact protected-main `dander-platform==0.9.0rc6` candidate. -- Completed OCI stage zero and migrated its state into the private versioned Object Storage bucket. -- Applied the RC5 OCI foundation and verified both stage-zero and foundation no-drift. -- Published the source-free RC5 runtime index in the disposable GCP proof registry. -- Corrected isolated OCIR promotion so Docker Desktop Buildx remains discoverable without copying - named-builder metadata into the temporary scoped-token configuration. +- Diagnosed live OCIR Buildx authentication without uploading an artifact. +- Confirmed the short-lived token and exact `pull,push` repository scope are valid. +- Changed isolated Docker credentials to OCIR's supported `BEARER_TOKEN` login form. +- Kept the scoped token out of command arguments and ephemeral after command exit. +- Covered both runtime promotion and controller publication projections. ## Try It -Run `pytest -q tests/bootstrap/test_oci_image_promotion.py`. +Run `pytest -q tests/bootstrap/test_oci_image_promotion.py tests/bootstrap/test_oci_controller_publication.py`. ## Checks -- Protected-main CI passed for the default-Vault correction and RC5 release. -- Protected-main CI passed for the isolated Docker-config correction before the RC6 cut. -- Live OCI stage zero and foundation both report no drift after RC5 apply. -- RC5 source-free GAR publication preserved the reviewed multi-platform digest and attestations. -- Isolated Docker-config regression coverage includes Docker Desktop context and plugin discovery. +- Live registry probes returned `403` for `identitytoken` and authenticated `not found` for the + same token in `BEARER_TOKEN` Basic form. +- Focused tests cover both OCIR publication paths and isolated Docker configuration. ## Decisions -- Keep Buildx available through only its non-secret plugin directory; do not copy named-builder - state into the temporary credential directory. +- Use Docker's ordinary `auth` field with the fixed OCIR access-token username. +- Preserve the one-use mode-`0600` Docker configuration and exact repository scope. - Keep OCI experimental until Phase 7 live acceptance and Phase 8 qualification pass. ## Remaining -- Merge and publish `v0.9.0rc6` through protected CI. -- Resume digest-preserving OCIR promotion from the accepted GCP runtime index. +- Merge this focused fix through protected CI and cut the next candidate. +- Resume digest-preserving OCIR promotion from the exact accepted runtime index. - Publish and deploy the exact-wheel OCI lifecycle controller. - Complete the bounded live lifecycle acceptance matrix. - Complete cleanup, no-drift evidence, and the binary Phase 7 recommendation. diff --git a/docs/decisions.md b/docs/decisions.md index c28ff571..2e58f571 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1441,3 +1441,16 @@ Resolves the "separate product decisions" the two entries above deferred, unbloc name, visibility, lifecycle, and immutability capability from the exact get response. - **Boundary:** Missing or inconsistent exact metadata still fails closed before registry-token creation or artifact copy; an omitted optional field in list summaries is not treated as false. + +## 2026-08-12 — OCIR scoped tokens use Docker's Basic credential shape + +- **Provider evidence:** The live OCIR distribution endpoint accepted the scoped token directly, + but Docker Buildx returned `403 Forbidden` when the token was stored as `identitytoken`. The same + token produced the expected authenticated `not found` response when supplied using the fixed + `BEARER_TOKEN` username. +- **Decision:** Encode `BEARER_TOKEN:` in Docker's standard `auth` entry inside the + mode-`0600` temporary configuration. Never pass the token on a command line or retain it after + promotion/controller publication exits. +- **Boundary:** This changes only Docker's credential representation; OCI session authentication, + repository scope, token lifetime validation, digest verification, and artifact behavior remain + unchanged. diff --git a/src/dander/bootstrap/oci_image.py b/src/dander/bootstrap/oci_image.py index 2a374849..fcab7cf2 100644 --- a/src/dander/bootstrap/oci_image.py +++ b/src/dander/bootstrap/oci_image.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import hashlib import json import os @@ -212,7 +213,7 @@ def _registry_token( @contextmanager -def _temporary_docker_config(*, host: str, identity_token: str) -> Iterator[Path]: +def _temporary_docker_config(*, host: str, access_token: str) -> Iterator[Path]: try: source_dir = Path(os.environ.get("DOCKER_CONFIG", Path.home() / ".docker")) source_path = source_dir / "config.json" @@ -239,7 +240,12 @@ def _temporary_docker_config(*, host: str, identity_token: str) -> Iterator[Path auths = document.get("auths", {}) if not isinstance(auths, dict): raise ProjectBootstrapError("Docker configuration has invalid registry entries") - document["auths"] = {**auths, host: {"identitytoken": identity_token}} + # OCI's scoped access-token endpoint returns a bearer token, but Docker + # interprets ``identitytoken`` as a token for its own registry OAuth + # exchange. OCIR instead accepts the scoped token through Docker's + # Basic-auth credential shape with the fixed BEARER_TOKEN username. + credential = base64.b64encode(f"BEARER_TOKEN:{access_token}".encode()).decode() + document["auths"] = {**auths, host: {"auth": credential}} with TemporaryDirectory(prefix="dander-ocir-") as directory: config_dir = Path(directory) config_path = config_dir / "config.json" @@ -373,7 +379,7 @@ def promote( namespace=namespace, repository_name=repository_name, ) - with _temporary_docker_config(host=host, identity_token=token) as config_dir: + with _temporary_docker_config(host=host, access_token=token) as config_dir: if created: self._runner( ( @@ -583,7 +589,7 @@ def publish( namespace=namespace, repository_name=repository_name, ) - with _temporary_docker_config(host=host, identity_token=token) as config_dir: + with _temporary_docker_config(host=host, access_token=token) as config_dir: if created: with TemporaryDirectory(prefix="dander-oci-controller-") as directory: context_dir = Path(directory) diff --git a/tests/bootstrap/test_oci_controller_publication.py b/tests/bootstrap/test_oci_controller_publication.py index 81967f0d..84b56f27 100644 --- a/tests/bootstrap/test_oci_controller_publication.py +++ b/tests/bootstrap/test_oci_controller_publication.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import hashlib import json import subprocess @@ -145,9 +146,11 @@ def __call__( config_path = Path(args[2]) / "config.json" self.config_paths.append(config_path) config = json.loads(config_path.read_text(encoding="utf-8")) - assert config["auths"]["ocir.us-ashburn-1.oci.oraclecloud.com"] == { - "identitytoken": _TOKEN - } + encoded = config["auths"]["ocir.us-ashburn-1.oci.oraclecloud.com"]["auth"] + assert base64.b64decode(encoded).decode() == f"BEARER_TOKEN:{_TOKEN}" + assert "identitytoken" not in config["auths"][ + "ocir.us-ashburn-1.oci.oraclecloud.com" + ] assert config["credHelpers"] == {"us-central1-docker.pkg.dev": "gcloud"} if "build" in args: assert cwd != self.project_dir diff --git a/tests/bootstrap/test_oci_image_promotion.py b/tests/bootstrap/test_oci_image_promotion.py index a8471613..bd592db0 100644 --- a/tests/bootstrap/test_oci_image_promotion.py +++ b/tests/bootstrap/test_oci_image_promotion.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import hashlib import json import subprocess @@ -163,9 +164,11 @@ def __call__( config_path = Path(args[2]) / "config.json" self.config_paths.append(config_path) config = json.loads(config_path.read_text(encoding="utf-8")) - assert config["auths"]["ocir.us-ashburn-1.oci.oraclecloud.com"] == { - "identitytoken": _TOKEN - } + encoded = config["auths"]["ocir.us-ashburn-1.oci.oraclecloud.com"]["auth"] + assert base64.b64decode(encoded).decode() == f"BEARER_TOKEN:{_TOKEN}" + assert "identitytoken" not in config["auths"][ + "ocir.us-ashburn-1.oci.oraclecloud.com" + ] assert config["credHelpers"] == {"us-central1-docker.pkg.dev": "gcloud"} assert "currentContext" not in config assert config["cliPluginsExtraDirs"][0] == str(Path(__file__).parent)