From befd32f97c7fd5d4c307c9981efc0a244f3c445a Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 30 Sep 2026 21:13:24 -0700 Subject: [PATCH 01/53] =?UTF-8?q?feat(aws-startup-advisor):=20artifact-con?= =?UTF-8?q?tract=20validator=20=E2=80=94=20check=20phase=20artifacts=20aga?= =?UTF-8?q?inst=20their=20published=20schemas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration skills pass ~17 JSON artifacts between phases, each with a contract published as a schema-*.md "shape" block (JSON/JSONC example) or, for two of them, a real JSON Schema under skills/shared/. Nothing checked that a producer writes what the next phase reads: a key could be renamed, emitted without being documented, or an enum widened, and every DSL _assert still passed because the model that writes the artifact evaluates the assertion. This is the precondition the backlog names for any engine consolidation or AI-engine extraction — moves across adapters have no drift detection without it. scripts/validate-artifacts.py (stdlib, Python 3.9+) - Parses each contract document's example blocks (JSONC comments, trailing commas, `...` ellipses, `a|b` enum strings, `// a | b | c` enum comments, `// REQUIRED` / `(REQUIRED)` markers, `""` keys = open object, `"": {...}` = template for every key) into a key-path contract; sub-section blocks (`## services[]`, `### metadata (REQUIRED)`) refine paths; manifest can pin a block by `@L` anchor, merge other documents' additions (graviton, workshop), or declare a row-annotation vocabulary (azure preferences' source/reason/context/...). - Validates real JSON Schemas with a draft-07 subset (type, required, properties, additionalProperties, enum, const, items, pattern, min/max, minItems, allOf/anyOf/oneOf, $ref). - Findings: UNKNOWN_KEY, MISSING_REQUIRED, TYPE_MISMATCH, ENUM_VIOLATION, SCHEMA_VIOLATION, SCHEMA_PARSE (the contract itself is broken), NO_CONTRACT (info). `_`-prefixed keys are annotations. - Modes: --self-check (contracts parse), --fixtures (CI; goldens against the baseline), --run-dir <.migration/run> --skill (users). - scripts/artifact-contracts.json maps artifact → contract per skill. - scripts/artifact-contracts-baseline.json: 21 explained entries covering 44 known gaps between what phases emit and what their contract documents say, each naming the document that must change. Stale entries are reported so the list only shrinks. Contract documents fixed where the document contradicted itself (its own prose named a key its example block omitted): - gcp schema-preferences.md: the shape block was not JSON (`{ "value", "chosen_by" }` set notation); rewritten as a JSONC row template. - gcp schema-estimate-infra.md: root block lacked complexity_tier, which estimation-infra.schema.json marks required. - gcp schema-discover-iac.md § Live discovery extensions: block now shows the metadata/resources[] additions the prose above it documents. - heroku schema-discover-heroku.md: resources[] example carries source/unmanaged_by_terraform/not_found_live (table at lines 93-95); addon config example carries the per-add-on and live-enrichment fields the bullets below it list. - azure schema-design-aws.md: services[] block carries routing_provenance/sizing_provenance/sizing_source/is_compute_unit/ hosted_app_azure_ids (the same document marks them REQUIRED); deferred[] carries routing_provenance. - azure schema-discover-azure.md: clusters[] carries pattern_status (discover-assemble.md § Status). - azure schema-preferences.md: top-level clarify_status (REQUIRED per clarify-assemble.md § clarify_status). Fixture: gcp-decision-gate's estimation-infra.json stub gains the four keys estimation-infra.schema.json requires; check_expected_decide.py still passes. Wiring: mise `lint:artifacts` task, added to `validate` (CI build runs it). README scripts/ entry updated. Tests: tests/test_validate_artifacts.py — 16 tests: JSONC stripping, template inference (enum/required/placeholder/any-key), each finding code, JSON Schema subset, baseline matching + stale detection, CLI self-check and fixtures pass on the real repo, and two negatives: an injected off-contract key and a schema enum violation both exit 1. --- mise.toml | 6 +- plugins/aws-startup-advisor/README.md | 2 +- .../estimation-infra.json | 11 +- .../scripts/artifact-contracts-baseline.json | 131 +++ .../scripts/artifact-contracts.json | 243 +++++ .../scripts/validate-artifacts.py | 888 ++++++++++++++++++ .../references/shared/schema-design-aws.md | 8 +- .../shared/schema-discover-azure.md | 3 +- .../references/shared/schema-preferences.md | 1 + .../references/shared/schema-discover-iac.md | 12 + .../shared/schema-estimate-infra.md | 1 + .../references/shared/schema-preferences.md | 11 +- .../shared/schema-discover-heroku.md | 18 +- .../tests/test_validate_artifacts.py | 214 +++++ 14 files changed, 1540 insertions(+), 9 deletions(-) create mode 100644 plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json create mode 100644 plugins/aws-startup-advisor/scripts/artifact-contracts.json create mode 100644 plugins/aws-startup-advisor/scripts/validate-artifacts.py create mode 100644 plugins/aws-startup-advisor/tests/test_validate_artifacts.py diff --git a/mise.toml b/mise.toml index cad129e3b..5aa686eff 100644 --- a/mise.toml +++ b/mise.toml @@ -27,9 +27,13 @@ run = "python3 tools/sync-plugin-skills.py --check" description = "Validate Agent Plugins v1.0.0 conformance (plugins/)" run = "uv run --with jsonschema --with pyyaml python tools/validate_spec.py" +[tasks."lint:artifacts"] +description = "Validate aws-startup-advisor fixture artifacts against their published schema contracts" +run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures" + [tasks.validate] description = "Run all validation" -depends = ["lint:manifests", "lint:skill-parity", "lint:spec"] +depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts"] [tasks.security] description = "Run security scans" diff --git a/plugins/aws-startup-advisor/README.md b/plugins/aws-startup-advisor/README.md index 0161445ee..15f17ae76 100644 --- a/plugins/aws-startup-advisor/README.md +++ b/plugins/aws-startup-advisor/README.md @@ -36,7 +36,7 @@ Alongside the skills, the plugin ships supporting material the skills load on de - [`skills/shared/`](skills/shared/) — Canonical reference material shared across skills rather than an invocable skill of its own: the phase-workflow interpreter contract, estimation schemas and complexity tiers, a snapshot of AWS infrastructure rates, phase-status schema, what-if workshop invariants, shared Clarify question fragments, and the AI-migration guardrails and Bedrock model references. Each consuming skill carries a byte-identical copy under its own `references/vendored/` so the skill folder stays self-contained. - [`agents/`](agents/) — Seven Claude Code subagent definitions: two generic, phase-agnostic migration phase workers (read/write and read/write/shell tiers) plus five specialists for the Bedrock rewrite flow (code analyzer, code rewriter, log ingestor, prompt evaluator, report generator). -- [`scripts/`](scripts/) — Three Python helpers that validate generated artifacts and emit run summaries: a migration report validator, a startup-program artifact check, and a plan summary emitter. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. +- [`scripts/`](scripts/) — Four Python helpers that validate generated artifacts and emit run summaries: a migration report validator, a startup-program artifact check, a plan summary emitter, and `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. - [`fixtures/`](fixtures/) — Reference and regression fixtures for the migration report and estimation artifacts, used to keep generated output within the documented contract. `fixtures/azure-iac-terraform/` additionally carries deterministic asserters that pin the Azure Discover, Clarify, Design, and Estimate contracts against a committed synthetic estate. ## MCP Servers diff --git a/plugins/aws-startup-advisor/fixtures/gcp-decision-gate/after-decide-complete/estimation-infra.json b/plugins/aws-startup-advisor/fixtures/gcp-decision-gate/after-decide-complete/estimation-infra.json index 8223f3baf..7ee9394b1 100644 --- a/plugins/aws-startup-advisor/fixtures/gcp-decision-gate/after-decide-complete/estimation-infra.json +++ b/plugins/aws-startup-advisor/fixtures/gcp-decision-gate/after-decide-complete/estimation-infra.json @@ -1,4 +1,9 @@ { + "_comment": "Minimal Estimate output for the decision-gate oracle. Carries the keys estimation-infra.schema.json marks required (phase, pricing_source.status, complexity_tier, recommendation.path) so scripts/validate-artifacts.py --fixtures keeps this golden on-contract; check_expected_decide.py reads only projected_costs, current_costs and recommendation.outcome*.", + "phase": "estimate", + "pricing_source": { + "status": "cached" + }, "projected_costs": { "aws_monthly_balanced": 155, "aws_monthly_premium": 212, @@ -9,6 +14,8 @@ }, "recommendation": { "outcome": "conditional_go", - "outcome_label": "Go, with conditions" - } + "outcome_label": "Go, with conditions", + "path": "migrate_optimized" + }, + "complexity_tier": "small" } diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json b/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json new file mode 100644 index 000000000..feed382c5 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json @@ -0,0 +1,131 @@ +{ + "_what_this_is": "Known findings from scripts/validate-artifacts.py --fixtures that do not fail the build YET. Every entry is a real gap between what a phase emits (as witnessed by a fixture golden) and what its contract document says, with a reason and the fix that would retire it. This file is a burn-down list, not an allow-list: an entry that stops matching anything is reported as STALE_BASELINE so it gets removed. Add an entry only with a reason that names the document or phase that must change.", + "entries": [ + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/gcp-*/*gcp-resource-inventory.json", + "path": "resources[[]*[]].id", + "reason": "schema-discover-iac.md:107 says the resource identifier is `address` — 'NOT `id`'. The live-capture seed and the workshop goldens carry `id` anyway. Either discover-live.md is emitting an off-contract key or the doc must admit `id` for live-sourced resources. Decide, fix one side, remove this entry." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/gcp-*/*gcp-resource-inventory.json", + "path": "edges", + "reason": "Top-level `edges[]` is produced by discover-live / clustering (schema-discover-iac.md:312 describes edges inside clusters only). Add a top-level `edges` entry to the inventory example or move the array under clusters." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/gcp-*/*gcp-resource-inventory.json", + "path": "migration_id", + "reason": "Discover stamps `migration_id` on the inventory root; schema-discover-iac.md does not list it. Add to the root example (it is useful provenance) or stop emitting it." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/gcp-*/*gcp-resource-inventory.json", + "path": "metadata.discovery_timestamp", + "reason": "Live discovery writes `metadata.discovery_timestamp`; the doc's metadata example has `report_date` only. Reconcile the two timestamp fields in schema-discover-iac.md § Live discovery extensions." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/gcp-*/*gcp-resource-inventory.json", + "path": "metadata.project", + "reason": "Live discovery writes `metadata.project` as well as `live_metadata.project`. Document one home for the GCP project id." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/heroku-*/*heroku-resource-inventory.json", + "path": "migration_id", + "reason": "Discover stamps `migration_id` on the inventory root; schema-discover-heroku.md does not list it. Add to the root example or stop emitting." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/heroku-*/*heroku-resource-inventory.json", + "path": "skill", + "reason": "Discover stamps `skill: heroku-to-aws` on the inventory root; schema-discover-heroku.md does not list it. Same fix as migration_id." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/heroku-*/*heroku-resource-inventory.json", + "path": "resources[[]*[]].config.config_var_keys", + "reason": "discover-live-capture.md captures config var KEY NAMES into app config; schema-discover-heroku.md's addon/formation config tables do not list `config_var_keys`. Add it to the `formation` (or app-level) config schema with the names-only rule stated." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/heroku-*/*heroku-resource-inventory.json", + "path": "resources[[]*[]].config.hostname", + "reason": "Domain resources carry `config.hostname`; schema-discover-heroku.md documents `domain` resource IDs (line 149) but has no `domain` config block. Add a `### domain config` block." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/heroku-*/*heroku-resource-inventory.json", + "path": "resources[[]*[]].config.sni_endpoint", + "reason": "Domain resources carry `config.sni_endpoint`; undocumented. Same `### domain config` block as hostname." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*aws-design.json", + "path": "services[[]*[]].sizing_note", + "reason": "Design writes a free-text `sizing_note` beside `sizing_source`; schema-design-aws.md § sizing_provenance does not mention it. Add to the services[] example or fold the text into `rationale`." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*aws-design.json", + "path": "target_region", + "reason": "Design copies `global.target_region` to the artifact root; schema-design-aws.md § Shape has no `target_region`. Add it (Estimate reads it) or have Estimate read preferences.json instead." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "app_service_plans[[]*[]].plan_name_expression", + "reason": "clarify-compute.md Q-C2 carries the plan's name expression, SKU and worker count on the isolation row so the sheet can render them; schema-preferences.md's app_service_plans[] example lists only plan_azure_id / hosted_app_count / isolation_split. Add the three fields to the example." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "app_service_plans[[]*[]].sku_name", + "reason": "See plan_name_expression." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "app_service_plans[[]*[]].worker_count", + "reason": "See plan_name_expression." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "licensing.blockers[[]*[]].severity", + "reason": "clarify-assemble.md § clarify_status requires `severity: \"blocker\"` on licensing.blockers[] entries; schema-preferences.md's blockers[] example has azure_id and code only. Add severity (and message/detail) to the example." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "licensing.blockers[[]*[]].message", + "reason": "See severity." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "baseline.azure_monthly_spend.currency", + "reason": "clarify-global.md Q-A5 records currency and period alongside the spend figure; schema-preferences.md's baseline example has the bare row. Add both." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "baseline.azure_monthly_spend.period", + "reason": "See currency." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "clusters[[]*[]].confirmed", + "reason": "The assembler writes `confirmed` and `pattern_status` on each clusters[] row (pattern confirmation, clarify.md § Orientation); schema-preferences.md's clusters[] example has pattern_id only. Add both." + }, + { + "code": "UNKNOWN_KEY", + "artifact": "fixtures/azure-*/*preferences.json", + "path": "clusters[[]*[]].pattern_status", + "reason": "See confirmed." + } + ] +} diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json new file mode 100644 index 000000000..99d48625a --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -0,0 +1,243 @@ +{ + "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (prefix match on the heading text, backticks ignored); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", + "fixture_prefixes": { + "gcp-": "gcp-to-aws", + "heroku-": "heroku-to-aws", + "azure-": "azure-to-aws" + }, + "shared": { + ".phase-status.json": { + "json_schema": "skills/shared/state/phase-status.schema.json" + } + }, + "skills": { + "gcp-to-aws": { + "gcp-resource-inventory.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-discover-iac.md", + "root_heading": "gcp-resource-inventory.json", + "sections": { + "Live discovery extensions": "", + "@L128": "resources[]" + }, + "ignore_headings": [ + "gcp-resource-clusters.json" + ], + "open_paths": [ + "resources[].config" + ], + "_open_paths_why": "schema-discover-iac.md documents one sample config; per-resource-type config shapes are not published, so config keys cannot be judged unknown.", + "_sections_why": "@L128 is the single-resource 'Example:' block under the inventory heading; it documents one resources[] item, not the root. 'Live discovery extensions' documents root-level additions (metadata, resources[], live_metadata), so it merges into the root.", + "extra_shapes": [ + { + "shape": "skills/gcp-to-aws/references/shared/schema-graviton.md", + "heading": "graviton_profile", + "path": "graviton_profile[]" + } + ] + }, + "gcp-resource-clusters.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-discover-iac.md", + "root_heading": "gcp-resource-clusters.json", + "ignore_headings": [ + "gcp-resource-inventory.json", + "Live discovery extensions" + ] + }, + "billing-profile.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-discover-billing.md", + "root_heading": "billing-profile.json" + }, + "ai-workload-profile.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-discover-ai.md", + "root_heading": "ai-workload-profile.json" + }, + "preferences.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-preferences.md", + "root_heading": "Top-level shape", + "extra_shapes": [ + { + "shape": "skills/gcp-to-aws/references/shared/schema-graviton.md", + "heading": "design_constraints.cpu_architecture", + "path": "design_constraints.cpu_architecture" + }, + { + "shape": "skills/gcp-to-aws/references/shared/schema-workshop-scenarios.md", + "heading": "preferences.json", + "path": "workshop" + } + ] + }, + "estimation-infra.json": { + "json_schema": "skills/shared/estimate/estimation-infra.schema.json", + "shape": "skills/gcp-to-aws/references/shared/schema-estimate-infra.md", + "root_heading": "estimation-infra.json schema", + "sections": { + "Observability Entry in": "projected_costs.breakdown[]", + "Security Baseline Entries in": "projected_costs.breakdown[]", + "architecture_comparison": "architecture_comparison" + }, + "ignore_headings": [ + "Cost tiers" + ] + }, + "scenarios/scenario-NNN.preferences.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-preferences.md", + "root_heading": "Top-level shape", + "extra_shapes": [ + { + "shape": "skills/gcp-to-aws/references/shared/schema-graviton.md", + "heading": "design_constraints.cpu_architecture", + "path": "design_constraints.cpu_architecture" + }, + { + "shape": "skills/gcp-to-aws/references/shared/schema-workshop-scenarios.md", + "heading": "preferences.json", + "path": "workshop" + } + ] + }, + "scenarios/scenario-NNN.estimation-infra.json": { + "json_schema": "skills/shared/estimate/estimation-infra.schema.json" + } + }, + "heroku-to-aws": { + "heroku-resource-inventory.json": { + "shape": "skills/heroku-to-aws/references/shared/schema-discover-heroku.md", + "root_heading": "heroku-resource-inventory.json", + "section_rules": [ + { + "heading_regex": "^`[a-z_]+` config$", + "path": "resources[].config" + } + ], + "sections": { + "Complete Example": "" + }, + "_sections_why": "'Complete Example' is a full artifact; merging it into the root makes keys it shows (resources[].source, unmanaged_by_terraform, …) part of the contract." + }, + "preferences.json": { + "shape": "skills/heroku-to-aws/references/phases/clarify/clarify-assemble.md", + "root_heading": "Step 4: Assemble and Write preferences.json", + "extra_shapes": [ + { + "shape": "skills/heroku-to-aws/references/shared/schema-workshop-scenarios.md", + "heading": "preferences.json", + "path": "workshop" + } + ] + }, + "estimation-infra.json": { + "json_schema": "skills/shared/estimate/estimation-infra.schema.json" + }, + "scenarios/index.json": { + "shape": "skills/heroku-to-aws/references/shared/schema-workshop-scenarios.md", + "root_heading": "scenarios/index.json", + "ignore_headings": [ + "preferences.json", + "scenarios/scenario-NNN.json" + ] + }, + "scenarios/scenario-NNN.json": { + "shape": "skills/heroku-to-aws/references/shared/schema-workshop-scenarios.md", + "root_heading": "scenarios/scenario-NNN.json", + "ignore_headings": [ + "preferences.json", + "scenarios/index.json" + ] + }, + "scenarios/scenario-NNN.preferences.json": { + "shape": "skills/heroku-to-aws/references/phases/clarify/clarify-assemble.md", + "root_heading": "Step 4: Assemble and Write preferences.json", + "extra_shapes": [ + { + "shape": "skills/heroku-to-aws/references/shared/schema-workshop-scenarios.md", + "heading": "preferences.json", + "path": "workshop" + } + ] + }, + "scenarios/scenario-NNN.estimation-infra.json": { + "json_schema": "skills/shared/estimate/estimation-infra.schema.json" + } + }, + "azure-to-aws": { + "azure-resource-inventory.json": { + "shape": "skills/azure-to-aws/references/shared/schema-discover-azure.md", + "root_heading": "azure-resource-inventory.json", + "sections": { + "Warnings": "warnings[]" + }, + "ignore_headings": [ + "azure-resource-clusters.json" + ], + "extra_shapes": [ + { + "shape": "skills/azure-to-aws/references/shared/schema-graviton.md", + "heading": "graviton_profile", + "path": "graviton_profile[]" + } + ] + }, + "azure-resource-clusters.json": { + "shape": "skills/azure-to-aws/references/shared/schema-discover-azure.md", + "root_heading": "azure-resource-clusters.json", + "ignore_headings": [ + "azure-resource-inventory.json", + "Warnings" + ] + }, + "ai-workload-profile.json": { + "shape": "skills/azure-to-aws/references/shared/schema-discover-ai.md", + "root_heading": "Shape" + }, + "preferences.json": { + "shape": "skills/azure-to-aws/references/shared/schema-preferences.md", + "root_heading": "Shape", + "allowed_anywhere": [ + "source", + "reason", + "context", + "note", + "unanswered", + "blocks_phase", + "conflict", + "forced_by", + "residency_warning", + "detail", + "default_basis", + "deferred_to_generate" + ], + "_allowed_anywhere_why": "schema-preferences.md defines these as row annotations usable on any row (§ 'The justification key is source' — REQUIRED on DETECTED; reason on N/A rows; context on ESSENTIAL rows; unanswered/blocks_phase/conflict per clarify-assemble.md § clarify_status; default_basis/deferred_to_generate per Q-D2). They are documented as a vocabulary, not per row, so the example block cannot list them on every row.", + "extra_shapes": [ + { + "shape": "skills/azure-to-aws/references/shared/schema-graviton.md", + "heading": "design_constraints.cpu_architecture", + "path": "design_constraints.cpu_architecture" + } + ] + }, + "aws-design.json": { + "shape": "skills/azure-to-aws/references/shared/schema-design-aws.md", + "root_heading": "Shape", + "extra_shapes": [ + { + "shape": "skills/azure-to-aws/references/shared/schema-graviton.md", + "heading": "graviton", + "path": "services[].graviton" + } + ] + }, + "aws-design-ai.json": { + "shape": "skills/azure-to-aws/references/shared/schema-design-aws-ai.md", + "root_heading": "Shape" + }, + "estimation-infra.json": { + "json_schema": "skills/shared/estimate/estimation-infra.schema.json" + }, + "scenarios/index.json": { + "shape": "skills/azure-to-aws/references/shared/schema-workshop-scenarios.md", + "root_heading": "scenarios/index.json" + } + } + } +} diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py new file mode 100644 index 000000000..16b9f5536 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -0,0 +1,888 @@ +#!/usr/bin/env python3 +"""Validate migration artifacts against the contracts the skills publish for them. + +Why this exists +--------------- +The migration skills (gcp-to-aws, heroku-to-aws, azure-to-aws) pass ~17 JSON artifacts +between phases: Discover writes an inventory, Clarify writes preferences, Design reads +both, and so on. Each artifact's contract is published as a `schema-*.md` document whose +"shape" is a JSON / JSONC example block, plus two real JSON Schemas under +`skills/shared/`. Those contracts are prose. Nothing checked that a producer actually +writes what the next phase reads, so a field could be renamed in one phase file, a new +key emitted without being documented, or a documented enum silently widened — and every +DSL `_assert` would still pass, because the model that writes the artifact is the model +that evaluates the assertion. + +This tool turns the published shape blocks into a key-path contract and checks real +artifacts against it. It is deliberately conservative: it enforces only what the +documents actually say (known keys, example types, `a | b | c` enums, keys the document +marks REQUIRED) and reports the rest as what it is — a gap in the contract, not in the +artifact. + +What it checks +-------------- +For every artifact it finds in a run directory (or in the fixture goldens): + + UNKNOWN_KEY a key the contract does not mention (off-contract emission — the + drift class that motivated this tool) + MISSING_REQUIRED a key the contract marks `(REQUIRED)` / `// REQUIRED` / `// ALWAYS + present` is absent + TYPE_MISMATCH the value's JSON type differs from the contract's example + ENUM_VIOLATION the value is outside an `a | b | c` set the contract spells out + SCHEMA_VIOLATION a real JSON Schema (draft-07 subset) rejected the value + SCHEMA_PARSE the contract document's own example block does not parse — the + contract is broken, independent of any artifact + NO_CONTRACT an artifact was found that no contract covers (informational) + +Keys beginning with `_` are treated as annotations (`_comment`, `_what_this_is`) and are +never reported as unknown. Placeholder values (`""`, `"..."`) accept any +string; a placeholder *key* (`""`, `"..."`) makes the object open. + +Usage +----- + python3 validate-artifacts.py --self-check + Parse every contract document and report SCHEMA_PARSE findings. No artifacts. + + python3 validate-artifacts.py --fixtures + Validate every golden artifact under fixtures/ against its skill's contracts, + applying the baseline file (known, explained findings that do not fail). + + python3 validate-artifacts.py --run-dir > --skill + Validate one real run. For users and for post-run checks in capability tests. + + --json machine-readable output + --baseline override the baseline file path (default: scripts/artifact-contracts-baseline.json) + --manifest override the contracts manifest (default: scripts/artifact-contracts.json) + +Exit 0 when no non-baselined ERROR findings; 1 otherwise. Stdlib only (Python 3.9+). +""" +from __future__ import annotations + +import argparse +import fnmatch +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +HERE = Path(__file__).resolve().parent +PLUGIN_ROOT = HERE.parent +DEFAULT_MANIFEST = HERE / "artifact-contracts.json" +DEFAULT_BASELINE = HERE / "artifact-contracts-baseline.json" + +ERROR_CODES = {"UNKNOWN_KEY", "MISSING_REQUIRED", "TYPE_MISMATCH", "ENUM_VIOLATION", + "SCHEMA_VIOLATION", "SCHEMA_PARSE"} +INFO_CODES = {"NO_CONTRACT"} + + +# --------------------------------------------------------------------------- findings + + +@dataclass +class Finding: + code: str + artifact: str # path of the artifact (relative to cwd when possible) + path: str # JSON path inside the artifact, e.g. "data.db_cutover" + message: str + contract: str = "" # which contract document/section produced it + baselined: bool = False + baseline_reason: str = "" + + @property + def severity(self) -> str: + return "error" if self.code in ERROR_CODES else "info" + + def as_dict(self) -> Dict[str, Any]: + d = {"code": self.code, "severity": self.severity, "artifact": self.artifact, + "path": self.path, "message": self.message, "contract": self.contract} + if self.baselined: + d["baselined"] = True + d["baseline_reason"] = self.baseline_reason + return d + + +# --------------------------------------------------------------------------- JSONC → JSON + + +_STRING_RE = re.compile(r'"(?:[^"\\]|\\.)*"') + + +def strip_jsonc(text: str) -> Tuple[str, Dict[int, str], Dict[int, str]]: + """Strip `//` comments outside strings and trailing commas. + + Returns (json_text, comment_by_line, key_by_line). Line numbers are 0-based into the + block. `key_by_line` holds the JSON key that opens on that line, so a trailing comment + can be attributed to the key it annotates. + """ + out_lines: List[str] = [] + comments: Dict[int, str] = {} + keys: Dict[int, str] = {} + for i, line in enumerate(text.splitlines()): + # find first `//` that is not inside a string + idx = None + in_str = False + esc = False + j = 0 + while j < len(line): + ch = line[j] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + else: + if ch == '"': + in_str = True + elif ch == "/" and j + 1 < len(line) and line[j + 1] == "/": + idx = j + break + j += 1 + if idx is not None: + comments[i] = line[idx + 2:].strip() + line = line[:idx].rstrip() + # the key a trailing comment annotates is the LAST `"key":` opened on the line + key_matches = re.findall(r'"((?:[^"\\]|\\.)*)"\s*:', line) + if key_matches: + keys[i] = key_matches[-1] + out_lines.append(line) + joined = "\n".join(out_lines) + # bare `...` ellipses in arrays/objects (`["Q1", "Q2", ...]`) → a placeholder string + joined = re.sub(r"(? bool: + return "any" in self.kinds or not self.kinds + + def merge(self, other: "Node") -> "Node": + """Union two templates for the same path (several documented variants).""" + self.kinds |= other.kinds + self.wildcard = self.wildcard or other.wildcard + self.required |= other.required + for k, v in other.keys.items(): + if k in self.keys: + self.keys[k].merge(v) + else: + self.keys[k] = v + if other.item is not None: + if self.item is None: + self.item = other.item + else: + self.item.merge(other.item) + if other.any_key is not None: + if self.any_key is None: + self.any_key = other.any_key + else: + self.any_key.merge(other.any_key) + if self.enum is not None and other.enum is not None: + self.enum = self.enum | other.enum + elif other.enum is None and self.enum is not None and "string" in other.kinds and not other.is_any(): + # the other variant documents a free string here — enum is not universal + self.enum = None + return self + + +_PLACEHOLDER_RE = re.compile(r"^<.*>$") +_ENUM_TOKEN_RE = re.compile(r"^[A-Za-z0-9_.:+\-/]+$") +_COMMENT_ENUM_RE = re.compile(r"(?:^|[—:\-]\s*)([A-Za-z0-9_.:+\-/]+(?:\s*\|\s*[A-Za-z0-9_.:+\-/]+)+)\s*(?:$|[—(\-])") + + +def _enum_from_string(s: str) -> Optional[Set[str]]: + if "|" not in s: + return None + toks = [t.strip() for t in s.split("|")] + if len(toks) >= 2 and all(_ENUM_TOKEN_RE.match(t) for t in toks): + return set(toks) + return None + + +def _enum_from_comment(c: str) -> Optional[Set[str]]: + m = _COMMENT_ENUM_RE.search(c) + if not m: + return None + toks = [t.strip() for t in m.group(1).split("|")] + if len(toks) >= 2 and all(_ENUM_TOKEN_RE.match(t) for t in toks): + return set(toks) + return None + + +def _is_required_comment(c: str) -> bool: + c_up = c.upper() + return ("REQUIRED" in c_up and "NOT REQUIRED" not in c_up and "REQUIRED WHEN" not in c_up + and "REQUIRED IF" not in c_up and "REQUIRED ONLY" not in c_up and "REQUIRED ON" not in c_up + and "REQUIRED WHENEVER" not in c_up) or "ALWAYS PRESENT" in c_up + + +def build_node(value: Any, key_comments: Dict[str, List[str]], source: str) -> Node: + """Infer a template node from an example value.""" + n = Node(source=source) + if isinstance(value, dict): + n.kinds.add("object") + for k, v in value.items(): + if k == "..." or _PLACEHOLDER_RE.match(k): + n.wildcard = True + if isinstance(v, (dict, list)) and v not in ({}, []) and not (isinstance(v, dict) and set(v) == {"..."}): + tmpl = build_node(v, key_comments, source) + n.any_key = tmpl if n.any_key is None else n.any_key.merge(tmpl) + continue + child = build_node(v, key_comments, source) + for c in key_comments.get(k, []): + if _is_required_comment(c): + n.required.add(k) + e = _enum_from_comment(c) + if e is not None and ("string" in child.kinds or "null" in child.kinds or child.is_any()): + # only trust a comment enum when the example value is consistent with it + if not isinstance(v, str) or v in e or _PLACEHOLDER_RE.match(v) or v == "...": + child.enum = e + child.kinds.add("string") + child.kinds.discard("any") + if "null" in c.lower() and "or null" in c.lower() or "nullable" in c.lower(): + child.kinds.add("null") + n.keys[k] = child + elif isinstance(value, list): + n.kinds.add("array") + item: Optional[Node] = None + for el in value: + en = build_node(el, key_comments, source) + item = en if item is None else item.merge(en) + n.item = item if item is not None else Node(kinds={"any"}, source=source) + elif isinstance(value, str): + if value == "..." or _PLACEHOLDER_RE.match(value): + # a placeholder like "" says nothing about the JSON type + n.kinds.add("any") + else: + e = _enum_from_string(value) + n.kinds.add("string") + if e is not None: + n.enum = e + elif isinstance(value, bool): + n.kinds.add("boolean") + elif isinstance(value, (int, float)): + n.kinds.add("number") + elif value is None: + n.kinds.add("null") + n.kinds.add("any") # a null example says nothing about the non-null type + return n + + +# --------------------------------------------------------------------------- contract documents + + +@dataclass +class Block: + heading: str + lang: str + text: str + line: int # 1-based line of the opening fence in the document + + +_FENCE_RE = re.compile(r"^```(\w+)?\s*$") + + +def extract_blocks(doc: Path) -> List[Block]: + blocks: List[Block] = [] + heading = "" + in_block = False + lang = "" + start = 0 + buf: List[str] = [] + for i, line in enumerate(doc.read_text(encoding="utf-8").splitlines(), start=1): + if not in_block: + if line.startswith("#"): + heading = line.lstrip("#").strip() + m = _FENCE_RE.match(line) + if m: + in_block = True + lang = (m.group(1) or "").lower() + start = i + buf = [] + else: + if line.startswith("```"): + in_block = False + if lang in ("json", "jsonc"): + blocks.append(Block(heading=heading, lang=lang, text="\n".join(buf), line=start)) + else: + buf.append(line) + return blocks + + +def _heading_matches(heading: str, wanted: str) -> bool: + h = heading.replace("`", "").strip().lower() + w = wanted.replace("`", "").strip().lower() + return h == w or h.startswith(w) + + +def _heading_token(heading: str) -> Optional[str]: + """The key path a sub-section heading documents, e.g. '## `services[]`' → 'services[]'.""" + m = re.search(r"`([^`]+)`", heading) + tok = m.group(1) if m else heading.split(" ")[0] + tok = tok.strip() + if not re.match(r"^[A-Za-z_][A-Za-z0-9_.\[\]]*$", tok): + return None + return tok + + +def parse_block(block: Block, doc: Path, findings: List[Finding]) -> Optional[Tuple[Any, Dict[str, List[str]]]]: + json_text, comments, keys = strip_jsonc(block.text) + # a lone `...` line inside an object means "more keys like these"; inside an array it + # means "more items". Try the object reading first, then the array reading, then the + # fragment reading (`"key": {...}` with no outer braces). + lone = re.compile(r'^(\s*)"\.\.\."\s*,?\s*$', re.M) + candidates = [lone.sub(r'\1"...": "..."', json_text), json_text, "{" + json_text + "}", + "{" + lone.sub(r'\1"...": "..."', json_text) + "}"] + value = None + first_error: Optional[json.JSONDecodeError] = None + for cand in candidates: + try: + value = json.loads(re.sub(r",(\s*[}\]])", r"\1", cand)) + break + except json.JSONDecodeError as e: + if first_error is None: + first_error = e + if value is None: + e = first_error + try: + value = json.loads("{" + json_text + "}") + except json.JSONDecodeError: + findings.append(Finding( + code="SCHEMA_PARSE", artifact=_rel(doc), path=f"L{block.line}", + message=f"example block under '{block.heading}' is not valid JSON/JSONC: {e.msg} " + f"(block line {e.lineno})", + contract=_rel(doc))) + return None + key_comments: Dict[str, List[str]] = {} + for ln, c in comments.items(): + k = keys.get(ln) + if k is not None: + key_comments.setdefault(k, []).append(c) + return value, key_comments + + +def _unwrap_to_path(value: Any, dotted: str) -> Any: + """A block may show its key path as wrapper objects — `{"design_constraints": + {"cpu_architecture": {...}}}` under a heading for `design_constraints.cpu_architecture`. + Peel single-key wrappers while they match the path components, so the block's own + keys land at the path and not one level below it.""" + comps = [c.rstrip("[]") for c in dotted.split(".") if c] + for c in comps: + if isinstance(value, dict) and len(value) == 1 and c in value: + value = value[c] + else: + break + # also peel when the wrapper is only the LAST component (common: {"metadata": {...}}) + if comps and isinstance(value, dict) and len(value) == 1 and comps[-1] in value \ + and isinstance(value[comps[-1]], (dict, list)): + value = value[comps[-1]] + return value + + +def _walk_path(root: Node, dotted: str) -> Optional[Node]: + """Find the node at 'a.b[].c'; create intermediate nodes as needed.""" + node = root + if dotted == "": + return root + for part in dotted.split("."): + is_array = part.endswith("[]") + key = part[:-2] if is_array else part + if key: + if key not in node.keys: + fresh = Node(kinds={"object"} if not is_array else {"array"}, source=root.source) + if node.any_key is not None: + fresh.merge(node.any_key) + node.keys[key] = fresh + node = node.keys[key] + if is_array: + node.kinds.add("array") + if node.item is None: + node.item = Node(kinds={"object"}, source=root.source) + node = node.item + return node + + +@dataclass +class ShapeContract: + doc: Path + root: Node + label: str + + +def build_shape_contract(spec: Dict[str, Any], findings: List[Finding]) -> Optional[ShapeContract]: + doc = PLUGIN_ROOT / spec["shape"] + if not doc.exists(): + findings.append(Finding("SCHEMA_PARSE", _rel(doc), "", "contract document not found", _rel(doc))) + return None + blocks = extract_blocks(doc) + root_heading = spec.get("root_heading") + root_block: Optional[Block] = None + for b in blocks: + if root_heading is None or _heading_matches(b.heading, root_heading): + root_block = b + break + if root_block is None: + findings.append(Finding("SCHEMA_PARSE", _rel(doc), "", + f"no JSON block found under heading '{root_heading}'", _rel(doc))) + return None + parsed = parse_block(root_block, doc, findings) + if parsed is None: + return None + value, kc = parsed + root = build_node(value, kc, f"{_rel(doc)}:L{root_block.line}") + if not isinstance(value, dict): + findings.append(Finding("SCHEMA_PARSE", _rel(doc), f"L{root_block.line}", + "root example block is not a JSON object", _rel(doc))) + return None + + # Sub-section blocks refine paths inside the root. Manifest `sections` wins; otherwise + # the heading's backticked token is the path ("services[]", "metadata", "a.b"). + explicit: Dict[str, str] = spec.get("sections", {}) # heading → path + rules: List[Dict[str, str]] = spec.get("section_rules", []) # {heading_regex, path} + ignore: List[str] = spec.get("ignore_headings", []) + anchors = {k for k in explicit if k.startswith("@L")} + seen_anchors: Set[str] = set() + for b in blocks: + if b is root_block: + continue + path: Optional[str] = None + anchor = f"@L{b.line}" + if anchor in explicit: + path = explicit[anchor] + seen_anchors.add(anchor) + elif root_heading is not None and _heading_matches(b.heading, root_heading): + # another example under the root heading (a per-item sample, a fragment) — + # only a line anchor in the manifest can say what it documents + continue + if path is None and any(_heading_matches(b.heading, ig) for ig in ignore): + continue + if path is None: + for h, p in explicit.items(): + if h.startswith("@L"): + continue + if _heading_matches(b.heading, h): + path = p + break + if path is None: + for r in rules: + if re.search(r["heading_regex"], b.heading): + path = r["path"] + break + if path is None: + tok = _heading_token(b.heading) + if tok is None: + continue + path = tok + parsed = parse_block(b, doc, findings) + if parsed is None: + continue + v, kc2 = parsed + v = _unwrap_to_path(v, path) + sub = build_node(v, kc2, f"{_rel(doc)}:L{b.line}") + target = _walk_path(root, path) + if target is None: + continue + target.merge(sub) + if "(REQUIRED)" in b.heading.upper(): + parent_path = ".".join(path.split(".")[:-1]) + parent = _walk_path(root, parent_path) if parent_path else root + if parent is not None: + parent.required.add(path.split(".")[-1].rstrip("[]")) + for a in anchors - seen_anchors: + findings.append(Finding("SCHEMA_PARSE", _rel(doc), a, + f"manifest section anchor {a} does not point at a JSON block start in this document " + f"(the document moved; update artifact-contracts.json)", _rel(doc))) + # Blocks in OTHER documents that add keys to this artifact (graviton, workshop, …) + for extra in spec.get("extra_shapes", []): + edoc = PLUGIN_ROOT / extra["shape"] + if not edoc.exists(): + findings.append(Finding("SCHEMA_PARSE", _rel(edoc), "", "extra_shapes document not found", _rel(edoc))) + continue + hit = None + for eb in extract_blocks(edoc): + if _heading_matches(eb.heading, extra["heading"]): + hit = eb + break + if hit is None: + findings.append(Finding("SCHEMA_PARSE", _rel(edoc), "", + f"no JSON block under heading '{extra['heading']}'", _rel(edoc))) + continue + parsed = parse_block(hit, edoc, findings) + if parsed is None: + continue + ev, ekc = parsed + epath = extra["path"] + ev = _unwrap_to_path(ev, epath) + enode = build_node(ev, ekc, f"{_rel(edoc)}:L{hit.line}") + target = _walk_path(root, epath) + if target is not None: + target.merge(enode) + for k in spec.get("required", []): + root.required.add(k) + for k in spec.get("open_paths", []): + n = _walk_path(root, k) + if n is not None: + n.wildcard = True + allowed = set(spec.get("allowed_anywhere", [])) + if allowed: + _mark_allowed_anywhere(root, allowed) + return ShapeContract(doc=doc, root=root, label=f"{_rel(doc)} § {root_block.heading}") + + +def _mark_allowed_anywhere(node: Node, allowed: Set[str]) -> None: + """Keys a document defines as a vocabulary usable on any row (e.g. azure preferences' + `source` / `reason` / `context`) are accepted at every object level without being + listed in each example row.""" + if "object" in node.kinds and node.keys: + for k in allowed: + node.keys.setdefault(k, Node(kinds={"any"}, source=node.source)) + for child in list(node.keys.values()): + _mark_allowed_anywhere(child, allowed) + if node.item is not None: + _mark_allowed_anywhere(node.item, allowed) + + +# --------------------------------------------------------------------------- shape validation + + +_KIND_OF = {dict: "object", list: "array", str: "string", bool: "boolean", int: "number", + float: "number", type(None): "null"} + + +def validate_shape(node: Node, value: Any, path: str, artifact: str, contract: str, out: List[Finding]) -> None: + kind = _KIND_OF.get(type(value), "any") + if not node.is_any() and kind not in node.kinds: + # tolerate null where the example never said — too many contracts write null for "absent" + if kind == "null": + return + out.append(Finding("TYPE_MISMATCH", artifact, path or "$", + f"is {kind}, contract example is {'/'.join(sorted(node.kinds))}", contract)) + return + if node.enum is not None and isinstance(value, str) and value not in node.enum: + out.append(Finding("ENUM_VIOLATION", artifact, path or "$", + f"'{value}' not in {{{', '.join(sorted(node.enum))}}}", contract)) + if isinstance(value, dict) and "object" in node.kinds: + for k in node.required: + if k not in value: + out.append(Finding("MISSING_REQUIRED", artifact, _join(path, k), + "contract marks this key REQUIRED / ALWAYS present", contract)) + for k, v in value.items(): + if k.startswith("_"): + continue + child = node.keys.get(k) + if child is None: + if node.any_key is not None: + validate_shape(node.any_key, v, _join(path, k), artifact, contract, out) + elif not node.wildcard and node.keys: + out.append(Finding("UNKNOWN_KEY", artifact, _join(path, k), + "key is not in the contract's example shape", contract)) + continue + validate_shape(child, v, _join(path, k), artifact, contract, out) + elif isinstance(value, list) and "array" in node.kinds and node.item is not None: + for i, el in enumerate(value): + validate_shape(node.item, el, f"{path}[{i}]", artifact, contract, out) + + +def _join(path: str, key: str) -> str: + return f"{path}.{key}" if path else key + + +# --------------------------------------------------------------------------- JSON Schema subset + + +def validate_json_schema(schema: Dict[str, Any], value: Any, path: str, artifact: str, contract: str, + out: List[Finding], root_schema: Optional[Dict[str, Any]] = None) -> None: + """Draft-07 subset: type, properties, required, additionalProperties, enum, const, items, + pattern, minimum, maximum, minItems, oneOf/anyOf/allOf, $ref (#/definitions/...).""" + root_schema = root_schema or schema + if "$ref" in schema: + ref = schema["$ref"] + if ref.startswith("#/"): + target: Any = root_schema + for part in ref[2:].split("/"): + target = target.get(part, {}) if isinstance(target, dict) else {} + validate_json_schema(target, value, path, artifact, contract, out, root_schema) + return + t = schema.get("type") + if t is not None: + types = t if isinstance(t, list) else [t] + kind = _KIND_OF.get(type(value), "any") + ok = kind in types or (kind == "number" and "integer" in types and isinstance(value, int) and not isinstance(value, bool)) + if not ok: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"is {kind}, schema type is {t}", contract)) + return + if "enum" in schema and value not in schema["enum"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"{value!r} not in enum {schema['enum']}", contract)) + if "const" in schema and value != schema["const"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"{value!r} != const {schema['const']!r}", contract)) + if isinstance(value, str) and "pattern" in schema and not re.search(schema["pattern"], value): + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"'{value}' does not match pattern {schema['pattern']}", contract)) + if isinstance(value, (int, float)) and not isinstance(value, bool): + if "minimum" in schema and value < schema["minimum"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{value} < minimum {schema['minimum']}", contract)) + if "maximum" in schema and value > schema["maximum"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{value} > maximum {schema['maximum']}", contract)) + if isinstance(value, dict): + props = schema.get("properties", {}) + for k in schema.get("required", []): + if k not in value: + out.append(Finding("SCHEMA_VIOLATION", artifact, _join(path, k), "required by schema", contract)) + addl = schema.get("additionalProperties", True) + for k, v in value.items(): + if k in props: + validate_json_schema(props[k], v, _join(path, k), artifact, contract, out, root_schema) + elif addl is False: + out.append(Finding("SCHEMA_VIOLATION", artifact, _join(path, k), + "additionalProperties is false", contract)) + elif isinstance(addl, dict): + validate_json_schema(addl, v, _join(path, k), artifact, contract, out, root_schema) + if isinstance(value, list): + if "minItems" in schema and len(value) < schema["minItems"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"{len(value)} items < minItems {schema['minItems']}", contract)) + items = schema.get("items") + if isinstance(items, dict): + for i, el in enumerate(value): + validate_json_schema(items, el, f"{path}[{i}]", artifact, contract, out, root_schema) + for combinator in ("allOf",): + for sub in schema.get(combinator, []): + validate_json_schema(sub, value, path, artifact, contract, out, root_schema) + for combinator in ("oneOf", "anyOf"): + subs = schema.get(combinator) + if subs: + passes = 0 + for sub in subs: + trial: List[Finding] = [] + validate_json_schema(sub, value, path, artifact, contract, trial, root_schema) + if not trial: + passes += 1 + if passes == 0 or (combinator == "oneOf" and passes > 1): + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"matches {passes} of {len(subs)} {combinator} branches", contract)) + + +# --------------------------------------------------------------------------- manifest + resolution + + +@dataclass +class Contract: + artifact_glob: str + shape: Optional[ShapeContract] = None + json_schema: Optional[Dict[str, Any]] = None + json_schema_path: str = "" + + +def load_manifest(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def build_contracts(manifest: Dict[str, Any], skill: str, findings: List[Finding]) -> List[Contract]: + specs: Dict[str, Any] = {} + specs.update(manifest.get("shared", {})) + specs.update(manifest.get("skills", {}).get(skill, {})) + contracts: List[Contract] = [] + for glob_, spec in specs.items(): + c = Contract(artifact_glob=glob_) + if "json_schema" in spec: + p = PLUGIN_ROOT / spec["json_schema"] + try: + c.json_schema = json.loads(p.read_text(encoding="utf-8")) + c.json_schema_path = _rel(p) + except (OSError, json.JSONDecodeError) as e: + findings.append(Finding("SCHEMA_PARSE", _rel(p), "", f"cannot load JSON Schema: {e}", _rel(p))) + if "shape" in spec: + c.shape = build_shape_contract(spec, findings) + contracts.append(c) + return contracts + + +def artifact_name(p: Path) -> str: + """Normalise 'scenarios/scenario-001.preferences.json' → 'scenarios/scenario-NNN.preferences.json'.""" + name = p.name + name = re.sub(r"scenario-\d+\.", "scenario-NNN.", name) + if p.parent.name == "scenarios": + return "scenarios/" + name + return name + + +def find_artifacts(run_dir: Path) -> List[Path]: + out: List[Path] = [] + for p in sorted(run_dir.rglob("*.json")): + rel = p.relative_to(run_dir) + parts = rel.parts + if any(part in ("live-capture", "workspace-terraform", ".terraform", "node_modules") for part in parts): + continue + if p.name.startswith("expected-") or p.name.startswith("clarify-answers"): + continue + out.append(p) + return out + + +def validate_run_dir(run_dir: Path, skill: str, manifest: Dict[str, Any], findings: List[Finding]) -> int: + contracts = build_contracts(manifest, skill, findings) + checked = 0 + for p in find_artifacts(run_dir): + name = artifact_name(p) + matched = [c for c in contracts if fnmatch.fnmatch(name, c.artifact_glob)] + rel = _rel(p) + if not matched: + findings.append(Finding("NO_CONTRACT", rel, "", f"no contract covers '{name}' for skill {skill}", "")) + continue + try: + value = json.loads(p.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + findings.append(Finding("TYPE_MISMATCH", rel, "$", f"artifact is not valid JSON: {e.msg}", "")) + continue + checked += 1 + for c in matched: + if c.json_schema is not None: + validate_json_schema(c.json_schema, value, "", rel, c.json_schema_path, findings) + if c.shape is not None: + validate_shape(c.shape.root, value, "", rel, c.shape.label, findings) + return checked + + +# --------------------------------------------------------------------------- baseline + + +def apply_baseline(findings: List[Finding], baseline: Dict[str, Any]) -> List[Dict[str, Any]]: + """Mark findings matched by a baseline entry. Returns entries that matched nothing.""" + entries = baseline.get("entries", []) + used = [False] * len(entries) + for f in findings: + for i, e in enumerate(entries): + if e.get("code") != f.code: + continue + if not fnmatch.fnmatch(f.artifact, e.get("artifact", "*")): + continue + if not fnmatch.fnmatch(f.path, e.get("path", "*")): + continue + f.baselined = True + f.baseline_reason = e.get("reason", "") + used[i] = True + break + return [e for e, u in zip(entries, used) if not u] + + +# --------------------------------------------------------------------------- CLI + + +def _rel(p: Path) -> str: + """Paths in findings are relative to the plugin root (stable across cwd, so the + baseline's artifact globs match whether the tool runs from the repo root or the + plugin directory); paths outside the plugin fall back to absolute.""" + try: + return str(p.resolve().relative_to(PLUGIN_ROOT)) + except ValueError: + return str(p.resolve()) + + +def skill_for_fixture(dirname: str, manifest: Dict[str, Any]) -> Optional[str]: + for prefix, skill in manifest.get("fixture_prefixes", {}).items(): + if dirname.startswith(prefix): + return skill + return None + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + mode = ap.add_mutually_exclusive_group(required=True) + mode.add_argument("--self-check", action="store_true", help="parse every contract document only") + mode.add_argument("--fixtures", action="store_true", help="validate the fixture goldens") + mode.add_argument("--run-dir", type=Path, help="a .migration/ directory to validate") + ap.add_argument("--skill", choices=["gcp-to-aws", "heroku-to-aws", "azure-to-aws"], + help="required with --run-dir") + ap.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + ap.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE) + ap.add_argument("--no-baseline", action="store_true", help="ignore the baseline file") + ap.add_argument("--json", action="store_true", help="machine-readable output") + args = ap.parse_args(argv) + + manifest = load_manifest(args.manifest) + findings: List[Finding] = [] + checked = 0 + + if args.self_check: + for skill in manifest.get("skills", {}): + build_contracts(manifest, skill, findings) + elif args.fixtures: + fixtures_root = PLUGIN_ROOT / "fixtures" + for d in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): + skill = skill_for_fixture(d.name, manifest) + if skill is None: + continue + checked += validate_run_dir(d, skill, manifest, findings) + else: + if not args.skill: + ap.error("--run-dir requires --skill") + if not args.run_dir.is_dir(): + ap.error(f"{args.run_dir} is not a directory") + checked += validate_run_dir(args.run_dir, args.skill, manifest, findings) + + stale: List[Dict[str, Any]] = [] + if not args.no_baseline and args.baseline.exists() and not args.self_check: + stale = apply_baseline(findings, json.loads(args.baseline.read_text(encoding="utf-8"))) + + # de-duplicate identical findings (several contracts can cover one artifact) + seen: Set[Tuple[str, str, str, str]] = set() + unique: List[Finding] = [] + for f in findings: + key = (f.code, f.artifact, f.path, f.message) + if key in seen: + continue + seen.add(key) + unique.append(f) + findings = unique + + errors = [f for f in findings if f.severity == "error" and not f.baselined] + baselined = [f for f in findings if f.baselined] + infos = [f for f in findings if f.severity == "info"] + + if args.json: + print(json.dumps({"checked_artifacts": checked, "errors": [f.as_dict() for f in errors], + "baselined": [f.as_dict() for f in baselined], "info": [f.as_dict() for f in infos], + "stale_baseline_entries": stale}, indent=2)) + else: + for f in errors: + print(f"ERROR {f.code:<17} {f.artifact} :: {f.path}\n {f.message}" + + (f"\n contract: {f.contract}" if f.contract else "")) + for f in infos: + print(f"info {f.code:<17} {f.artifact} :: {f.message}") + if baselined: + by_code: Dict[str, int] = {} + for f in baselined: + by_code[f.code] = by_code.get(f.code, 0) + 1 + print(f"baselined (known, explained, not failing): " + + ", ".join(f"{k}={v}" for k, v in sorted(by_code.items()))) + for e in stale: + print(f"warn STALE_BASELINE entry matched nothing — remove it: {json.dumps(e)}") + print(f"\n{'FAIL' if errors else 'PASS'} — {checked} artifact(s) checked, " + f"{len(errors)} error(s), {len(baselined)} baselined, {len(infos)} info") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws.md index cc3852add..05b96c979 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws.md @@ -46,6 +46,11 @@ A resource with an AWS target. "confidence": "deterministic", // deterministic | measured | inferred | billing_inferred "fast_path_row": "Microsoft.Cache/Redis", // REQUIRED when confidence is deterministic "rubric_applied": "compute.md", // REQUIRED when pass 2 ran + "routing_provenance": "table", // REQUIRED — table | index_md | child_type_rule | namespace_rule | model_category — see § routing_provenance + "is_compute_unit": false, // true on the entry that carries a cluster's compute sizing + "sizing_source": {}, // REQUIRED when is_compute_unit — see § sizing_provenance + "sizing_provenance": "table", // REQUIRED whenever aws_config carries a size — see § sizing_provenance + "hosted_app_azure_ids": [], // REQUIRED on a Microsoft.Web/serverfarms entry — the apps folded into this plan "rationale": "" } ``` @@ -104,7 +109,8 @@ A specialist gate fired. See `design-refs/specialist-gates.md`. "azure_type": "Microsoft.Sql/managedInstances", "aws_service": "Deferred — specialist engagement", "reason": "", - "recommendation": "Engage your AWS account team; this workload needs a migration assessment." + "recommendation": "Engage your AWS account team; this workload needs a migration assessment.", + "routing_provenance": "index_md" // how the entry was routed to the gate — same vocabulary as services[] } ``` diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-azure.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-azure.md index ca87be564..d2c5c7422 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-azure.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-azure.md @@ -198,7 +198,8 @@ Design writes its own `warnings[]` with a separate vocabulary; see "justification": "seed:resource_group", // REQUIRED — see below "edges": [], // the edge set that JUSTIFIED this grouping "pattern_id": "unclassified", - "pattern_confidence": "inferred" + "pattern_confidence": "inferred", + "pattern_status": "catalog_absent" // catalog_absent until patterns.md exists — a defined state, not a gap (discover-assemble.md § Status) } ], "unclustered": [] diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md index c9be15130..9a925bbdc 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md @@ -53,6 +53,7 @@ licensing exposure and found none" is a different statement from silence. ```jsonc { "phase": "clarify", + "clarify_status": "COMPLETE", // REQUIRED — COMPLETE | BLOCKED_ON_ESSENTIAL — the phase's own verdict (clarify-assemble.md § clarify_status) "global": { "target_region": { "disposition": "DETECTED", "value": "eu-west-1", "default": "eu-west-1" }, "user_geography": { diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md index 57b4724c2..7a42f2460 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md @@ -175,6 +175,18 @@ Live gcloud discovery produces the same inventory/cluster schemas with these add ```json { + "metadata": { + "discovery_sources": ["terraform", "live"], + "clustering_mode": "simplified" + }, + "resources": [ + { + "address": "google_sql_database_instance.db", + "source": "live+terraform", + "unmanaged_by_terraform": false, + "not_found_live": false + } + ], "live_metadata": { "found": true, "captured_at": "2026-07-20T18:20:00Z", diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-estimate-infra.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-estimate-infra.md index 35525f12b..3c326781a 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-estimate-infra.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-estimate-infra.md @@ -61,6 +61,7 @@ Validation: when all three tiers are present, `scenario_deltas.premium` and `sce "services_with_missing_fallback": [] }, "accuracy_confidence": "±5-10%|±15-25%", + "complexity_tier": "small|medium|large", "current_costs": { "source": "billing_data|inventory_estimate|preferences|user_provided|unavailable", diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md index 298ece803..d9727a2eb 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md @@ -39,7 +39,16 @@ Use the **Recommendation Impact** row for the selected answer from the category ```json { "metadata": { "...": "..." }, - "design_constraints": { "": { "value", "chosen_by", "prompt", "design_consequence", "question_id?", "source?" } }, + "design_constraints": { + "": { + "value": "", + "chosen_by": "user|default|extracted", + "prompt": "", + "design_consequence": "", + "question_id": "", + "source": "" + } + }, "ai_constraints": { "...": "..." }, "startup_constraints": { "...": "..." } } diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/shared/schema-discover-heroku.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/shared/schema-discover-heroku.md index eaa5009be..818b13b79 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/shared/schema-discover-heroku.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/shared/schema-discover-heroku.md @@ -38,7 +38,10 @@ Complete inventory of discovered Heroku resources. Uses a **flat resource model* "resource_id": "formation:my-web-app:web", "resource_type": "formation", "heroku_app": "my-web-app", - "config": {} + "config": {}, + "source": "terraform|live|live+terraform", + "unmanaged_by_terraform": false, + "not_found_live": false } ], "billing_profile": {}, @@ -178,7 +181,18 @@ Deterministic ID format per resource type: "addon_service": "heroku-postgresql", "plan": "standard-0", "provider": "heroku", - "connection_pooling": true + "connection_pooling": true, + "ha_enabled": false, + "encryption_in_transit": true, + "redis_version": "7.0", + "topic_count": 4, + "partitions_per_topic": 8, + "replication_factor": 3, + "monthly_price_usd": 50, + "pg_version": "16", + "data_size_gb": 2.1, + "table_count": 42, + "maxmemory_policy": "noeviction" } ``` diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py new file mode 100644 index 000000000..677e50668 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -0,0 +1,214 @@ +"""Tests for scripts/validate-artifacts.py — the artifact-contract validator. + +Two layers: unit tests on the shape-template builder / validator (so a contract-document +convention that stops parsing is caught here, not in CI noise), and CLI tests against the +real repository (self-check passes, fixtures pass with the baseline, and an injected +off-contract key FAILS — the regression this tool exists to catch). +""" +from __future__ import annotations + +import importlib.util +import json +import shutil +import subprocess +import sys +from pathlib import Path + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("validate_artifacts", SCRIPT) + mod = importlib.util.module_from_spec(spec) + sys.modules["validate_artifacts"] = mod + spec.loader.exec_module(mod) + return mod + + +va = _load() + + +# --------------------------------------------------------------------------- JSONC stripping + + +def test_strip_jsonc_removes_comments_outside_strings_only(): + text = '{\n "url": "https://x/y", // keep the url\n "n": 1, // REQUIRED\n}' + out, comments, keys = va.strip_jsonc(text) + v = json.loads(out) + assert v == {"url": "https://x/y", "n": 1} + assert comments[1] == "keep the url" and keys[1] == "url" + assert comments[2] == "REQUIRED" and keys[2] == "n" + + +def test_strip_jsonc_handles_ellipsis_and_bool_alternation(): + out, _, _ = va.strip_jsonc('{"a": ["Q1", ...], "b": true|false}') + assert json.loads(out) == {"a": ["Q1", "..."], "b": True} + + +# --------------------------------------------------------------------------- template inference + + +def _node(example: str): + text, comments, keys = va.strip_jsonc(example) + kc = {} + for ln, c in comments.items(): + k = keys.get(ln) + if k: + kc.setdefault(k, []).append(c) + return va.build_node(json.loads(text), kc, "test") + + +def test_enum_from_pipe_string_and_from_comment(): + n = _node('{"status": "full|reduced", "tier": "compute", // network | data | compute\n}') + assert n.keys["status"].enum == {"full", "reduced"} + assert n.keys["tier"].enum == {"network", "data", "compute"} + + +def test_required_from_comment_and_placeholder_is_any_type(): + n = _node('{"id": "", // REQUIRED\n "n": ""}') + assert "id" in n.required + assert n.keys["n"].is_any() # a placeholder says nothing about the JSON type + + +def test_placeholder_key_makes_object_open_and_sets_any_key_template(): + n = _node('{"": {"value": 1, "chosen_by": "user|default"}}') + assert n.wildcard and n.any_key is not None + assert set(n.any_key.keys) == {"value", "chosen_by"} + + +# --------------------------------------------------------------------------- shape validation + + +def _validate(example: str, instance, allowed_anywhere=()): + n = _node(example) + if allowed_anywhere: + va._mark_allowed_anywhere(n, set(allowed_anywhere)) + out = [] + va.validate_shape(n, instance, "", "art.json", "contract", out) + return {(f.code, f.path) for f in out} + + +def test_unknown_key_is_reported_but_underscore_keys_are_not(): + found = _validate('{"a": 1, "b": "x"}', {"a": 1, "b": "x", "c": 2, "_comment": "ok"}) + assert found == {("UNKNOWN_KEY", "c")} + + +def test_missing_required_type_mismatch_enum_violation(): + ex = '{"a": 1, // REQUIRED\n "s": "full|reduced", "arr": [{"k": true}]}' + found = _validate(ex, {"s": "partial", "arr": [{"k": "no"}]}) + assert ("MISSING_REQUIRED", "a") in found + assert ("ENUM_VIOLATION", "s") in found + assert ("TYPE_MISMATCH", "arr[0].k") in found + + +def test_null_is_tolerated_and_open_objects_accept_anything(): + found = _validate('{"a": 1, "cfg": {"...": "..."}}', {"a": None, "cfg": {"anything": [1, 2]}}) + assert found == set() + + +def test_any_key_template_applies_to_named_children(): + ex = '{"rows": {"": {"value": "x", "chosen_by": "user|default"}}}' + found = _validate(ex, {"rows": {"q1": {"value": "y", "chosen_by": "user", "extra": 1}}}) + assert found == {("UNKNOWN_KEY", "rows.q1.extra")} + + +def test_allowed_anywhere_vocabulary(): + ex = '{"g": {"r": {"value": 1}}}' + assert _validate(ex, {"g": {"r": {"value": 1, "source": "live"}}}) == {("UNKNOWN_KEY", "g.r.source")} + assert _validate(ex, {"g": {"r": {"value": 1, "source": "live"}}}, allowed_anywhere=["source"]) == set() + + +# --------------------------------------------------------------------------- JSON Schema subset + + +def test_json_schema_subset(): + schema = { + "type": "object", + "required": ["phase"], + "additionalProperties": False, + "properties": { + "phase": {"const": "estimate"}, + "tier": {"type": "string", "enum": ["small", "medium"]}, + "items": {"type": "array", "items": {"type": "integer", "minimum": 0}}, + "id": {"type": "string", "pattern": "^[a-z]+$"}, + }, + } + out = [] + va.validate_json_schema(schema, {"phase": "design", "tier": "huge", "items": [-1, "x"], "id": "A1", "zzz": 1}, + "", "a.json", "s", out) + codes = {(f.path, f.message.split(",")[0]) for f in out} + assert ("phase", "'design' != const 'estimate'") in codes + assert any(p == "tier" for p, _ in codes) + assert any(p == "items[0]" for p, _ in codes) and any(p == "items[1]" for p, _ in codes) + assert any(p == "id" for p, _ in codes) + assert ("zzz", "additionalProperties is false") in codes + ok = [] + va.validate_json_schema(schema, {"phase": "estimate", "tier": "small", "items": [1], "id": "ab"}, "", "a", "s", ok) + assert ok == [] + + +# --------------------------------------------------------------------------- baseline + + +def test_baseline_marks_matches_and_reports_stale_entries(): + f1 = va.Finding("UNKNOWN_KEY", "fixtures/x/a.json", "resources[3].id", "m") + f2 = va.Finding("UNKNOWN_KEY", "fixtures/x/a.json", "other", "m") + stale = va.apply_baseline([f1, f2], {"entries": [ + {"code": "UNKNOWN_KEY", "artifact": "fixtures/x/*", "path": "resources[[]*[]].id", "reason": "r"}, + {"code": "UNKNOWN_KEY", "artifact": "fixtures/x/*", "path": "never.matches", "reason": "r"}, + ]}) + assert f1.baselined and f1.baseline_reason == "r" + assert not f2.baselined + assert len(stale) == 1 and stale[0]["path"] == "never.matches" + + +# --------------------------------------------------------------------------- CLI against the real repo + + +def _run(*args: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, cwd=PLUGIN_ROOT) + + +def test_self_check_passes_every_contract_document_parses(): + r = _run("--self-check") + assert r.returncode == 0, r.stdout + r.stderr + + +def test_fixtures_pass_with_baseline_and_no_stale_entries(): + r = _run("--fixtures", "--json") + assert r.returncode == 0, r.stdout + r.stderr + report = json.loads(r.stdout) + assert report["checked_artifacts"] > 0 + assert report["errors"] == [] + assert report["stale_baseline_entries"] == [], "remove baseline entries that no longer match anything" + + +def test_injected_off_contract_key_fails(tmp_path: Path): + """The regression this tool exists for: a producer emits a key its contract never + documented. Copy a golden run, plant the key, expect exit 1 naming the path.""" + src = PLUGIN_ROOT / "fixtures" / "azure-iac-terraform" / "after-discover" + run = tmp_path / "run" + shutil.copytree(src, run) + inv = run / "azure-resource-inventory.json" + data = json.loads(inv.read_text()) + data["resources"][0]["invented_by_a_phase_file"] = True + inv.write_text(json.dumps(data)) + r = _run("--run-dir", str(run), "--skill", "azure-to-aws", "--no-baseline", "--json") + assert r.returncode == 1 + report = json.loads(r.stdout) + assert any(e["code"] == "UNKNOWN_KEY" and e["path"] == "resources[0].invented_by_a_phase_file" + for e in report["errors"]), report["errors"] + + +def test_real_json_schema_violation_fails(tmp_path: Path): + src = PLUGIN_ROOT / "fixtures" / "azure-iac-terraform" / "after-estimate" + run = tmp_path / "run" + shutil.copytree(src, run) + p = run / "estimation-infra.json" + data = json.loads(p.read_text()) + data["complexity_tier"] = "enormous" + p.write_text(json.dumps(data)) + r = _run("--run-dir", str(run), "--skill", "azure-to-aws", "--no-baseline", "--json") + assert r.returncode == 1 + assert any(e["code"] == "SCHEMA_VIOLATION" and e["path"] == "complexity_tier" for e in json.loads(r.stdout)["errors"]) From 17ce32934c79f5c05347870beca216b9f3cb947a Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 30 Sep 2026 21:38:37 -0700 Subject: [PATCH 02/53] =?UTF-8?q?fix(aws-startup-advisor):=20validate-arti?= =?UTF-8?q?facts=20=E2=80=94=20enforce=20minProperties=20(and=20reject=20u?= =?UTF-8?q?nknown=20schema=20keywords);=20stale=20baseline=20fails?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #385, both should-fix findings valid: 1. [Propagation] phase-status.schema.json has `minProperties: 1` on `phases`; the draft-07 subset did not implement it, so `phases: {}` passed. Fixed the instance AND the class: the subset now enforces minProperties, maxProperties, exclusiveMinimum/Maximum, maxItems, uniqueItems, minLength/maxLength and `not`; and `unsupported_keywords()` walks every loaded JSON Schema at contract-build time and emits SCHEMA_PARSE for any validation keyword that is neither enforced nor a known annotation. A constraint can no longer be skipped silently — adding a keyword to a schema without teaching the validator fails the gate. 2. [Completeness] a baseline entry that matched nothing was printed as a warning and exited 0, so the burn-down list only shrank if someone read the log. STALE_BASELINE is now an error code: `--fixtures` (what `mise run lint:artifacts` runs) exits 1 on any stale entry. Tests (19): empty `phases` fails with minProperties named; the two real schemas use only enforced/annotation keywords and an invented `contentEncoding` / `dependencies` schema is rejected; a never-matching baseline entry exits 1 with STALE_BASELINE. --- .../scripts/validate-artifacts.py | 88 +++++++++++++++++-- .../tests/test_validate_artifacts.py | 44 ++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 16b9f5536..7482f4108 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -32,6 +32,9 @@ SCHEMA_VIOLATION a real JSON Schema (draft-07 subset) rejected the value SCHEMA_PARSE the contract document's own example block does not parse — the contract is broken, independent of any artifact + STALE_BASELINE a baseline entry matched no finding — the gap it described is fixed + (remove the entry) or the entry is mistyped (it hides nothing today and + could hide a real finding tomorrow); fails the run either way NO_CONTRACT an artifact was found that no contract covers (informational) Keys beginning with `_` are treated as annotations (`_comment`, `_what_this_is`) and are @@ -73,7 +76,7 @@ DEFAULT_BASELINE = HERE / "artifact-contracts-baseline.json" ERROR_CODES = {"UNKNOWN_KEY", "MISSING_REQUIRED", "TYPE_MISMATCH", "ENUM_VIOLATION", - "SCHEMA_VIOLATION", "SCHEMA_PARSE"} + "SCHEMA_VIOLATION", "SCHEMA_PARSE", "STALE_BASELINE"} INFO_CODES = {"NO_CONTRACT"} @@ -609,10 +612,49 @@ def _join(path: str, key: str) -> str: # --------------------------------------------------------------------------- JSON Schema subset +# Keywords the subset validator ENFORCES. Anything else that is a validation keyword is +# reported by `unsupported_keywords` so a schema cannot silently lose a constraint. +_ENFORCED_KEYWORDS = { + "type", "properties", "required", "additionalProperties", "enum", "const", "items", + "pattern", "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "minItems", + "maxItems", "uniqueItems", "minProperties", "maxProperties", "minLength", "maxLength", + "allOf", "anyOf", "oneOf", "not", "$ref", +} +# Annotations / structure that carry no constraint of their own. +_ANNOTATION_KEYWORDS = { + "$schema", "$id", "$comment", "title", "description", "default", "examples", + "definitions", "$defs", "format", # format is intentionally not enforced (annotation-level) +} + + +def unsupported_keywords(schema: Any, ptr: str = "#") -> List[Tuple[str, str]]: + """Walk a JSON Schema and return (json-pointer, keyword) pairs for every keyword the + subset neither enforces nor recognises as an annotation. Called once per schema load.""" + found: List[Tuple[str, str]] = [] + if isinstance(schema, dict): + for k, v in schema.items(): + if k in ("properties", "definitions", "$defs", "patternProperties"): + if isinstance(v, dict): + for name, sub in v.items(): + found.extend(unsupported_keywords(sub, f"{ptr}/{k}/{name}")) + if k == "patternProperties": + found.append((ptr, k)) + continue + if k in ("items", "additionalProperties", "not"): + found.extend(unsupported_keywords(v, f"{ptr}/{k}")) + elif k in ("allOf", "anyOf", "oneOf") and isinstance(v, list): + for i, sub in enumerate(v): + found.extend(unsupported_keywords(sub, f"{ptr}/{k}/{i}")) + if k not in _ENFORCED_KEYWORDS and k not in _ANNOTATION_KEYWORDS: + found.append((ptr, k)) + return found + + def validate_json_schema(schema: Dict[str, Any], value: Any, path: str, artifact: str, contract: str, out: List[Finding], root_schema: Optional[Dict[str, Any]] = None) -> None: - """Draft-07 subset: type, properties, required, additionalProperties, enum, const, items, - pattern, minimum, maximum, minItems, oneOf/anyOf/allOf, $ref (#/definitions/...).""" + """Draft-07 subset — exactly the keywords in `_ENFORCED_KEYWORDS`. A schema using any + other validation keyword is rejected at load time (see `unsupported_keywords`), so a + constraint can never be skipped silently.""" root_schema = root_schema or schema if "$ref" in schema: ref = schema["$ref"] @@ -640,12 +682,27 @@ def validate_json_schema(schema: Dict[str, Any], value: Any, path: str, artifact if isinstance(value, str) and "pattern" in schema and not re.search(schema["pattern"], value): out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"'{value}' does not match pattern {schema['pattern']}", contract)) + if isinstance(value, str): + if "minLength" in schema and len(value) < schema["minLength"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"length {len(value)} < minLength {schema['minLength']}", contract)) + if "maxLength" in schema and len(value) > schema["maxLength"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"length {len(value)} > maxLength {schema['maxLength']}", contract)) if isinstance(value, (int, float)) and not isinstance(value, bool): if "minimum" in schema and value < schema["minimum"]: out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{value} < minimum {schema['minimum']}", contract)) if "maximum" in schema and value > schema["maximum"]: out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{value} > maximum {schema['maximum']}", contract)) + if "exclusiveMinimum" in schema and value <= schema["exclusiveMinimum"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{value} <= exclusiveMinimum {schema['exclusiveMinimum']}", contract)) + if "exclusiveMaximum" in schema and value >= schema["exclusiveMaximum"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{value} >= exclusiveMaximum {schema['exclusiveMaximum']}", contract)) if isinstance(value, dict): + if "minProperties" in schema and len(value) < schema["minProperties"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"{len(value)} properties < minProperties {schema['minProperties']}", contract)) + if "maxProperties" in schema and len(value) > schema["maxProperties"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"{len(value)} properties > maxProperties {schema['maxProperties']}", contract)) props = schema.get("properties", {}) for k in schema.get("required", []): if k not in value: @@ -663,6 +720,11 @@ def validate_json_schema(schema: Dict[str, Any], value: Any, path: str, artifact if "minItems" in schema and len(value) < schema["minItems"]: out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", f"{len(value)} items < minItems {schema['minItems']}", contract)) + if "maxItems" in schema and len(value) > schema["maxItems"]: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", + f"{len(value)} items > maxItems {schema['maxItems']}", contract)) + if schema.get("uniqueItems") and len({json.dumps(v, sort_keys=True) for v in value}) != len(value): + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", "items are not unique (uniqueItems)", contract)) items = schema.get("items") if isinstance(items, dict): for i, el in enumerate(value): @@ -670,6 +732,11 @@ def validate_json_schema(schema: Dict[str, Any], value: Any, path: str, artifact for combinator in ("allOf",): for sub in schema.get(combinator, []): validate_json_schema(sub, value, path, artifact, contract, out, root_schema) + if isinstance(schema.get("not"), dict): + trial_not: List[Finding] = [] + validate_json_schema(schema["not"], value, path, artifact, contract, trial_not, root_schema) + if not trial_not: + out.append(Finding("SCHEMA_VIOLATION", artifact, path or "$", "matches a `not` schema", contract)) for combinator in ("oneOf", "anyOf"): subs = schema.get(combinator) if subs: @@ -713,6 +780,13 @@ def build_contracts(manifest: Dict[str, Any], skill: str, findings: List[Finding c.json_schema_path = _rel(p) except (OSError, json.JSONDecodeError) as e: findings.append(Finding("SCHEMA_PARSE", _rel(p), "", f"cannot load JSON Schema: {e}", _rel(p))) + else: + for ptr, kw in unsupported_keywords(c.json_schema): + findings.append(Finding( + "SCHEMA_PARSE", _rel(p), ptr, + f"JSON Schema keyword '{kw}' is not implemented by this validator's subset — " + f"it would be skipped silently. Add it to _ENFORCED_KEYWORDS (with a check) or " + f"_ANNOTATION_KEYWORDS (if it carries no constraint).", _rel(p))) if "shape" in spec: c.shape = build_shape_contract(spec, findings) contracts.append(c) @@ -845,6 +919,12 @@ def main(argv: Optional[List[str]] = None) -> int: stale: List[Dict[str, Any]] = [] if not args.no_baseline and args.baseline.exists() and not args.self_check: stale = apply_baseline(findings, json.loads(args.baseline.read_text(encoding="utf-8"))) + for e in stale: + # a baseline entry that matches nothing is either fixed (remove it) or mistyped + # (it is hiding nothing and would hide a future finding by accident) — fail either way + findings.append(Finding("STALE_BASELINE", _rel(args.baseline), e.get("path", "*"), + f"entry matched no finding — remove it, or fix its code/artifact/path: {json.dumps(e)}", + "")) # de-duplicate identical findings (several contracts can cover one artifact) seen: Set[Tuple[str, str, str, str]] = set() @@ -877,8 +957,6 @@ def main(argv: Optional[List[str]] = None) -> int: by_code[f.code] = by_code.get(f.code, 0) + 1 print(f"baselined (known, explained, not failing): " + ", ".join(f"{k}={v}" for k, v in sorted(by_code.items()))) - for e in stale: - print(f"warn STALE_BASELINE entry matched nothing — remove it: {json.dumps(e)}") print(f"\n{'FAIL' if errors else 'PASS'} — {checked} artifact(s) checked, " f"{len(errors)} error(s), {len(baselined)} baselined, {len(infos)} info") return 1 if errors else 0 diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 677e50668..61df9fb53 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -201,6 +201,50 @@ def test_injected_off_contract_key_fails(tmp_path: Path): for e in report["errors"]), report["errors"] +def test_empty_phases_fails_min_properties(tmp_path: Path): + """Review finding on #385: phase-status.schema.json has `minProperties: 1` on `phases` + and the subset silently skipped it, so `phases: {}` passed.""" + run = tmp_path / "run" + run.mkdir() + (run / ".phase-status.json").write_text(json.dumps( + {"migration_id": "x", "last_updated": "2026-01-01T00:00:00Z", "phases": {}})) + r = _run("--run-dir", str(run), "--skill", "gcp-to-aws", "--no-baseline", "--json") + assert r.returncode == 1 + errs = json.loads(r.stdout)["errors"] + assert any(e["code"] == "SCHEMA_VIOLATION" and e["path"] == "phases" and "minProperties" in e["message"] + for e in errs), errs + + +def test_schema_keyword_the_subset_does_not_implement_is_rejected_at_load(): + """The class behind the minProperties hole: a keyword the subset does not know must + surface as SCHEMA_PARSE, never be skipped.""" + schema = {"type": "object", "properties": {"a": {"type": "string", "contentEncoding": "base64"}}, + "dependencies": {"a": ["b"]}} + found = va.unsupported_keywords(schema) + assert ("#/properties/a", "contentEncoding") in found + assert ("#", "dependencies") in found + # every keyword the two real schemas use is either enforced or an annotation + for p in (PLUGIN_ROOT / "skills/shared/state/phase-status.schema.json", + PLUGIN_ROOT / "skills/shared/estimate/estimation-infra.schema.json"): + assert va.unsupported_keywords(json.loads(p.read_text())) == [], p.name + + +def test_stale_baseline_entry_fails_the_gate(tmp_path: Path): + """Review finding on #385: a baseline entry matching nothing printed a warning and + exited 0, so the burn-down list could only shrink if someone read the log.""" + src = PLUGIN_ROOT / "fixtures" / "azure-iac-terraform" / "after-discover" + run = tmp_path / "run" + shutil.copytree(src, run) + baseline = tmp_path / "baseline.json" + baseline.write_text(json.dumps({"entries": [ + {"code": "UNKNOWN_KEY", "artifact": "*", "path": "never.matches.anything", "reason": "stale"}]})) + r = _run("--run-dir", str(run), "--skill", "azure-to-aws", "--baseline", str(baseline), "--json") + assert r.returncode == 1 + report = json.loads(r.stdout) + assert any(e["code"] == "STALE_BASELINE" for e in report["errors"]), report["errors"] + assert report["stale_baseline_entries"][0]["path"] == "never.matches.anything" + + def test_real_json_schema_violation_fails(tmp_path: Path): src = PLUGIN_ROOT / "fixtures" / "azure-iac-terraform" / "after-estimate" run = tmp_path / "run" From 462d1a118adbc767d457892e048a73637993afca Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 30 Sep 2026 21:30:32 -0700 Subject: [PATCH 03/53] =?UTF-8?q?ci(aws-startup-advisor):=20vendored-parit?= =?UTF-8?q?y=20gate=20=E2=80=94=20skills'=20references/vendored/=20must=20?= =?UTF-8?q?match=20skills/shared/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each migration skill is installable on its own (`npx skills add … --skill gcp-to-aws`), so shared material (DSL interpreter, pricing caches, estimate schemas, …) is vendored into skills//references/vendored/ from the canonical skills/shared/. The vendored READMEs said: "This repository has no automated sync task for these copies — keeping them byte-identical is part of the change that touches the canonical file." All 44 copies are identical today; nothing stopped the next edit to skills/shared/ from leaving a skill running stale logic. - tools/sync-vendored.py (stdlib): same shape as sync-plugin-skills.py. Default mode copies skills/shared/ over every vendored copy that differs; --check exits 1 listing any copy that differs or has no canonical source (a vendored file nobody owns). README.md in a vendored dir is explanatory and skipped. The vendored tree is a subset of shared/, so files in shared/ that no skill vendors are fine. - mise `lint:vendored-parity`, added to `validate` (the CI build runs it). - The four vendored READMEs replace the manual cp + md5sum instructions with the tool and the CI rule. - tools/test_sync_vendored.py: 4 tests on a throwaway tree — drift and orphan detection (README excluded), sync repairs drift and leaves orphans as errors, CLI exit codes — plus the real repository is in sync. No skill content changes. Stacked on #385 only because both add a task to the same `validate` depends line. --- mise.toml | 6 +- .../references/vendored/README.md | 9 +- .../references/vendored/README.md | 15 +- .../gcp-to-aws/references/vendored/README.md | 15 +- .../references/vendored/README.md | 15 +- tools/sync-vendored.py | 144 ++++++++++++++++++ tools/test_sync_vendored.py | 82 ++++++++++ 7 files changed, 253 insertions(+), 33 deletions(-) create mode 100644 tools/sync-vendored.py create mode 100644 tools/test_sync_vendored.py diff --git a/mise.toml b/mise.toml index 5aa686eff..99c613ef1 100644 --- a/mise.toml +++ b/mise.toml @@ -31,9 +31,13 @@ run = "uv run --with jsonschema --with pyyaml python tools/validate_spec.py" description = "Validate aws-startup-advisor fixture artifacts against their published schema contracts" run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures" +[tasks."lint:vendored-parity"] +description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" +run = "python3 tools/sync-vendored.py --check" + [tasks.validate] description = "Run all validation" -depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts"] +depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts", "lint:vendored-parity"] [tasks.security] description = "Run security scans" diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md index 5ce67362e..79599ac62 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md @@ -6,11 +6,10 @@ so the skill folder is **self-contained** — it runs standalone (lifted out, zi or used on its own) without reaching outside its own directory. **Do not hand-edit anything in this directory.** Edit the canonical source instead, -then re-copy it over every skill's `references/vendored/` copy of the same file. - -Every copy must stay **byte-identical** to the canonical source. A stale copy means -this skill and the canonical source disagree, so verify the copies match (for example -with `md5sum`) after editing the canonical file. +then run `python3 tools/sync-vendored.py` from the repository root to bring every +skill's `references/vendored/` copy back in sync. Every copy must stay **byte-identical** +to the canonical source; CI runs `python3 tools/sync-vendored.py --check` +(`mise run lint:vendored-parity`) and fails on any drift or any copy with no canonical source. | Vendored path | Canonical source | | -------------------- | ---------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index ee0e3cc85..d6b99a6b1 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -6,19 +6,16 @@ so the skill folder is **self-contained** — it runs standalone (lifted out, zi or used on its own) without reaching outside its own directory. **Do not hand-edit anything in this directory.** Edit the canonical source instead, -then copy the changed file over every vendored copy in the same change so the -copies stay byte-identical: +then bring every vendored copy back in sync: ```sh -# from the repository root, for each vendored path listed below -cp plugins/aws-startup-advisor/skills/shared/ \ - plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ +# from the repository root +python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` -This repository has no automated sync task for these copies — keeping them -byte-identical is part of the change that touches the canonical file. Verify with -`md5sum` (or `md5 -q`) over the canonical file and every vendored copy before -opening a pull request; the hashes must match. +CI fails when any vendored copy differs from its canonical source or has no canonical +source at all, so a change to `skills/shared/` that forgets the copies cannot merge. | Vendored path | Canonical source | | --------------------------------------- | ----------------------------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index baf06d5f6..1cc40a378 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -6,19 +6,16 @@ so the skill folder is **self-contained** — it runs standalone (lifted out, zi or used on its own) without reaching outside its own directory. **Do not hand-edit anything in this directory.** Edit the canonical source instead, -then copy the changed file over every vendored copy in the same change so the -copies stay byte-identical: +then bring every vendored copy back in sync: ```sh -# from the repository root, for each vendored path listed below -cp plugins/aws-startup-advisor/skills/shared/ \ - plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ +# from the repository root +python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` -This repository has no automated sync task for these copies — keeping them -byte-identical is part of the change that touches the canonical file. Verify with -`md5sum` (or `md5 -q`) over the canonical file and every vendored copy before -opening a pull request; the hashes must match. +CI fails when any vendored copy differs from its canonical source or has no canonical +source at all, so a change to `skills/shared/` that forgets the copies cannot merge. | Vendored path | Canonical source | | --------------------------------------- | ----------------------------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md index 33807b7e9..366570549 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md @@ -6,19 +6,16 @@ so the skill folder is **self-contained** — it runs standalone (lifted out, zi or used on its own) without reaching outside its own directory. **Do not hand-edit anything in this directory.** Edit the canonical source instead, -then copy the changed file over every vendored copy in the same change so the -copies stay byte-identical: +then bring every vendored copy back in sync: ```sh -# from the repository root, for each vendored path listed below -cp plugins/aws-startup-advisor/skills/shared/ \ - plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/ +# from the repository root +python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` -This repository has no automated sync task for these copies — keeping them -byte-identical is part of the change that touches the canonical file. Verify with -`md5sum` (or `md5 -q`) over the canonical file and every vendored copy before -opening a pull request; the hashes must match. +CI fails when any vendored copy differs from its canonical source or has no canonical +source at all, so a change to `skills/shared/` that forgets the copies cannot merge. | Vendored path | Canonical source | | --------------------------------------- | ----------------------------------------------------- | diff --git a/tools/sync-vendored.py b/tools/sync-vendored.py new file mode 100644 index 000000000..57ca5bf63 --- /dev/null +++ b/tools/sync-vendored.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Keep each skill's `references/vendored/` copies byte-identical to `skills/shared/`. + +Why +--- +The aws-startup-advisor migration skills are installable one at a time +(`npx skills add … --skill gcp-to-aws`), so each skill folder must be self-contained. +Shared material (the DSL interpreter, pricing caches, estimate schemas, …) is therefore +*vendored* — copied — into `skills//references/vendored/` from the +canonical `skills/shared/`. Until now the copies were kept in sync by hand +("copy the changed file over every vendored copy in the same change"), with no gate. +This tool is the gate, in the same shape as `sync-plugin-skills.py`. + +Rule +---- +For every file under `plugins//skills//references/vendored/` +(README.md excluded), `plugins//skills/shared/` must exist and be +byte-identical. The vendored tree is a subset of shared/ — a skill vendors only +what it loads — so files present in shared/ but not vendored are fine. + +Usage +----- + python3 tools/sync-vendored.py # copy shared/ over every vendored copy + python3 tools/sync-vendored.py --check # exit 1 if any copy differs / has no source + python3 tools/sync-vendored.py --plugin aws-startup-advisor + +Exit 0 when in sync (or after syncing), 1 on --check failure. Stdlib only. +""" +from __future__ import annotations + +import argparse +import filecmp +import shutil +import sys +from pathlib import Path +from typing import List, Tuple + +REPO_ROOT = Path(__file__).resolve().parent.parent +PLUGINS_ROOT = REPO_ROOT / "plugins" +VENDORED_SUBDIR = Path("references") / "vendored" +SHARED_DIRNAME = "shared" +SKIP_NAMES = {"README.md"} + + +def _rel(p: Path) -> str: + """Repo-relative for display; absolute when the tree lives elsewhere (tests).""" + try: + return str(p.resolve().relative_to(REPO_ROOT)) + except ValueError: + return str(p) + + +def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: + """Yield (vendored_file, canonical_file, rel_path) triples for one plugin.""" + skills_dir = plugin_dir / "skills" + shared = skills_dir / SHARED_DIRNAME + out: List[Tuple[Path, Path, Path]] = [] + if not skills_dir.is_dir(): + return out + for skill in sorted(skills_dir.iterdir()): + vend = skill / VENDORED_SUBDIR + if skill.name == SHARED_DIRNAME or not vend.is_dir(): + continue + for f in sorted(p for p in vend.rglob("*") if p.is_file()): + if f.name in SKIP_NAMES: + continue + rel = f.relative_to(vend) + out.append((f, shared / rel, rel)) + return out + + +def check_plugin(plugin_dir: Path) -> List[str]: + errors: List[str] = [] + for vend, canon, rel in vendored_files(plugin_dir): + if not canon.is_file(): + errors.append( + f"{_rel(vend)}: no canonical source at " + f"{_rel(canon)} — either add the file to skills/shared/ " + f"or stop vendoring it" + ) + elif not filecmp.cmp(vend, canon, shallow=False): + errors.append( + f"{_rel(vend)}: differs from " + f"{_rel(canon)} — run `python3 tools/sync-vendored.py`" + ) + return errors + + +def sync_plugin(plugin_dir: Path) -> Tuple[int, List[str]]: + """Copy canonical over vendored where they differ. Returns (copied, errors).""" + copied = 0 + errors: List[str] = [] + for vend, canon, rel in vendored_files(plugin_dir): + if not canon.is_file(): + errors.append( + f"{_rel(vend)}: no canonical source at " + f"{_rel(canon)} — cannot sync; add it to skills/shared/ or delete the copy" + ) + continue + if not filecmp.cmp(vend, canon, shallow=False): + shutil.copyfile(canon, vend) + copied += 1 + print(f" synced {_rel(vend)} <- {_rel(canon)}") + return copied, errors + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Keep skills' references/vendored/ copies identical to skills/shared/") + ap.add_argument("--plugin", help="Only this plugin (directory name under plugins/)") + ap.add_argument("--check", action="store_true", help="Report drift and exit 1; change nothing") + ap.add_argument("--plugins-root", type=Path, default=PLUGINS_ROOT, help=argparse.SUPPRESS) + args = ap.parse_args(argv) + + plugins = [args.plugins_root / args.plugin] if args.plugin else sorted( + p for p in args.plugins_root.iterdir() if p.is_dir()) + total_files = 0 + all_errors: List[str] = [] + copied = 0 + for plugin_dir in plugins: + files = vendored_files(plugin_dir) + total_files += len(files) + if not files: + continue + if args.check: + all_errors.extend(check_plugin(plugin_dir)) + else: + n, errs = sync_plugin(plugin_dir) + copied += n + all_errors.extend(errs) + + if all_errors: + print(f"FAIL — {len(all_errors)} vendored file(s) out of contract:") + for e in all_errors: + print(f" - {e}") + return 1 + if args.check: + print(f"PASS — {total_files} vendored file(s) byte-identical to skills/shared/") + else: + print(f"OK — {total_files} vendored file(s) checked, {copied} synced") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_sync_vendored.py b/tools/test_sync_vendored.py new file mode 100644 index 000000000..d9799ff5a --- /dev/null +++ b/tools/test_sync_vendored.py @@ -0,0 +1,82 @@ +"""Tests for tools/sync-vendored.py — the vendored-copy parity gate. + +Builds a throwaway plugins tree so the tests do not depend on (or mutate) the real +skills, then checks the real repository is clean as the last assertion. +""" +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = REPO_ROOT / "tools" / "sync-vendored.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("sync_vendored", SCRIPT) + mod = importlib.util.module_from_spec(spec) + sys.modules["sync_vendored"] = mod + spec.loader.exec_module(mod) + return mod + + +sv = _load() + + +def _tree(tmp_path: Path) -> Path: + plugins = tmp_path / "plugins" + shared = plugins / "p" / "skills" / "shared" + (shared / "dsl").mkdir(parents=True) + (shared / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") + (shared / "pricing.json").write_text('{"a": 1}\n') + vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + (vend / "dsl").mkdir(parents=True) + (vend / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") # in sync + (vend / "pricing.json").write_text('{"a": 0}\n') # drifted + (vend / "orphan.md").write_text("no source\n") # no canonical + (vend / "README.md").write_text("explanatory, never compared\n") + # a skill that vendors nothing, and the shared dir itself, are both ignored + (plugins / "p" / "skills" / "beta").mkdir() + return plugins + + +def test_check_reports_drift_and_orphans_but_not_readme(tmp_path: Path): + plugins = _tree(tmp_path) + errors = sv.check_plugin(plugins / "p") + joined = "\n".join(errors) + assert "pricing.json: differs from" in joined + assert "orphan.md: no canonical source" in joined + assert "INTERPRETER.md" not in joined + assert "README.md" not in joined + assert len(errors) == 2 + + +def test_sync_copies_canonical_over_drifted_and_leaves_orphans_as_errors(tmp_path: Path): + plugins = _tree(tmp_path) + copied, errors = sv.sync_plugin(plugins / "p") + assert copied == 1 + vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + assert (vend / "pricing.json").read_text() == '{"a": 1}\n' + assert len(errors) == 1 and "orphan.md" in errors[0] + # after sync, only the orphan remains + assert [e for e in sv.check_plugin(plugins / "p") if "orphan" not in e] == [] + + +def test_cli_check_exit_codes(tmp_path: Path): + plugins = _tree(tmp_path) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + capture_output=True, text=True) + assert r.returncode == 1 and "FAIL" in r.stdout + (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() + subprocess.run([sys.executable, str(SCRIPT), "--plugins-root", str(plugins)], check=True, capture_output=True) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + capture_output=True, text=True) + assert r.returncode == 0 and r.stdout.startswith("PASS") + + +def test_real_repository_is_in_sync(): + r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=REPO_ROOT) + assert r.returncode == 0, r.stdout + assert "vendored file(s) byte-identical" in r.stdout From 0b53a4940e01d80b3f4d354250aef8ab17cddc32 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 30 Sep 2026 21:47:23 -0700 Subject: [PATCH 04/53] =?UTF-8?q?fix(aws-startup-advisor):=20sync-vendored?= =?UTF-8?q?=20=E2=80=94=20README=20table=20is=20the=20manifest;=20deleted?= =?UTF-8?q?=20copies=20fail;=20READMEs=20say=20what=20the=20gate=20sees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #386 (Contradictions / Completeness, should-fix): the three migration READMEs claimed "a change to skills/shared/ that forgets the copies cannot merge", but the check only walked files that still existed — a deleted vendored copy, or a new shared file never vendored, passed. Rather than only weakening the sentence, the gate now uses the record of intent each vendored README already carries: its `| Vendored path | Canonical source |` table. - check: every row must exist on disk (deleted copy → FAIL), every file on disk must have a row (README stays honest), and each row's canonical path must be skills/shared/. A README with no table is an error. - sync: restores a listed-but-missing copy from canonical before comparing. - What the gate still cannot see — a NEW skills/shared/ file no skill vendors — is now stated in the tool docstring and in all four READMEs, with the instruction "add the copy AND a table row". The three READMEs were already complete tables (gcp 14/14, heroku 8/8, azure 21/21, agent-advisor 1/1); the real repo passes unchanged. Tests (7): deleted copy caught via the table and restored by sync; unlisted file and wrong canonical path caught; a new shared file nobody vendors is NOT an error (pins the documented limit); CLI exit codes; real repo in sync. Reviewer probes re-run on a throwaway copy of the plugin: deleting agent-advisor's INTERPRETER.md → FAIL naming it; default mode restores it; adding skills/shared/brand-new.md → no error, as documented. --- .../references/vendored/README.md | 4 +- .../references/vendored/README.md | 7 +- .../gcp-to-aws/references/vendored/README.md | 7 +- .../references/vendored/README.md | 7 +- tools/sync-vendored.py | 100 ++++++++++++++++-- tools/test_sync_vendored.py | 54 +++++++++- 6 files changed, 159 insertions(+), 20 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md index 79599ac62..16a0e196d 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md @@ -9,7 +9,9 @@ or used on its own) without reaching outside its own directory. then run `python3 tools/sync-vendored.py` from the repository root to bring every skill's `references/vendored/` copy back in sync. Every copy must stay **byte-identical** to the canonical source; CI runs `python3 tools/sync-vendored.py --check` -(`mise run lint:vendored-parity`) and fails on any drift or any copy with no canonical source. +(`mise run lint:vendored-parity`) and fails on any drift, any copy with no canonical source, +any row in the table below with no file on disk, and any file on disk with no row. It cannot +see a new `skills/shared/` file no skill vendors yet — adding one here means the copy **and** a row. | Vendored path | Canonical source | | -------------------- | ---------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index d6b99a6b1..b5b09aa75 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -14,8 +14,11 @@ python3 tools/sync-vendored.py # copies skills/shared/ over every python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` -CI fails when any vendored copy differs from its canonical source or has no canonical -source at all, so a change to `skills/shared/` that forgets the copies cannot merge. +CI fails when a vendored copy differs from its canonical source, has no canonical source, +is listed in the table below but missing on disk, or is on disk but not listed below. So an +edit to a `skills/shared/` file this skill vendors, or a deleted copy, cannot merge +unnoticed. What CI cannot see is a **new** `skills/shared/` file that no skill vendors yet — +if this skill needs one, add the copy **and** a row in the table below. | Vendored path | Canonical source | | --------------------------------------- | ----------------------------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 1cc40a378..38c4356e0 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -14,8 +14,11 @@ python3 tools/sync-vendored.py # copies skills/shared/ over every python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` -CI fails when any vendored copy differs from its canonical source or has no canonical -source at all, so a change to `skills/shared/` that forgets the copies cannot merge. +CI fails when a vendored copy differs from its canonical source, has no canonical source, +is listed in the table below but missing on disk, or is on disk but not listed below. So an +edit to a `skills/shared/` file this skill vendors, or a deleted copy, cannot merge +unnoticed. What CI cannot see is a **new** `skills/shared/` file that no skill vendors yet — +if this skill needs one, add the copy **and** a row in the table below. | Vendored path | Canonical source | | --------------------------------------- | ----------------------------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md index 366570549..ab9ef3a05 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md @@ -14,8 +14,11 @@ python3 tools/sync-vendored.py # copies skills/shared/ over every python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` -CI fails when any vendored copy differs from its canonical source or has no canonical -source at all, so a change to `skills/shared/` that forgets the copies cannot merge. +CI fails when a vendored copy differs from its canonical source, has no canonical source, +is listed in the table below but missing on disk, or is on disk but not listed below. So an +edit to a `skills/shared/` file this skill vendors, or a deleted copy, cannot merge +unnoticed. What CI cannot see is a **new** `skills/shared/` file that no skill vendors yet — +if this skill needs one, add the copy **and** a row in the table below. | Vendored path | Canonical source | | --------------------------------------- | ----------------------------------------------------- | diff --git a/tools/sync-vendored.py b/tools/sync-vendored.py index 57ca5bf63..1ec0058fe 100644 --- a/tools/sync-vendored.py +++ b/tools/sync-vendored.py @@ -11,12 +11,19 @@ ("copy the changed file over every vendored copy in the same change"), with no gate. This tool is the gate, in the same shape as `sync-plugin-skills.py`. -Rule ----- -For every file under `plugins//skills//references/vendored/` -(README.md excluded), `plugins//skills/shared/` must exist and be -byte-identical. The vendored tree is a subset of shared/ — a skill vendors only -what it loads — so files present in shared/ but not vendored are fine. +Rules +----- +1. Every file under `plugins//skills//references/vendored/` + (README.md excluded) has a canonical `plugins//skills/shared/` and is + byte-identical to it. +2. Each vendored directory's README.md carries a `| Vendored path | Canonical source |` + table. Every row must exist on disk (a deleted copy is caught), every file on disk must + have a row (the README stays honest), and the row's canonical source must be + `skills/shared/`. + +What this gate cannot see: a NEW file added to skills/shared/ that no skill vendors yet. +The vendored tree is a subset of shared/ by design — a skill vendors only what it loads — +so adding a shared file a skill needs means adding the copy AND the README row. Usage ----- @@ -30,10 +37,11 @@ import argparse import filecmp +import re import shutil import sys from pathlib import Path -from typing import List, Tuple +from typing import Dict, List, Tuple REPO_ROOT = Path(__file__).resolve().parent.parent PLUGINS_ROOT = REPO_ROOT / "plugins" @@ -50,6 +58,57 @@ def _rel(p: Path) -> str: return str(p) +_ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|\s*$", re.M) + + +def readme_manifest(vend_dir: Path) -> Tuple[Dict[str, str], List[str]]: + """Parse the `| Vendored path | Canonical source |` table in a vendored README. + + Returns ({vendored_rel_path: canonical_path_as_written}, errors). A README without the + table is an error: the table is the only record of what the skill is MEANT to vendor, + which is what lets a deleted copy be detected at all.""" + readme = vend_dir / "README.md" + if not readme.is_file(): + return {}, [f"{_rel(vend_dir)}: no README.md — the vendored-path table lives there"] + rows = {} + text = readme.read_text(encoding="utf-8") + for vend_rel, canon in _ROW_RE.findall(text): + rows[vend_rel.strip()] = canon.strip() + if not rows: + return {}, [f"{_rel(readme)}: no `| Vendored path | Canonical source |` table rows found"] + return rows, [] + + +def check_manifest(plugin_dir: Path) -> List[str]: + """Rule 2: README table ↔ files on disk, both directions, with correct canonical paths.""" + errors: List[str] = [] + skills_dir = plugin_dir / "skills" + if not skills_dir.is_dir(): + return errors + for skill in sorted(skills_dir.iterdir()): + vend = skill / VENDORED_SUBDIR + if skill.name == SHARED_DIRNAME or not vend.is_dir(): + continue + rows, errs = readme_manifest(vend) + errors.extend(errs) + if not rows: + continue + on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if p.is_file() and p.name not in SKIP_NAMES} + for rel in sorted(set(rows) - on_disk): + errors.append( + f"{_rel(vend / rel)}: listed in {_rel(vend / 'README.md')} but missing on disk — " + f"the copy was deleted; restore it with `python3 tools/sync-vendored.py` or remove the row") + for rel in sorted(on_disk - set(rows)): + errors.append( + f"{_rel(vend / rel)}: on disk but not listed in {_rel(vend / 'README.md')} — add a row") + for rel, canon in sorted(rows.items()): + expected = f"skills/{SHARED_DIRNAME}/{rel}" + if canon != expected: + errors.append( + f"{_rel(vend / 'README.md')}: row for `{rel}` names canonical `{canon}`, expected `{expected}`") + return errors + + def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: """Yield (vendored_file, canonical_file, rel_path) triples for one plugin.""" skills_dir = plugin_dir / "skills" @@ -87,9 +146,26 @@ def check_plugin(plugin_dir: Path) -> List[str]: def sync_plugin(plugin_dir: Path) -> Tuple[int, List[str]]: - """Copy canonical over vendored where they differ. Returns (copied, errors).""" + """Copy canonical over vendored where they differ, and restore copies the README + lists that are missing on disk. Returns (copied, errors).""" copied = 0 errors: List[str] = [] + skills_dir = plugin_dir / "skills" + shared = skills_dir / SHARED_DIRNAME + if skills_dir.is_dir(): + for skill in sorted(skills_dir.iterdir()): + vend = skill / VENDORED_SUBDIR + if skill.name == SHARED_DIRNAME or not vend.is_dir(): + continue + rows, _ = readme_manifest(vend) + for rel in sorted(rows): + target = vend / rel + canon = shared / rel + if not target.exists() and canon.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(canon, target) + copied += 1 + print(f" restored {_rel(target)} <- {_rel(canon)}") for vend, canon, rel in vendored_files(plugin_dir): if not canon.is_file(): errors.append( @@ -119,14 +195,18 @@ def main(argv=None) -> int: for plugin_dir in plugins: files = vendored_files(plugin_dir) total_files += len(files) - if not files: + if not files and not any((skill / VENDORED_SUBDIR).is_dir() + for skill in (plugin_dir / "skills").iterdir() + if (plugin_dir / "skills").is_dir() and skill.is_dir()): continue if args.check: all_errors.extend(check_plugin(plugin_dir)) + all_errors.extend(check_manifest(plugin_dir)) else: n, errs = sync_plugin(plugin_dir) copied += n all_errors.extend(errs) + all_errors.extend(check_manifest(plugin_dir)) if all_errors: print(f"FAIL — {len(all_errors)} vendored file(s) out of contract:") @@ -134,7 +214,7 @@ def main(argv=None) -> int: print(f" - {e}") return 1 if args.check: - print(f"PASS — {total_files} vendored file(s) byte-identical to skills/shared/") + print(f"PASS — {total_files} vendored file(s) byte-identical to skills/shared/ and matching their README tables") else: print(f"OK — {total_files} vendored file(s) checked, {copied} synced") return 0 diff --git a/tools/test_sync_vendored.py b/tools/test_sync_vendored.py index d9799ff5a..4d996852b 100644 --- a/tools/test_sync_vendored.py +++ b/tools/test_sync_vendored.py @@ -36,7 +36,13 @@ def _tree(tmp_path: Path) -> Path: (vend / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") # in sync (vend / "pricing.json").write_text('{"a": 0}\n') # drifted (vend / "orphan.md").write_text("no source\n") # no canonical - (vend / "README.md").write_text("explanatory, never compared\n") + (vend / "README.md").write_text( + "# Vendored\n\n" + "| Vendored path | Canonical source |\n" + "| --- | --- |\n" + "| `dsl/INTERPRETER.md` | `skills/shared/dsl/INTERPRETER.md` |\n" + "| `pricing.json` | `skills/shared/pricing.json` |\n" + "| `orphan.md` | `skills/shared/orphan.md` |\n") # a skill that vendors nothing, and the shared dir itself, are both ignored (plugins / "p" / "skills" / "beta").mkdir() return plugins @@ -64,12 +70,54 @@ def test_sync_copies_canonical_over_drifted_and_leaves_orphans_as_errors(tmp_pat assert [e for e in sv.check_plugin(plugins / "p") if "orphan" not in e] == [] +def test_deleted_copy_is_caught_via_readme_table(tmp_path: Path): + """Review probe on #386: deleting a vendored copy left check_plugin at 0 errors because + it only walked files that still existed. The README table is the record of intent.""" + plugins = _tree(tmp_path) + vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + (vend / "dsl" / "INTERPRETER.md").unlink() + errors = sv.check_manifest(plugins / "p") + assert any("dsl/INTERPRETER.md" in e and "missing on disk" in e for e in errors), errors + # sync restores it from canonical + copied, _ = sv.sync_plugin(plugins / "p") + assert (vend / "dsl" / "INTERPRETER.md").read_text() == "canonical v2\n" + assert copied >= 1 + assert not any("missing on disk" in e for e in sv.check_manifest(plugins / "p")) + + +def test_unlisted_copy_and_wrong_canonical_path_are_caught(tmp_path: Path): + plugins = _tree(tmp_path) + vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + (vend / "extra.md").write_text("x\n") + readme = vend / "README.md" + readme.write_text(readme.read_text().replace("`skills/shared/pricing.json`", "`skills/other/pricing.json`")) + errors = sv.check_manifest(plugins / "p") + assert any("extra.md" in e and "not listed" in e for e in errors), errors + assert any("pricing.json" in e and "expected `skills/shared/pricing.json`" in e for e in errors), errors + + +def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): + """Documented limit, pinned so the README wording stays honest: a brand-new + skills/shared/ file that no skill vendors is NOT drift.""" + plugins = _tree(tmp_path) + (plugins / "p" / "skills" / "shared" / "brand-new.md").write_text("new\n") + (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() + readme = plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "pricing.json").write_text('{"a": 1}\n') + assert sv.check_plugin(plugins / "p") == [] + assert sv.check_manifest(plugins / "p") == [] + + def test_cli_check_exit_codes(tmp_path: Path): plugins = _tree(tmp_path) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], capture_output=True, text=True) assert r.returncode == 1 and "FAIL" in r.stdout - (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() + vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") subprocess.run([sys.executable, str(SCRIPT), "--plugins-root", str(plugins)], check=True, capture_output=True) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], capture_output=True, text=True) @@ -79,4 +127,4 @@ def test_cli_check_exit_codes(tmp_path: Path): def test_real_repository_is_in_sync(): r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=REPO_ROOT) assert r.returncode == 0, r.stdout - assert "vendored file(s) byte-identical" in r.stdout + assert "vendored file(s) byte-identical" in r.stdout and "README tables" in r.stdout From 1e6bc621dddd7e6668a02d28ad0fd43fcd3716a0 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 30 Sep 2026 22:35:49 -0700 Subject: [PATCH 05/53] =?UTF-8?q?ci(aws-startup-advisor):=20reference-path?= =?UTF-8?q?=20lint=20=E2=80=94=20every=20file=20path=20a=20skill=20mention?= =?UTF-8?q?s=20must=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skills are prose state machines: a phase file says "Load `references/phases/design/design-ai.md`" or "run `python3 \"/scripts/x.py\"`" and the agent follows the string. Nothing checked the target exists. A renamed or never-written file is a dead end the user hits mid-migration, and any file move during consolidation would break references silently. This is the third gate in the family (validate-artifacts.py: artifact contracts; sync-vendored.py: vendored parity) and the precondition for moving phase files between skills. tools/check-skill-refs.py (stdlib) - Scans skills/**/*.md and agents/*.md for path tokens into the skill tree (references/, knowledge/, scripts/, skills/, agents/, data/, and the agent-advisor short forms phases/, shared/, design-refs/) with optional $PLUGIN_ROOT / ${CLAUDE_PLUGIN_ROOT} / $SKILL_BASE / / $GCP_BASE / ./ / ../ prefixes. - Resolution: explicit prefix → skill-relative → file-relative → skill/references/ → plugin/repo-relative. Library-origin files (skills/shared/*, */references/vendored/*, */references/shared/*) speak from the consuming skill's point of view and resolve against skills/shared and every skill; agent-advisor and llm-to-bedrock may name gcp-to-aws's tree (documented inline execution); agents/ resolve against llm-to-bedrock. - Skips run-artifact paths ($MIGRATION_DIR/…, .migration/…, /…), templated tokens (, NNN, *), URLs, and ./ or ../ tokens with no skill-tree root in them (generated outputs like ./deploy.sh). - tools/skill-refs-baseline.json: `ignore` rules (permanent policy — scripts Generate writes into $MIGRATION_DIR/scripts/, an example path in the user's app) and `entries` (known dead references with the fix; STALE when they stop matching, failing the gate so the list only shrinks). 1118 references checked on the real repo; 7 dead ones baselined, each a real user-facing defect: azure discover-iac.md points Bicep/ARM input at extract-bicep.md / extract-arm.md that do not exist (so Bicep halts); design-refs/patterns.md referenced from two azure files before it exists; contextual-offers SKILL.md names references/offers.md unqualified when the file lives in knowledge-base-for-startups; gcp design-ai.md:424 points at shared/ai-model-lifecycle.md, which in gcp is vendored at references/vendored/ai/. mise `lint:skill-refs`, added to `validate`. Tests (8): each resolution rule, each skip rule, library-origin resolution, gcp-dependent and agents fallbacks, ignore vs entries vs stale, CLI exit codes, real repo PASS. --- mise.toml | 6 +- tools/check-skill-refs.py | 317 +++++++++++++++++++++++++++++++++ tools/skill-refs-baseline.json | 47 +++++ tools/test_check_skill_refs.py | 152 ++++++++++++++++ 4 files changed, 521 insertions(+), 1 deletion(-) create mode 100644 tools/check-skill-refs.py create mode 100644 tools/skill-refs-baseline.json create mode 100644 tools/test_check_skill_refs.py diff --git a/mise.toml b/mise.toml index 99c613ef1..bf8120065 100644 --- a/mise.toml +++ b/mise.toml @@ -35,9 +35,13 @@ run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtu description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" run = "python3 tools/sync-vendored.py --check" +[tasks."lint:skill-refs"] +description = "Check every file path the aws-startup-advisor skills reference exists" +run = "python3 tools/check-skill-refs.py --check" + [tasks.validate] description = "Run all validation" -depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts", "lint:vendored-parity"] +depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts", "lint:vendored-parity", "lint:skill-refs"] [tasks.security] description = "Run security scans" diff --git a/tools/check-skill-refs.py b/tools/check-skill-refs.py new file mode 100644 index 000000000..cb9a29691 --- /dev/null +++ b/tools/check-skill-refs.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Check that every file path a skill's instructions mention actually exists. + +Why +--- +The aws-startup-advisor skills are prose state machines: a phase file says "Load +`references/phases/design/design-ai.md`" or "run `python3 \"/scripts/x.py\"`" +and the agent follows the string. Nothing checked that the target exists. A renamed or +never-written file is a dead end the user hits mid-migration (azure-to-aws halts on Bicep +input because `discover-iac.md` points at `extract-bicep.md`, which does not exist), and +any file move during consolidation would break references silently. This is the gate that +makes a move provable, in the same family as `validate-artifacts.py` (artifact contracts) +and `sync-vendored.py` (vendored parity). + +What counts as a reference +-------------------------- +A token in a skill file that looks like a repository path to an instruction/data file: +`references/…`, `knowledge/…`, `scripts/…`, `skills/…`, `agents/…`, `data/…`, and the +agent-advisor short forms `phases/…`, `shared/…`, `design-refs/…`, with an extension in +md/json/py/sh/tf/yml/yaml/html/txt. Optional prefixes `$PLUGIN_ROOT/`, +`${CLAUDE_PLUGIN_ROOT}/`, `$SKILL_BASE/`, `/`, `$GCP_BASE/`, `./`, `../`. + +Not references (skipped): run-artifact paths (`$MIGRATION_DIR/…`, `$RUN_DIR/…`, `$REPO/…`, +`.migration/…`, `/…`, `.saws-migrate/…`, `terraform/…`), templated tokens +(``, `{name}`, `*`, `NNN`), and URLs. + +Resolution order for a bare path `P` found in `skills//…/file.md`: + 1. explicit prefix → plugin root / skill root / gcp-to-aws root / referencing file's dir + 2. `skills//P` (skill-relative — the documented convention) + 3. `/P` (file-relative) + 4. `skills//references/P` (agent-advisor short forms: phases/, shared/, design-refs/) + 5. `/P` and `/P` (plugin- or repo-relative) +A path that resolves nowhere is MISSING_REF. + +Usage +----- + python3 tools/check-skill-refs.py --check # CI: exit 1 on any non-baselined MISSING_REF + python3 tools/check-skill-refs.py --check --json + python3 tools/check-skill-refs.py --list # print every resolved reference (debug) + --baseline tools/skill-refs-baseline.json # known dead refs with reasons; stale entries fail + +Stdlib only (Python 3.9+). +""" +from __future__ import annotations + +import argparse +import fnmatch +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PLUGIN = REPO_ROOT / "plugins" / "aws-startup-advisor" +DEFAULT_BASELINE = REPO_ROOT / "tools" / "skill-refs-baseline.json" + +EXTS = r"(?:md|json|py|sh|tf|ya?ml|html|txt|toml)" +ROOTS = r"(?:references|knowledge|scripts|skills|agents|data|phases|shared|design-refs|fixtures|tools)" +PREFIX = r"(?:\$\{?[A-Z_]+\}?/|<[A-Z_]+>/|\./|(?:\.\./)+)*" +# the path token: optional variable/relative prefix, then seg/seg/…/file.ext. A token with no +# prefix must start with a known root dir, so prose like "foo/bar.md" is not a reference. +_TOKEN_RE = re.compile( + rf"(?P{PREFIX})(?P[A-Za-z0-9_.\-]+(?:/[A-Za-z0-9_.\-]+)*\.{EXTS})(?![A-Za-z0-9_.])" +) +_ROOTS_RE = re.compile(rf"^{ROOTS}/") +# the vendored/shared library speaks from the CONSUMING skill's point of view; its references +# resolve against skills/shared and against any skill that vendors it +_LIBRARY_ORIGIN_RE = re.compile(r"/skills/(shared/|[^/]+/references/(vendored|shared)/)") +# skills that execute gcp-to-aws's phase files inline (agent-advisor) or by Skill invocation +# (llm-to-bedrock) legitimately name paths inside gcp-to-aws's tree +_GCP_DEPENDENTS = {"agent-advisor", "llm-to-bedrock"} +# anything carrying a run-artifact prefix or templating is not a repo path +_SKIP_PREFIX_RE = re.compile( + r"(\$\{?(MIGRATION_DIR|RUN_DIR|REPO|PHASE_DIR|PLAN_DIR|OUT|TARGET|APP|WORKSPACE|TMP)\}?/" + r"|<(PLAN_DIR|MIGRATION_DIR|RUN_DIR|REPO|run_dir|migration_dir|path|repo)>/" + r"|\.migration/|\.saws-migrate/|terraform/|ai-migration/|poc/|scaffold/|\.github/)" +) +_URL_RE = re.compile(r"https?://\S*") + + +@dataclass +class Ref: + source: Path # file containing the reference + line: int + raw: str # the token as written + resolved: Optional[Path] + + @property + def skill(self) -> str: + parts = self.source.parts + if "skills" in parts: + i = parts.index("skills") + if i + 1 < len(parts): + return parts[i + 1] + if "agents" in parts: + return "agents" + return "?" + + +def _skill_root(source: Path, plugin: Path) -> Optional[Path]: + try: + rel = source.relative_to(plugin / "skills") + except ValueError: + return None + return plugin / "skills" / rel.parts[0] + + +def _candidates(prefix: str, path: str, source: Path, plugin: Path) -> List[Path]: + """Ordered list of filesystem locations the token could mean.""" + skill = _skill_root(source, plugin) + here = source.parent + cands: List[Path] = [] + p = prefix + if "PLUGIN_ROOT" in p: + cands.append(plugin / path) + elif "GCP_BASE" in p: + cands.append(plugin / "skills" / "gcp-to-aws" / path) + elif "SKILL_BASE" in p or "SKILL_DIR" in p: + if skill is not None: + cands.append(skill / path) + else: + cands.append(here / path) + elif "HELPERS" in p and skill is not None: + cands.append(skill / "references" / "helpers" / path) + elif "SCRIPTS" in p and skill is not None: + cands.append(skill / "scripts" / path) + elif p.startswith("./") or p.startswith("../"): + cands.append((here / p / path)) + if skill is not None: + cands.append(skill / path) + # `/../gcp-to-aws/SKILL.md` style: prefix carries `../` + if "../" in p and skill is not None and "SKILL_BASE" in p: + up = skill + for _ in range(p.count("../")): + up = up.parent + cands.append(up / path) + if not cands or not p: + if skill is not None: + cands.append(skill / path) + cands.append(here / path) + if skill is not None: + cands.append(skill / "references" / path) + cands.append(plugin / path) + cands.append(REPO_ROOT / path) + src = str(source) + if _LIBRARY_ORIGIN_RE.search(src): + shared = plugin / "skills" / "shared" + stripped = re.sub(r"^(references/)?(vendored|shared)/", "", path) + cands.append(shared / stripped) + # a shared doc may name a file by its vendored-tree path with a flattened dir + # (`shared/ai-migration-guardrails.md` → skills/shared/ai/ai-migration-guardrails.md) + for hit in shared.rglob(Path(stripped).name): + if str(hit).endswith(stripped.split("/")[-1]) and hit.is_file(): + cands.append(hit) + for other in sorted((plugin / "skills").iterdir()): + if other.is_dir(): + cands.append(other / path) + cands.append(other / "references" / path) + if skill is not None and skill.name in _GCP_DEPENDENTS: + gcp = plugin / "skills" / "gcp-to-aws" + cands.append(gcp / path) + cands.append(gcp / "references" / path) + if skill is None and "agents" in source.parts: + # agents are llm-to-bedrock's subagents; their paths are relative to that skill's + # scripts/ or helper dirs, named through variables the agent defines at runtime + for other in sorted((plugin / "skills").iterdir()): + if other.is_dir(): + cands.append(other / path) + for hit in other.rglob(Path(path).name): + if str(hit).endswith(path) and hit.is_file(): + cands.append(hit) + return cands + + +def _resolve(prefix: str, path: str, source: Path, plugin: Path) -> Optional[Path]: + for c in _candidates(prefix, path, source, plugin): + try: + if c.resolve().is_file(): + return c.resolve() + except OSError: + continue + return None + + +def _is_templated(token: str) -> bool: + body = re.sub(r"^(?:<[A-Z_]+>/|\$\{?[A-Z_]+\}?/)+", "", token) + return any(ch in body for ch in "<>{}*[]") or "NNN" in body or "/…" in body + + +def scan_file(source: Path, plugin: Path) -> List[Ref]: + refs: List[Ref] = [] + try: + text = source.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return refs + for i, line in enumerate(text.splitlines(), start=1): + line_wo_urls = _URL_RE.sub("", line) + for m in _TOKEN_RE.finditer(line_wo_urls): + start = m.start() + # what immediately precedes the token decides whether it is a repo path + before = line_wo_urls[max(0, start - 40):start] + full = m.group(0) + if _SKIP_PREFIX_RE.search(before + full[: len(m.group("prefix"))]): + continue + if _is_templated(full): + continue + pfx = m.group("prefix") + if not pfx and not _ROOTS_RE.match(m.group("path")): + continue + # `./x.sh`, `../plan.md`: a relative token with no known root dir anywhere in it is a + # generated-output or working-tree path, not a reference into the skill tree + if pfx and re.fullmatch(r"(\./|\.\./)+", pfx) and not re.search(rf"(^|/){ROOTS}/", m.group("path")): + continue + # a token glued to a preceding word char or '/' is part of a longer identifier/path + if start > 0 and (line_wo_urls[start - 1].isalnum() or line_wo_urls[start - 1] in "_-/"): + continue + resolved = _resolve(m.group("prefix"), m.group("path"), source, plugin) + refs.append(Ref(source=source, line=i, raw=full, resolved=resolved)) + return refs + + +def scan(plugin: Path) -> List[Ref]: + files: List[Path] = [] + for sub in ("skills", "agents"): + root = plugin / sub + if root.is_dir(): + files.extend(p for p in root.rglob("*.md") if p.is_file()) + out: List[Ref] = [] + for f in sorted(files): + if any(part in ("node_modules", ".git", "fixtures") for part in f.parts): + continue + out.extend(scan_file(f, plugin)) + return out + + +def _rel(p: Path) -> str: + try: + return str(p.resolve().relative_to(REPO_ROOT)) + except ValueError: + return str(p) + + +def apply_baseline(missing: List[Ref], baseline: Dict) -> Tuple[List[Ref], List[Ref], List[Dict]]: + """Split missing refs into (unbaselined, baselined); return stale entries too. + + Two kinds of rule live in the baseline file: + ignore[] — permanent policy: tokens that are not repository references by nature (paths + the skill GENERATES, example paths in a user's app). Never stale. + entries[] — known dead references, each with the fix that retires it. STALE when they + stop matching, so the list only shrinks.""" + for rule in baseline.get("ignore", []): + missing = [r for r in missing + if not (fnmatch.fnmatch(_rel(r.source), rule.get("source", "*")) and fnmatch.fnmatch(r.raw, rule.get("ref", "*")))] + entries = baseline.get("entries", []) + used = [False] * len(entries) + un: List[Ref] = [] + bl: List[Ref] = [] + for r in missing: + hit = False + for i, e in enumerate(entries): + if fnmatch.fnmatch(_rel(r.source), e.get("source", "*")) and fnmatch.fnmatch(r.raw, e.get("ref", "*")): + used[i] = True + hit = True + break + (bl if hit else un).append(r) + stale = [e for e, u in zip(entries, used) if not u] + return un, bl, stale + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--plugin", type=Path, default=DEFAULT_PLUGIN, help=argparse.SUPPRESS) + ap.add_argument("--check", action="store_true", help="exit 1 on any non-baselined missing reference") + ap.add_argument("--list", action="store_true", help="print every reference and where it resolved") + ap.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE) + ap.add_argument("--no-baseline", action="store_true") + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + if not (args.check or args.list): + ap.error("pass --check or --list") + + refs = scan(args.plugin) + if args.list: + for r in refs: + print(f"{_rel(r.source)}:{r.line} {r.raw} -> {_rel(r.resolved) if r.resolved else 'MISSING'}") + return 0 + + missing = [r for r in refs if r.resolved is None] + stale: List[Dict] = [] + baselined: List[Ref] = [] + if not args.no_baseline and args.baseline.exists(): + missing, baselined, stale = apply_baseline(missing, json.loads(args.baseline.read_text(encoding="utf-8"))) + + if args.json: + print(json.dumps({ + "references": len(refs), + "missing": [{"source": _rel(r.source), "line": r.line, "ref": r.raw, "skill": r.skill} for r in missing], + "baselined": [{"source": _rel(r.source), "line": r.line, "ref": r.raw} for r in baselined], + "stale_baseline_entries": stale, + }, indent=2)) + else: + for r in missing: + print(f"ERROR MISSING_REF {_rel(r.source)}:{r.line} `{r.raw}`") + for e in stale: + print(f"ERROR STALE_BASELINE entry matched nothing — remove it: {json.dumps(e)}") + if baselined: + print(f"baselined (known dead references, explained in {_rel(args.baseline)}): {len(baselined)}") + ok = not missing and not stale + print(f"\n{'PASS' if ok else 'FAIL'} — {len(refs)} reference(s) checked, " + f"{len(missing)} missing, {len(baselined)} baselined, {len(stale)} stale baseline entr{'y' if len(stale) == 1 else 'ies'}") + return 0 if (not missing and not stale) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/skill-refs-baseline.json b/tools/skill-refs-baseline.json new file mode 100644 index 000000000..5ede122f0 --- /dev/null +++ b/tools/skill-refs-baseline.json @@ -0,0 +1,47 @@ +{ + "_what_this_is": "Policy and burn-down for tools/check-skill-refs.py. `ignore` rules are permanent: the matched tokens are not references into the repository (they are paths the skill GENERATES into a run directory, or example paths in a user's own app). `entries` are known dead references with the fix that retires each one; an entry that stops matching is reported STALE and fails the gate, so this list only shrinks.", + "ignore": [ + { + "source": "plugins/aws-startup-advisor/skills/*/references/phases/generate/*.md", + "ref": "*scripts/*.sh", + "reason": "Generate writes these scripts into $MIGRATION_DIR/scripts/ (01-validate-prerequisites.sh … 05-validate-migration.sh, migrate-postgres.sh, migrate-redis.sh). The phase files name them as OUTPUTS to produce and as commands for MIGRATION_GUIDE.md, not as repository files to load." + }, + { + "source": "plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md", + "ref": "fixtures/data.json", + "reason": "Example of a path inside the USER's application (`Path(__file__).resolve().parent / \"fixtures/data.json\"`) used to illustrate a rewrite rule; not a plugin file." + } + ], + "entries": [ + { + "source": "plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-iac.md", + "ref": "references/shared/extract-bicep.md", + "reason": "Bicep extraction is not implemented; discover-iac.md:55 says 'HALT if a dialect is present and its ref is not on disk', so a Bicep workspace halts with a path that will never exist. Fix: either write extract-bicep.md or replace the row with an explicit 'Bicep not yet supported — export to Terraform or use live `az` discovery' message that does not point at a file." + }, + { + "source": "plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-iac.md", + "ref": "references/shared/extract-arm.md", + "reason": "Same as extract-bicep.md: ARM templates halt against a reference that does not exist." + }, + { + "source": "plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md", + "ref": "design-refs/patterns.md", + "reason": "Pattern catalog not yet written; every cluster carries pattern_status: catalog_absent by design (discover-assemble.md § Status). Intentional forward reference. Retire when patterns.md lands or the gate wording stops naming a file." + }, + { + "source": "plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws.md", + "ref": "design-refs/patterns.md", + "reason": "See design.md entry — same forward reference, in the pattern_status vocabulary table." + }, + { + "source": "plugins/aws-startup-advisor/skills/contextual-offers-for-startups/SKILL.md", + "ref": "references/offers.md", + "reason": "The file is `../knowledge-base-for-startups/references/offers.md` (SKILL.md:29 has the correct path). Lines 3 and 149 use the unqualified short form, which resolves nowhere inside this skill. Fix: qualify both mentions." + }, + { + "source": "plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md", + "ref": "shared/ai-model-lifecycle.md", + "reason": "Wrong path: gcp-to-aws has no references/shared/ai-model-lifecycle.md; the file is vendored at references/vendored/ai/ai-model-lifecycle.md (canonical skills/shared/ai/). design-ai.md:424 should say `references/vendored/ai/ai-model-lifecycle.md`." + } + ] +} diff --git a/tools/test_check_skill_refs.py b/tools/test_check_skill_refs.py new file mode 100644 index 000000000..8663b594a --- /dev/null +++ b/tools/test_check_skill_refs.py @@ -0,0 +1,152 @@ +"""Tests for tools/check-skill-refs.py — the reference-path lint. + +A throwaway plugin tree exercises each resolution rule and each skip rule; the last test +runs the real repository and requires PASS with no stale baseline entries. +""" +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = REPO_ROOT / "tools" / "check-skill-refs.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("check_skill_refs", SCRIPT) + mod = importlib.util.module_from_spec(spec) + sys.modules["check_skill_refs"] = mod + spec.loader.exec_module(mod) + return mod + + +csr = _load() + + +def _plugin(tmp_path: Path) -> Path: + plugin = tmp_path / "plugins" / "p" + sk = plugin / "skills" + (sk / "alpha" / "references" / "phases").mkdir(parents=True) + (sk / "alpha" / "scripts").mkdir() + (sk / "alpha" / "references" / "vendored" / "dsl").mkdir(parents=True) + (sk / "shared" / "dsl").mkdir(parents=True) + (sk / "shared" / "ai").mkdir() + (sk / "gcp-to-aws" / "references" / "design-refs").mkdir(parents=True) + (sk / "agent-advisor").mkdir() + (sk / "beta" / "references").mkdir(parents=True) + (plugin / "agents").mkdir() + # targets + (sk / "alpha" / "references" / "phases" / "design.md").write_text("x") + (sk / "alpha" / "scripts" / "tool.py").write_text("x") + (sk / "shared" / "dsl" / "INTERPRETER.md").write_text("x") + (sk / "alpha" / "references" / "vendored" / "dsl" / "INTERPRETER.md").write_text( + "see `references/vendored/state/phase-status.schema.json` and `shared/ai-guardrails.md`") + (sk / "shared" / "ai" / "ai-guardrails.md").write_text("x") + (sk / "shared" / "state.md").write_text("vendored at `references/vendored/state.md`") + (sk / "gcp-to-aws" / "references" / "design-refs" / "ai.md").write_text("x") + (sk / "beta" / "references" / "offers.md").write_text("x") + (plugin / "agents" / "worker.md").write_text("validate with `scripts/tool.py`") + return plugin + + +def _refs(plugin: Path, skill: str, text: str, name: str = "SKILL.md"): + f = plugin / "skills" / skill / name + f.write_text(text) + return {r.raw: (r.resolved is not None) for r in csr.scan_file(f, plugin)} + + +def test_skill_relative_and_variable_prefixes_resolve(tmp_path: Path): + plugin = _plugin(tmp_path) + got = _refs(plugin, "alpha", + "Load `references/phases/design.md`. Run `python3 \"/scripts/tool.py\"`. " + "Also `$SKILL_BASE/scripts/tool.py`, `./scripts/tool.py`, `$PLUGIN_ROOT/skills/shared/dsl/INTERPRETER.md`, " + "`${CLAUDE_PLUGIN_ROOT}/skills/alpha/scripts/tool.py`, and `../beta/references/offers.md`.") + assert got == { + "references/phases/design.md": True, "/scripts/tool.py": True, + "$SKILL_BASE/scripts/tool.py": True, "./scripts/tool.py": True, + "$PLUGIN_ROOT/skills/shared/dsl/INTERPRETER.md": True, + "${CLAUDE_PLUGIN_ROOT}/skills/alpha/scripts/tool.py": True, + "../beta/references/offers.md": True, + } + + +def test_missing_reference_is_reported(tmp_path: Path): + plugin = _plugin(tmp_path) + got = _refs(plugin, "alpha", "Load `references/phases/nope.md` then `scripts/gone.py`.") + assert got == {"references/phases/nope.md": False, "scripts/gone.py": False} + + +def test_run_artifacts_templates_urls_and_relative_outputs_are_not_references(tmp_path: Path): + plugin = _plugin(tmp_path) + got = _refs(plugin, "alpha", + "Write `$MIGRATION_DIR/scripts/02-migrate-data.sh`; read `.migration//references/x.md`; " + "see https://example.com/references/phases/design.md ; open `references/offers/.md`; " + "then `./deploy.sh` and `../plan.md` and `scenarios/scenario-NNN.json`.") + assert got == {} + + +def test_library_origin_files_resolve_against_shared(tmp_path: Path): + """shared/ and vendored copies speak from the consuming skill's point of view.""" + plugin = _plugin(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" / "dsl" / "INTERPRETER.md" + got = {r.raw: r.resolved for r in csr.scan_file(vend, plugin)} + # `shared/ai-guardrails.md` → skills/shared/ai/ai-guardrails.md (flattened dir); the + # state schema is genuinely absent everywhere → MISSING + assert got["shared/ai-guardrails.md"] is not None + assert got["references/vendored/state/phase-status.schema.json"] is None + shared_doc = plugin / "skills" / "shared" / "state.md" + got2 = {r.raw: r.resolved for r in csr.scan_file(shared_doc, plugin)} + assert got2["references/vendored/state.md"] is not None + + +def test_gcp_dependents_may_name_gcp_tree_and_agents_search_skills(tmp_path: Path): + plugin = _plugin(tmp_path) + got = _refs(plugin, "agent-advisor", "remap `design-refs/ai.md` → `$GCP_BASE/references/design-refs/ai.md`") + assert got == {"design-refs/ai.md": True, "$GCP_BASE/references/design-refs/ai.md": True} + agent = plugin / "agents" / "worker.md" + got2 = {r.raw: (r.resolved is not None) for r in csr.scan_file(agent, plugin)} + assert got2 == {"scripts/tool.py": True} + + +def test_baseline_ignore_vs_entries_and_stale(tmp_path: Path): + plugin = _plugin(tmp_path) + f = plugin / "skills" / "alpha" / "SKILL.md" + f.write_text("`references/phases/nope.md` and `scripts/gone.py`") + missing = [r for r in csr.scan_file(f, plugin) if r.resolved is None] + un, bl, stale = csr.apply_baseline(missing, { + "ignore": [{"source": "*", "ref": "scripts/gone.py", "reason": "generated"}], + "entries": [{"source": "*alpha/SKILL.md", "ref": "references/phases/nope.md", "reason": "todo"}, + {"source": "*", "ref": "never.md", "reason": "stale"}], + }) + assert [r.raw for r in un] == [] + assert [r.raw for r in bl] == ["references/phases/nope.md"] + assert len(stale) == 1 and stale[0]["ref"] == "never.md" + + +def _run(*args: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, cwd=REPO_ROOT) + + +def test_cli_exit_codes(tmp_path: Path): + plugin = _plugin(tmp_path) + (plugin / "skills" / "alpha" / "SKILL.md").write_text("`references/phases/nope.md`") + r = _run("--check", "--plugin", str(plugin), "--no-baseline", "--json") + assert r.returncode == 1 + assert json.loads(r.stdout)["missing"][0]["ref"] == "references/phases/nope.md" + (plugin / "skills" / "alpha" / "SKILL.md").write_text("`references/phases/design.md`") + # the fixture tree's vendored INTERPRETER.md carries one deliberately dead reference + (plugin / "skills" / "alpha" / "references" / "vendored" / "dsl" / "INTERPRETER.md").unlink() + r = _run("--check", "--plugin", str(plugin), "--no-baseline") + assert r.returncode == 0 and r.stdout.strip().endswith("0 stale baseline entries") + + +def test_real_repository_passes_with_no_stale_entries(): + r = _run("--check", "--json") + assert r.returncode == 0, r.stdout + report = json.loads(r.stdout) + assert report["missing"] == [] + assert report["stale_baseline_entries"] == [] + assert report["references"] > 500 From 990fd390add04c979fc4ac0059067c7a9276830e Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Thu, 1 Oct 2026 09:09:10 -0700 Subject: [PATCH 06/53] =?UTF-8?q?fix(aws-startup-advisor):=20check-skill-r?= =?UTF-8?q?efs=20=E2=80=94=20agents/=20resolve=20only=20against=20llm-to-b?= =?UTF-8?q?edrock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #387 (Completeness / Contradictions, should-fix): the agents/ branch of the resolver searched every skill and kept the first hit, so agents/llm2bedrock-prompt-evaluator.md:233 `scripts/pyproject.toml` resolved to skills/agent-advisor/scripts/pyproject.toml (sorts first) and a tree with the file only under agent-advisor still passed — deleting the llm-to-bedrock copy would not have failed the gate. The comment above the loop said llm-to-bedrock; the loop did not. - Agent references now resolve ONLY against skills/llm-to-bedrock (_AGENTS_OWNER), including the -style rglob, which is scoped to that skill. - Tests: scripts/pyproject.toml present only under agent-advisor is MISSING; present under llm-to-bedrock resolves to llm-to-bedrock's copy even when agent-advisor also has it; the fixture tree's agents/worker.md reference to another skill's file is now correctly MISSING. - Real repo unchanged: 1118 references, 0 missing, 7 baselined. --- tools/check-skill-refs.py | 21 +++++++++++++-------- tools/test_check_skill_refs.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/tools/check-skill-refs.py b/tools/check-skill-refs.py index cb9a29691..82055226a 100644 --- a/tools/check-skill-refs.py +++ b/tools/check-skill-refs.py @@ -71,6 +71,8 @@ # skills that execute gcp-to-aws's phase files inline (agent-advisor) or by Skill invocation # (llm-to-bedrock) legitimately name paths inside gcp-to-aws's tree _GCP_DEPENDENTS = {"agent-advisor", "llm-to-bedrock"} +# the skill whose subagents live in plugins//agents/ +_AGENTS_OWNER = "llm-to-bedrock" # anything carrying a run-artifact prefix or templating is not a repo path _SKIP_PREFIX_RE = re.compile( r"(\$\{?(MIGRATION_DIR|RUN_DIR|REPO|PHASE_DIR|PLAN_DIR|OUT|TARGET|APP|WORKSPACE|TMP)\}?/" @@ -163,14 +165,17 @@ def _candidates(prefix: str, path: str, source: Path, plugin: Path) -> List[Path cands.append(gcp / path) cands.append(gcp / "references" / path) if skill is None and "agents" in source.parts: - # agents are llm-to-bedrock's subagents; their paths are relative to that skill's - # scripts/ or helper dirs, named through variables the agent defines at runtime - for other in sorted((plugin / "skills").iterdir()): - if other.is_dir(): - cands.append(other / path) - for hit in other.rglob(Path(path).name): - if str(hit).endswith(path) and hit.is_file(): - cands.append(hit) + # agents/ holds llm-to-bedrock's subagents (llm2bedrock-*) and two generic phase + # workers; their paths are relative to llm-to-bedrock's scripts/ or helper dirs, + # named through variables (, $HELPERS) the agent defines at runtime. + # Resolve ONLY against that skill — a same-named file in another skill must not + # make a dead llm-to-bedrock path look alive. + owner = plugin / "skills" / _AGENTS_OWNER + cands.append(owner / path) + if owner.is_dir(): + for hit in owner.rglob(Path(path).name): + if str(hit).endswith(path) and hit.is_file(): + cands.append(hit) return cands diff --git a/tools/test_check_skill_refs.py b/tools/test_check_skill_refs.py index 8663b594a..bc7a0d031 100644 --- a/tools/test_check_skill_refs.py +++ b/tools/test_check_skill_refs.py @@ -108,7 +108,32 @@ def test_gcp_dependents_may_name_gcp_tree_and_agents_search_skills(tmp_path: Pat assert got == {"design-refs/ai.md": True, "$GCP_BASE/references/design-refs/ai.md": True} agent = plugin / "agents" / "worker.md" got2 = {r.raw: (r.resolved is not None) for r in csr.scan_file(agent, plugin)} - assert got2 == {"scripts/tool.py": True} + # the fixture's agents/worker.md names scripts/tool.py, which exists only under alpha — + # agents resolve against llm-to-bedrock ONLY, so this is MISSING + assert got2 == {"scripts/tool.py": False} + + +def test_agents_resolve_only_against_llm_to_bedrock(tmp_path: Path): + """Review finding on #387: the agent loop searched every skill and kept the first hit, so + `scripts/pyproject.toml` present only under agent-advisor made a dead llm-to-bedrock + path look alive.""" + plugin = _plugin(tmp_path) + (plugin / "skills" / "agent-advisor" / "scripts").mkdir(parents=True) + (plugin / "skills" / "agent-advisor" / "scripts" / "pyproject.toml").write_text("x") + agent = plugin / "agents" / "evaluator.md" + agent.write_text("run with `scripts/pyproject.toml` and `/references/openai-to-bedrock.md`") + got = {r.raw: r.resolved for r in csr.scan_file(agent, plugin)} + assert got["scripts/pyproject.toml"] is None # only agent-advisor has it -> MISSING + assert got["/references/openai-to-bedrock.md"] is None + l2b = plugin / "skills" / "llm-to-bedrock" + (l2b / "scripts").mkdir(parents=True) + (l2b / "scripts" / "pyproject.toml").write_text("x") + (l2b / "references" / "helpers" / "bdd" / "references").mkdir(parents=True) + (l2b / "references" / "helpers" / "bdd" / "references" / "openai-to-bedrock.md").write_text("x") + got = {r.raw: r.resolved for r in csr.scan_file(agent, plugin)} + assert got["scripts/pyproject.toml"] == (l2b / "scripts" / "pyproject.toml").resolve() + assert got["/references/openai-to-bedrock.md"] is not None + assert "llm-to-bedrock" in str(got["/references/openai-to-bedrock.md"]) def test_baseline_ignore_vs_entries_and_stale(tmp_path: Path): @@ -135,10 +160,13 @@ def test_cli_exit_codes(tmp_path: Path): (plugin / "skills" / "alpha" / "SKILL.md").write_text("`references/phases/nope.md`") r = _run("--check", "--plugin", str(plugin), "--no-baseline", "--json") assert r.returncode == 1 - assert json.loads(r.stdout)["missing"][0]["ref"] == "references/phases/nope.md" + assert "references/phases/nope.md" in {m["ref"] for m in json.loads(r.stdout)["missing"]} (plugin / "skills" / "alpha" / "SKILL.md").write_text("`references/phases/design.md`") - # the fixture tree's vendored INTERPRETER.md carries one deliberately dead reference + # the fixture tree carries two deliberately dead references: the vendored INTERPRETER.md's + # state schema, and agents/worker.md's scripts/tool.py (agents resolve only against + # llm-to-bedrock, which the fixture does not have) (plugin / "skills" / "alpha" / "references" / "vendored" / "dsl" / "INTERPRETER.md").unlink() + (plugin / "agents" / "worker.md").unlink() r = _run("--check", "--plugin", str(plugin), "--no-baseline") assert r.returncode == 0 and r.stdout.strip().endswith("0 stale baseline entries") From b59f58c5f34ca3057f51e05238511d9120a4ef3d Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 10:47:43 -0700 Subject: [PATCH 07/53] =?UTF-8?q?fix(aws-startup-advisor):=20validate-arti?= =?UTF-8?q?facts=20=E2=80=94=20fixture=20baseline=20only=20for=20--fixture?= =?UTF-8?q?s;=20enums=20survive=20example=20merges;=20required=20table=20h?= =?UTF-8?q?eadings;=20real=20estimate/graviton/workloads=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 2 on #385 (seven P2s, plus the Azure composite-source enum raised on #386/#387 against the same code): - The shipped baseline describes the fixture corpus (every glob is fixtures/...), so it now applies by default only to --fixtures. `--run-dir --skill ` runs with no baseline unless --baseline names one, and then every entry must still match (a valid real run previously failed with 21 STALE_BASELINE errors). - Node.merge keeps a declared enum when another example shows a concrete member (`"confidence": "full"` no longer erases `full|reduced`); a concrete token the document shows joins the set, only a free-text example clears it. Adopted children are copied, so refining `design_constraints.cpu_architecture` no longer widens the generic `` template's `chosen_by`. GCP schema-preferences.md example now lists `derived`. - `(or a "+"-joined set)` on an enum comment sets Node.enum_join: `live+terraform` is valid, `terraform+bogus` is not, closed enums elsewhere stay closed. - Heading match is exact or word-boundary prefix: `graviton` no longer selects `graviton_profile`; the Azure design manifest names `graviton block` explicitly. - Wrapped arrays (`{"workloads": [...]}` under a `workloads[]` heading) merge at the array level, so workload key/type drift is detected; bare item examples still merge into the item. - `(REQUIRED)` headings are read from every heading, not only fenced ones: Heroku `metadata` / `apps[]` / `resources[]` are now required; OPTIONAL sections unchanged. Two baseline entries record the hand-authored Heroku seed inventories that lack them. - GCP estimate: scenario_deltas, breakdown.observability, breakdown.security_baseline and breakdown.security_baseline_compliance bind to their object keys (line anchors separate the two Security Baseline blocks); nested drift inside them is still caught. - Heroku workshop `preferences_subset` is the documented open knob map (open_paths). 15 regression tests added (unit + CLI against the real contracts); each fails on the previous head. Fixture gate: 50 artifacts, 0 errors, 50 baselined, 0 stale. --- .../scripts/artifact-contracts-baseline.json | 12 + .../scripts/artifact-contracts.json | 23 +- .../scripts/validate-artifacts.py | 194 +++++++---- .../references/shared/schema-preferences.md | 2 +- .../tests/test_validate_artifacts.py | 313 ++++++++++++++++++ 5 files changed, 473 insertions(+), 71 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json b/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json index feed382c5..dd0a5c9d8 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts-baseline.json @@ -126,6 +126,18 @@ "artifact": "fixtures/azure-*/*preferences.json", "path": "clusters[[]*[]].pattern_status", "reason": "See confirmed." + }, + { + "code": "MISSING_REQUIRED", + "artifact": "fixtures/heroku-*/*heroku-resource-inventory.json", + "path": "apps", + "reason": "schema-discover-heroku.md § `apps[]` (REQUIRED): every Discover output carries one apps[] row per Heroku app. The heroku-nonweb-scaling (seed, after-design) and heroku-workshop (seed, after-arm64-reprice) inventories are hand-authored seeds for Design/Workshop fixtures and carry only resources[] (+ billing_profile). Add an apps[] row for each heroku_app the resources reference (app_name, heroku_generation, generation_action, discovery_status, failure_reason), then remove this entry." + }, + { + "code": "MISSING_REQUIRED", + "artifact": "fixtures/heroku-workshop/*heroku-resource-inventory.json", + "path": "metadata", + "reason": "schema-discover-heroku.md § `metadata` (REQUIRED). The heroku-workshop seed and after-arm64-reprice inventories have no metadata block at all (migration_id/skill on the root only). Add metadata (report_date, discovery_sources, confidence) to both goldens, then remove this entry." } ] } diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 99d48625a..a4eda814e 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -1,5 +1,5 @@ { - "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (prefix match on the heading text, backticks ignored); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", + "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (exact or word-boundary prefix match on the heading text, backticks ignored — 'graviton' matches '`graviton` block …' but not '`graviton_profile` …'); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", "fixture_prefixes": { "gcp-": "gcp-to-aws", "heroku-": "heroku-to-aws", @@ -72,13 +72,13 @@ "shape": "skills/gcp-to-aws/references/shared/schema-estimate-infra.md", "root_heading": "estimation-infra.json schema", "sections": { - "Observability Entry in": "projected_costs.breakdown[]", - "Security Baseline Entries in": "projected_costs.breakdown[]", + "Cost tiers": "projected_costs.scenario_deltas", + "Observability Entry in": "projected_costs.breakdown.observability", + "@L373": "projected_costs.breakdown.security_baseline", + "@L390": "projected_costs.breakdown.security_baseline_compliance", "architecture_comparison": "architecture_comparison" }, - "ignore_headings": [ - "Cost tiers" - ] + "_sections_why": "`projected_costs.breakdown` is a keyed object (shared estimation-infra.schema.json; emit-plan-json.py and validate-migration-report.py read it by key), so the Observability / Security Baseline sections document the named `observability`, `security_baseline` and `security_baseline_compliance` members — not items of an array. The two Security Baseline blocks share one heading, so line anchors tell them apart (the first is `security_baseline`, the second — with `emission_reason` — is `security_baseline_compliance`). 'Cost tiers' carries the `scenario_deltas` block that estimate-infra.md requires alongside the three tiers." }, "scenarios/scenario-NNN.preferences.json": { "shape": "skills/gcp-to-aws/references/shared/schema-preferences.md", @@ -143,7 +143,11 @@ "ignore_headings": [ "preferences.json", "scenarios/index.json" - ] + ], + "open_paths": [ + "preferences_subset" + ], + "_open_paths_why": "schema-workshop-scenarios.md defines `preferences_subset` as the map of dot-path knobs that differ from the baseline scenario (workshop-refresh.md step 4; workshop-invariants.md). The example shows two knobs; the workshop sheet also emits availability, database/Redis HA, compute-target and cost-optimization deltas, so the keys are the open knob vocabulary, not the two in the example." }, "scenarios/scenario-NNN.preferences.json": { "shape": "skills/heroku-to-aws/references/phases/clarify/clarify-assemble.md", @@ -222,8 +226,9 @@ "extra_shapes": [ { "shape": "skills/azure-to-aws/references/shared/schema-graviton.md", - "heading": "graviton", - "path": "services[].graviton" + "heading": "graviton block", + "path": "services[].graviton", + "_heading_why": "schema-graviton.md has both '`graviton_profile` (emitted by Discover…)' and '`graviton` block (added to each compute service…)'. Heading matches end on a word boundary so 'graviton' alone no longer selects graviton_profile, but the full 'graviton block' names the design section explicitly." } ] }, diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 7482f4108..0247fda70 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -48,13 +48,17 @@ python3 validate-artifacts.py --fixtures Validate every golden artifact under fixtures/ against its skill's contracts, - applying the baseline file (known, explained findings that do not fail). + applying the shipped baseline file (known, explained findings that do not fail). python3 validate-artifacts.py --run-dir > --skill Validate one real run. For users and for post-run checks in capability tests. + No baseline is applied unless `--baseline` names one: the shipped baseline + describes gaps in the fixture corpus (its globs are `fixtures/...`), so applying + it to a run outside that corpus would only report every entry as stale. --json machine-readable output - --baseline override the baseline file path (default: scripts/artifact-contracts-baseline.json) + --baseline apply this baseline file (default: scripts/artifact-contracts-baseline.json + for --fixtures; none for --run-dir). Every entry must match a finding. --manifest override the contracts manifest (default: scripts/artifact-contracts.json) Exit 0 when no non-baselined ERROR findings; 1 otherwise. Stdlib only (Python 3.9+). @@ -62,6 +66,7 @@ from __future__ import annotations import argparse +import copy import fnmatch import json import re @@ -174,13 +179,20 @@ class Node: any_key: Optional["Node"] = None # template every key under this object must satisfy item: Optional["Node"] = None enum: Optional[Set[str]] = None + enum_join: Optional[str] = None # "+" when the contract allows a joined set of enum members + examples: Set[str] = field(default_factory=set) # concrete (non-enum, non-placeholder) string values shown source: str = "" def is_any(self) -> bool: return "any" in self.kinds or not self.kinds def merge(self, other: "Node") -> "Node": - """Union two templates for the same path (several documented variants).""" + """Union two templates for the same path (several documented variants). + + Children adopted from `other` are copied, never shared: a later refinement of a + named child (`design_constraints.cpu_architecture`) must not mutate the generic + template (`design_constraints.`) it was seeded from. + """ self.kinds |= other.kinds self.wildcard = self.wildcard or other.wildcard self.required |= other.required @@ -188,28 +200,38 @@ def merge(self, other: "Node") -> "Node": if k in self.keys: self.keys[k].merge(v) else: - self.keys[k] = v + self.keys[k] = copy.deepcopy(v) if other.item is not None: if self.item is None: - self.item = other.item + self.item = copy.deepcopy(other.item) else: self.item.merge(other.item) if other.any_key is not None: if self.any_key is None: - self.any_key = other.any_key + self.any_key = copy.deepcopy(other.any_key) else: self.any_key.merge(other.any_key) - if self.enum is not None and other.enum is not None: - self.enum = self.enum | other.enum - elif other.enum is None and self.enum is not None and "string" in other.kinds and not other.is_any(): - # the other variant documents a free string here — enum is not universal - self.enum = None + self.enum_join = self.enum_join or other.enum_join + self.examples |= other.examples + if self.enum is not None or other.enum is not None: + # A declared enum survives a merge with a concrete example: `"confidence": "full"` + # demonstrates one member of `full|reduced`, it does not widen the field to any + # string. A concrete token the document shows that is not yet in the enum is + # documented by that very example, so it joins the set. Only a free-text + # example (not a token — spaces, punctuation) proves the field is not an enum. + merged = (self.enum or set()) | (other.enum or set()) + if any(not _ENUM_TOKEN_RE.match(e) for e in self.examples): + self.enum = None + else: + self.enum = merged | self.examples return self _PLACEHOLDER_RE = re.compile(r"^<.*>$") _ENUM_TOKEN_RE = re.compile(r"^[A-Za-z0-9_.:+\-/]+$") _COMMENT_ENUM_RE = re.compile(r"(?:^|[—:\-]\s*)([A-Za-z0-9_.:+\-/]+(?:\s*\|\s*[A-Za-z0-9_.:+\-/]+)+)\s*(?:$|[—(\-])") +# `terraform | live | billing (or a "+"-joined set)` — members may be combined with "+" +_JOINED_SET_RE = re.compile(r"[\"'`]?\+[\"'`]?[\s-]*joined|joined\s+(?:by|with|on)\s+[\"'`]?\+", re.I) def _enum_from_string(s: str) -> Optional[Set[str]]: @@ -261,6 +283,8 @@ def build_node(value: Any, key_comments: Dict[str, List[str]], source: str) -> N child.enum = e child.kinds.add("string") child.kinds.discard("any") + if _JOINED_SET_RE.search(c): + child.enum_join = "+" if "null" in c.lower() and "or null" in c.lower() or "nullable" in c.lower(): child.kinds.add("null") n.keys[k] = child @@ -280,6 +304,8 @@ def build_node(value: Any, key_comments: Dict[str, List[str]], source: str) -> N n.kinds.add("string") if e is not None: n.enum = e + else: + n.examples.add(value) elif isinstance(value, bool): n.kinds.add("boolean") elif isinstance(value, (int, float)): @@ -305,7 +331,19 @@ class Block: def extract_blocks(doc: Path) -> List[Block]: + return _scan_doc(doc)[0] + + +def extract_headings(doc: Path) -> List[str]: + """Every heading in document order, whether or not a JSON block follows it. A heading + such as '### `metadata` (REQUIRED)' is a contract statement even when the section body + is a table, so requiredness cannot be read from fenced blocks alone.""" + return _scan_doc(doc)[1] + + +def _scan_doc(doc: Path) -> Tuple[List[Block], List[str]]: blocks: List[Block] = [] + headings: List[str] = [] heading = "" in_block = False lang = "" @@ -315,6 +353,7 @@ def extract_blocks(doc: Path) -> List[Block]: if not in_block: if line.startswith("#"): heading = line.lstrip("#").strip() + headings.append(heading) m = _FENCE_RE.match(line) if m: in_block = True @@ -328,13 +367,20 @@ def extract_blocks(doc: Path) -> List[Block]: blocks.append(Block(heading=heading, lang=lang, text="\n".join(buf), line=start)) else: buf.append(line) - return blocks + return blocks, headings def _heading_matches(heading: str, wanted: str) -> bool: + """Exact match, or a prefix match that ends on a word boundary — `graviton` matches + '`graviton` block (added to …)' but not '`graviton_profile` (emitted by …)'.""" h = heading.replace("`", "").strip().lower() w = wanted.replace("`", "").strip().lower() - return h == w or h.startswith(w) + if h == w: + return True + if not h.startswith(w): + return False + nxt = h[len(w)] + return not (nxt.isalnum() or nxt == "_") def _heading_token(heading: str) -> Optional[str]: @@ -401,6 +447,23 @@ def _unwrap_to_path(value: Any, dotted: str) -> Any: return value +def _merge_example_at(root: Node, path: str, value: Any, key_comments: Dict[str, List[str]], source: str) -> None: + """Merge an example block into the template at `path`, at the level the example + actually shows. A path ending in `[]` names the array's ITEM; a block for it may be a + bare item (`{"name": …}`, merged into the item) or the whole array (`[{…}, {…}]` after + `{"workloads": [...]}` was unwrapped — merged into the array node, one level up). + Merging an array example into the item node would nest a phantom item level whose + object has no declared keys, so none of the item's field checks would ever run.""" + value = _unwrap_to_path(value, path) + node = build_node(value, key_comments, source) + if path.endswith("[]") and isinstance(value, list): + target = _walk_path(root, path[:-2]) + else: + target = _walk_path(root, path) + if target is not None: + target.merge(node) + + def _walk_path(root: Node, dotted: str) -> Optional[Node]: """Find the node at 'a.b[].c'; create intermediate nodes as needed.""" node = root @@ -436,7 +499,7 @@ def build_shape_contract(spec: Dict[str, Any], findings: List[Finding]) -> Optio if not doc.exists(): findings.append(Finding("SCHEMA_PARSE", _rel(doc), "", "contract document not found", _rel(doc))) return None - blocks = extract_blocks(doc) + blocks, headings = _scan_doc(doc) root_heading = spec.get("root_heading") root_block: Optional[Block] = None for b in blocks: @@ -464,52 +527,52 @@ def build_shape_contract(spec: Dict[str, Any], findings: List[Finding]) -> Optio ignore: List[str] = spec.get("ignore_headings", []) anchors = {k for k in explicit if k.startswith("@L")} seen_anchors: Set[str] = set() + + def section_path(heading: str) -> Optional[str]: + """The key path a (non-anchored) section heading documents, or None to skip it.""" + if root_heading is not None and _heading_matches(heading, root_heading): + # another example under the root heading (a per-item sample, a fragment) — + # only a line anchor in the manifest can say what it documents + return None + if any(_heading_matches(heading, ig) for ig in ignore): + return None + for h, p in explicit.items(): + if not h.startswith("@L") and _heading_matches(heading, h): + return p + for r in rules: + if re.search(r["heading_regex"], heading): + return r["path"] + return _heading_token(heading) + for b in blocks: if b is root_block: continue - path: Optional[str] = None anchor = f"@L{b.line}" if anchor in explicit: - path = explicit[anchor] + path: Optional[str] = explicit[anchor] seen_anchors.add(anchor) - elif root_heading is not None and _heading_matches(b.heading, root_heading): - # another example under the root heading (a per-item sample, a fragment) — - # only a line anchor in the manifest can say what it documents - continue - if path is None and any(_heading_matches(b.heading, ig) for ig in ignore): - continue - if path is None: - for h, p in explicit.items(): - if h.startswith("@L"): - continue - if _heading_matches(b.heading, h): - path = p - break - if path is None: - for r in rules: - if re.search(r["heading_regex"], b.heading): - path = r["path"] - break + else: + path = section_path(b.heading) if path is None: - tok = _heading_token(b.heading) - if tok is None: - continue - path = tok + continue parsed = parse_block(b, doc, findings) if parsed is None: continue v, kc2 = parsed - v = _unwrap_to_path(v, path) - sub = build_node(v, kc2, f"{_rel(doc)}:L{b.line}") - target = _walk_path(root, path) - if target is None: + _merge_example_at(root, path, v, kc2, f"{_rel(doc)}:L{b.line}") + # A heading marked `(REQUIRED)` is a contract statement on its own, whether the section + # body is a JSON block or a table (`### \`metadata\` (REQUIRED)` in the Heroku inventory + # contract is a table). Read requiredness from every heading, not only fenced ones. + for heading in headings: + if "(REQUIRED)" not in heading.upper(): + continue + path = section_path(heading) + if not path: continue - target.merge(sub) - if "(REQUIRED)" in b.heading.upper(): - parent_path = ".".join(path.split(".")[:-1]) - parent = _walk_path(root, parent_path) if parent_path else root - if parent is not None: - parent.required.add(path.split(".")[-1].rstrip("[]")) + parent_path = ".".join(path.split(".")[:-1]) + parent = _walk_path(root, parent_path) if parent_path else root + if parent is not None: + parent.required.add(path.split(".")[-1].rstrip("[]")) for a in anchors - seen_anchors: findings.append(Finding("SCHEMA_PARSE", _rel(doc), a, f"manifest section anchor {a} does not point at a JSON block start in this document " @@ -533,12 +596,7 @@ def build_shape_contract(spec: Dict[str, Any], findings: List[Finding]) -> Optio if parsed is None: continue ev, ekc = parsed - epath = extra["path"] - ev = _unwrap_to_path(ev, epath) - enode = build_node(ev, ekc, f"{_rel(edoc)}:L{hit.line}") - target = _walk_path(root, epath) - if target is not None: - target.merge(enode) + _merge_example_at(root, extra["path"], ev, ekc, f"{_rel(edoc)}:L{hit.line}") for k in spec.get("required", []): root.required.add(k) for k in spec.get("open_paths", []): @@ -580,9 +638,15 @@ def validate_shape(node: Node, value: Any, path: str, artifact: str, contract: s out.append(Finding("TYPE_MISMATCH", artifact, path or "$", f"is {kind}, contract example is {'/'.join(sorted(node.kinds))}", contract)) return - if node.enum is not None and isinstance(value, str) and value not in node.enum: - out.append(Finding("ENUM_VIOLATION", artifact, path or "$", - f"'{value}' not in {{{', '.join(sorted(node.enum))}}}", contract)) + if node.enum is not None and isinstance(value, str): + # a joined set (`terraform+live`) is valid when every member is; an unknown member + # is still a violation + members = value.split(node.enum_join) if node.enum_join else [value] + if not members or any(m not in node.enum for m in members): + allowed = ", ".join(sorted(node.enum)) + joined = f" (or a '{node.enum_join}'-joined set of them)" if node.enum_join else "" + out.append(Finding("ENUM_VIOLATION", artifact, path or "$", + f"'{value}' not in {{{allowed}}}{joined}", contract)) if isinstance(value, dict) and "object" in node.kinds: for k in node.required: if k not in value: @@ -890,7 +954,9 @@ def main(argv: Optional[List[str]] = None) -> int: ap.add_argument("--skill", choices=["gcp-to-aws", "heroku-to-aws", "azure-to-aws"], help="required with --run-dir") ap.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) - ap.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE) + ap.add_argument("--baseline", type=Path, default=None, + help="baseline file to apply; defaults to the shipped fixture baseline for --fixtures " + "and to none for --run-dir (the shipped entries describe the fixture corpus)") ap.add_argument("--no-baseline", action="store_true", help="ignore the baseline file") ap.add_argument("--json", action="store_true", help="machine-readable output") args = ap.parse_args(argv) @@ -916,13 +982,19 @@ def main(argv: Optional[List[str]] = None) -> int: ap.error(f"{args.run_dir} is not a directory") checked += validate_run_dir(args.run_dir, args.skill, manifest, findings) + # The shipped baseline is the fixture corpus' burn-down list (every entry's artifact + # glob is `fixtures/...`), so it applies by default only to --fixtures. A real run gets + # a baseline only when the caller names one — and then every entry must match. + baseline_path: Optional[Path] = args.baseline + if baseline_path is None and args.fixtures: + baseline_path = DEFAULT_BASELINE stale: List[Dict[str, Any]] = [] - if not args.no_baseline and args.baseline.exists() and not args.self_check: - stale = apply_baseline(findings, json.loads(args.baseline.read_text(encoding="utf-8"))) + if not args.no_baseline and baseline_path is not None and baseline_path.exists() and not args.self_check: + stale = apply_baseline(findings, json.loads(baseline_path.read_text(encoding="utf-8"))) for e in stale: # a baseline entry that matches nothing is either fixed (remove it) or mistyped # (it is hiding nothing and would hide a future finding by accident) — fail either way - findings.append(Finding("STALE_BASELINE", _rel(args.baseline), e.get("path", "*"), + findings.append(Finding("STALE_BASELINE", _rel(baseline_path), e.get("path", "*"), f"entry matched no finding — remove it, or fix its code/artifact/path: {json.dumps(e)}", "")) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md index d9727a2eb..3b8e5c0f4 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md @@ -42,7 +42,7 @@ Use the **Recommendation Impact** row for the selected answer from the category "design_constraints": { "": { "value": "", - "chosen_by": "user|default|extracted", + "chosen_by": "user|default|extracted|derived", "prompt": "", "design_consequence": "", "question_id": "", diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 61df9fb53..fb08c9c4b 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -119,6 +119,108 @@ def test_allowed_anywhere_vocabulary(): assert _validate(ex, {"g": {"r": {"value": 1, "source": "live"}}}, allowed_anywhere=["source"]) == set() +# --------------------------------------------------------------------------- template merging + + +def test_merge_keeps_declared_enum_when_a_concrete_example_shows_one_member(): + """Review finding on #385: merging `"confidence": "full"` (a Complete Example) into + `"confidence": "full|reduced"` cleared the enum, so any string passed.""" + declared = _node('{"confidence": "full|reduced"}') + declared.merge(_node('{"confidence": "full"}')) + assert declared.keys["confidence"].enum == {"full", "reduced"} + # the other direction too (the example is seen before the declaration) + example = _node('{"confidence": "full"}') + example.merge(_node('{"confidence": "full|reduced"}')) + assert example.keys["confidence"].enum == {"full", "reduced"} + # a concrete token the document shows that the enum omitted is documented by that example + declared.merge(_node('{"confidence": "partial"}')) + assert declared.keys["confidence"].enum == {"full", "reduced", "partial"} + # only a free-text example (not a token) proves the field is not an enum + declared.merge(_node('{"confidence": "Using cached prices from 2026-03-04 (±5%)"}')) + assert declared.keys["confidence"].enum is None + + +def test_merge_copies_adopted_children_so_a_named_refinement_does_not_leak_into_the_generic_template(): + """Review finding on #385/#387: `design_constraints.cpu_architecture` was seeded from the + `` template by reference, so refining it widened `chosen_by` on every row.""" + root = _node('{"design_constraints": {"": {"value": "", "chosen_by": "user|default|extracted"}}}') + target = va._walk_path(root, "design_constraints.cpu_architecture") + target.merge(_node('{"value": "graviton", "chosen_by": "default"}')) + generic = root.keys["design_constraints"].any_key.keys["chosen_by"] + assert generic.enum == {"user", "default", "extracted"} + assert target.keys["chosen_by"].enum == {"user", "default", "extracted"} + out = [] + va.validate_shape(root, {"design_constraints": {"target_region": {"value": "us-east-1", "chosen_by": "invalid-origin"}, + "cpu_architecture": {"value": "graviton", "chosen_by": "bogus"}}}, + "", "a", "c", out) + assert {(f.code, f.path) for f in out} == {("ENUM_VIOLATION", "design_constraints.target_region.chosen_by"), + ("ENUM_VIOLATION", "design_constraints.cpu_architecture.chosen_by")} + + +def test_joined_set_qualifier_allows_plus_joined_members_but_not_unknown_ones(): + """Review finding on #386/#387: `terraform | live | billing (or a "+"-joined set)` was read + as a closed enum, so the documented `live+terraform` provenance failed.""" + ex = ('{"source": "terraform", // terraform | bicep | arm | live | rdfa | billing (or a "+"-joined set)\n' + ' "prov": "table", // table | rubric\n}') + n = _node(ex) + assert n.keys["source"].enum_join == "+" and n.keys["prov"].enum_join is None + assert _validate(ex, {"source": "live+terraform", "prov": "table"}) == set() + assert _validate(ex, {"source": "terraform", "prov": "table"}) == set() + assert _validate(ex, {"source": "terraform+bogus", "prov": "table"}) == {("ENUM_VIOLATION", "source")} + assert _validate(ex, {"source": "live", "prov": "table+rubric"}) == {("ENUM_VIOLATION", "prov")} + + +def test_wrapped_array_example_merges_at_the_array_level(tmp_path: Path): + """Review finding on #385: `{"workloads": [...]}` under a `workloads[]` heading was merged + into the ITEM node, nesting a phantom level whose object had no keys — so no workload + field was ever checked.""" + root = _node('{"metadata": {"a": 1}}') + text, comments, keys = va.strip_jsonc('{"workloads": [{"workload_id": "wl_1", "structured_output": false, "call_sites": [{"file": "a", "line": 1}]}]}') + va._merge_example_at(root, "workloads[]", json.loads(text), {}, "t") + item = root.keys["workloads"].item + assert set(item.keys) == {"workload_id", "structured_output", "call_sites"} + # a bare per-item example still lands on the item + va._merge_example_at(root, "workloads[]", {"model_id": "m"}, {}, "t") + assert "model_id" in root.keys["workloads"].item.keys + out = [] + va.validate_shape(root, {"metadata": {"a": 1}, "workloads": [{"workload_id": "x", "model_name": "m", "structured_output": "yes", "call_sites": "a.ts"}]}, + "", "a", "c", out) + assert {(f.code, f.path) for f in out} == {("UNKNOWN_KEY", "workloads[0].model_name"), + ("TYPE_MISMATCH", "workloads[0].structured_output"), + ("TYPE_MISMATCH", "workloads[0].call_sites")} + ok = [] + va.validate_shape(root, {"metadata": {"a": 1}, "workloads": []}, "", "a", "c", ok) + assert ok == [] + + +def test_heading_match_is_exact_or_word_boundary_prefix(): + """Review finding on #385: `graviton` selected '`graviton_profile` (emitted by …)' by prefix.""" + assert va._heading_matches("`graviton` block (added to each compute service in `aws-design.json`)", "graviton") + assert not va._heading_matches("`graviton_profile` (emitted by Discover, one entry per compute service)", "graviton") + assert va._heading_matches("`graviton_profile` (emitted by Discover)", "graviton_profile") + assert va._heading_matches("Shape", "Shape") and not va._heading_matches("Shapes", "Shape") + assert va._heading_matches("Cost tiers (`projected_costs` / `cost_comparison`)", "Cost tiers") + + +def test_required_heading_with_a_table_body_is_enforced(tmp_path: Path): + """Review finding on #385: requiredness was read only from headings that had a JSON block, + so '### `apps[]` (REQUIRED)' followed by a table never reached `root.required`.""" + doc = tmp_path / "schema.md" + doc.write_text("# Contract\n\n## thing.json\n\n```json\n{\"metadata\": {\"x\": 1}, \"apps\": [], \"opt\": {}}\n```\n\n" + "### `metadata` (REQUIRED)\n\n| Field | Type |\n|---|---|\n| `x` | number |\n\n" + "### `apps[]` (REQUIRED)\n\n| Field | Type |\n|---|---|\n\n" + "### `opt` (OPTIONAL — present when available)\n\n| Field | Type |\n|---|---|\n") + old_root = va.PLUGIN_ROOT + va.PLUGIN_ROOT = tmp_path + try: + findings = [] + c = va.build_shape_contract({"shape": "schema.md", "root_heading": "thing.json"}, findings) + finally: + va.PLUGIN_ROOT = old_root + assert findings == [] and c is not None + assert c.root.required == {"metadata", "apps"} + + # --------------------------------------------------------------------------- JSON Schema subset @@ -256,3 +358,214 @@ def test_real_json_schema_violation_fails(tmp_path: Path): r = _run("--run-dir", str(run), "--skill", "azure-to-aws", "--no-baseline", "--json") assert r.returncode == 1 assert any(e["code"] == "SCHEMA_VIOLATION" and e["path"] == "complexity_tier" for e in json.loads(r.stdout)["errors"]) + + +# --------------------------------------------------------------------------- review findings (#385 round 2) against the real contracts + + +def _errors(r: subprocess.CompletedProcess): + assert r.stdout, r.stderr + return {(e["code"], e["path"]) for e in json.loads(r.stdout)["errors"]} + + +def _write_run(tmp_path: Path, name: str, artifact: str, data) -> Path: + run = tmp_path / name + (run / Path(artifact).parent).mkdir(parents=True, exist_ok=True) + (run / artifact).write_text(json.dumps(data)) + return run + + +def _doc_example(doc: Path, heading: str): + """The first JSON block under `heading`, with `a|b` alternations resolved to `a`.""" + block = next(b for b in va.extract_blocks(doc) if va._heading_matches(b.heading, heading)) + findings = [] + value, _ = va.parse_block(block, doc, findings) # also reads `"key": {...}` fragments + assert findings == [] + + def pick(o): + if isinstance(o, dict): + return {k: pick(v) for k, v in o.items()} + if isinstance(o, list): + return [pick(x) for x in o] + if isinstance(o, str) and "|" in o and va._enum_from_string(o): + return o.split("|")[0] + return o + return pick(value) + + +def test_documented_default_run_dir_command_passes_for_a_valid_run(tmp_path: Path): + """Review finding on #385: `--run-dir --skill gcp-to-aws` applied the shipped + fixture baseline, so a valid real run failed with 21 STALE_BASELINE errors.""" + status = json.loads((PLUGIN_ROOT / "fixtures/gcp-decision-gate/after-decide-complete/.phase-status.json").read_text()) + run = _write_run(tmp_path, "run", ".phase-status.json", status) + r = _run("--run-dir", str(run), "--skill", "gcp-to-aws", "--json") + assert r.returncode == 0, r.stdout + r.stderr + report = json.loads(r.stdout) + assert report["errors"] == [] and report["stale_baseline_entries"] == [] + # the real fixture run passes the same way when validated as a run directory + r = _run("--run-dir", str(PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-discover"), "--skill", "azure-to-aws", "--json") + assert r.returncode == 0, r.stdout + # the shipped baseline still applies to --fixtures (and every entry must match there): + # test_fixtures_pass_with_baseline_and_no_stale_entries. An explicit --baseline on a run + # is still audited for stale entries: test_stale_baseline_entry_fails_the_gate. + + +def test_heroku_complete_example_enums_are_enforced_after_merge(tmp_path: Path): + """Review finding on #385: merging the Heroku Complete Example cleared the declared enums + on confidence / heroku_generation / discovery_status / dyno_type.""" + doc = PLUGIN_ROOT / "skills/heroku-to-aws/references/shared/schema-discover-heroku.md" + good = _doc_example(doc, "Complete Example") + r = _run("--run-dir", str(_write_run(tmp_path, "ok", "heroku-resource-inventory.json", good)), + "--skill", "heroku-to-aws", "--no-baseline", "--json") + assert r.returncode == 0, r.stdout + formation = next(i for i, x in enumerate(good["resources"]) if x["resource_type"] == "formation") + mutants = { + "metadata.confidence": lambda d: d["metadata"].__setitem__("confidence", "bogus"), + "apps[0].heroku_generation": lambda d: d["apps"][0].__setitem__("heroku_generation", "mars"), + "apps[0].discovery_status": lambda d: d["apps"][0].__setitem__("discovery_status", "whatever"), + f"resources[{formation}].config.dyno_type": lambda d: d["resources"][formation]["config"].__setitem__("dyno_type", "giga-9000"), + } + for path, mutate in mutants.items(): + bad = json.loads(json.dumps(good)) + mutate(bad) + r = _run("--run-dir", str(_write_run(tmp_path, path, "heroku-resource-inventory.json", bad)), + "--skill", "heroku-to-aws", "--no-baseline", "--json") + assert r.returncode == 1 and ("ENUM_VIOLATION", path) in _errors(r), (path, r.stdout) + + +def test_heroku_required_table_sections_are_enforced(tmp_path: Path): + """Review finding on #385: `metadata`, `apps[]`, `resources[]` are headed `(REQUIRED)` but + documented as tables, so `{}` passed as heroku-resource-inventory.json.""" + doc = PLUGIN_ROOT / "skills/heroku-to-aws/references/shared/schema-discover-heroku.md" + good = _doc_example(doc, "Complete Example") + for key in ("metadata", "apps", "resources"): + bad = json.loads(json.dumps(good)) + bad.pop(key) + r = _run("--run-dir", str(_write_run(tmp_path, "no-" + key, "heroku-resource-inventory.json", bad)), + "--skill", "heroku-to-aws", "--no-baseline", "--json") + assert r.returncode == 1 and _errors(r) == {("MISSING_REQUIRED", key)}, (key, r.stdout) + for key in ("billing_profile", "terraform_metadata"): # OPTIONAL sections stay optional + ok = json.loads(json.dumps(good)) + ok.pop(key) + r = _run("--run-dir", str(_write_run(tmp_path, "opt-" + key, "heroku-resource-inventory.json", ok)), + "--skill", "heroku-to-aws", "--no-baseline", "--json") + assert r.returncode == 0, (key, r.stdout) + r = _run("--run-dir", str(_write_run(tmp_path, "empty", "heroku-resource-inventory.json", {})), + "--skill", "heroku-to-aws", "--no-baseline", "--json") + assert _errors(r) == {("MISSING_REQUIRED", "metadata"), ("MISSING_REQUIRED", "apps"), ("MISSING_REQUIRED", "resources")} + + +def test_gcp_estimate_extensions_bind_to_their_object_keys(tmp_path: Path): + """Review finding on #385: scenario_deltas and the observability / security_baseline / + security_baseline_compliance breakdown members were bound to a `breakdown[]` item (the + schema types breakdown as an object), so a complete estimate failed with UNKNOWN_KEY.""" + doc = PLUGIN_ROOT / "skills/gcp-to-aws/references/shared/schema-estimate-infra.md" + est = json.loads((PLUGIN_ROOT / "fixtures/gcp-decision-gate/after-decide-complete/estimation-infra.json").read_text()) + sec_blocks = [b for b in va.extract_blocks(doc) if va._heading_matches(b.heading, "Security Baseline Entries in")] + assert len(sec_blocks) == 2, "manifest anchors @L373/@L390 expect two Security Baseline blocks" + est["projected_costs"]["scenario_deltas"] = _doc_example(doc, "Cost tiers")["scenario_deltas"] + est["projected_costs"]["breakdown"] = { + "compute": {"service": "Fargate", "monthly": 71}, + "observability": _doc_example(doc, "Observability Entry in"), + "security_baseline": json.loads(va.strip_jsonc(sec_blocks[0].text)[0]), + "security_baseline_compliance": json.loads(va.strip_jsonc(sec_blocks[1].text)[0]), + } + r = _run("--run-dir", str(_write_run(tmp_path, "ok", "estimation-infra.json", est)), "--skill", "gcp-to-aws", "--json") + assert r.returncode == 0, r.stdout + bad = json.loads(json.dumps(est)) + bad["projected_costs"]["breakdown"]["observability"]["components"]["bogus"] = 1 + bad["projected_costs"]["breakdown"]["security_baseline"]["invented"] = 1 + bad["projected_costs"]["scenario_deltas"]["weird"] = [] + bad["projected_costs"]["unrelated"] = 1 + r = _run("--run-dir", str(_write_run(tmp_path, "bad", "estimation-infra.json", bad)), "--skill", "gcp-to-aws", "--json") + assert r.returncode == 1 + assert _errors(r) == {("UNKNOWN_KEY", "projected_costs.breakdown.observability.components.bogus"), + ("UNKNOWN_KEY", "projected_costs.breakdown.security_baseline.invented"), + ("UNKNOWN_KEY", "projected_costs.scenario_deltas.weird"), + ("UNKNOWN_KEY", "projected_costs.unrelated")} + + +def test_azure_design_graviton_block_is_the_design_section_not_the_discovery_profile(tmp_path: Path): + """Review finding on #385: the `graviton` heading matched `graviton_profile` by prefix, so + services[].graviton rejected the documented `compatibility` field.""" + src = PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-design" + control = _errors(_run("--run-dir", str(src), "--skill", "azure-to-aws", "--no-baseline", "--json")) + design = json.loads((src / "aws-design.json").read_text()) + design["services"][0]["graviton"] = {"compatibility": "ready", "target_architecture": "arm64", "caveats": []} + r = _run("--run-dir", str(_write_run(tmp_path, "grav", "aws-design.json", design)), "--skill", "azure-to-aws", "--no-baseline", "--json") + assert _errors(r) - control == set(), r.stdout + design["services"][0]["graviton"] = {"compatibility": "ready", "tier": "ready", "service_name": "x"} # discovery-profile fields + r = _run("--run-dir", str(_write_run(tmp_path, "mixed", "aws-design.json", design)), "--skill", "azure-to-aws", "--no-baseline", "--json") + assert _errors(r) - control == {("UNKNOWN_KEY", "services[0].graviton.tier"), ("UNKNOWN_KEY", "services[0].graviton.service_name")} + # the discovery profile is still selected for the inventory + inv = json.loads((PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-discover/azure-resource-inventory.json").read_text()) + inv["graviton_profile"] = [{"service_name": "api", "tier": "ready", "target_architecture": "arm64", "signals": [], "caveats": [], "source": "app_code"}] + r = _run("--run-dir", str(_write_run(tmp_path, "inv", "azure-resource-inventory.json", inv)), "--skill", "azure-to-aws", "--no-baseline", "--json") + assert not any(p.startswith("graviton_profile") for _, p in _errors(r)), r.stdout + + +def test_gcp_ai_workloads_fields_are_checked(tmp_path: Path): + """Review finding on #385: the wrapped `{"workloads": [...]}` example merged one level too + deep, so key and type drift inside a workload never fired.""" + doc = PLUGIN_ROOT / "skills/gcp-to-aws/references/shared/schema-discover-ai.md" + profile = _doc_example(doc, "ai-workload-profile.json") + profile["workloads"] = _doc_example(doc, "workloads[]")["workloads"] + r = _run("--run-dir", str(_write_run(tmp_path, "ok", "ai-workload-profile.json", profile)), "--skill", "gcp-to-aws", "--json") + assert r.returncode == 0, r.stdout + empty = dict(profile, workloads=[]) + r = _run("--run-dir", str(_write_run(tmp_path, "empty", "ai-workload-profile.json", empty)), "--skill", "gcp-to-aws", "--json") + assert r.returncode == 0, r.stdout + bad = json.loads(json.dumps(profile)) + w = bad["workloads"][0] + w["model_name"] = w.pop("model_id") + w["structured_output"] = "yes" + w["call_sites"] = "lib/gemini.ts" + r = _run("--run-dir", str(_write_run(tmp_path, "bad", "ai-workload-profile.json", bad)), "--skill", "gcp-to-aws", "--json") + assert r.returncode == 1 + assert _errors(r) == {("UNKNOWN_KEY", "workloads[0].model_name"), + ("TYPE_MISMATCH", "workloads[0].structured_output"), + ("TYPE_MISMATCH", "workloads[0].call_sites")} + + +def test_heroku_workshop_preferences_subset_is_an_open_knob_map(tmp_path: Path): + """Review finding on #385: `preferences_subset` was closed to the two knobs in the example, + so a cost-optimization or availability delta failed with UNKNOWN_KEY.""" + sc = json.loads((PLUGIN_ROOT / "fixtures/heroku-workshop/after-arm64-reprice/scenarios/scenario-002.json").read_text()) + sc["preferences_subset"].update({"operational.cost_optimization": "aggressive", "availability.multi_az": True, + "database.ha": "multi_az", "compute.target": "fargate"}) + r = _run("--run-dir", str(_write_run(tmp_path, "ok", "scenarios/scenario-002.json", sc)), "--skill", "heroku-to-aws", "--json") + assert r.returncode == 0, r.stdout + sc["invented_root_key"] = 1 # the manifest itself is still closed + r = _run("--run-dir", str(_write_run(tmp_path, "bad", "scenarios/scenario-002.json", sc)), "--skill", "heroku-to-aws", "--json") + assert _errors(r) == {("UNKNOWN_KEY", "invented_root_key")} + + +def test_azure_inventory_source_accepts_the_documented_joined_set(tmp_path: Path): + """Review finding on #386/#387: `source` is documented as `terraform | bicep | arm | live | + rdfa | billing (or a "+"-joined set)` and discover-live.md emits `live+terraform`.""" + inv = json.loads((PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-discover/azure-resource-inventory.json").read_text()) + for val, expect in (("live+terraform", set()), ("terraform+live", set()), ("terraform", set()), + ("terraform+bogus", {("ENUM_VIOLATION", "resources[0].source")}), + ("bogus", {("ENUM_VIOLATION", "resources[0].source")})): + inv["resources"][0]["source"] = val + r = _run("--run-dir", str(_write_run(tmp_path, val, "azure-resource-inventory.json", inv)), "--skill", "azure-to-aws", "--json") + assert _errors(r) == expect, (val, r.stdout) + inv["resources"][0]["source"] = "terraform" + inv["resources"][0]["azure_type_provenance"] = "terraform+live" # a genuinely closed enum stays closed + r = _run("--run-dir", str(_write_run(tmp_path, "prov", "azure-resource-inventory.json", inv)), "--skill", "azure-to-aws", "--json") + assert _errors(r) == {("ENUM_VIOLATION", "resources[0].azure_type_provenance")}, r.stdout + + +def test_gcp_preferences_chosen_by_enum_is_enforced_including_derived(tmp_path: Path): + """Review finding on #385/#387: the Graviton refinement cleared `chosen_by` for every row; + restoring the enum must keep the AI-only route's `derived` (schema-preferences.md § Wrapper).""" + pref = json.loads((PLUGIN_ROOT / "fixtures/gcp-workshop/after-graviton-reprice/preferences.json").read_text()) + for val, expect in (("derived", set()), ("default", set()), + ("invalid-origin", {("ENUM_VIOLATION", "design_constraints.target_region.chosen_by")})): + pref["design_constraints"]["target_region"]["chosen_by"] = val + r = _run("--run-dir", str(_write_run(tmp_path, val, "preferences.json", pref)), "--skill", "gcp-to-aws", "--json") + assert _errors(r) == expect, (val, r.stdout) + pref["design_constraints"]["target_region"]["chosen_by"] = "user" + pref["design_constraints"]["cpu_architecture"] = {"value": "graviton", "chosen_by": "bogus"} + r = _run("--run-dir", str(_write_run(tmp_path, "cpu", "preferences.json", pref)), "--skill", "gcp-to-aws", "--json") + assert _errors(r) == {("ENUM_VIOLATION", "design_constraints.cpu_architecture.chosen_by")}, r.stdout From 509f1656086566c65e049b21917b78387065745e Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 10:53:50 -0700 Subject: [PATCH 08/53] =?UTF-8?q?fix(aws-startup-advisor):=20check-skill-r?= =?UTF-8?q?efs=20=E2=80=94=20run-artifact=20skip=20is=20per=20token;=20exp?= =?UTF-8?q?licit=20bases=20are=20authoritative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 2 on #387 (the two findings unique to this tool): - The run-artifact skip looked 40 characters back, so an earlier, separate token (`$MIGRATION_DIR/`) swallowed the schema reference that followed it on the same line (azure design-ai.md:197) — renaming that schema passed CI. The skip now reads only the token's own contiguous prefix; `/…`, `.migration//…` and `$MIGRATION_DIR/…` forms are still skipped. Real repo: 1157 references (2 previously swallowed, both resolve). - An explicit base (`/`, `$PLUGIN_ROOT/`, `$GCP_BASE/`, `./`, `../`) is now authoritative: no library-origin / gcp-dependent / agents fallbacks after it, so a same-named file under another skill cannot make a dead `/scripts/x.py` look alive. Unqualified references keep every fallback; `/../gcp-to-aws/SKILL.md` traversal still resolves. Two regression tests added; each fails on the previous head. Real repo: PASS, 0 missing, 7 baselined, 0 stale. --- tools/check-skill-refs.py | 31 +++++++++++++-------- tools/test_check_skill_refs.py | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/tools/check-skill-refs.py b/tools/check-skill-refs.py index 82055226a..606440319 100644 --- a/tools/check-skill-refs.py +++ b/tools/check-skill-refs.py @@ -25,7 +25,9 @@ (``, `{name}`, `*`, `NNN`), and URLs. Resolution order for a bare path `P` found in `skills//…/file.md`: - 1. explicit prefix → plugin root / skill root / gcp-to-aws root / referencing file's dir + 1. explicit prefix → plugin root / skill root / gcp-to-aws root / referencing file's dir. + An explicit prefix is authoritative: the agent runs `/scripts/x.py` as + written, so none of the fallbacks below apply to it. 2. `skills//P` (skill-relative — the documented convention) 3. `/P` (file-relative) 4. `skills//references/P` (agent-advisor short forms: phases/, shared/, design-refs/) @@ -138,14 +140,18 @@ def _candidates(prefix: str, path: str, source: Path, plugin: Path) -> List[Path for _ in range(p.count("../")): up = up.parent cands.append(up / path) - if not cands or not p: - if skill is not None: - cands.append(skill / path) - cands.append(here / path) - if skill is not None: - cands.append(skill / "references" / path) - cands.append(plugin / path) - cands.append(REPO_ROOT / path) + if cands: + # An explicit base (`/`, `$PLUGIN_ROOT/`, `$GCP_BASE/`, `./`, `../`) names + # the location. The agent runs the command as written, so a same-named file under + # another skill does not make the named one exist — no cross-skill fallbacks here. + return cands + if skill is not None: + cands.append(skill / path) + cands.append(here / path) + if skill is not None: + cands.append(skill / "references" / path) + cands.append(plugin / path) + cands.append(REPO_ROOT / path) src = str(source) if _LIBRARY_ORIGIN_RE.search(src): shared = plugin / "skills" / "shared" @@ -204,8 +210,11 @@ def scan_file(source: Path, plugin: Path) -> List[Ref]: line_wo_urls = _URL_RE.sub("", line) for m in _TOKEN_RE.finditer(line_wo_urls): start = m.start() - # what immediately precedes the token decides whether it is a repo path - before = line_wo_urls[max(0, start - 40):start] + # Only the token's own contiguous prefix decides whether it is a run-artifact + # path (`/references/x.md`, `.migration//scripts/y.sh`). An + # earlier, separate token on the line — "write to `$MIGRATION_DIR/` per + # `references/shared/schema.md`" — says nothing about this one. + before = re.search(r"[^\s`]*$", line_wo_urls[:start]).group(0) full = m.group(0) if _SKIP_PREFIX_RE.search(before + full[: len(m.group("prefix"))]): continue diff --git a/tools/test_check_skill_refs.py b/tools/test_check_skill_refs.py index bc7a0d031..9cc83eda8 100644 --- a/tools/test_check_skill_refs.py +++ b/tools/test_check_skill_refs.py @@ -136,6 +136,55 @@ def test_agents_resolve_only_against_llm_to_bedrock(tmp_path: Path): assert "llm-to-bedrock" in str(got["/references/openai-to-bedrock.md"]) +def test_run_artifact_skip_is_limited_to_the_current_token(tmp_path: Path): + """Review finding on #387: a 40-character lookbehind let an earlier `$MIGRATION_DIR/` + token swallow the separate schema reference that followed it (azure design-ai.md:197), + so a renamed schema file was never reported.""" + plugin = _plugin(tmp_path) + missing = "references/shared/schema-design-aws-ai.md" # not in the fixture tree + got = _refs(plugin, "alpha", + f"Alone: `{missing}`.\n" + f"Write `aws-design-ai.json` to `$MIGRATION_DIR/` per `{missing}`.\n" + f"Emit `$MIGRATION_DIR/scripts/02-migrate-data.sh`\n`{missing}`\n" + f"`/{missing}` and `.migration//{missing}` and `$MIGRATION_DIR/{missing}`", + name="design-ai.md") + # the schema reference is seen (and MISSING) alone, after an output path, and after a + # newline; the genuinely run-artifact-prefixed forms are still skipped + assert got == {missing: False} + f = plugin / "skills" / "alpha" / "design-ai.md" + assert sorted(r.line for r in csr.scan_file(f, plugin)) == [1, 2, 4] + + +def test_explicit_base_prefixes_do_not_fall_back_to_other_skills(tmp_path: Path): + """Review finding on #387: library-origin and gcp-dependent fallbacks were appended even + after an explicit base named the target, so `/scripts/validate.py` with no + helper under alpha resolved to beta's copy. The command as written would still fail.""" + plugin = _plugin(tmp_path) + (plugin / "skills" / "beta" / "scripts").mkdir() + (plugin / "skills" / "beta" / "scripts" / "validate.py").write_text("x") + (plugin / "skills" / "alpha" / "references" / "shared").mkdir() + contract = plugin / "skills" / "alpha" / "references" / "shared" / "contract.md" # library origin + contract.write_text("run `/scripts/validate.py`, `$PLUGIN_ROOT/scripts/validate.py`, " + "`$GCP_BASE/scripts/validate.py`, `./scripts/validate.py`; bare `scripts/validate.py`; " + "see `/../gcp-to-aws/SKILL.md` and `/scripts/tool.py`") + (plugin / "skills" / "gcp-to-aws" / "SKILL.md").write_text("x") + got = {r.raw: r.resolved for r in csr.scan_file(contract, plugin)} + assert got["/scripts/validate.py"] is None + assert got["$PLUGIN_ROOT/scripts/validate.py"] is None + assert got["$GCP_BASE/scripts/validate.py"] is None + assert got["./scripts/validate.py"] is None + # an unqualified reference from a library-origin file may still mean any vendoring skill + assert got["scripts/validate.py"] == (plugin / "skills" / "beta" / "scripts" / "validate.py").resolve() + # explicit bases that DO name an existing file keep resolving, including `../` traversal + assert got["/../gcp-to-aws/SKILL.md"] == (plugin / "skills" / "gcp-to-aws" / "SKILL.md").resolve() + assert got["/scripts/tool.py"] == (plugin / "skills" / "alpha" / "scripts" / "tool.py").resolve() + # a gcp dependent naming `$GCP_BASE/…` resolves there and only there + (plugin / "skills" / "agent-advisor" / "references").mkdir() + (plugin / "skills" / "agent-advisor" / "references" / "ai.md").write_text("x") + got2 = _refs(plugin, "agent-advisor", "`$GCP_BASE/references/design-refs/ai.md` `$GCP_BASE/references/ai.md`") + assert got2 == {"$GCP_BASE/references/design-refs/ai.md": True, "$GCP_BASE/references/ai.md": False} + + def test_baseline_ignore_vs_entries_and_stale(tmp_path: Path): plugin = _plugin(tmp_path) f = plugin / "skills" / "alpha" / "SKILL.md" From f7a3ee35b4c20aad9b61d8c6e1ff7382ec777c66 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 18:07:42 -0700 Subject: [PATCH 09/53] fix(aws-startup-advisor): admit the unfired licensing stub in the preferences example The fast-path fixture writes licensing as the N/A stub, and the artifact check was rejecting those keys. Co-authored-by: Cursor --- .../azure-to-aws/references/shared/schema-preferences.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md index a05fe2208..c4375737c 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-preferences.md @@ -109,6 +109,9 @@ licensing exposure and found none" is a different statement from silence. "default": "identity_center_reinvite" }, "licensing": { + "disposition": "N/A", // unfired stub only — clarify-licensing.md § Step 3; absent when the per-model rows below are present + "value": null, + "default": null, "windows_model": { "disposition": "ESSENTIAL", "value": "license_included", From 35a55d4c3bd702dec25d5f85ba19fb8d65eb27fc Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 18:18:26 -0700 Subject: [PATCH 10/53] fix(aws-startup-advisor): close three artifact and vendored-parity gaps An empty-array example no longer leaves the item typed as any, quoted comment enums are enforced, and a plugin without skills/ no longer crashes the vendored scan. Co-authored-by: Cursor --- .../scripts/validate-artifacts.py | 51 ++++++- .../tests/test_validate_artifacts.py | 127 ++++++++++++++++++ tools/sync-vendored.py | 9 +- tools/test_sync_vendored.py | 25 ++++ 4 files changed, 207 insertions(+), 5 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 0247fda70..d947b9083 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -186,13 +186,46 @@ class Node: def is_any(self) -> bool: return "any" in self.kinds or not self.kinds + def _is_unspecified_any(self) -> bool: + """An empty-array seed (`"services": []`) or a bare placeholder. It names no type, + no keys, and no enum — unlike a null example, which is `{null, any}` on purpose.""" + return ( + self.kinds == {"any"} + and not self.keys + and self.item is None + and self.any_key is None + and self.enum is None + and not self.examples + and not self.wildcard + and not self.required + ) + def merge(self, other: "Node") -> "Node": """Union two templates for the same path (several documented variants). Children adopted from `other` are copied, never shared: a later refinement of a named child (`design_constraints.cpu_architecture`) must not mutate the generic template (`design_constraints.`) it was seeded from. + + An unspecified `any` (the item of `"services": []`) is replaced when a concrete + shape arrives. Unioning would keep `any` in the kinds and skip type checks for + good. A later empty array does not put `any` back onto a shape already learned. """ + if self._is_unspecified_any() and not other._is_unspecified_any(): + concrete = copy.deepcopy(other) + self.kinds = concrete.kinds + self.wildcard = concrete.wildcard + self.required = concrete.required + self.keys = concrete.keys + self.item = concrete.item + self.any_key = concrete.any_key + self.enum = concrete.enum + self.enum_join = concrete.enum_join + self.examples = concrete.examples + self.source = concrete.source or self.source + return self + if other._is_unspecified_any() and not self._is_unspecified_any(): + return self self.kinds |= other.kinds self.wildcard = self.wildcard or other.wildcard self.required |= other.required @@ -229,7 +262,12 @@ def merge(self, other: "Node") -> "Node": _PLACEHOLDER_RE = re.compile(r"^<.*>$") _ENUM_TOKEN_RE = re.compile(r"^[A-Za-z0-9_.:+\-/]+$") -_COMMENT_ENUM_RE = re.compile(r"(?:^|[—:\-]\s*)([A-Za-z0-9_.:+\-/]+(?:\s*\|\s*[A-Za-z0-9_.:+\-/]+)+)\s*(?:$|[—(\-])") +# A member may be a bare token or a quoted one (`"fast_path" | "wizard"`). Quotes are +# stripped before the token check, so a quoted phrase with spaces stays free text. +_ENUM_ALT = r"(?:`[^`]+`|\"[^\"]+\"|'[^']+'|[A-Za-z0-9_.:+\-/]+)" +_COMMENT_ENUM_RE = re.compile( + rf"(?:^|[—:\-]\s*)({_ENUM_ALT}(?:\s*\|\s*{_ENUM_ALT})+)\s*(?:$|[—(\-])" +) # `terraform | live | billing (or a "+"-joined set)` — members may be combined with "+" _JOINED_SET_RE = re.compile(r"[\"'`]?\+[\"'`]?[\s-]*joined|joined\s+(?:by|with|on)\s+[\"'`]?\+", re.I) @@ -243,12 +281,19 @@ def _enum_from_string(s: str) -> Optional[Set[str]]: return None +def _enum_token(raw: str) -> Optional[str]: + t = raw.strip() + if len(t) >= 2 and t[0] == t[-1] and t[0] in "\"'`": + t = t[1:-1] + return t if _ENUM_TOKEN_RE.match(t) else None + + def _enum_from_comment(c: str) -> Optional[Set[str]]: m = _COMMENT_ENUM_RE.search(c) if not m: return None - toks = [t.strip() for t in m.group(1).split("|")] - if len(toks) >= 2 and all(_ENUM_TOKEN_RE.match(t) for t in toks): + toks = [_enum_token(t) for t in m.group(1).split("|")] + if len(toks) >= 2 and all(t is not None for t in toks): return set(toks) return None diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index fb08c9c4b..78bdcee96 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -65,6 +65,16 @@ def test_enum_from_pipe_string_and_from_comment(): assert n.keys["tier"].enum == {"network", "data", "compute"} +def test_quoted_comment_enum_members_are_recognized(): + """Review finding on #386: `"fast_path" | "wizard"` was invisible to the bare-token regex.""" + n = _node('{"clarify_mode": "wizard", // "fast_path" | "wizard" — which flow\n' + ' "size_coverage": "complete", // "complete" | "partial" | "unknown"\n' + ' "note": "x", // "not a token" | "also free"\n}') + assert n.keys["clarify_mode"].enum == {"fast_path", "wizard"} + assert n.keys["size_coverage"].enum == {"complete", "partial", "unknown"} + assert n.keys["note"].enum is None # quoted phrases with spaces stay free text + + def test_required_from_comment_and_placeholder_is_any_type(): n = _node('{"id": "", // REQUIRED\n "n": ""}') assert "id" in n.required @@ -122,6 +132,39 @@ def test_allowed_anywhere_vocabulary(): # --------------------------------------------------------------------------- template merging +def test_empty_array_item_is_replaced_when_a_concrete_shape_arrives(): + """Review finding on #386: `"services": []` seeded an `any` item, and merging the + later object kept `any`, so a scalar member passed.""" + seed = _node('{"services": [], "warnings": []}') + seed.merge(_node('{"services": [{"routing_provenance": "table", // REQUIRED\n "n": 1}]}')) + item = seed.keys["services"].item + assert not item.is_any() and "object" in item.kinds + assert "routing_provenance" in item.required + # a later empty array must not put `any` back + seed.merge(_node('{"services": []}')) + assert not seed.keys["services"].item.is_any() + ex = '{"services": [{"routing_provenance": "table", // REQUIRED\n "n": 1}]}' + # rebuild the same way the contract does: empty root, then the section + root = _node('{"services": []}') + root.merge(_node(ex)) + def check(value): + out = [] + va.validate_shape(root, value, "", "a", "c", out) + return {(f.code, f.path) for f in out} + assert check({"services": []}) == set() + assert check({"services": [{"routing_provenance": "table", "n": 1}]}) == set() + assert check({"services": [None]}) == set() # null stays compatible + assert ("TYPE_MISMATCH", "services[0]") in check({"services": [42]}) + assert ("TYPE_MISMATCH", "services[0]") in check({"services": [["nested"]]}) + assert ("MISSING_REQUIRED", "services[0].routing_provenance") in check({"services": [{}]}) + # a placeholder-only array is still open: the document never supplied a type + ph = _node('{"ids": [""]}') + assert ph.keys["ids"].item.is_any() + out = [] + va.validate_shape(ph, {"ids": [1]}, "", "a", "c", out) + assert out == [] + + def test_merge_keeps_declared_enum_when_a_concrete_example_shows_one_member(): """Review finding on #385: merging `"confidence": "full"` (a Complete Example) into `"confidence": "full|reduced"` cleared the enum, so any string passed.""" @@ -569,3 +612,87 @@ def test_gcp_preferences_chosen_by_enum_is_enforced_including_derived(tmp_path: pref["design_constraints"]["cpu_architecture"] = {"value": "graviton", "chosen_by": "bogus"} r = _run("--run-dir", str(_write_run(tmp_path, "cpu", "preferences.json", pref)), "--skill", "gcp-to-aws", "--json") assert _errors(r) == {("ENUM_VIOLATION", "design_constraints.cpu_architecture.chosen_by")}, r.stdout + + +def test_azure_design_arrays_reject_a_scalar_after_the_empty_seed(tmp_path: Path): + """Review finding on #386: services/clusters/deferred are seeded `[]` and refined by a + later object section. A scalar or nested list must fail; an empty array and a real + member stay valid. Null stays compatible.""" + src = PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-design/aws-design.json" + design = json.loads(src.read_text()) + control = _errors(_run("--run-dir", str(src.parent), "--skill", "azure-to-aws", "--no-baseline", "--json")) + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "aws-design.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + for key in ("services", "clusters", "deferred"): + scalar = json.loads(json.dumps(design)) + scalar[key] = [42] + assert ("TYPE_MISMATCH", f"{key}[0]") in run(key + "-scalar", scalar) - control + nested = json.loads(json.dumps(design)) + nested[key] = [["nope"]] + assert ("TYPE_MISMATCH", f"{key}[0]") in run(key + "-list", nested) - control + empty = json.loads(json.dumps(design)) + empty[key] = [] + assert ("TYPE_MISMATCH", f"{key}[0]") not in run(key + "-empty", empty) + blank = json.loads(json.dumps(design)) + blank[key] = [{}] + errs = run(key + "-obj", blank) + assert ("TYPE_MISMATCH", f"{key}[0]") not in errs + blank = json.loads(json.dumps(design)) + blank["services"] = [{}] + assert ("MISSING_REQUIRED", "services[0].routing_provenance") in run("svc-obj", blank) + nulled = json.loads(json.dumps(design)) + nulled["services"] = [None] + assert ("TYPE_MISMATCH", "services[0]") not in run("null-item", nulled) + + +def test_gcp_cluster_edges_reject_a_scalar_member(tmp_path: Path): + """The clusters example seeds `edges: []` on one cluster and a typed edge on the next. + Merging those must keep the edge object shape.""" + doc = PLUGIN_ROOT / "skills/gcp-to-aws/references/shared/schema-discover-iac.md" + clusters = _doc_example(doc, "gcp-resource-clusters.json") + r = _run("--run-dir", str(_write_run(tmp_path, "ok", "gcp-resource-clusters.json", clusters)), + "--skill", "gcp-to-aws", "--no-baseline", "--json") + assert r.returncode == 0, r.stdout + bad = json.loads(json.dumps(clusters)) + bad["clusters"][1]["edges"] = [42] + r = _run("--run-dir", str(_write_run(tmp_path, "scalar", "gcp-resource-clusters.json", bad)), + "--skill", "gcp-to-aws", "--no-baseline", "--json") + assert ("TYPE_MISMATCH", "clusters[1].edges[0]") in _errors(r), r.stdout + bad["clusters"][1]["edges"] = [] + r = _run("--run-dir", str(_write_run(tmp_path, "empty", "gcp-resource-clusters.json", bad)), + "--skill", "gcp-to-aws", "--no-baseline", "--json") + assert r.returncode == 0, r.stdout + + +def test_azure_quoted_preference_enums_reject_unknown_values(tmp_path: Path): + """`metadata.clarify_mode` and `data.db_cutover.size_coverage` are quoted comment enums.""" + src = PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-clarify-fast-path/preferences.json" + pref = json.loads(src.read_text()) + control = _errors(_run("--run-dir", str(src.parent), "--skill", "azure-to-aws", "--no-baseline", "--json")) + assert control == set(), control + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "preferences.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + for mode in ("fast_path", "wizard"): + ok = json.loads(json.dumps(pref)) + ok["metadata"]["clarify_mode"] = mode + assert run("mode-" + mode, ok) == set() + bad = json.loads(json.dumps(pref)) + bad["metadata"]["clarify_mode"] = "invented-mode" + assert run("mode-bad", bad) == {("ENUM_VIOLATION", "metadata.clarify_mode")} + for coverage in ("complete", "partial", "unknown"): + ok = json.loads(json.dumps(pref)) + ok["data"]["db_cutover"]["size_coverage"] = coverage + assert run("cov-" + coverage, ok) == set(), coverage + bad = json.loads(json.dumps(pref)) + bad["data"]["db_cutover"]["size_coverage"] = "invented-coverage" + assert run("cov-bad", bad) == {("ENUM_VIOLATION", "data.db_cutover.size_coverage")} + # the unquoted clarify_status control still rejects an unknown value + bad = json.loads(json.dumps(pref)) + bad["clarify_status"] = "invented-status" + assert ("ENUM_VIOLATION", "clarify_status") in run("status-bad", bad) diff --git a/tools/sync-vendored.py b/tools/sync-vendored.py index 1ec0058fe..763ec452f 100644 --- a/tools/sync-vendored.py +++ b/tools/sync-vendored.py @@ -195,9 +195,14 @@ def main(argv=None) -> int: for plugin_dir in plugins: files = vendored_files(plugin_dir) total_files += len(files) + skills_dir = plugin_dir / "skills" + # An MCP-only plugin has no skills/ and still passes the spec validator. + # iterdir() on the missing directory raises before the is_dir() filter can run. + if not skills_dir.is_dir(): + continue if not files and not any((skill / VENDORED_SUBDIR).is_dir() - for skill in (plugin_dir / "skills").iterdir() - if (plugin_dir / "skills").is_dir() and skill.is_dir()): + for skill in skills_dir.iterdir() + if skill.is_dir()): continue if args.check: all_errors.extend(check_plugin(plugin_dir)) diff --git a/tools/test_sync_vendored.py b/tools/test_sync_vendored.py index 4d996852b..a960221f1 100644 --- a/tools/test_sync_vendored.py +++ b/tools/test_sync_vendored.py @@ -124,6 +124,31 @@ def test_cli_check_exit_codes(tmp_path: Path): assert r.returncode == 0 and r.stdout.startswith("PASS") +def test_plugin_without_a_skills_directory_is_skipped(tmp_path: Path): + """Review finding on #386: `--check` called skills.iterdir() before the is_dir() filter, + so an MCP-only plugin (no skills/) raised FileNotFoundError.""" + plugins = _tree(tmp_path) + (plugins / "mcp-only").mkdir() + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + capture_output=True, text=True) + assert "Traceback" not in r.stderr, r.stderr + assert r.returncode == 1 and "FAIL" in r.stdout + # missing-copy still fails with the MCP-only plugin beside it + vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + (vend / "dsl" / "INTERPRETER.md").unlink() + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + capture_output=True, text=True) + assert r.returncode == 1 and "dsl/INTERPRETER.md" in r.stdout and "missing on disk" in r.stdout + # a clean tree plus the skills-less plugin passes + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + subprocess.run([sys.executable, str(SCRIPT), "--plugins-root", str(plugins)], check=True, capture_output=True) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + capture_output=True, text=True) + assert r.returncode == 0 and r.stdout.startswith("PASS"), r.stdout + r.stderr + + def test_real_repository_is_in_sync(): r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=REPO_ROOT) assert r.returncode == 0, r.stdout From e086f5108fa1355b686a09370bfa9f1a8c0c6f85 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 20:41:00 -0700 Subject: [PATCH 11/53] fix(aws-startup-advisor): reject refined empty-array items and honor relative prefixes An empty example array kept item type any after a later object section, and a short relative path still matched the unprefixed file under the skill root. Co-authored-by: Cursor --- .../scripts/validate-artifacts.py | 33 ++++++++++ .../tests/test_validate_artifacts.py | 63 +++++++++++++++++++ tools/check-skill-refs.py | 13 ++-- tools/test_check_skill_refs.py | 23 +++++++ 4 files changed, 127 insertions(+), 5 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 0247fda70..18ec3e898 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -186,13 +186,46 @@ class Node: def is_any(self) -> bool: return "any" in self.kinds or not self.kinds + def _is_unspecified_any(self) -> bool: + """An empty-array seed (`"services": []`) or a bare placeholder. It names no type, + no keys, and no enum — unlike a null example, which is `{null, any}` on purpose.""" + return ( + self.kinds == {"any"} + and not self.keys + and self.item is None + and self.any_key is None + and self.enum is None + and not self.examples + and not self.wildcard + and not self.required + ) + def merge(self, other: "Node") -> "Node": """Union two templates for the same path (several documented variants). Children adopted from `other` are copied, never shared: a later refinement of a named child (`design_constraints.cpu_architecture`) must not mutate the generic template (`design_constraints.`) it was seeded from. + + An unspecified `any` (the item of `"services": []`) is replaced when a concrete + shape arrives. Unioning would keep `any` in the kinds and skip type checks for + good. A later empty array does not put `any` back onto a shape already learned. """ + if self._is_unspecified_any() and not other._is_unspecified_any(): + concrete = copy.deepcopy(other) + self.kinds = concrete.kinds + self.wildcard = concrete.wildcard + self.required = concrete.required + self.keys = concrete.keys + self.item = concrete.item + self.any_key = concrete.any_key + self.enum = concrete.enum + self.enum_join = concrete.enum_join + self.examples = concrete.examples + self.source = concrete.source or self.source + return self + if other._is_unspecified_any() and not self._is_unspecified_any(): + return self self.kinds |= other.kinds self.wildcard = self.wildcard or other.wildcard self.required |= other.required diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index fb08c9c4b..b3f056ff1 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -71,6 +71,40 @@ def test_required_from_comment_and_placeholder_is_any_type(): assert n.keys["n"].is_any() # a placeholder says nothing about the JSON type +def test_empty_array_item_is_replaced_when_a_concrete_shape_arrives(): + """Review finding on #387: `"services": []` seeded an `any` item, and merging the + later object kept `any`, so a scalar or nested list passed.""" + seed = _node('{"services": [], "warnings": []}') + seed.merge(_node('{"services": [{"routing_provenance": "table", // REQUIRED\n "n": 1}]}')) + item = seed.keys["services"].item + assert not item.is_any() and "object" in item.kinds + assert "routing_provenance" in item.required + seed.merge(_node('{"services": []}')) + assert not seed.keys["services"].item.is_any() + ex = '{"services": [{"routing_provenance": "table", // REQUIRED\n "n": 1}]}' + root = _node('{"services": []}') + root.merge(_node(ex)) + + def check(value): + out = [] + va.validate_shape(root, value, "", "a", "c", out) + return {(f.code, f.path) for f in out} + + assert check({"services": []}) == set() + assert check({"services": [{"routing_provenance": "table", "n": 1}]}) == set() + assert check({"services": [None]}) == set() + assert ("TYPE_MISMATCH", "services[0]") in check({"services": ["not-an-object"]}) + assert ("TYPE_MISMATCH", "services[0]") in check({"services": [17]}) + assert ("TYPE_MISMATCH", "services[0]") in check({"services": [False]}) + assert ("TYPE_MISMATCH", "services[0]") in check({"services": [[]]}) + assert ("MISSING_REQUIRED", "services[0].routing_provenance") in check({"services": [{}]}) + ph = _node('{"ids": [""]}') + assert ph.keys["ids"].item.is_any() + out = [] + va.validate_shape(ph, {"ids": [1, "x", False]}, "", "a", "c", out) + assert out == [] + + def test_placeholder_key_makes_object_open_and_sets_any_key_template(): n = _node('{"": {"value": 1, "chosen_by": "user|default"}}') assert n.wildcard and n.any_key is not None @@ -569,3 +603,32 @@ def test_gcp_preferences_chosen_by_enum_is_enforced_including_derived(tmp_path: pref["design_constraints"]["cpu_architecture"] = {"value": "graviton", "chosen_by": "bogus"} r = _run("--run-dir", str(_write_run(tmp_path, "cpu", "preferences.json", pref)), "--skill", "gcp-to-aws", "--json") assert _errors(r) == {("ENUM_VIOLATION", "design_constraints.cpu_architecture.chosen_by")}, r.stdout + + +def test_azure_design_arrays_reject_a_scalar_after_the_empty_seed(tmp_path: Path): + """Review finding on #387: services, clusters, deferred, and pending_rubric are seeded + `[]` and refined by a later object section. A scalar, string, boolean, or nested list + must fail. An empty array and an object stay valid as types. An array whose example + never names an item type stays open.""" + src = PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-design/aws-design.json" + design = json.loads(src.read_text()) + control = _errors(_run("--run-dir", str(src.parent), "--skill", "azure-to-aws", "--no-baseline", "--json")) + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "aws-design.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + for key in ("services", "clusters", "deferred", "pending_rubric"): + for label, member in (("str", "not-an-object"), ("num", 17), ("bool", False), ("list", [])): + bad = json.loads(json.dumps(design)) + bad[key] = [member] + assert ("TYPE_MISMATCH", f"{key}[0]") in run(f"{key}-{label}", bad) - control + empty = json.loads(json.dumps(design)) + empty[key] = [] + assert ("TYPE_MISMATCH", f"{key}[0]") not in run(key + "-empty", empty) + blank = json.loads(json.dumps(design)) + blank[key] = [{}] + assert ("TYPE_MISMATCH", f"{key}[0]") not in run(key + "-obj", blank) + blank = json.loads(json.dumps(design)) + blank["services"] = [{}] + assert ("MISSING_REQUIRED", "services[0].routing_provenance") in run("svc-obj", blank) diff --git a/tools/check-skill-refs.py b/tools/check-skill-refs.py index 606440319..65fb4bc6a 100644 --- a/tools/check-skill-refs.py +++ b/tools/check-skill-refs.py @@ -26,8 +26,10 @@ Resolution order for a bare path `P` found in `skills//…/file.md`: 1. explicit prefix → plugin root / skill root / gcp-to-aws root / referencing file's dir. - An explicit prefix is authoritative: the agent runs `/scripts/x.py` as - written, so none of the fallbacks below apply to it. + An explicit prefix is authoritative: the agent runs `/scripts/x.py`, + `./scripts/x.py`, or `../../knowledge/estimate/estimate-defaults.json` as written. + A same-named file under the skill root does not make a wrong relative path exist, + and none of the fallbacks below apply. 2. `skills//P` (skill-relative — the documented convention) 3. `/P` (file-relative) 4. `skills//references/P` (agent-advisor short forms: phases/, shared/, design-refs/) @@ -131,9 +133,10 @@ def _candidates(prefix: str, path: str, source: Path, plugin: Path) -> List[Path elif "SCRIPTS" in p and skill is not None: cands.append(skill / "scripts" / path) elif p.startswith("./") or p.startswith("../"): - cands.append((here / p / path)) - if skill is not None: - cands.append(skill / path) + # The relative prefix is the whole location. Do not also try `skill / path`: + # that unprefixed tail is how `../../knowledge/estimate/estimate-defaults.json` + # looked found when the literal target did not exist. + cands.append(here / p / path) # `/../gcp-to-aws/SKILL.md` style: prefix carries `../` if "../" in p and skill is not None and "SKILL_BASE" in p: up = skill diff --git a/tools/test_check_skill_refs.py b/tools/test_check_skill_refs.py index 9cc83eda8..986f0e5fc 100644 --- a/tools/test_check_skill_refs.py +++ b/tools/test_check_skill_refs.py @@ -185,6 +185,29 @@ def test_explicit_base_prefixes_do_not_fall_back_to_other_skills(tmp_path: Path) assert got2 == {"$GCP_BASE/references/design-refs/ai.md": True, "$GCP_BASE/references/ai.md": False} +def test_relative_prefixes_do_not_fall_back_to_the_unprefixed_skill_path(tmp_path: Path): + """Review finding on #387: `./` and `../` still tried `skill / path` after the literal + target missed. Azure estimate-infra.md:370's link is + `../../../knowledge/estimate/estimate-defaults.json`; dropping one `../` leaves a + nonexistent literal, but the unprefixed tail exists under the skill root.""" + plugin = _plugin(tmp_path) + dest = plugin / "skills" / "alpha" / "knowledge" / "estimate" + dest.mkdir(parents=True) + (dest / "estimate-defaults.json").write_text("{}") + nested = plugin / "skills" / "alpha" / "references" / "phases" / "estimate" + nested.mkdir(parents=True) + f = nested / "estimate-infra.md" + f.write_text( + "see [`knowledge/estimate/estimate-defaults.json`](../../../knowledge/estimate/estimate-defaults.json).\n" + "short [`knowledge/estimate/estimate-defaults.json`](../../knowledge/estimate/estimate-defaults.json).\n" + "dot [`knowledge/estimate/estimate-defaults.json`](./knowledge/estimate/estimate-defaults.json).\n" + ) + got = {r.raw: r.resolved for r in csr.scan_file(f, plugin)} + assert got["../../../knowledge/estimate/estimate-defaults.json"] == (dest / "estimate-defaults.json").resolve() + assert got["../../knowledge/estimate/estimate-defaults.json"] is None + assert got["./knowledge/estimate/estimate-defaults.json"] is None + + def test_baseline_ignore_vs_entries_and_stale(tmp_path: Path): plugin = _plugin(tmp_path) f = plugin / "skills" / "alpha" / "SKILL.md" From de89ec195fc9fc86f8e4ab7b1c3e791895557fb6 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 20:47:48 -0700 Subject: [PATCH 12/53] fix(aws-startup-advisor): require the Azure discovery verdict and check estimate snapshots A standalone REQUIRED comment was dropped, so a discovery inventory without clarify_fast_path passed. Scenario estimate copies also skipped the active estimate shape, so an unknown pricing bucket failed on the live file and passed on the snapshot. Co-authored-by: Cursor --- .../scripts/artifact-contracts.json | 12 ++++- .../scripts/validate-artifacts.py | 10 ++++ .../tests/test_validate_artifacts.py | 52 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index a4eda814e..6e6747758 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -97,7 +97,17 @@ ] }, "scenarios/scenario-NNN.estimation-infra.json": { - "json_schema": "skills/shared/estimate/estimation-infra.schema.json" + "json_schema": "skills/shared/estimate/estimation-infra.schema.json", + "shape": "skills/gcp-to-aws/references/shared/schema-estimate-infra.md", + "root_heading": "estimation-infra.json schema", + "sections": { + "Cost tiers": "projected_costs.scenario_deltas", + "Observability Entry in": "projected_costs.breakdown.observability", + "@L373": "projected_costs.breakdown.security_baseline", + "@L390": "projected_costs.breakdown.security_baseline_compliance", + "architecture_comparison": "architecture_comparison" + }, + "_sections_why": "Workshop baseline capture copies the active estimation-infra.json into scenarios/scenario-NNN.estimation-infra.json. The snapshot is the same artifact, so it uses the same Markdown shape, root heading, and section refinements — not the JSON Schema alone." } }, "heroku-to-aws": { diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index d947b9083..0eb02453d 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -467,8 +467,18 @@ def parse_block(block: Block, doc: Path, findings: List[Finding]) -> Optional[Tu contract=_rel(doc))) return None key_comments: Dict[str, List[str]] = {} + raw_lines = block.text.splitlines() for ln, c in comments.items(): k = keys.get(ln) + if k is None: + # A comment on its own line inside an object belongs to the key that opened + # it. `metadata.clarify_fast_path`'s REQUIRED marker is written that way, and + # dropping it left the verdict optional. + prev = ln - 1 + while prev >= 0 and not raw_lines[prev].split("//", 1)[0].strip(): + prev -= 1 + if prev >= 0 and prev in keys and raw_lines[prev].split("//", 1)[0].rstrip().endswith("{"): + k = keys[prev] if k is not None: key_comments.setdefault(k, []).append(c) return value, key_comments diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 78bdcee96..f452d9b41 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -696,3 +696,55 @@ def run(name, data): bad = json.loads(json.dumps(pref)) bad["clarify_status"] = "invented-status" assert ("ENUM_VIOLATION", "clarify_status") in run("status-bad", bad) + + +def test_azure_discovery_verdict_is_required(tmp_path: Path): + """Review finding on #386: `metadata.clarify_fast_path` is REQUIRED on a comment inside + the object, which the parser dropped, so a discovery inventory without the verdict passed.""" + src = PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-discover/azure-resource-inventory.json" + inv = json.loads(src.read_text()) + assert inv["metadata"]["clarify_fast_path"]["eligible"] is False + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "azure-resource-inventory.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + assert ("MISSING_REQUIRED", "metadata.clarify_fast_path") not in run("kept", inv) + fresh = json.loads(json.dumps(inv)) + del fresh["metadata"]["clarify_fast_path"] + assert ("MISSING_REQUIRED", "metadata.clarify_fast_path") in run("omitted", fresh) + still = json.loads(json.dumps(inv)) + still["metadata"]["clarify_fast_path"] = {"eligible": False, "reasons_ineligible": ["has_vm"]} + assert ("MISSING_REQUIRED", "metadata.clarify_fast_path") not in run("ineligible", still) + # the inline resource requirement was already enforced and stays enforced + dropped = json.loads(json.dumps(inv)) + del dropped["resources"][0]["azure_type_provenance"] + assert ("MISSING_REQUIRED", "resources[0].azure_type_provenance") in run("provenance", dropped) + + +def test_gcp_estimate_snapshots_use_the_active_estimate_shape(tmp_path: Path): + """Review finding on #386: scenario snapshots used only the JSON Schema, so an unknown + pricing bucket failed on estimation-infra.json and passed on the snapshot copy.""" + est = json.loads((PLUGIN_ROOT / "fixtures/gcp-decision-gate/after-decide-complete/estimation-infra.json").read_text()) + doc = PLUGIN_ROOT / "skills/gcp-to-aws/references/shared/schema-estimate-infra.md" + sec_blocks = [b for b in va.extract_blocks(doc) if va._heading_matches(b.heading, "Security Baseline Entries in")] + est["projected_costs"]["scenario_deltas"] = _doc_example(doc, "Cost tiers")["scenario_deltas"] + est["projected_costs"]["breakdown"] = { + "compute": {"service": "Fargate", "monthly": 71}, + "observability": _doc_example(doc, "Observability Entry in"), + "security_baseline": json.loads(va.strip_jsonc(sec_blocks[0].text)[0]), + "security_baseline_compliance": json.loads(va.strip_jsonc(sec_blocks[1].text)[0]), + } + + def run(name, artifact, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, artifact, data)), + "--skill", "gcp-to-aws", "--no-baseline", "--json")) + + est["pricing_source"]["services_by_source"] = { + "cached": ["Fargate"], "fallback": ["NAT Gateway"], "estimated": [], + } + for artifact in ("estimation-infra.json", "scenarios/scenario-017.estimation-infra.json"): + assert run("ok-" + artifact, artifact, est) == set(), artifact + bad = json.loads(json.dumps(est)) + bad["pricing_source"]["services_by_source"]["live"] = ["Fargate"] + assert ("UNKNOWN_KEY", "pricing_source.services_by_source.live") in run("bad-" + artifact, artifact, bad) From a9e625914984e513b3c950bc3912ce4f450dc51b Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 2 Oct 2026 21:59:04 -0700 Subject: [PATCH 13/53] fix(aws-startup-advisor): accept each Azure preference route and typed drift values An AI-only preferences file was checked against the infra shape, numeric GCP drift failed as a string, and a plugin with no skills directory crashed the vendored-copy gate. --- .../scripts/artifact-contracts.json | 19 ++++ .../scripts/validate-artifacts.py | 25 +++++- .../phases/clarify/clarify-ai-only.md | 89 +++++++++++++++++++ .../references/phases/clarify/clarify-ai.md | 19 ++++ .../shared/schema-workshop-scenarios.md | 3 + .../references/shared/schema-discover-iac.md | 4 +- .../tests/test_validate_artifacts.py | 86 ++++++++++++++++++ tools/sync-vendored.py | 11 ++- tools/test_sync_vendored.py | 12 +++ 9 files changed, 262 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index a4eda814e..ad2101864 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -217,6 +217,25 @@ "shape": "skills/azure-to-aws/references/shared/schema-graviton.md", "heading": "design_constraints.cpu_architecture", "path": "design_constraints.cpu_architecture" + }, + { + "shape": "skills/azure-to-aws/references/phases/clarify/clarify-ai.md", + "heading": "Step 3: Rows returned", + "path": "" + }, + { + "shape": "skills/azure-to-aws/references/phases/clarify/clarify-ai.md", + "heading": "workloads[]", + "path": "workloads[]" + } + ], + "route_shapes": [ + { + "when_path": "metadata.migration_type", + "when_value": "ai-only", + "shape": "skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md", + "root_heading": "preferences.json (AI-only)", + "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required." } ] }, diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 18ec3e898..03ef685bb 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -857,6 +857,9 @@ class Contract: shape: Optional[ShapeContract] = None json_schema: Optional[Dict[str, Any]] = None json_schema_path: str = "" + # (json path, expected value, shape). The first match replaces the default shape, + # so requiredness follows the producer route that wrote the file. + routes: List[Tuple[str, Any, ShapeContract]] = field(default_factory=list) def load_manifest(path: Path) -> Dict[str, Any]: @@ -886,10 +889,23 @@ def build_contracts(manifest: Dict[str, Any], skill: str, findings: List[Finding f"_ANNOTATION_KEYWORDS (if it carries no constraint).", _rel(p))) if "shape" in spec: c.shape = build_shape_contract(spec, findings) + for route in spec.get("route_shapes", []): + built = build_shape_contract(route, findings) + if built is not None: + c.routes.append((route["when_path"], route.get("when_value"), built)) contracts.append(c) return contracts +def _value_at(value: Any, dotted: str) -> Any: + cur = value + for part in dotted.split("."): + if not isinstance(cur, dict) or part not in cur: + return None + cur = cur[part] + return cur + + def artifact_name(p: Path) -> str: """Normalise 'scenarios/scenario-001.preferences.json' → 'scenarios/scenario-NNN.preferences.json'.""" name = p.name @@ -931,8 +947,13 @@ def validate_run_dir(run_dir: Path, skill: str, manifest: Dict[str, Any], findin for c in matched: if c.json_schema is not None: validate_json_schema(c.json_schema, value, "", rel, c.json_schema_path, findings) - if c.shape is not None: - validate_shape(c.shape.root, value, "", rel, c.shape.label, findings) + shape = c.shape + for path, expect, route in c.routes: + if _value_at(value, path) == expect: + shape = route + break + if shape is not None: + validate_shape(shape.root, value, "", rel, shape.label, findings) return checked diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index dda2a91ae..263f0b618 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -149,6 +149,95 @@ Each constraint carries the full clarify field shape (`value`, `chosen_by` ∈ `user|extracted|default|derived`, `prompt`, `design_consequence`, `source`/`question_id`). No nulls. Delete `preferences-draft.json` after writing. +## preferences.json (AI-only) + +```jsonc +{ + "metadata": { // REQUIRED + "migration_type": "ai-only", // REQUIRED — downstream skips infra phases + "discovery_artifacts": ["ai-workload-profile.json"], + "clarify_mode": "full", // fast_path | full + "questions_asked": [], + "questions_defaulted": [], + "questions_extracted": [] + }, + "design_constraints": { // REQUIRED — region and compliance only; no infra constraints + "target_region": { // REQUIRED + "value": "us-east-1", + "chosen_by": "user|extracted|default|derived", + "prompt": "", + "design_consequence": "", + "question_id": "Q4" + }, + "compliance": { // REQUIRED + "value": [], + "chosen_by": "user|extracted|default|derived", + "prompt": "", + "design_consequence": "", + "question_id": "Q1.5" + } + }, + "startup_constraints": { // REQUIRED + "startup_program_status": { // REQUIRED + "value": "", + "chosen_by": "user|extracted|default|derived", + "prompt": "", + "design_consequence": "", + "question_id": "Q11" + } + }, + "ai_constraints": { // REQUIRED + "ai_framework": { // REQUIRED + "value": "direct", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" + }, + "ai_priority": { // REQUIRED + "value": "balanced", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q17" + }, + "ai_monthly_spend": { // REQUIRED + "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18" + }, + "cross_cloud": { // REQUIRED + "value": "latency-acceptable", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q4" + }, + "ai_model_baseline": { // REQUIRED + "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19" + }, + "ai_vision": { // REQUIRED + "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20" + }, + "ai_token_volume": { // REQUIRED + "value": "low", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q21" + }, + "ai_latency": { // REQUIRED + "value": "standard", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q22" + }, + "ai_complexity": { // REQUIRED + "value": "moderate", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q23" + }, + "ai_critical_feature": { // REQUIRED + "value": "", "chosen_by": "user", "prompt": "", "design_consequence": "", "question_id": "Q24" + }, + "ai_capabilities_required": { // REQUIRED + "value": [], "chosen_by": "derived", "prompt": "", "design_consequence": "", "question_id": "Q25" + } + }, + "workloads": [ // REQUIRED — [] when none; design-ai reads this array, not the profile + { + "workload_id": "wl-1", // REQUIRED + "model_id": "gpt-4o", // REQUIRED + "sdk_method": "chat.completions", // REQUIRED + "capability": "chat", // REQUIRED + "capability_confidence": "high", // REQUIRED + "structured_output": false, // REQUIRED + "call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED + "target_bedrock_model": "anthropic.claude", // REQUIRED + "priority": "balanced", // REQUIRED + "latency_tier": "standard" // REQUIRED + } + ] +} +``` + ## Step 4: Output gate - `preferences.json` exists and `metadata.migration_type == "ai-only"`. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md index 090c08df6..a4bfe41bf 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md @@ -127,6 +127,25 @@ Q16–Q22, and write `"workloads": []` when none. A value taken from its default stays **PROPOSED** — never promoted to DETECTED (DETECTED means read from the estate; Design's rationale and the report distinguish "you chose" from "we assumed"). +### `workloads[]` + +```jsonc +[ + { + "workload_id": "wl-1", // REQUIRED + "model_id": "gpt-4o", // REQUIRED + "sdk_method": "chat.completions", // REQUIRED + "capability": "chat", // REQUIRED + "capability_confidence": "high", // REQUIRED + "structured_output": false, // REQUIRED + "call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED + "target_bedrock_model": "anthropic.claude", // REQUIRED + "priority": "balanced", // REQUIRED + "latency_tier": "standard" // REQUIRED + } +] +``` + ## Who consumes these | Field | Consumer | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-workshop-scenarios.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-workshop-scenarios.md index d6e0acbdb..06a998adb 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-workshop-scenarios.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-workshop-scenarios.md @@ -11,6 +11,9 @@ with this file. { "phase": "workshop", "baseline_scenario_id": "baseline", + "active_scenario_id": "baseline", // workshop-refresh.md sets this to the snapshot Apply/resume is on; equals baseline at capture + "max_scenarios": 5, // the five-scenario cap from the shared workshop invariants + "inventory_fingerprint": "", // frozen at baseline capture; refresh aborts when the inventory bytes change "scenarios": [ { "scenario_id": "baseline", diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md index 7a42f2460..7a11ce962 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-iac.md @@ -203,8 +203,8 @@ Live gcloud discovery produces the same inventory/cluster schemas with these add { "address": "google_sql_database_instance.db", "field": "settings.tier", - "terraform_value": "db-f1-micro", - "live_value": "db-custom-2-8192" + "terraform_value": "", // the overwritten value as JSON: a string tier, a numeric disk size, or a boolean flag + "live_value": "" } ] } diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index b3f056ff1..fb1a0bf17 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -632,3 +632,89 @@ def run(name, data): blank = json.loads(json.dumps(design)) blank["services"] = [{}] assert ("MISSING_REQUIRED", "services[0].routing_provenance") in run("svc-obj", blank) + + +def test_azure_preferences_follow_the_producer_route(tmp_path: Path): + """Review finding on #387: the infra shape was the only preferences contract, so an + AI-only file failed clarify_status and the mixed flow's AI rows were unknown keys.""" + infra = json.loads((PLUGIN_ROOT / "fixtures/azure-iac-terraform/after-clarify/preferences.json").read_text()) + rows = _doc_example(PLUGIN_ROOT / "skills/azure-to-aws/references/phases/clarify/clarify-ai.md", + "Step 3: Rows returned") + mixed = json.loads(json.dumps(infra)) + mixed.update(rows) + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "preferences.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + assert ("MISSING_REQUIRED", "clarify_status") not in run("infra", infra) + assert run("mixed", mixed) - run("infra", infra) == set() + bad = json.loads(json.dumps(mixed)) + bad["not_a_preference"] = 1 + assert ("UNKNOWN_KEY", "not_a_preference") in run("mixed-bad", bad) + bad = json.loads(json.dumps(mixed)) + bad["workloads"] = [42] + assert ("TYPE_MISMATCH", "workloads[0]") in run("mixed-scalar", bad) + + ai = _doc_example(PLUGIN_ROOT / "skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md", + "preferences.json (AI-only)") + assert run("ai", ai) == set(), run("ai", ai) + omitted = json.loads(json.dumps(ai)) + del omitted["workloads"] + assert ("MISSING_REQUIRED", "workloads") in run("ai-omit", omitted) + dropped = json.loads(json.dumps(ai)) + del dropped["ai_constraints"]["ai_framework"] + assert ("MISSING_REQUIRED", "ai_constraints.ai_framework") in run("ai-framework", dropped) + invented = json.loads(json.dumps(ai)) + invented["metadata"]["invented"] = 1 + assert ("UNKNOWN_KEY", "metadata.invented") in run("ai-invented", invented) + # the infra verdict stays required on the infra route + no_status = json.loads(json.dumps(infra)) + no_status.pop("clarify_status", None) + assert ("MISSING_REQUIRED", "clarify_status") in run("infra-status", no_status) + + +def test_gcp_drift_values_keep_their_json_types(tmp_path: Path): + """Review finding on #387: the live-discovery example typed drift values as strings, + so a numeric or boolean config change failed.""" + inv = json.loads((PLUGIN_ROOT / "fixtures/gcp-workshop/seed/gcp-resource-inventory.json").read_text()) + inv["live_metadata"] = { + "found": True, + "drift": {"config_conflicts": [{ + "address": "google_compute_disk.d", + "field": "size_gb", + "terraform_value": 10, + "live_value": 20, + }]}, + } + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "gcp-resource-inventory.json", data)), + "--skill", "gcp-to-aws", "--json")) + + assert ("TYPE_MISMATCH", "live_metadata.drift.config_conflicts[0].terraform_value") not in run("num", inv) + inv["live_metadata"]["drift"]["config_conflicts"][0]["terraform_value"] = False + inv["live_metadata"]["drift"]["config_conflicts"][0]["live_value"] = True + assert ("TYPE_MISMATCH", "live_metadata.drift.config_conflicts[0].live_value") not in run("bool", inv) + inv["live_metadata"]["drift"]["config_conflicts"][0]["terraform_value"] = "db-f1-micro" + assert ("TYPE_MISMATCH", "live_metadata.drift.config_conflicts[0].terraform_value") not in run("str", inv) + inv["live_metadata"]["drift"]["config_conflicts"][0]["address"] = 10 + assert ("TYPE_MISMATCH", "live_metadata.drift.config_conflicts[0].address") in run("address", inv) + + +def test_azure_workshop_index_records_capture_state(tmp_path: Path): + """Review finding on #387: the index example omitted the fields workshop-refresh writes.""" + doc = PLUGIN_ROOT / "skills/azure-to-aws/references/shared/schema-workshop-scenarios.md" + index = _doc_example(doc, "scenarios/index.json") + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "scenarios/index.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + assert run("baseline", index) == set(), index + active = json.loads(json.dumps(index)) + active["active_scenario_id"] = "scenario-002" + assert run("active", active) == set() + bad = json.loads(json.dumps(index)) + bad["invented"] = 1 + assert ("UNKNOWN_KEY", "invented") in run("bad", bad) diff --git a/tools/sync-vendored.py b/tools/sync-vendored.py index 1ec0058fe..7492d3f42 100644 --- a/tools/sync-vendored.py +++ b/tools/sync-vendored.py @@ -193,11 +193,18 @@ def main(argv=None) -> int: all_errors: List[str] = [] copied = 0 for plugin_dir in plugins: + skills_dir = plugin_dir / "skills" + # An MCP-only plugin has no skills directory. iterdir() on the missing + # path raises FileNotFoundError, and mise validate runs this task over + # every plugin. Skip before iterating. README checks still run for a + # plugin that has skills and a vendored directory. + if not skills_dir.is_dir(): + continue files = vendored_files(plugin_dir) total_files += len(files) if not files and not any((skill / VENDORED_SUBDIR).is_dir() - for skill in (plugin_dir / "skills").iterdir() - if (plugin_dir / "skills").is_dir() and skill.is_dir()): + for skill in skills_dir.iterdir() + if skill.is_dir()): continue if args.check: all_errors.extend(check_plugin(plugin_dir)) diff --git a/tools/test_sync_vendored.py b/tools/test_sync_vendored.py index 4d996852b..33c5cd8cb 100644 --- a/tools/test_sync_vendored.py +++ b/tools/test_sync_vendored.py @@ -124,6 +124,18 @@ def test_cli_check_exit_codes(tmp_path: Path): assert r.returncode == 0 and r.stdout.startswith("PASS") +def test_plugin_without_a_skills_directory_is_skipped(tmp_path: Path): + """Review finding on #387: iterating a missing skills/ raised FileNotFoundError.""" + plugins = tmp_path / "plugins" + (plugins / "mcp-only").mkdir(parents=True) + (plugins / "mcp-only" / "manifest.json").write_text("{}\n") + for flag in (["--check"], []): + r = subprocess.run([sys.executable, str(SCRIPT), *flag, "--plugins-root", str(plugins)], + capture_output=True, text=True) + assert r.returncode == 0, (flag, r.stdout, r.stderr) + assert "FileNotFoundError" not in r.stderr + + def test_real_repository_is_in_sync(): r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=REPO_ROOT) assert r.returncode == 0, r.stdout From 5c6eeb197683c8319863dece19508fa14a1cf466 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 13:06:37 -0700 Subject: [PATCH 14/53] fix: ignore Bedrock run state references --- tools/check-skill-refs.py | 2 +- tools/test_check_skill_refs.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/check-skill-refs.py b/tools/check-skill-refs.py index 65fb4bc6a..ab9359d49 100644 --- a/tools/check-skill-refs.py +++ b/tools/check-skill-refs.py @@ -79,7 +79,7 @@ _AGENTS_OWNER = "llm-to-bedrock" # anything carrying a run-artifact prefix or templating is not a repo path _SKIP_PREFIX_RE = re.compile( - r"(\$\{?(MIGRATION_DIR|RUN_DIR|REPO|PHASE_DIR|PLAN_DIR|OUT|TARGET|APP|WORKSPACE|TMP)\}?/" + r"(\$\{?(MIGRATION_DIR|RUN_DIR|BEDROCK_RUN_DIR|REPO|PHASE_DIR|PLAN_DIR|OUT|TARGET|APP|WORKSPACE|TMP)\}?/" r"|<(PLAN_DIR|MIGRATION_DIR|RUN_DIR|REPO|run_dir|migration_dir|path|repo)>/" r"|\.migration/|\.saws-migrate/|terraform/|ai-migration/|poc/|scaffold/|\.github/)" ) diff --git a/tools/test_check_skill_refs.py b/tools/test_check_skill_refs.py index 986f0e5fc..9c1b1de0e 100644 --- a/tools/test_check_skill_refs.py +++ b/tools/test_check_skill_refs.py @@ -82,7 +82,8 @@ def test_missing_reference_is_reported(tmp_path: Path): def test_run_artifacts_templates_urls_and_relative_outputs_are_not_references(tmp_path: Path): plugin = _plugin(tmp_path) got = _refs(plugin, "alpha", - "Write `$MIGRATION_DIR/scripts/02-migrate-data.sh`; read `.migration//references/x.md`; " + "Write `$MIGRATION_DIR/scripts/02-migrate-data.sh`; read `$BEDROCK_RUN_DIR/.phase-status.json`; " + "read `.migration//references/x.md`; " "see https://example.com/references/phases/design.md ; open `references/offers/.md`; " "then `./deploy.sh` and `../plan.md` and `scenarios/scenario-NNN.json`.") assert got == {} From b5f91914f09123dd23ee4134d6a8211f4fe4c7e5 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 19:21:07 -0700 Subject: [PATCH 15/53] fix(aws-startup-advisor): keep vendored-parity tooling plugin-local Move tools/sync-vendored.py and its test into plugins/aws-startup-advisor/tools/, since the vendored/shared pattern it gates is unique to this plugin (no other plugin uses references/vendored/). - Script now resolves PLUGIN_ROOT from its own location instead of scanning every plugins/ subdirectory; dropped the now-unneeded --plugin/--plugins-root multi-plugin CLI surface in favor of a single --plugin-root override for tests. - Updated the test suite to match (plugin-root fixtures instead of a multi-plugin tree; the MCP-only-plugin skip test now asserts a clean exit for a plugin root with no skills/ at all). - Updated mise.toml's lint:vendored-parity task and the four vendored/ README.md usage snippets (agent-advisor, heroku-to-aws, gcp-to-aws, azure-to-aws) to the new path. --- mise.toml | 2 +- .../references/vendored/README.md | 4 +- .../references/vendored/README.md | 4 +- .../gcp-to-aws/references/vendored/README.md | 4 +- .../references/vendored/README.md | 4 +- .../tools}/sync-vendored.py | 58 +++++----- .../tools}/test_sync_vendored.py | 101 ++++++++---------- 7 files changed, 79 insertions(+), 98 deletions(-) rename {tools => plugins/aws-startup-advisor/tools}/sync-vendored.py (81%) rename {tools => plugins/aws-startup-advisor/tools}/test_sync_vendored.py (56%) diff --git a/mise.toml b/mise.toml index 99c613ef1..c5b6d26aa 100644 --- a/mise.toml +++ b/mise.toml @@ -33,7 +33,7 @@ run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtu [tasks."lint:vendored-parity"] description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" -run = "python3 tools/sync-vendored.py --check" +run = "python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check" [tasks.validate] description = "Run all validation" diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md index 16a0e196d..c84f897ef 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md @@ -6,9 +6,9 @@ so the skill folder is **self-contained** — it runs standalone (lifted out, zi or used on its own) without reaching outside its own directory. **Do not hand-edit anything in this directory.** Edit the canonical source instead, -then run `python3 tools/sync-vendored.py` from the repository root to bring every +then run `python3 plugins/aws-startup-advisor/tools/sync-vendored.py` from the repository root to bring every skill's `references/vendored/` copy back in sync. Every copy must stay **byte-identical** -to the canonical source; CI runs `python3 tools/sync-vendored.py --check` +to the canonical source; CI runs `python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check` (`mise run lint:vendored-parity`) and fails on any drift, any copy with no canonical source, any row in the table below with no file on disk, and any file on disk with no row. It cannot see a new `skills/shared/` file no skill vendors yet — adding one here means the copy **and** a row. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index b5b09aa75..a5e154143 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -10,8 +10,8 @@ then bring every vendored copy back in sync: ```sh # from the repository root -python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` CI fails when a vendored copy differs from its canonical source, has no canonical source, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 38c4356e0..01040332d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -10,8 +10,8 @@ then bring every vendored copy back in sync: ```sh # from the repository root -python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` CI fails when a vendored copy differs from its canonical source, has no canonical source, diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md index ab9ef3a05..cb53a9dea 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md @@ -10,8 +10,8 @@ then bring every vendored copy back in sync: ```sh # from the repository root -python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` CI fails when a vendored copy differs from its canonical source, has no canonical source, diff --git a/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py similarity index 81% rename from tools/sync-vendored.py rename to plugins/aws-startup-advisor/tools/sync-vendored.py index 763ec452f..26d038ce6 100644 --- a/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -25,11 +25,15 @@ The vendored tree is a subset of shared/ by design — a skill vendors only what it loads — so adding a shared file a skill needs means adding the copy AND the README row. +This gate is owned by, and only ever scans, the aws-startup-advisor plugin — it is the +only plugin in this repository using the vendored/shared pattern. Everything this script +reads lives inside this plugin directory (`plugins/aws-startup-advisor/`); it does not +depend on, or write to, anything at the repository root or any other plugin. + Usage ----- - python3 tools/sync-vendored.py # copy shared/ over every vendored copy - python3 tools/sync-vendored.py --check # exit 1 if any copy differs / has no source - python3 tools/sync-vendored.py --plugin aws-startup-advisor + python3 plugins/aws-startup-advisor/tools/sync-vendored.py # sync + python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # exit 1 on drift Exit 0 when in sync (or after syncing), 1 on --check failure. Stdlib only. """ @@ -43,17 +47,17 @@ from pathlib import Path from typing import Dict, List, Tuple -REPO_ROOT = Path(__file__).resolve().parent.parent -PLUGINS_ROOT = REPO_ROOT / "plugins" +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS_DIR = PLUGIN_ROOT / "skills" VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} def _rel(p: Path) -> str: - """Repo-relative for display; absolute when the tree lives elsewhere (tests).""" + """Plugin-relative for display; absolute when the tree lives elsewhere (tests).""" try: - return str(p.resolve().relative_to(REPO_ROOT)) + return str(p.resolve().relative_to(PLUGIN_ROOT)) except ValueError: return str(p) @@ -181,37 +185,27 @@ def sync_plugin(plugin_dir: Path) -> Tuple[int, List[str]]: def main(argv=None) -> int: - ap = argparse.ArgumentParser(description="Keep skills' references/vendored/ copies identical to skills/shared/") - ap.add_argument("--plugin", help="Only this plugin (directory name under plugins/)") + ap = argparse.ArgumentParser( + description="Keep aws-startup-advisor skills' references/vendored/ copies identical to skills/shared/") ap.add_argument("--check", action="store_true", help="Report drift and exit 1; change nothing") - ap.add_argument("--plugins-root", type=Path, default=PLUGINS_ROOT, help=argparse.SUPPRESS) + ap.add_argument("--plugin-root", type=Path, default=PLUGIN_ROOT, help=argparse.SUPPRESS) args = ap.parse_args(argv) - plugins = [args.plugins_root / args.plugin] if args.plugin else sorted( - p for p in args.plugins_root.iterdir() if p.is_dir()) + plugin_dir = args.plugin_root total_files = 0 all_errors: List[str] = [] copied = 0 - for plugin_dir in plugins: - files = vendored_files(plugin_dir) - total_files += len(files) - skills_dir = plugin_dir / "skills" - # An MCP-only plugin has no skills/ and still passes the spec validator. - # iterdir() on the missing directory raises before the is_dir() filter can run. - if not skills_dir.is_dir(): - continue - if not files and not any((skill / VENDORED_SUBDIR).is_dir() - for skill in skills_dir.iterdir() - if skill.is_dir()): - continue - if args.check: - all_errors.extend(check_plugin(plugin_dir)) - all_errors.extend(check_manifest(plugin_dir)) - else: - n, errs = sync_plugin(plugin_dir) - copied += n - all_errors.extend(errs) - all_errors.extend(check_manifest(plugin_dir)) + + files = vendored_files(plugin_dir) + total_files += len(files) + if args.check: + all_errors.extend(check_plugin(plugin_dir)) + all_errors.extend(check_manifest(plugin_dir)) + else: + n, errs = sync_plugin(plugin_dir) + copied += n + all_errors.extend(errs) + all_errors.extend(check_manifest(plugin_dir)) if all_errors: print(f"FAIL — {len(all_errors)} vendored file(s) out of contract:") diff --git a/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py similarity index 56% rename from tools/test_sync_vendored.py rename to plugins/aws-startup-advisor/tools/test_sync_vendored.py index a960221f1..f7188b952 100644 --- a/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -1,7 +1,7 @@ -"""Tests for tools/sync-vendored.py — the vendored-copy parity gate. +"""Tests for sync-vendored.py — the vendored-copy parity gate. -Builds a throwaway plugins tree so the tests do not depend on (or mutate) the real -skills, then checks the real repository is clean as the last assertion. +Builds a throwaway plugin tree so the tests do not depend on (or mutate) the real +skills, then checks the real plugin is clean as the last assertion. """ from __future__ import annotations @@ -10,8 +10,8 @@ import sys from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parent.parent -SCRIPT = REPO_ROOT / "tools" / "sync-vendored.py" +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = PLUGIN_ROOT / "tools" / "sync-vendored.py" def _load(): @@ -26,12 +26,12 @@ def _load(): def _tree(tmp_path: Path) -> Path: - plugins = tmp_path / "plugins" - shared = plugins / "p" / "skills" / "shared" + plugin = tmp_path / "p" + shared = plugin / "skills" / "shared" (shared / "dsl").mkdir(parents=True) (shared / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") (shared / "pricing.json").write_text('{"a": 1}\n') - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "dsl").mkdir(parents=True) (vend / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") # in sync (vend / "pricing.json").write_text('{"a": 0}\n') # drifted @@ -44,13 +44,13 @@ def _tree(tmp_path: Path) -> Path: "| `pricing.json` | `skills/shared/pricing.json` |\n" "| `orphan.md` | `skills/shared/orphan.md` |\n") # a skill that vendors nothing, and the shared dir itself, are both ignored - (plugins / "p" / "skills" / "beta").mkdir() - return plugins + (plugin / "skills" / "beta").mkdir() + return plugin def test_check_reports_drift_and_orphans_but_not_readme(tmp_path: Path): - plugins = _tree(tmp_path) - errors = sv.check_plugin(plugins / "p") + plugin = _tree(tmp_path) + errors = sv.check_plugin(plugin) joined = "\n".join(errors) assert "pricing.json: differs from" in joined assert "orphan.md: no canonical source" in joined @@ -60,38 +60,38 @@ def test_check_reports_drift_and_orphans_but_not_readme(tmp_path: Path): def test_sync_copies_canonical_over_drifted_and_leaves_orphans_as_errors(tmp_path: Path): - plugins = _tree(tmp_path) - copied, errors = sv.sync_plugin(plugins / "p") + plugin = _tree(tmp_path) + copied, errors = sv.sync_plugin(plugin) assert copied == 1 - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + vend = plugin / "skills" / "alpha" / "references" / "vendored" assert (vend / "pricing.json").read_text() == '{"a": 1}\n' assert len(errors) == 1 and "orphan.md" in errors[0] # after sync, only the orphan remains - assert [e for e in sv.check_plugin(plugins / "p") if "orphan" not in e] == [] + assert [e for e in sv.check_plugin(plugin) if "orphan" not in e] == [] def test_deleted_copy_is_caught_via_readme_table(tmp_path: Path): """Review probe on #386: deleting a vendored copy left check_plugin at 0 errors because it only walked files that still existed. The README table is the record of intent.""" - plugins = _tree(tmp_path) - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "dsl" / "INTERPRETER.md").unlink() - errors = sv.check_manifest(plugins / "p") + errors = sv.check_manifest(plugin) assert any("dsl/INTERPRETER.md" in e and "missing on disk" in e for e in errors), errors # sync restores it from canonical - copied, _ = sv.sync_plugin(plugins / "p") + copied, _ = sv.sync_plugin(plugin) assert (vend / "dsl" / "INTERPRETER.md").read_text() == "canonical v2\n" assert copied >= 1 - assert not any("missing on disk" in e for e in sv.check_manifest(plugins / "p")) + assert not any("missing on disk" in e for e in sv.check_manifest(plugin)) def test_unlisted_copy_and_wrong_canonical_path_are_caught(tmp_path: Path): - plugins = _tree(tmp_path) - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "extra.md").write_text("x\n") readme = vend / "README.md" readme.write_text(readme.read_text().replace("`skills/shared/pricing.json`", "`skills/other/pricing.json`")) - errors = sv.check_manifest(plugins / "p") + errors = sv.check_manifest(plugin) assert any("extra.md" in e and "not listed" in e for e in errors), errors assert any("pricing.json" in e and "expected `skills/shared/pricing.json`" in e for e in errors), errors @@ -99,57 +99,44 @@ def test_unlisted_copy_and_wrong_canonical_path_are_caught(tmp_path: Path): def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): """Documented limit, pinned so the README wording stays honest: a brand-new skills/shared/ file that no skill vendors is NOT drift.""" - plugins = _tree(tmp_path) - (plugins / "p" / "skills" / "shared" / "brand-new.md").write_text("new\n") - (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() - readme = plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "README.md" + plugin = _tree(tmp_path) + (plugin / "skills" / "shared" / "brand-new.md").write_text("new\n") + (plugin / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() + readme = plugin / "skills" / "alpha" / "references" / "vendored" / "README.md" readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") - (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "pricing.json").write_text('{"a": 1}\n') - assert sv.check_plugin(plugins / "p") == [] - assert sv.check_manifest(plugins / "p") == [] + (plugin / "skills" / "alpha" / "references" / "vendored" / "pricing.json").write_text('{"a": 1}\n') + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] def test_cli_check_exit_codes(tmp_path: Path): - plugins = _tree(tmp_path) - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + plugin = _tree(tmp_path) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], capture_output=True, text=True) assert r.returncode == 1 and "FAIL" in r.stdout - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "orphan.md").unlink() readme = vend / "README.md" readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") - subprocess.run([sys.executable, str(SCRIPT), "--plugins-root", str(plugins)], check=True, capture_output=True) - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + subprocess.run([sys.executable, str(SCRIPT), "--plugin-root", str(plugin)], check=True, capture_output=True) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], capture_output=True, text=True) assert r.returncode == 0 and r.stdout.startswith("PASS") def test_plugin_without_a_skills_directory_is_skipped(tmp_path: Path): """Review finding on #386: `--check` called skills.iterdir() before the is_dir() filter, - so an MCP-only plugin (no skills/) raised FileNotFoundError.""" - plugins = _tree(tmp_path) - (plugins / "mcp-only").mkdir() - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + so a plugin root with no skills/ raised FileNotFoundError. Pinned behaviorally: an + empty plugin root (no skills/ at all) must pass cleanly, not crash.""" + plugin = tmp_path / "mcp-only" + plugin.mkdir() + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], capture_output=True, text=True) assert "Traceback" not in r.stderr, r.stderr - assert r.returncode == 1 and "FAIL" in r.stdout - # missing-copy still fails with the MCP-only plugin beside it - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" - (vend / "dsl" / "INTERPRETER.md").unlink() - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], - capture_output=True, text=True) - assert r.returncode == 1 and "dsl/INTERPRETER.md" in r.stdout and "missing on disk" in r.stdout - # a clean tree plus the skills-less plugin passes - (vend / "orphan.md").unlink() - readme = vend / "README.md" - readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") - subprocess.run([sys.executable, str(SCRIPT), "--plugins-root", str(plugins)], check=True, capture_output=True) - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], - capture_output=True, text=True) - assert r.returncode == 0 and r.stdout.startswith("PASS"), r.stdout + r.stderr + assert r.returncode == 0 and "PASS" in r.stdout -def test_real_repository_is_in_sync(): - r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=REPO_ROOT) +def test_real_plugin_is_in_sync(): + r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=PLUGIN_ROOT) assert r.returncode == 0, r.stdout assert "vendored file(s) byte-identical" in r.stdout and "README tables" in r.stdout From 9e60fce74ff65ef4752dd2c4a1ae0a0bca6007a8 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 19:26:27 -0700 Subject: [PATCH 16/53] fix(aws-startup-advisor): keep skill-refs and vendored-parity tooling plugin-local MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move tools/check-skill-refs.py, tools/skill-refs-baseline.json, and tools/sync-vendored.py (plus their tests) into plugins/aws-startup-advisor/tools/. Both gates are owned by, and only ever scan, this plugin — check-skill-refs.py was already hardcoded to DEFAULT_PLUGIN=aws-startup-advisor, and sync-vendored.py's vendored/shared pattern has no other consumer in this repository. - check-skill-refs.py and its baseline/test now resolve relative to the script's own plugin-local position (PLUGIN_ROOT) instead of a root-level tools/ directory; REPO_ROOT is still derived (two levels further up) for the one genuinely repo-relative fallback candidate and for display paths. - sync-vendored.py: same single-plugin fix as PR #386 (PLUGIN_ROOT instead of scanning every plugins/ subdirectory; --plugin-root replaces the removed multi-plugin --plugin/--plugins-root surface). - Updated mise.toml's lint:skill-refs and lint:vendored-parity tasks and the four vendored/README.md usage snippets to the new paths. - Updated the baseline file's _what_this_is pointer to the new script path. --- mise.toml | 4 +- .../references/vendored/README.md | 4 +- .../references/vendored/README.md | 4 +- .../gcp-to-aws/references/vendored/README.md | 4 +- .../references/vendored/README.md | 4 +- .../tools}/check-skill-refs.py | 19 ++-- .../tools}/skill-refs-baseline.json | 2 +- .../tools}/sync-vendored.py | 60 +++++------- .../tools}/test_check_skill_refs.py | 10 +- .../tools}/test_sync_vendored.py | 98 +++++++++---------- 10 files changed, 103 insertions(+), 106 deletions(-) rename {tools => plugins/aws-startup-advisor/tools}/check-skill-refs.py (94%) rename {tools => plugins/aws-startup-advisor/tools}/skill-refs-baseline.json (87%) rename {tools => plugins/aws-startup-advisor/tools}/sync-vendored.py (80%) rename {tools => plugins/aws-startup-advisor/tools}/test_check_skill_refs.py (97%) rename {tools => plugins/aws-startup-advisor/tools}/test_sync_vendored.py (60%) diff --git a/mise.toml b/mise.toml index bf8120065..1a8eec4d9 100644 --- a/mise.toml +++ b/mise.toml @@ -33,11 +33,11 @@ run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtu [tasks."lint:vendored-parity"] description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" -run = "python3 tools/sync-vendored.py --check" +run = "python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check" [tasks."lint:skill-refs"] description = "Check every file path the aws-startup-advisor skills reference exists" -run = "python3 tools/check-skill-refs.py --check" +run = "python3 plugins/aws-startup-advisor/tools/check-skill-refs.py --check" [tasks.validate] description = "Run all validation" diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md index 16a0e196d..c84f897ef 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md @@ -6,9 +6,9 @@ so the skill folder is **self-contained** — it runs standalone (lifted out, zi or used on its own) without reaching outside its own directory. **Do not hand-edit anything in this directory.** Edit the canonical source instead, -then run `python3 tools/sync-vendored.py` from the repository root to bring every +then run `python3 plugins/aws-startup-advisor/tools/sync-vendored.py` from the repository root to bring every skill's `references/vendored/` copy back in sync. Every copy must stay **byte-identical** -to the canonical source; CI runs `python3 tools/sync-vendored.py --check` +to the canonical source; CI runs `python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check` (`mise run lint:vendored-parity`) and fails on any drift, any copy with no canonical source, any row in the table below with no file on disk, and any file on disk with no row. It cannot see a new `skills/shared/` file no skill vendors yet — adding one here means the copy **and** a row. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index b5b09aa75..a5e154143 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -10,8 +10,8 @@ then bring every vendored copy back in sync: ```sh # from the repository root -python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` CI fails when a vendored copy differs from its canonical source, has no canonical source, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 38c4356e0..01040332d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -10,8 +10,8 @@ then bring every vendored copy back in sync: ```sh # from the repository root -python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` CI fails when a vendored copy differs from its canonical source, has no canonical source, diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md index ab9ef3a05..cb53a9dea 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md @@ -10,8 +10,8 @@ then bring every vendored copy back in sync: ```sh # from the repository root -python3 tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) ``` CI fails when a vendored copy differs from its canonical source, has no canonical source, diff --git a/tools/check-skill-refs.py b/plugins/aws-startup-advisor/tools/check-skill-refs.py similarity index 94% rename from tools/check-skill-refs.py rename to plugins/aws-startup-advisor/tools/check-skill-refs.py index ab9359d49..656c8ab16 100644 --- a/tools/check-skill-refs.py +++ b/plugins/aws-startup-advisor/tools/check-skill-refs.py @@ -36,12 +36,16 @@ 5. `/P` and `/P` (plugin- or repo-relative) A path that resolves nowhere is MISSING_REF. +This gate is plugin-local: it scans and is invoked only against aws-startup-advisor. +Everything it reads lives inside this plugin directory (`plugins/aws-startup-advisor/`); +it does not depend on, or write to, anything at the repository root. + Usage ----- - python3 tools/check-skill-refs.py --check # CI: exit 1 on any non-baselined MISSING_REF - python3 tools/check-skill-refs.py --check --json - python3 tools/check-skill-refs.py --list # print every resolved reference (debug) - --baseline tools/skill-refs-baseline.json # known dead refs with reasons; stale entries fail + python3 plugins/aws-startup-advisor/tools/check-skill-refs.py --check # CI: exit 1 on any non-baselined MISSING_REF + python3 plugins/aws-startup-advisor/tools/check-skill-refs.py --check --json + python3 plugins/aws-startup-advisor/tools/check-skill-refs.py --list # print every resolved reference (debug) + --baseline plugins/aws-startup-advisor/tools/skill-refs-baseline.json # known dead refs with reasons; stale entries fail Stdlib only (Python 3.9+). """ @@ -56,9 +60,10 @@ from pathlib import Path from typing import Dict, List, Optional, Set, Tuple -REPO_ROOT = Path(__file__).resolve().parent.parent -DEFAULT_PLUGIN = REPO_ROOT / "plugins" / "aws-startup-advisor" -DEFAULT_BASELINE = REPO_ROOT / "tools" / "skill-refs-baseline.json" +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +REPO_ROOT = PLUGIN_ROOT.parent.parent +DEFAULT_PLUGIN = PLUGIN_ROOT +DEFAULT_BASELINE = PLUGIN_ROOT / "tools" / "skill-refs-baseline.json" EXTS = r"(?:md|json|py|sh|tf|ya?ml|html|txt|toml)" ROOTS = r"(?:references|knowledge|scripts|skills|agents|data|phases|shared|design-refs|fixtures|tools)" diff --git a/tools/skill-refs-baseline.json b/plugins/aws-startup-advisor/tools/skill-refs-baseline.json similarity index 87% rename from tools/skill-refs-baseline.json rename to plugins/aws-startup-advisor/tools/skill-refs-baseline.json index 5ede122f0..4a5101450 100644 --- a/tools/skill-refs-baseline.json +++ b/plugins/aws-startup-advisor/tools/skill-refs-baseline.json @@ -1,5 +1,5 @@ { - "_what_this_is": "Policy and burn-down for tools/check-skill-refs.py. `ignore` rules are permanent: the matched tokens are not references into the repository (they are paths the skill GENERATES into a run directory, or example paths in a user's own app). `entries` are known dead references with the fix that retires each one; an entry that stops matching is reported STALE and fails the gate, so this list only shrinks.", + "_what_this_is": "Policy and burn-down for plugins/aws-startup-advisor/tools/check-skill-refs.py. `ignore` rules are permanent: the matched tokens are not references into the repository (they are paths the skill GENERATES into a run directory, or example paths in a user's own app). `entries` are known dead references with the fix that retires each one; an entry that stops matching is reported STALE and fails the gate, so this list only shrinks.", "ignore": [ { "source": "plugins/aws-startup-advisor/skills/*/references/phases/generate/*.md", diff --git a/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py similarity index 80% rename from tools/sync-vendored.py rename to plugins/aws-startup-advisor/tools/sync-vendored.py index 7492d3f42..26d038ce6 100644 --- a/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -25,11 +25,15 @@ The vendored tree is a subset of shared/ by design — a skill vendors only what it loads — so adding a shared file a skill needs means adding the copy AND the README row. +This gate is owned by, and only ever scans, the aws-startup-advisor plugin — it is the +only plugin in this repository using the vendored/shared pattern. Everything this script +reads lives inside this plugin directory (`plugins/aws-startup-advisor/`); it does not +depend on, or write to, anything at the repository root or any other plugin. + Usage ----- - python3 tools/sync-vendored.py # copy shared/ over every vendored copy - python3 tools/sync-vendored.py --check # exit 1 if any copy differs / has no source - python3 tools/sync-vendored.py --plugin aws-startup-advisor + python3 plugins/aws-startup-advisor/tools/sync-vendored.py # sync + python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # exit 1 on drift Exit 0 when in sync (or after syncing), 1 on --check failure. Stdlib only. """ @@ -43,17 +47,17 @@ from pathlib import Path from typing import Dict, List, Tuple -REPO_ROOT = Path(__file__).resolve().parent.parent -PLUGINS_ROOT = REPO_ROOT / "plugins" +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS_DIR = PLUGIN_ROOT / "skills" VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} def _rel(p: Path) -> str: - """Repo-relative for display; absolute when the tree lives elsewhere (tests).""" + """Plugin-relative for display; absolute when the tree lives elsewhere (tests).""" try: - return str(p.resolve().relative_to(REPO_ROOT)) + return str(p.resolve().relative_to(PLUGIN_ROOT)) except ValueError: return str(p) @@ -181,39 +185,27 @@ def sync_plugin(plugin_dir: Path) -> Tuple[int, List[str]]: def main(argv=None) -> int: - ap = argparse.ArgumentParser(description="Keep skills' references/vendored/ copies identical to skills/shared/") - ap.add_argument("--plugin", help="Only this plugin (directory name under plugins/)") + ap = argparse.ArgumentParser( + description="Keep aws-startup-advisor skills' references/vendored/ copies identical to skills/shared/") ap.add_argument("--check", action="store_true", help="Report drift and exit 1; change nothing") - ap.add_argument("--plugins-root", type=Path, default=PLUGINS_ROOT, help=argparse.SUPPRESS) + ap.add_argument("--plugin-root", type=Path, default=PLUGIN_ROOT, help=argparse.SUPPRESS) args = ap.parse_args(argv) - plugins = [args.plugins_root / args.plugin] if args.plugin else sorted( - p for p in args.plugins_root.iterdir() if p.is_dir()) + plugin_dir = args.plugin_root total_files = 0 all_errors: List[str] = [] copied = 0 - for plugin_dir in plugins: - skills_dir = plugin_dir / "skills" - # An MCP-only plugin has no skills directory. iterdir() on the missing - # path raises FileNotFoundError, and mise validate runs this task over - # every plugin. Skip before iterating. README checks still run for a - # plugin that has skills and a vendored directory. - if not skills_dir.is_dir(): - continue - files = vendored_files(plugin_dir) - total_files += len(files) - if not files and not any((skill / VENDORED_SUBDIR).is_dir() - for skill in skills_dir.iterdir() - if skill.is_dir()): - continue - if args.check: - all_errors.extend(check_plugin(plugin_dir)) - all_errors.extend(check_manifest(plugin_dir)) - else: - n, errs = sync_plugin(plugin_dir) - copied += n - all_errors.extend(errs) - all_errors.extend(check_manifest(plugin_dir)) + + files = vendored_files(plugin_dir) + total_files += len(files) + if args.check: + all_errors.extend(check_plugin(plugin_dir)) + all_errors.extend(check_manifest(plugin_dir)) + else: + n, errs = sync_plugin(plugin_dir) + copied += n + all_errors.extend(errs) + all_errors.extend(check_manifest(plugin_dir)) if all_errors: print(f"FAIL — {len(all_errors)} vendored file(s) out of contract:") diff --git a/tools/test_check_skill_refs.py b/plugins/aws-startup-advisor/tools/test_check_skill_refs.py similarity index 97% rename from tools/test_check_skill_refs.py rename to plugins/aws-startup-advisor/tools/test_check_skill_refs.py index 9c1b1de0e..9a16e532b 100644 --- a/tools/test_check_skill_refs.py +++ b/plugins/aws-startup-advisor/tools/test_check_skill_refs.py @@ -1,7 +1,7 @@ -"""Tests for tools/check-skill-refs.py — the reference-path lint. +"""Tests for check-skill-refs.py — the reference-path lint. A throwaway plugin tree exercises each resolution rule and each skip rule; the last test -runs the real repository and requires PASS with no stale baseline entries. +runs the real aws-startup-advisor plugin and requires PASS with no stale baseline entries. """ from __future__ import annotations @@ -11,8 +11,8 @@ import sys from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parent.parent -SCRIPT = REPO_ROOT / "tools" / "check-skill-refs.py" +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = PLUGIN_ROOT / "tools" / "check-skill-refs.py" def _load(): @@ -225,7 +225,7 @@ def test_baseline_ignore_vs_entries_and_stale(tmp_path: Path): def _run(*args: str) -> subprocess.CompletedProcess: - return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, cwd=REPO_ROOT) + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, cwd=PLUGIN_ROOT) def test_cli_exit_codes(tmp_path: Path): diff --git a/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py similarity index 60% rename from tools/test_sync_vendored.py rename to plugins/aws-startup-advisor/tools/test_sync_vendored.py index 33c5cd8cb..f7188b952 100644 --- a/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -1,7 +1,7 @@ -"""Tests for tools/sync-vendored.py — the vendored-copy parity gate. +"""Tests for sync-vendored.py — the vendored-copy parity gate. -Builds a throwaway plugins tree so the tests do not depend on (or mutate) the real -skills, then checks the real repository is clean as the last assertion. +Builds a throwaway plugin tree so the tests do not depend on (or mutate) the real +skills, then checks the real plugin is clean as the last assertion. """ from __future__ import annotations @@ -10,8 +10,8 @@ import sys from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parent.parent -SCRIPT = REPO_ROOT / "tools" / "sync-vendored.py" +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = PLUGIN_ROOT / "tools" / "sync-vendored.py" def _load(): @@ -26,12 +26,12 @@ def _load(): def _tree(tmp_path: Path) -> Path: - plugins = tmp_path / "plugins" - shared = plugins / "p" / "skills" / "shared" + plugin = tmp_path / "p" + shared = plugin / "skills" / "shared" (shared / "dsl").mkdir(parents=True) (shared / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") (shared / "pricing.json").write_text('{"a": 1}\n') - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "dsl").mkdir(parents=True) (vend / "dsl" / "INTERPRETER.md").write_text("canonical v2\n") # in sync (vend / "pricing.json").write_text('{"a": 0}\n') # drifted @@ -44,13 +44,13 @@ def _tree(tmp_path: Path) -> Path: "| `pricing.json` | `skills/shared/pricing.json` |\n" "| `orphan.md` | `skills/shared/orphan.md` |\n") # a skill that vendors nothing, and the shared dir itself, are both ignored - (plugins / "p" / "skills" / "beta").mkdir() - return plugins + (plugin / "skills" / "beta").mkdir() + return plugin def test_check_reports_drift_and_orphans_but_not_readme(tmp_path: Path): - plugins = _tree(tmp_path) - errors = sv.check_plugin(plugins / "p") + plugin = _tree(tmp_path) + errors = sv.check_plugin(plugin) joined = "\n".join(errors) assert "pricing.json: differs from" in joined assert "orphan.md: no canonical source" in joined @@ -60,38 +60,38 @@ def test_check_reports_drift_and_orphans_but_not_readme(tmp_path: Path): def test_sync_copies_canonical_over_drifted_and_leaves_orphans_as_errors(tmp_path: Path): - plugins = _tree(tmp_path) - copied, errors = sv.sync_plugin(plugins / "p") + plugin = _tree(tmp_path) + copied, errors = sv.sync_plugin(plugin) assert copied == 1 - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + vend = plugin / "skills" / "alpha" / "references" / "vendored" assert (vend / "pricing.json").read_text() == '{"a": 1}\n' assert len(errors) == 1 and "orphan.md" in errors[0] # after sync, only the orphan remains - assert [e for e in sv.check_plugin(plugins / "p") if "orphan" not in e] == [] + assert [e for e in sv.check_plugin(plugin) if "orphan" not in e] == [] def test_deleted_copy_is_caught_via_readme_table(tmp_path: Path): """Review probe on #386: deleting a vendored copy left check_plugin at 0 errors because it only walked files that still existed. The README table is the record of intent.""" - plugins = _tree(tmp_path) - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "dsl" / "INTERPRETER.md").unlink() - errors = sv.check_manifest(plugins / "p") + errors = sv.check_manifest(plugin) assert any("dsl/INTERPRETER.md" in e and "missing on disk" in e for e in errors), errors # sync restores it from canonical - copied, _ = sv.sync_plugin(plugins / "p") + copied, _ = sv.sync_plugin(plugin) assert (vend / "dsl" / "INTERPRETER.md").read_text() == "canonical v2\n" assert copied >= 1 - assert not any("missing on disk" in e for e in sv.check_manifest(plugins / "p")) + assert not any("missing on disk" in e for e in sv.check_manifest(plugin)) def test_unlisted_copy_and_wrong_canonical_path_are_caught(tmp_path: Path): - plugins = _tree(tmp_path) - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "extra.md").write_text("x\n") readme = vend / "README.md" readme.write_text(readme.read_text().replace("`skills/shared/pricing.json`", "`skills/other/pricing.json`")) - errors = sv.check_manifest(plugins / "p") + errors = sv.check_manifest(plugin) assert any("extra.md" in e and "not listed" in e for e in errors), errors assert any("pricing.json" in e and "expected `skills/shared/pricing.json`" in e for e in errors), errors @@ -99,44 +99,44 @@ def test_unlisted_copy_and_wrong_canonical_path_are_caught(tmp_path: Path): def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): """Documented limit, pinned so the README wording stays honest: a brand-new skills/shared/ file that no skill vendors is NOT drift.""" - plugins = _tree(tmp_path) - (plugins / "p" / "skills" / "shared" / "brand-new.md").write_text("new\n") - (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() - readme = plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "README.md" + plugin = _tree(tmp_path) + (plugin / "skills" / "shared" / "brand-new.md").write_text("new\n") + (plugin / "skills" / "alpha" / "references" / "vendored" / "orphan.md").unlink() + readme = plugin / "skills" / "alpha" / "references" / "vendored" / "README.md" readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") - (plugins / "p" / "skills" / "alpha" / "references" / "vendored" / "pricing.json").write_text('{"a": 1}\n') - assert sv.check_plugin(plugins / "p") == [] - assert sv.check_manifest(plugins / "p") == [] + (plugin / "skills" / "alpha" / "references" / "vendored" / "pricing.json").write_text('{"a": 1}\n') + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] def test_cli_check_exit_codes(tmp_path: Path): - plugins = _tree(tmp_path) - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + plugin = _tree(tmp_path) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], capture_output=True, text=True) assert r.returncode == 1 and "FAIL" in r.stdout - vend = plugins / "p" / "skills" / "alpha" / "references" / "vendored" + vend = plugin / "skills" / "alpha" / "references" / "vendored" (vend / "orphan.md").unlink() readme = vend / "README.md" readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") - subprocess.run([sys.executable, str(SCRIPT), "--plugins-root", str(plugins)], check=True, capture_output=True) - r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugins-root", str(plugins)], + subprocess.run([sys.executable, str(SCRIPT), "--plugin-root", str(plugin)], check=True, capture_output=True) + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], capture_output=True, text=True) assert r.returncode == 0 and r.stdout.startswith("PASS") def test_plugin_without_a_skills_directory_is_skipped(tmp_path: Path): - """Review finding on #387: iterating a missing skills/ raised FileNotFoundError.""" - plugins = tmp_path / "plugins" - (plugins / "mcp-only").mkdir(parents=True) - (plugins / "mcp-only" / "manifest.json").write_text("{}\n") - for flag in (["--check"], []): - r = subprocess.run([sys.executable, str(SCRIPT), *flag, "--plugins-root", str(plugins)], - capture_output=True, text=True) - assert r.returncode == 0, (flag, r.stdout, r.stderr) - assert "FileNotFoundError" not in r.stderr - - -def test_real_repository_is_in_sync(): - r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=REPO_ROOT) + """Review finding on #386: `--check` called skills.iterdir() before the is_dir() filter, + so a plugin root with no skills/ raised FileNotFoundError. Pinned behaviorally: an + empty plugin root (no skills/ at all) must pass cleanly, not crash.""" + plugin = tmp_path / "mcp-only" + plugin.mkdir() + r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], + capture_output=True, text=True) + assert "Traceback" not in r.stderr, r.stderr + assert r.returncode == 0 and "PASS" in r.stdout + + +def test_real_plugin_is_in_sync(): + r = subprocess.run([sys.executable, str(SCRIPT), "--check"], capture_output=True, text=True, cwd=PLUGIN_ROOT) assert r.returncode == 0, r.stdout assert "vendored file(s) byte-identical" in r.stdout and "README tables" in r.stdout From fb83e9033e5891689f61664a56f389ac5be492e6 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 21:33:28 -0700 Subject: [PATCH 17/53] test(aws-startup-advisor): pin AI migration matrix Validation:\n- uv run --with pytest python -m pytest plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py (6 passed)\n- mise run lint:artifacts (passed)\n- mise run lint:vendored-parity (passed)\n- mise run lint:skill-refs (passed)\n- mise run lint:md (passed)\n- mise run validate (passed)\n- mise run build (passed) --- .../aws-startup-advisor/fixtures/README.md | 1 + .../ai-only-gemini/.phase-status.json | 1 + .../ai-only-gemini/expected-artifacts.json | 19 ++ .../ai-only-openai/.phase-status.json | 1 + .../ai-only-openai/expected-artifacts.json | 19 ++ .../ai-only-openrouter/.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../combined-azure-gemini/.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../integration-validation.json | 1 + .../combined-azure-openai/.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../integration-validation.json | 1 + .../.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../integration-validation.json | 1 + .../combined-gcp-gemini/.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../integration-validation.json | 1 + .../combined-gcp-openai/.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../integration-validation.json | 1 + .../.phase-status.json | 1 + .../expected-artifacts.json | 19 ++ .../integration-validation.json | 1 + .../ai-migration-matrix/expected-matrix.json | 17 ++ .../infra-only-azure/.phase-status.json | 1 + .../infra-only-azure/expected-artifacts.json | 19 ++ .../infra-only-gcp/.phase-status.json | 1 + .../infra-only-gcp/expected-artifacts.json | 19 ++ .../scripts/artifact-contracts.json | 6 +- .../integration-validation.schema.json | 55 ++++++ .../test_ai_migration_matrix_contracts.py | 176 ++++++++++++++++++ 33 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json create mode 100644 plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 4783bd047..8250ad48e 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -2,6 +2,7 @@ Also see: +- `ai-migration-matrix/` — target-only contract snapshots for infrastructure-only, AI-only, and combined migration modes. These fixtures do not claim current runtime support. They keep infrastructure source, AI provider/model family, and router/gateway independent; combined cases use one `.phase-status.json` and a separate `integration-validation.json` cross-track verdict. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/.phase-status.json new file mode 100644 index 000000000..101a8743c --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"ai-only-gemini","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000003","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json new file mode 100644 index 000000000..43c796d13 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "ai-only-gemini", + "mode": "ai_only", + "migration_id": "ai-only-gemini", + "run_id": "00000000-0000-4000-8000-000000000003", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": null, + "ai_provider": "gcp_ai", + "model_families": ["gemini"], + "gateway_type": null, + "upstream_provider_signals": ["gcp_ai"] + }, + "outcomes": { + "infrastructure": null, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/.phase-status.json new file mode 100644 index 000000000..e7dbc68ae --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"ai-only-openai","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000004","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json new file mode 100644 index 000000000..60a4b8abc --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "ai-only-openai", + "mode": "ai_only", + "migration_id": "ai-only-openai", + "run_id": "00000000-0000-4000-8000-000000000004", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": null, + "ai_provider": "openai", + "model_families": ["gpt"], + "gateway_type": null, + "upstream_provider_signals": ["openai", "azure_openai"] + }, + "outcomes": { + "infrastructure": null, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/.phase-status.json new file mode 100644 index 000000000..d884e9fec --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"ai-only-openrouter","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000005","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json new file mode 100644 index 000000000..51a5df9cb --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "ai-only-openrouter", + "mode": "ai_only", + "migration_id": "ai-only-openrouter", + "run_id": "00000000-0000-4000-8000-000000000005", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": null, + "ai_provider": "openai", + "model_families": ["gpt"], + "gateway_type": "llm_router", + "upstream_provider_signals": ["openai"] + }, + "outcomes": { + "infrastructure": null, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/.phase-status.json new file mode 100644 index 000000000..bf745ec21 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-azure-gemini","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000009","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json new file mode 100644 index 000000000..60c37b691 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "combined-azure-gemini", + "mode": "combined", + "migration_id": "combined-azure-gemini", + "run_id": "00000000-0000-4000-8000-000000000009", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "azure", + "ai_provider": "gcp_ai", + "model_families": ["gemini"], + "gateway_type": null, + "upstream_provider_signals": ["gcp_ai"] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/integration-validation.json new file mode 100644 index 000000000..cbf8437b9 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000009","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/.phase-status.json new file mode 100644 index 000000000..53945b59c --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-azure-openai","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000007","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json new file mode 100644 index 000000000..38e1ad6c1 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "combined-azure-openai", + "mode": "combined", + "migration_id": "combined-azure-openai", + "run_id": "00000000-0000-4000-8000-000000000007", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "azure", + "ai_provider": "azure_openai", + "model_families": ["gpt"], + "gateway_type": null, + "upstream_provider_signals": ["azure_openai"] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/integration-validation.json new file mode 100644 index 000000000..737328506 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000007","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/.phase-status.json new file mode 100644 index 000000000..b127de2c4 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-azure-openrouter","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000011","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json new file mode 100644 index 000000000..691798b5a --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "combined-azure-openrouter", + "mode": "combined", + "migration_id": "combined-azure-openrouter", + "run_id": "00000000-0000-4000-8000-000000000011", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "azure", + "ai_provider": "anthropic", + "model_families": ["claude"], + "gateway_type": "llm_router", + "upstream_provider_signals": ["anthropic"] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/integration-validation.json new file mode 100644 index 000000000..0ddbd7102 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000011","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/.phase-status.json new file mode 100644 index 000000000..4d9dccd71 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-gcp-gemini","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000006","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json new file mode 100644 index 000000000..e4f8febb5 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "combined-gcp-gemini", + "mode": "combined", + "migration_id": "combined-gcp-gemini", + "run_id": "00000000-0000-4000-8000-000000000006", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "gcp", + "ai_provider": "gcp_ai", + "model_families": ["gemini"], + "gateway_type": null, + "upstream_provider_signals": ["gcp_ai"] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/integration-validation.json new file mode 100644 index 000000000..cdf6f9723 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000006","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/.phase-status.json new file mode 100644 index 000000000..679535473 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-gcp-openai","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000008","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json new file mode 100644 index 000000000..acffaa389 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "combined-gcp-openai", + "mode": "combined", + "migration_id": "combined-gcp-openai", + "run_id": "00000000-0000-4000-8000-000000000008", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "gcp", + "ai_provider": "openai", + "model_families": ["gpt"], + "gateway_type": null, + "upstream_provider_signals": ["openai", "azure_openai"] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/integration-validation.json new file mode 100644 index 000000000..a24ba3c4c --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000008","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/.phase-status.json new file mode 100644 index 000000000..ca5a50dec --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-gcp-openrouter","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000010","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json new file mode 100644 index 000000000..c89e9c313 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "combined-gcp-openrouter", + "mode": "combined", + "migration_id": "combined-gcp-openrouter", + "run_id": "00000000-0000-4000-8000-000000000010", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "gcp", + "ai_provider": "openai", + "model_families": ["gpt"], + "gateway_type": "llm_router", + "upstream_provider_signals": ["openai"] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/integration-validation.json new file mode 100644 index 000000000..de7c05a81 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000010","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json new file mode 100644 index 000000000..024f87cc5 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json @@ -0,0 +1,17 @@ +{ + "contract_version": "1.0", + "description": "Target migration-mode contracts only; these fixtures do not assert current runtime support.", + "cases": [ + {"id": "infra-only-gcp", "snapshot": "infra-only-gcp/expected-artifacts.json"}, + {"id": "infra-only-azure", "snapshot": "infra-only-azure/expected-artifacts.json"}, + {"id": "ai-only-gemini", "snapshot": "ai-only-gemini/expected-artifacts.json"}, + {"id": "ai-only-openai", "snapshot": "ai-only-openai/expected-artifacts.json"}, + {"id": "ai-only-openrouter", "snapshot": "ai-only-openrouter/expected-artifacts.json"}, + {"id": "combined-gcp-gemini", "snapshot": "combined-gcp-gemini/expected-artifacts.json"}, + {"id": "combined-azure-openai", "snapshot": "combined-azure-openai/expected-artifacts.json"}, + {"id": "combined-gcp-openai", "snapshot": "combined-gcp-openai/expected-artifacts.json"}, + {"id": "combined-azure-gemini", "snapshot": "combined-azure-gemini/expected-artifacts.json"}, + {"id": "combined-gcp-openrouter", "snapshot": "combined-gcp-openrouter/expected-artifacts.json"}, + {"id": "combined-azure-openrouter", "snapshot": "combined-azure-openrouter/expected-artifacts.json"} + ] +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/.phase-status.json new file mode 100644 index 000000000..de21224a7 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"infra-only-azure","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000002","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json new file mode 100644 index 000000000..5835e00d6 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "infra-only-azure", + "mode": "infrastructure_only", + "migration_id": "infra-only-azure", + "run_id": "00000000-0000-4000-8000-000000000002", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "azure", + "ai_provider": null, + "model_families": [], + "gateway_type": null, + "upstream_provider_signals": [] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": null, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/.phase-status.json new file mode 100644 index 000000000..e497909d8 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"infra-only-gcp","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000001","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json new file mode 100644 index 000000000..cb105b2df --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json @@ -0,0 +1,19 @@ +{ + "id": "infra-only-gcp", + "mode": "infrastructure_only", + "migration_id": "infra-only-gcp", + "run_id": "00000000-0000-4000-8000-000000000001", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "gcp", + "ai_provider": null, + "model_families": [], + "gateway_type": null, + "upstream_provider_signals": [] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": null, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index ad2101864..bf5fa1fd9 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -3,11 +3,15 @@ "fixture_prefixes": { "gcp-": "gcp-to-aws", "heroku-": "heroku-to-aws", - "azure-": "azure-to-aws" + "azure-": "azure-to-aws", + "ai-migration-matrix": "gcp-to-aws" }, "shared": { ".phase-status.json": { "json_schema": "skills/shared/state/phase-status.schema.json" + }, + "integration-validation.json": { + "json_schema": "scripts/contracts/integration-validation.schema.json" } }, "skills": { diff --git a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json new file mode 100644 index 000000000..308cac515 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:integration-validation", + "title": "integration-validation.json", + "description": "Target snapshot contract for cross-track validation in a combined infrastructure and AI migration.", + "type": "object", + "required": ["schema_version", "run_id", "verdict", "track_results", "checks"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "1.0"}, + "run_id": {"type": "string", "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"}, + "verdict": {"enum": ["passed", "failed"]}, + "track_results": { + "type": "object", + "required": ["infrastructure", "ai"], + "additionalProperties": false, + "properties": { + "infrastructure": {"$ref": "#/definitions/result"}, + "ai": {"$ref": "#/definitions/result"} + } + }, + "checks": { + "type": "object", + "required": ["networking", "iam_and_secrets", "data_flow", "observability", "deployment_order", "cutover", "rollback", "cost_estimate", "validation_dependencies"], + "additionalProperties": false, + "properties": { + "networking": {"$ref": "#/definitions/result"}, + "iam_and_secrets": {"$ref": "#/definitions/result"}, + "data_flow": {"$ref": "#/definitions/result"}, + "observability": {"$ref": "#/definitions/result"}, + "deployment_order": {"$ref": "#/definitions/result"}, + "cutover": {"$ref": "#/definitions/result"}, + "rollback": {"$ref": "#/definitions/result"}, + "cost_estimate": {"$ref": "#/definitions/result"}, + "validation_dependencies": {"$ref": "#/definitions/result"} + } + } + }, + "definitions": { + "result": { + "type": "object", + "required": ["verdict", "references"], + "additionalProperties": false, + "properties": { + "verdict": {"enum": ["passed", "failed"]}, + "references": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + } + } +} diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py new file mode 100644 index 000000000..e41de9170 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -0,0 +1,176 @@ +"""Contract tests for target AI-only and combined migration modes. + +These tests intentionally validate deterministic fixtures rather than runtime routing. +They pin the contract that later implementation PRs must satisfy without claiming that +all scenarios are supported by the current skills. +""" +from __future__ import annotations + +import json +from pathlib import Path + +FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "ai-migration-matrix" +INFRA_ARTIFACTS = { + "aws-design.json", + "estimation-infra.json", + "generation-infra.json", + "validation-report.json", +} +AI_ARTIFACTS = { + "ai-workload-profile.json", + "aws-design-ai.json", + "estimation-ai.json", + "generation-ai.json", +} +INTEGRATION_CHECKS = { + "networking", + "iam_and_secrets", + "data_flow", + "observability", + "deployment_order", + "cutover", + "rollback", + "cost_estimate", + "validation_dependencies", +} +EXPECTED_CASES = { + "infra-only-gcp", + "infra-only-azure", + "ai-only-gemini", + "ai-only-openai", + "ai-only-openrouter", + "combined-gcp-gemini", + "combined-azure-openai", + "combined-gcp-openai", + "combined-azure-gemini", + "combined-gcp-openrouter", + "combined-azure-openrouter", +} +EXPECTED_COMBINATIONS = { + ("gcp", "gcp_ai", None), + ("azure", "azure_openai", None), + ("gcp", "openai", None), + ("azure", "gcp_ai", None), + ("gcp", "openai", "llm_router"), + ("azure", "anthropic", "llm_router"), +} + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _cases() -> dict[str, tuple[Path, dict]]: + manifest = _load(FIXTURE_ROOT / "expected-matrix.json") + assert manifest["contract_version"] == "1.0" + loaded = {} + for entry in manifest["cases"]: + path = FIXTURE_ROOT / entry["snapshot"] + snapshot = _load(path) + assert snapshot["id"] == entry["id"] + loaded[entry["id"]] = (path.parent, snapshot) + return loaded + + +def test_manifest_has_complete_scenario_coverage() -> None: + cases = _cases() + assert set(cases) == EXPECTED_CASES + assert len(cases) == len(EXPECTED_CASES) + assert {case["mode"] for _, case in cases.values()} == { + "infrastructure_only", + "ai_only", + "combined", + } + + +def test_infrastructure_ai_and_gateway_are_independent_dimensions() -> None: + cases = _cases() + for _, snapshot in cases.values(): + dimensions = snapshot["dimensions"] + assert set(dimensions) == {"infrastructure_source", "ai_provider", "model_families", "gateway_type", "upstream_provider_signals"} + assert dimensions["infrastructure_source"] in {None, "gcp", "azure"} + assert dimensions["ai_provider"] in {None, "gcp_ai", "openai", "azure_openai", "anthropic"} + assert dimensions["gateway_type"] in {None, "llm_router"} + assert "openrouter" not in {dimensions["infrastructure_source"], dimensions["ai_provider"]} + if dimensions["gateway_type"] == "llm_router": + assert dimensions["ai_provider"] is not None + assert dimensions["model_families"] + assert dimensions["upstream_provider_signals"] + + combined = { + ( + snapshot["dimensions"]["infrastructure_source"], + snapshot["dimensions"]["ai_provider"], + snapshot["dimensions"]["gateway_type"], + ) + for _, snapshot in cases.values() + if snapshot["mode"] == "combined" + } + assert combined == EXPECTED_COMBINATIONS + + +def test_ai_only_origins_require_no_infrastructure() -> None: + cases = _cases() + ai_only = [snapshot for _, snapshot in cases.values() if snapshot["mode"] == "ai_only"] + assert {(case["dimensions"]["ai_provider"], case["dimensions"]["gateway_type"]) for case in ai_only} == { + ("gcp_ai", None), + ("openai", None), + ("openai", "llm_router"), + } + openai = cases["ai-only-openai"][1] + assert openai["dimensions"]["upstream_provider_signals"] == ["openai", "azure_openai"] + for snapshot in ai_only: + assert snapshot["dimensions"]["infrastructure_source"] is None + assert snapshot["outcomes"]["infrastructure"] is None + assert snapshot["outcomes"]["integration_validation"] is None + + +def test_each_case_has_one_shared_run_state() -> None: + for case_dir, snapshot in _cases().values(): + assert snapshot["state"] == ".phase-status.json" + state = _load(case_dir / snapshot["state"]) + assert state["migration_id"] == snapshot["migration_id"] + assert state["run_id"] == snapshot["run_id"] + assert set(state["phases"].values()) == {"completed"} + + +def test_required_artifacts_and_verdicts_match_each_mode() -> None: + for case_dir, snapshot in _cases().values(): + outcomes = snapshot["outcomes"] + infra = outcomes["infrastructure"] + ai = outcomes["ai"] + integration = outcomes["integration_validation"] + + if snapshot["mode"] in {"infrastructure_only", "combined"}: + assert infra == {"verdict": "passed", "artifacts": sorted(INFRA_ARTIFACTS)} + else: + assert infra is None + if snapshot["mode"] in {"ai_only", "combined"}: + assert ai == {"verdict": "passed", "artifacts": sorted(AI_ARTIFACTS)} + else: + assert ai is None + + integration_path = case_dir / "integration-validation.json" + if snapshot["mode"] == "combined": + assert integration == {"verdict": "passed", "artifact": "integration-validation.json"} + assert integration_path.is_file() + else: + assert integration is None + assert not integration_path.exists() + + +def test_combined_validation_references_both_tracks_and_all_categories() -> None: + for case_dir, snapshot in _cases().values(): + if snapshot["mode"] != "combined": + continue + validation = _load(case_dir / "integration-validation.json") + assert validation["schema_version"] == "1.0" + assert validation["run_id"] == snapshot["run_id"] + assert validation["verdict"] == "passed" + assert set(validation["track_results"]) == {"infrastructure", "ai"} + assert set(validation["track_results"]["infrastructure"]["references"]) == INFRA_ARTIFACTS + assert set(validation["track_results"]["ai"]["references"]) == AI_ARTIFACTS + assert set(validation["checks"]) == INTEGRATION_CHECKS + for result in [*validation["track_results"].values(), *validation["checks"].values()]: + assert result["verdict"] == "passed" + assert result["references"] From e68f84d21b9c46b456f71a8bc9a886d493fdcc86 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 21:37:52 -0700 Subject: [PATCH 18/53] fix(aws-startup-advisor): fail closed integration verdict Validation:\n- uv run --with pytest python -m pytest plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py (7 passed)\n- mise run lint:artifacts (70 checked, 0 errors)\n- mise run lint:vendored-parity (44 files passed)\n- mise run lint:skill-refs (1160 refs passed)\n- mise run lint:md (2475 files, 0 issues)\n- mise run validate (passed)\n- mise run build (passed; gitleaks found no leaks)\n- git diff --check (passed) --- .../integration-validation.schema.json | 31 +++++++++++++ .../test_ai_migration_matrix_contracts.py | 43 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json index 308cac515..492b6fa17 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json @@ -36,6 +36,37 @@ } } }, + "oneOf": [ + { + "properties": { + "verdict": {"const": "passed"}, + "track_results": { + "properties": { + "infrastructure": {"properties": {"verdict": {"const": "passed"}}}, + "ai": {"properties": {"verdict": {"const": "passed"}}} + } + }, + "checks": { + "properties": { + "networking": {"properties": {"verdict": {"const": "passed"}}}, + "iam_and_secrets": {"properties": {"verdict": {"const": "passed"}}}, + "data_flow": {"properties": {"verdict": {"const": "passed"}}}, + "observability": {"properties": {"verdict": {"const": "passed"}}}, + "deployment_order": {"properties": {"verdict": {"const": "passed"}}}, + "cutover": {"properties": {"verdict": {"const": "passed"}}}, + "rollback": {"properties": {"verdict": {"const": "passed"}}}, + "cost_estimate": {"properties": {"verdict": {"const": "passed"}}}, + "validation_dependencies": {"properties": {"verdict": {"const": "passed"}}} + } + } + } + }, + { + "properties": { + "verdict": {"const": "failed"} + } + } + ], "definitions": { "result": { "type": "object", diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py index e41de9170..1955ab0da 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -7,9 +7,14 @@ from __future__ import annotations import json +import subprocess +import sys from pathlib import Path FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "ai-migration-matrix" +PLUGIN_ROOT = FIXTURE_ROOT.parent.parent +ARTIFACT_VALIDATOR = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +ARTIFACT_MANIFEST = PLUGIN_ROOT / "scripts" / "artifact-contracts.json" INFRA_ARTIFACTS = { "aws-design.json", "estimation-infra.json", @@ -174,3 +179,41 @@ def test_combined_validation_references_both_tracks_and_all_categories() -> None for result in [*validation["track_results"].values(), *validation["checks"].values()]: assert result["verdict"] == "passed" assert result["references"] + + +def test_success_verdict_rejects_any_failed_child(tmp_path: Path) -> None: + source = FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json" + failed_children = [ + ("track_results", "infrastructure"), + ("track_results", "ai"), + ("checks", "iam_and_secrets"), + ] + + for section, child in failed_children: + invalid = _load(source) + invalid[section][child]["verdict"] = "failed" + case_dir = tmp_path / f"{section}-{child}" + case_dir.mkdir() + (case_dir / "integration-validation.json").write_text(json.dumps(invalid)) + + result = subprocess.run( + [ + sys.executable, + str(ARTIFACT_VALIDATOR), + "--run-dir", + str(case_dir), + "--skill", + "gcp-to-aws", + "--manifest", + str(ARTIFACT_MANIFEST), + "--no-baseline", + "--json", + ], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 1 + findings = json.loads(result.stdout)["errors"] + assert any(finding["code"] == "SCHEMA_VIOLATION" for finding in findings) From eac1d7758088e96ec7613cb6c451be1a4a87c1b2 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 21:55:20 -0700 Subject: [PATCH 19/53] refactor(aws-startup-advisor): separate Bedrock pricing cache Extract provider-neutral Bedrock rates into the shared AI canonical/vendor pattern and repoint GCP, llm-to-bedrock, and agent-advisor consumers without changing pricing behavior. Validation: parity script PASS (54 model keys, 24 static fallbacks, representative estimates unchanged); llm pricing 21 passed; agent-advisor 323 passed; Azure estimate fixture PASS; mise run build PASS. --- .../scripts/check-bedrock-pricing-parity.py | 180 +++++++ .../scripts/model-id-lint.py | 2 +- .../references/decision-refs/freshness.md | 2 +- .../models/anthropic-bedrock-2026-07-21.json | 2 +- .../references/phases/estimate/estimate.md | 2 +- .../references/vendored/README.md | 10 +- .../vendored/ai/bedrock-pricing-cache.md | 449 ++++++++++++++++++ .../references/vendored/README.md | 1 + .../vendored/ai/ai-migration-guardrails.md | 2 +- .../vendored/ai/ai-model-lifecycle.md | 4 +- .../vendored/ai/ai-openai-to-bedrock.md | 2 +- .../vendored/ai/bedrock-pricing-cache.md | 449 ++++++++++++++++++ .../skills/gcp-to-aws/SKILL.md | 8 +- .../design-refs/ai-gemini-to-bedrock.md | 4 +- .../references/phases/design/design-ai.md | 4 +- .../phases/discover/discover-preview.md | 2 +- .../references/phases/estimate/estimate-ai.md | 15 +- .../references/phases/estimate/estimate.md | 20 +- .../references/shared/openai-on-bedrock.md | 4 +- .../references/shared/pricing-cache.md | 439 +---------------- .../references/shared/pricing-fallback.md | 2 +- .../gcp-to-aws/references/vendored/README.md | 1 + .../vendored/ai/ai-migration-guardrails.md | 2 +- .../vendored/ai/ai-model-lifecycle.md | 4 +- .../vendored/ai/ai-openai-to-bedrock.md | 2 +- .../vendored/ai/bedrock-pricing-cache.md | 449 ++++++++++++++++++ .../llm-to-bedrock/scripts/bedrock_pricing.py | 2 +- .../scripts/test_bedrock_pricing.py | 2 +- .../skills/shared/ai/README.md | 11 +- .../shared/ai/ai-migration-guardrails.md | 2 +- .../skills/shared/ai/ai-model-lifecycle.md | 4 +- .../skills/shared/ai/ai-openai-to-bedrock.md | 2 +- .../skills/shared/ai/bedrock-pricing-cache.md | 449 ++++++++++++++++++ .../tests/test_model_id_lint.py | 2 +- 34 files changed, 2044 insertions(+), 491 deletions(-) create mode 100755 plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py create mode 100644 plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md create mode 100644 plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md create mode 100644 plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md create mode 100644 plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md diff --git a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py new file mode 100755 index 000000000..1f2c87a58 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Prove the Bedrock pricing split preserves the pre-split data and behavior. + +Run from anywhere: + python3 plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py + +The immutable pre-split comparison point is the stacked PR #387 head on which this +branch was created. The check uses only the standard library and local git data. +""" +from __future__ import annotations + +import __future__ +import json +import re +import subprocess +import tempfile +import types +from pathlib import Path + +BASE_SHA = "9e60fce74ff65ef4752dd2c4a1ae0a0bca6007a8" +PLUGIN = Path(__file__).resolve().parent.parent +REPO = PLUGIN.parents[1] +OLD_CACHE = "plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md" +OLD_SCRIPT = "plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py" +CANONICAL = PLUGIN / "skills/shared/ai/bedrock-pricing-cache.md" +GCP_CACHE = PLUGIN / "skills/gcp-to-aws/references/shared/pricing-cache.md" +VENDORED = ( + PLUGIN / "skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md", + PLUGIN / "skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md", + PLUGIN / "skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md", +) + + +def git_show(path: str) -> str: + return subprocess.run( + ["git", "show", f"{BASE_SHA}:{path}"], + cwd=REPO, + check=True, + capture_output=True, + text=True, + ).stdout + + +def section(text: str, heading: str) -> str: + match = re.search(rf"(?m)^## {re.escape(heading)}.*$", text) + assert match, f"missing section: {heading}" + end = re.search(r"(?m)^## ", text[match.end() :]) + stop = match.end() + end.start() if end else len(text) + return text[match.start() : stop] + + +def metadata(text: str) -> dict[str, str]: + fields = {} + for key in ("Last updated", "Region", "Currency", "Accuracy"): + match = re.search(rf"(?m)^\*\*{re.escape(key)}:\*\*\s*(.+)$", text) + assert match, f"missing metadata: {key}" + fields[key] = match.group(1) + return fields + + +def model_rows(text: str) -> dict[str, tuple[str, str]]: + rows = {} + for line in section(text, "Bedrock Models (On-Demand)").splitlines(): + if not line.startswith("|"): + continue + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if len(cells) < 5 or "." not in cells[1]: + continue + if re.fullmatch(r"\d+(?:\.\d+)?", cells[3]) and re.fullmatch(r"\d+(?:\.\d+)?", cells[4]): + rows[cells[1]] = (cells[3], cells[4]) + return rows + + +def load_module(path: Path, name: str): + module = types.ModuleType(name) + code = compile( + path.read_text(), + str(path), + "exec", + flags=__future__.annotations.compiler_flag, + ) + exec(code, module.__dict__) + return module + + +def pricing_behavior(module) -> dict: + fragment = { + "terms": {"OnDemand": {"x": {"priceDimensions": { + "input": {"pricePerUnit": {"USD": "0.003"}, "description": "Input tokens"}, + "output": {"pricePerUnit": {"USD": "0.015"}, "description": "Output tokens"}, + "cache": {"pricePerUnit": {"USD": "0.0003"}, "description": "Cache read input tokens"}, + }}}}} + probes = sorted(module.STATIC_FALLBACK) + [ + "anthropic.claude-opus-4-8" + "-20250610-v1:0", + "us.anthropic.claude-sonnet-4-6" + "-20250514-v1:0", + "openai.gpt-5.6", + ] + return { + "table": module.STATIC_FALLBACK, + "parse": module.parse_price_dimensions(fragment), + "unavailable": module.unavailable("parity probe"), + "display": module.display_name_guess("us.anthropic.claude-haiku-4-5-20251001-v1:0"), + "mantle": [module.is_mantle_gpt(x) for x in ("openai.gpt-5.6-luna", "openai.gpt-oss-120b-1:0")], + "fallbacks": {probe: module._static_fallback(probe) for probe in probes}, + "lookups": {probe: module.lookup("us-east-1", probe) for probe in probes if module._static_fallback(probe)}, + } + + +def main() -> int: + old = git_show(OLD_CACHE) + canonical = CANONICAL.read_text() + remaining = GCP_CACHE.read_text() + + old_bedrock = section(old, "Bedrock Models (On-Demand)") + new_bedrock = section(canonical, "Bedrock Models (On-Demand)") + assert new_bedrock.rstrip() == old_bedrock.rstrip(), "Bedrock section changed during extraction" + assert metadata(canonical) == metadata(old), "freshness/source metadata changed" + + for heading in ( + "Compute", "Database", "Storage", "Networking", "Supporting Services", + "Analytics", "Source Provider Pricing (for Migration Comparison)", "Security Baseline", + ): + assert section(remaining, heading) == section(old, heading), f"non-Bedrock section changed: {heading}" + assert "## Bedrock Models (On-Demand)" not in remaining + assert "references/vendored/ai/bedrock-pricing-cache.md" in remaining + + canonical_bytes = CANONICAL.read_bytes() + for copy in VENDORED: + assert copy.read_bytes() == canonical_bytes, f"vendored drift: {copy}" + + old_rows = model_rows(old) + new_rows = model_rows(canonical) + assert new_rows == old_rows and new_rows, "model lookup keys or quick-reference rates changed" + volumes = (60, 40) + representative = ( + "anthropic.claude-sonnet-5", + "amazon.nova-pro-v1:0", + "openai.gpt-5.6-luna", + ) + old_estimates = { + model: volumes[0] * float(old_rows[model][0]) + volumes[1] * float(old_rows[model][1]) + for model in representative + } + new_estimates = { + model: volumes[0] * float(new_rows[model][0]) + volumes[1] * float(new_rows[model][1]) + for model in representative + } + assert new_estimates == old_estimates, "representative estimate outputs changed" + + current_script = PLUGIN / "skills/llm-to-bedrock/scripts/bedrock_pricing.py" + with tempfile.TemporaryDirectory() as tmp: + old_script = Path(tmp) / "bedrock_pricing_before.py" + old_script.write_text(git_show(OLD_SCRIPT)) + before = load_module(old_script, "bedrock_pricing_before") + after = load_module(current_script, "bedrock_pricing_after") + assert pricing_behavior(after) == pricing_behavior(before), "pricing parser/lookup behavior changed" + + forbidden = "skills/gcp-to-aws/references/shared/pricing-cache.md" + neutral_consumers = ( + PLUGIN / "skills/llm-to-bedrock/scripts/bedrock_pricing.py", + PLUGIN / "skills/llm-to-bedrock/scripts/test_bedrock_pricing.py", + PLUGIN / "skills/agent-advisor/references/phases/estimate/estimate.md", + PLUGIN / "skills/agent-advisor/references/decision-refs/freshness.md", + ) + for consumer in neutral_consumers: + assert forbidden not in consumer.read_text(), f"neutral GCP pricing dependency remains: {consumer}" + + print(json.dumps({ + "status": "PASS", + "bedrock_model_lookup_keys": len(new_rows), + "static_fallback_keys": len(pricing_behavior(after)["table"]), + "representative_monthly_estimates": new_estimates, + "vendored_copies": len(VENDORED), + "baseline": BASE_SHA, + }, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/aws-startup-advisor/scripts/model-id-lint.py b/plugins/aws-startup-advisor/scripts/model-id-lint.py index 98890a754..c0c16eef4 100644 --- a/plugins/aws-startup-advisor/scripts/model-id-lint.py +++ b/plugins/aws-startup-advisor/scripts/model-id-lint.py @@ -54,7 +54,7 @@ # canonical shared path (below), so only canonical paths are listed. CATALOG_ALLOWLIST = { "skills/shared/ai/ai-model-lifecycle.md", - "skills/gcp-to-aws/references/shared/pricing-cache.md", + "skills/shared/ai/bedrock-pricing-cache.md", "skills/azure-to-aws/references/shared/pricing-cache.md", } diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/freshness.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/freshness.md index 5f1e144af..b846ed0b8 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/freshness.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/freshness.md @@ -16,7 +16,7 @@ or the run directory in an outbound request — the answer never depends on it. (`registry_regions` in `references/runtimes/agentcore.json`; see the procedure below) - Lambda MicroVMs launch TPS (5, not adjustable) - FedRAMP certification status for AgentCore and Lambda MicroVMs -- Any Bedrock model price (defer to the `gcp-to-aws` pricing cache; never hardcode here) +- Any Bedrock model price (defer to `references/vendored/ai/bedrock-pricing-cache.md`; never hardcode here) ## Temporal (design.md — Freshness, temporal units only) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/models/anthropic-bedrock-2026-07-21.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/models/anthropic-bedrock-2026-07-21.json index 8cc8f4972..b1093cb88 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/models/anthropic-bedrock-2026-07-21.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/models/anthropic-bedrock-2026-07-21.json @@ -4,7 +4,7 @@ "verified_at": "2026-07-21", "verified_region": "us-east-1", "source": "AWS Bedrock userguide Anthropic model cards (retrieved 2026-07-21) for Claude Sonnet 5; internal migration reference, Anthropic path (2026-06-24 live probes) for the path metadata and the Opus 4.8 / Haiku 4.5 entries", - "disclaimer": "Point-in-time capability data. Re-probe every selected model and API path in the target account and region before implementation. Claude Sonnet 5 (launched 2026-06-30, after the reference snapshot) replaces Sonnet 4.6 in the recommendation pool: per its AWS model card it supports Messages on bedrock-mantle as well as Converse/Invoke on bedrock-runtime, with 1M context / 128K output and Global+geo CRIS. Claude Fable 5 / Fable 5.1 and Claude Mythos 5 / Mythos 5.1 — and any later Fable or Mythos release — are deliberately NOT pool candidates (Status `restricted` in pricing-cache.md). Fable 5 and 5.1 are Covered Models that require the account-level aws_review data-retention opt-in (prompts retained up to 30 days for AWS review) — a compliance decision, not a default recommendation; Fable 5.1 (GA 2026-09-01) is additionally CRIS-only in commercial regions. Mythos 5 and 5.1 are gated Previews for vetted cybersecurity and life-sciences research organizations (provider_data_share retention) — no account-level opt-in reaches them. Evidence: AWS model cards model-card-anthropic-claude-fable-5, -fable-5-1, -mythos-5, -mythos-5-1, read 2026-09-03; this file's 2026-07-21 snapshot predates Fable 5.1.", + "disclaimer": "Point-in-time capability data. Re-probe every selected model and API path in the target account and region before implementation. Claude Sonnet 5 (launched 2026-06-30, after the reference snapshot) replaces Sonnet 4.6 in the recommendation pool: per its AWS model card it supports Messages on bedrock-mantle as well as Converse/Invoke on bedrock-runtime, with 1M context / 128K output and Global+geo CRIS. Claude Fable 5 / Fable 5.1 and Claude Mythos 5 / Mythos 5.1 — and any later Fable or Mythos release — are deliberately NOT pool candidates (Status `restricted` in references/vendored/ai/bedrock-pricing-cache.md). Fable 5 and 5.1 are Covered Models that require the account-level aws_review data-retention opt-in (prompts retained up to 30 days for AWS review) — a compliance decision, not a default recommendation; Fable 5.1 (GA 2026-09-01) is additionally CRIS-only in commercial regions. Mythos 5 and 5.1 are gated Previews for vetted cybersecurity and life-sciences research organizations (provider_data_share retention) — no account-level opt-in reaches them. Evidence: AWS model cards model-card-anthropic-claude-fable-5, -fable-5-1, -mythos-5, -mythos-5-1, read 2026-09-03; this file's 2026-07-21 snapshot predates Fable 5.1.", "paths": { "mantle_messages": { "sdk": "anthropic[bedrock]", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/estimate/estimate.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/estimate/estimate.md index 9e65edc89..afd12d791 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/estimate/estimate.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/estimate/estimate.md @@ -62,7 +62,7 @@ Cached anchors (order-of-magnitude, us-east-1, verify; last updated 2025-07-14): for that component with a note to verify the EC2 rate before committing. - Lambda MicroVMs: ~$0.0997/vCPU-hour, ~$0.0132/GB-hour - Fargate: ~$0.04048/vCPU-hour, ~$0.004445/GB-hour -- Bedrock model token rates: defer to the `gcp-to-aws` pricing cache (`skills/gcp-to-aws/references/shared/pricing-cache.md`) for exact figures +- Bedrock model token rates: defer to `references/vendored/ai/bedrock-pricing-cache.md` for exact figures ## Step 3 — Produce a magnitude per unit, not a quote diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md index c84f897ef..c8944c37b 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md @@ -13,11 +13,13 @@ to the canonical source; CI runs `python3 plugins/aws-startup-advisor/tools/sync any row in the table below with no file on disk, and any file on disk with no row. It cannot see a new `skills/shared/` file no skill vendors yet — adding one here means the copy **and** a row. -| Vendored path | Canonical source | -| -------------------- | ---------------------------------- | -| `dsl/INTERPRETER.md` | `skills/shared/dsl/INTERPRETER.md` | +| Vendored path | Canonical source | +| -------------------------------- | ---------------------------------------------- | +| `ai/bedrock-pricing-cache.md` | `skills/shared/ai/bedrock-pricing-cache.md` | +| `dsl/INTERPRETER.md` | `skills/shared/dsl/INTERPRETER.md` | -This skill vendors ONLY the DSL interpreter contract. It does not vendor the shared +This skill vendors the DSL interpreter contract and the provider-neutral Bedrock pricing +cache. It does not vendor the shared state schema (`skills/shared/state/phase-status.schema.json`) — agent-advisor's `.phase-status.json` carries advisor-specific keys and statuses, declared in SKILL.md § State file per INTERPRETER.md § Skill bindings. diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md new file mode 100644 index 000000000..b1f94f147 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md @@ -0,0 +1,449 @@ +# Amazon Bedrock Pricing Cache + +**Last updated:** 2026-09-03 +**Region:** us-east-1 +**Currency:** USD +**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) + +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> +> **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. + +--- +## Bedrock Models (On-Demand) + +**Anthropic Claude (Standard on-demand)** figures below match **US East (N. Virginia)** on [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) as of cache refresh. **Recommend defaults (new migrations):** Claude Sonnet 5 (flagship), Claude Opus 4.8 (hardest reasoning), Claude Haiku 4.5 (cost/speed). Do not default to any Claude Fable / Mythos frontier model (Fable 5, Fable 5.1, and successors). **Claude Fable 5** and **Claude Fable 5.1** (GA Sep 1, 2026) are the most expensive Anthropic models at $10/$50 per 1M tokens (Mythos-class); 5.1's on-demand rate difference is cache read at $0.25 (0.025x) vs $1.00; Bedrock lists no batch rate for either. Both are Anthropic **Covered Models**: using them requires opting the account into the `aws_review` data-retention mode, and Fable 5.1 is served commercially only via `us.`/`global.` CRIS profiles (no in-region — the bare `anthropic.claude-fable-5-1` id is not invokable on `bedrock-runtime`, hence the `us.` form in the table), with in-region access in AWS GovCloud (US) only. Claude Mythos 5.1 is a gated Preview for vetted cyber/bio research organizations and is not a migration target. **Claude Opus 4.8** keeps the same $5/$25 rate as Opus 4.6/4.7. **Claude Sonnet 5** launched June 30 at $2/$10; that launch rate became the standard price on Sep 1, 2026 (the scheduled increase to $3/$15 was cancelled), so Sonnet 5 is now both newer and cheaper than Sonnet 4.6 ($3/$15). **Claude Opus 4.7** lists the same headline on-demand input/output as **Opus 4.6** on that page; confirm **batch** availability per model (Opus 4.7 batch was **not** listed on the global cross-region table when this row was added). **Claude Opus 4.1** entered **Legacy** on Jul 8, 2026 (EOL Jan 8, 2027). **Batch**, **prompt cache** (5m / 1h write + cache read), and **geo / in-region cross-region** rows on that page can differ; e.g. **US East (Ohio)** cross-region inference for Claude Sonnet 4.6 is listed at **$3.30 / $16.50** per 1M input/output (≈10% above N. Virginia). Long-context SKUs **do not** all use the same multiplier: **Sonnet 4.6** and **Opus 4.6** long-context modes share the same on-demand rates as the non–long-context rows on the standard table; **Sonnet 4.5** and **Sonnet 4** long-context rows are priced higher on that same table. + +### Multi-provider quick reference (per 1M tokens) + +See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommend Legacy models for new migrations.** Rows whose Status is `restricted (…)` are never `recommended_model` / `backup_model` and never a default (Covered Models, gated previews). + +| Model | Model ID | Provider | Input $/1M | Output $/1M | Context | Tier | Status | +| -------------------------------- | ---------------------------------------- | --------- | ---------- | ----------- | ------- | --------- | ------------------------------------------------------------ | +| Claude Fable 5 | anthropic.claude-fable-5 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in) | +| Claude Fable 5.1 | us.anthropic.claude-fable-5-1 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in; CRIS only) | +| Claude Sonnet 5 | anthropic.claude-sonnet-5 | Anthropic | 2.00 | 10.00 | 1M | flagship | active ($2/$10 made the standard price Sep 1, 2026) | +| Claude Opus 4.8 | anthropic.claude-opus-4-8 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.6 | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.6 — Long Context | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K+ | flagship | active | +| Claude Opus 4.6 | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Opus 4.6 — Long Context | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K+ | premium | active | +| Claude Opus 4.5 | — | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.5 | — | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.5 — Long Context | — | Anthropic | 6.00 | 22.50 | 200K+ | flagship | active | +| Claude Haiku 4.5 | anthropic.claude-haiku-4-5-20251001-v1:0 | Anthropic | 1.00 | 5.00 | 200K | fast | active | +| Claude Opus 4.1 | anthropic.claude-opus-4-1-20250805-v1:0 | Anthropic | 15.00 | 75.00 | 200K | premium | legacy (EOL Jan 8, 2027) | +| Claude Sonnet 4 | anthropic.claude-sonnet-4-20250514-v1:0 | Anthropic | 3.00 | 15.00 | 200K | flagship | excluded (EOL Oct 14, 2026) | +| Llama 4 Maverick | meta.llama4-maverick-17b-instruct-v1:0 | Meta | 0.24 | 0.97 | 1M | mid | active | +| Llama 4 Scout | meta.llama4-scout-17b-instruct-v1:0 | Meta | 0.17 | 0.66 | 10M | efficient | active | +| Llama 3.3 70B | meta.llama3-3-70b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | active | +| Llama 3.2 90B | meta.llama3-2-90b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | EOL (Jul 7, 2026) | +| Nova 2 Lite | amazon.nova-2-lite-v1:0 | Amazon | 0.33 | 2.75 | 1M | mid | active | +| Nova 2 Pro (Preview) | amazon.nova-2-pro-v1:0 | Amazon | 1.375 | 11.00 | 1M | flagship | active | +| Nova Pro | amazon.nova-pro-v1:0 | Amazon | 0.80 | 3.20 | 300K | mid | active | +| Nova Pro (latency optimized) | — | Amazon | 1.00 | 4.00 | 300K | mid | active | +| Nova Lite | amazon.nova-lite-v1:0 | Amazon | 0.06 | 0.24 | 300K | fast | active | +| Nova Micro | amazon.nova-micro-v1:0 | Amazon | 0.035 | 0.14 | 128K | budget | active | +| Nova Premier | amazon.nova-premier-v1:0 | Amazon | 2.50 | 12.50 | 1M | reasoning | EOL (2026-09-14) | +| Mistral Large 3 | mistral.mistral-large-3-675b-instruct | Mistral | 0.50 | 1.50 | 256K | flagship | active | +| DeepSeek-R1 | deepseek.r1-v1:0 | DeepSeek | 1.35 | 5.40 | 128K | reasoning | active | +| DeepSeek-V3.1 | — | DeepSeek | 0.58 | 1.68 | — | mid | active (Sydney only) | +| gpt-oss-20b | openai.gpt-oss-20b-1:0 | OpenAI | 0.07 | 0.30 | 128K | budget | active | +| gpt-oss-120b | openai.gpt-oss-120b-1:0 | OpenAI | 0.15 | 0.60 | 128K | efficient | active | +| GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | +| GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | +| GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | +| Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | +| Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | +| Gemma 3 4B IT | google.gemma-3-4b-it | Google | 0.04 | 0.08 | 128K | budget | active | +| Gemma 3 12B IT | google.gemma-3-12b-it | Google | 0.09 | 0.29 | 128K | budget | active | +| Gemma 3 27B IT | google.gemma-3-27b-it | Google | 0.23 | 0.38 | 128K | efficient | active | +| MiniMax M2 | minimax.minimax-m2 | MiniMax | 0.30 | 1.20 | 1M | mid | active | +| MiniMax M2.1 | minimax.minimax-m2.1 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| MiniMax M2.5 | minimax.minimax-m2.5 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| Jamba 1.5 Large | ai21.jamba-1-5-large-v1:0 | AI21 Labs | 2.00 | 8.00 | — | mid | excluded (EOL 2026-11-26) | +| Jamba 1.5 Mini | ai21.jamba-1-5-mini-v1:0 | AI21 Labs | 0.20 | 0.40 | — | efficient | excluded (EOL 2026-11-26) | +| Jurassic-2 Mid | — | AI21 Labs | 12.50 | 12.50 | — | legacy | legacy | +| Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | +| Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | + +### Stability AI — Image Generation (per image, US East) + +Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. + +| Model | Model ID | Price/image | Resolution | Tier | Status | +| -------------------------- | --------------------------------- | ----------- | ---------- | -------- | ------ | +| Stable Image Ultra | stability.stable-image-ultra-v1:0 | $0.08 | up to 4MP | premium | active | +| Stable Diffusion 3.5 Large | stability.sd3-5-large-v1:0 | $0.065 | up to 1MP | flagship | active | +| Stable Image Core | stability.stable-image-core-v1:0 | $0.04 | up to 1MP | fast | active | + +Image editing services (inpaint, erase, upscale, etc.) are priced at $0.03–$0.60/operation. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full list. + +> **Cost comparison note:** DALL-E 3 (OpenAI) charges $0.04–$0.12/image depending on resolution. Google Imagen charges per character of prompt. When comparing, use per-image cost directly — do not convert Stability AI prices to per-token equivalents. + +### Anthropic Claude — batch & prompt cache (Standard, US East N. Virginia) + +Per 1M tokens unless noted. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full regional and tier tables. + +> **Batch support verified 2026-09-02** against the +> [supported Regions and models for batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) +> table: **Opus 4.6, Sonnet 4.6, Opus 4.5, Sonnet 4.5, Sonnet 4.5 — LC, and Haiku 4.5 are listed** (via cross-region +> inference profiles; batch is 50% of on-demand per the pricing page, matching the rows below). **Fable 5, Sonnet 5, +> and Opus 4.8 are NOT on that table** — Sonnet 5 and Opus 4.8's batch cells below are **unverified (‡)**: do not build a batch-discount +> TCO on them; price on-demand and note batch as a possible future saving pending the docs table listing them. +> Fable 5 / 5.1 instead carry **N/A** per the separate pricing-page verification below. +> Prompt-cache columns are unaffected (caching support is independent of batch). +> **Batch rate vs. cross-region inference (CRIS):** the confirmed rows show 50% of the **N. Virginia** on-demand +> price. Where batch is only reachable through a CRIS profile that lists above N. Virginia (e.g. Sonnet 4.6 in US East +> (Ohio) at ≈10% higher — see the on-demand section), the effective batch rate is 50% of that CRIS price, i.e. ≈10% +> above the cells here. Re-derive from the profile's on-demand rate for a region-specific quote. +> **Do not cross-check these rows against the AWS Pricing Calculator** — as of Sep 2026 the calculator carries no +> batch metered IDs for Opus 4.6 / Sonnet 4.6 and collapses other batch/cache SKUs (known calculator defect, not a +> pricing-page signal). The pricing page and the docs table above are the sources of truth. + +| Model | Batch in | Batch out | 5m cache write | 1h cache write | Cache read | +| ------------------------ | -------- | --------- | -------------- | -------------- | ---------- | +| Claude Fable 5 | N/A | N/A | 12.50 | 20.00 | 1.00 | +| Claude Fable 5.1 | N/A | N/A | 12.50 | 20.00 | 0.25 | +| Claude Sonnet 5 | 1.00 ‡ | 5.00 ‡ | 2.50 | 4.00 | 0.20 | +| Claude Opus 4.8 | 2.50 ‡ | 12.50 ‡ | 6.25 | 10.00 | 0.50 | +| Claude Sonnet 4.6 (+ LC) | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Opus 4.6 (+ LC) | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Opus 4.5 | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Haiku 4.5 | 0.50 | 2.50 | 1.25 | 2.00 | 0.10 | +| Claude Sonnet 4.5 | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Sonnet 4.5 — LC | 3.00 | 11.25 | 7.50 | 12.00 | 0.60 | + +_Batch: the Bedrock pricing page lists **N/A** for Claude Fable 5 and Fable 5.1 in both the Global and the Geo / In-region Anthropic tables (read 2026-09-03). Anthropic's first-party API offers a 50% batch discount on these models; Bedrock does not list one — do not assume a batch rate for Fable-class models._ + +‡ Not listed on the batch-supported models table as of 2026-09-02 — rate is the standard 50%-of-on-demand +projection, not a confirmed SKU. Mark these batch cells `_unverified_` in estimate output; the `unverified` gate in +`estimate-ai.md` (§ Pricing source caveat) treats any `_unverified_` row as blocking for a quoted figure — resolve +from the Bedrock pricing page first. + +### AI21 Labs + +**On-demand**, **US East (N. Virginia)** per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/). Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ---------------- | ---------- | ----------- | +| Jamba 1.5 Large | 2.00 | 8.00 | +| Jamba 1.5 Mini | 0.20 | 0.40 | +| Jurassic-2 Mid | 12.50 | 12.50 | +| Jurassic-2 Ultra | 18.80 | 18.80 | +| Jamba-Instruct | 0.50 | 0.70 | + +_Quick-reference rows use **—** for **model ID** and **context**; resolve in the Bedrock console or AWS model documentation._ + +### Mistral AI + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Mistral AI). **Priority** tier ≈ **+75%** vs Standard; **Flex** ≈ **−50%** vs Standard on the same page. + +#### US East (Ohio) — Pixtral (on-demand + batch header from page) + +| Model | Input $/1M | Output $/1M | Batch in | Batch out | +| --------------------- | ---------- | ----------- | -------- | --------- | +| Pixtral Large (25.02) | 2.00 | 6.00 | N/A | N/A | + +#### On-demand — US East (N. Virginia), US East (Ohio), US West (Oregon) + +Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Devstral 2 123B | 0.40 | 2.00 | +| Magistral Small 1.2 | 0.50 | 1.50 | +| Voxtral Mini 1.0 | 0.04 | 0.04 | +| Voxtral Small 1.0 | 0.10 | 0.30 | +| Ministral 3B 3.0 | 0.10 | 0.10 | +| Ministral 8B 3.0 | 0.15 | 0.15 | +| Ministral 14B 3.0 | 0.20 | 0.20 | +| Mistral Large 3 | 0.50 | 1.50 | + +_Rates differ in **Asia Pacific (Mumbai)**, **São Paulo**, **Tokyo**, **Europe (Ireland/Milan/London)**, **Sydney**, **Jakarta**, **Frankfurt**, **Stockholm** (e.g. Mistral Large 3 Mumbai **$0.59 / $1.76**). **Devstral 2 123B** appears in Jakarta / Frankfurt / Stockholm at **$0.48 / $2.40** on the same page._ + +### DeepSeek (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (DeepSeek). + +#### US East (Ohio) — Standard tier + +> **Note:** DeepSeek-V3.1 is no longer listed for US East regions on the Bedrock pricing page as of May 2026. Use DeepSeek-V3.2 for US East deployments. DeepSeek-V3.1 remains available in Asia Pacific (Sydney). + +| Model | Input $/1M | Output $/1M | +| ----------- | ---------- | ----------- | +| DeepSeek-R1 | 1.35 | 5.40 | + +#### US East (N. Virginia), US East (Ohio), US West (Oregon) + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| DeepSeek v3.2 | 0.62 | 1.85 | + +#### Other regions (examples on same page) + +- **Asia Pacific (Mumbai)**, **São Paulo**, **Jakarta**, **Tokyo**, **Stockholm**: DeepSeek v3.2 **$0.74 / $2.22** per 1M. +- **Asia Pacific (Sydney)**: DeepSeek v3.1 **$0.5974 / $1.7304**; v3.2 **$0.6386 / $1.9055** (and **Priority** / **Flex** tiers listed separately on the page). + +### OpenAI on Bedrock (GPT-5.6 and gpt-oss) + +See `shared/openai-on-bedrock.md` for model IDs, endpoint paths, regions, quotas, and caching rules. **Bedrock +in-region is priced at parity with OpenAI's _data residency_ tier — exactly 1.10x OpenAI's standard list price** — so +a same-model migration costs about 10% MORE, not the same. Do not describe it as cost-neutral. + +#### Proprietary GPT models — short context (272K), in-region + +Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally carries Terra, Luna and GPT-5.4; AWS GovCloud us-gov-west-1 / us-gov-east-1 carry Terra and Luna, us-gov-west-1 also GPT-5.4 (at its own rate below). Rates reflect the July 30, 2026 reduction (Luna −80%, Terra −20%) and the Aug 21, 2026 Sol reduction (−20% input / −33% output, promotional through at least Nov 21, 2026). + +| Model | Input $/1M | Output $/1M | Cache write $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | ---------------- | --------------- | +| GPT-5.6 Sol | 4.40 | 22.00 | 5.50 | 0.44 | +| GPT-5.6 Terra | 2.20 | 13.20 | 2.75 | 0.22 | +| GPT-5.6 Luna | 0.22 | 1.32 | 0.275 | 0.022 | +| GPT-5.5 | 5.50 | 33.00 | — | 0.55 | +| GPT-5.4 | 2.75 | 16.50 | — | 0.275 | + +#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| GPT-5.6 Sol | 8.80 | 33.00 | +| GPT-5.6 Terra | 4.40 | 19.80 | +| GPT-5.6 Luna | 0.44 | 1.98 | + +A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so +their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — +see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. + +> **One source conflict.** These models are absent +> from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and +> filtering on `GPT-5` or a `gpt-5` usage type returns zero rows (price-list publication 2026-08-04). An empty +> price-list result is **not** evidence the model is unavailable. (An earlier revision also +> flagged the AWS News Blog's Luna 0.20 / 1.20 as conflicting; it is the Global CRIS rate, not an error.) + +**Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum +1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. +GPT-5.5 and GPT-5.4 do not list prompt caching support. + +#### Open-weight gpt-oss + +| Model | Region | Input $/1M | Output $/1M | +| ------------ | --------------------- | ---------- | ----------- | +| gpt-oss-20b | US East (Ohio) | 0.07 | 0.30 | +| gpt-oss-120b | US East (Ohio) | 0.15 | 0.60 | +| gpt-oss-20b | Asia Pacific (Sydney) | 0.0721 | 0.3090 | +| gpt-oss-120b | Asia Pacific (Sydney) | 0.1545 | 0.6180 | + +_The Bedrock page also lists **Flex**, **Priority**, **Batch**, **GPT OSS Safeguard**, and **model customization** pricing for other regions._ + +### Meta Llama (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Meta). Prices per 1M input / output tokens unless noted. + +#### Llama 4 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------- | ------------ | ------------- | -------- | --------- | +| Llama 4 Maverick 17B | 0.24 | 0.97 | 0.12 | 0.485 | +| Llama 4 Scout 17B | 0.17 | 0.66 | 0.085 | 0.33 | + +#### Llama 3.3 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.3 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | + +#### Llama 3.2 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** All Llama 3.2 Instruct models reached **EOL on Jul 7, 2026** and are no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Scout / Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.2 Instruct (1B) | 0.10 | 0.10 | N/A | N/A | +| Llama 3.2 Instruct (3B) | 0.15 | 0.15 | N/A | N/A | +| Llama 3.2 Instruct (11B) | 0.16 | 0.16 | N/A | N/A | +| Llama 3.2 Instruct (90B) | 0.72 | 0.72 | N/A | N/A | + +#### Llama 3.2 — customization & provisioned throughput (US West Oregon) + +**Fine-tuning** (per 1M tokens trained / month storage / no-commit PT per model unit-hour): + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.2 Instruct (1B) | $0.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (3B) | $1.10 | $1.95 | $23.00 | +| Llama 3.2 Instruct (11B) | $3.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (90B) | $7.90 | $1.95 | $30.00 | + +**Provisioned throughput** ($/hour per model unit): + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.2 Instruct (1B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (3B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (11B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (90B) | $30.00 | $26.48 | $16.35 | + +#### Llama 3.1 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** Llama 3.1 405B Instruct reached **EOL on Jul 7, 2026** and is no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------------------------------------- | ------------ | ------------- | -------- | --------- | +| Llama 3.1 Instruct (8B) | 0.22 | 0.22 | 0.11 | 0.11 | +| Llama 3.1 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | +| Llama 3.1 Instruct (405B) | 2.40 | 2.40 | 1.20 | 1.20 | +| Llama 3.1 Instruct (70B) (latency optimized inf.) | 0.90 | 0.90 | N/A | N/A | +| Llama 3.1 Instruct (405B) (latency optimized inf.) | 3.00 | 3.00 | N/A | N/A | + +#### Llama 3.1 — customization & provisioned throughput (US West Oregon) + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.1 Instruct (8B) | $1.49 | $1.95 | $24.00 | +| Llama 3.1 Instruct (70B) | $7.99 | $1.95 | $24.00 | + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.1 Instruct (8B) | $24.00 | $21.18 | $13.08 | +| Llama 3.1 Instruct (70B) | $24.00 | $21.18 | $13.08 | + +#### Llama 3 — US East (N. Virginia), on-demand + +| Model | Input $/1M | Output $/1M | +| ---------------------- | ---------- | ----------- | +| Llama 3 Instruct (8B) | 0.30 | 0.60 | +| Llama 3 Instruct (70B) | 2.65 | 3.50 | + +#### Llama 2 — US East (N. Virginia) and US West (Oregon), on-demand + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Llama 2 Chat (13B) | 0.75 | 1.00 | +| Llama 2 Chat (70B) | 1.95 | 2.56 | + +#### Llama 2 — customization & provisioned throughput + +**Fine-tuning:** Llama 2 Pretrained (13B) **$1.49** per 1M tokens trained, **$1.95**/month storage, **$23.50**/unit-hr no-commit infer; (70B) **$7.99** / **$1.95** / **$23.50**. + +**Provisioned throughput** (per model unit-hour): 13B and 70B — **$21.18** (1-mo commit), **$13.08** (6-mo commit). Pretrained Llama 2 is **provisioned throughput only after customization** per AWS. + +### Amazon Nova + +All figures from [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Amazon Nova sections). Nova pricing depends on **inference deployment** (e.g. **Global cross-region** vs **Geo cross-region and in-region**), **tier** (Standard, Priority, Flex, Batch), **modality** (text / image / video / audio), and **AWS Region**. The quick-reference table above uses **Geo cross-region inference and in-region**, **Standard tier**, **US East (Ohio)**, **text + image + video** rates where a single input price applies to all three. Confirm the page before estimates. + +**Cache read (Nova):** AWS states cache read input tokens are **75% below** the on-demand input price for the applicable tier/modality. + +#### US East (Ohio) — Standard tier, text + image + video (Geo cross-region inference and in-region) + +Per 1M input / output tokens unless noted. + +| Model | Input $/1M | Output $/1M | +| ---------------------------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.33 | 2.75 | +| Amazon Nova Micro | 0.035 | 0.14 | +| Amazon Nova Lite | 0.06 | 0.24 | +| Amazon Nova Pro | 0.80 | 3.20 | +| Amazon Nova Pro (latency optimized inf.) | 1.00 | 4.00 | +| Amazon Nova Premier | 2.50 | 12.50 | + +#### US East (Ohio) — Standard tier, Global cross-region inference (text + image + video) + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.30 | 2.50 | + +#### US East (Ohio) — Standard tier, text / image / video / audio (Geo cross-region and in-region) + +Per 1M tokens. **Nova 2 Omni** and **Nova 2 Pro** are **Preview**. Image column is **output** image pricing where listed. + +| Model | Text in | Image in | Video in | Audio in | Text out | Image out | +| ---------------------------- | ------- | -------- | -------- | -------- | -------- | --------- | +| Amazon Nova 2 Omni (Preview) | 0.30 | 0.30 | 0.30 | 1.10 | 2.80 | 44.00 | +| Amazon Nova 2 Pro (Preview) | 1.375 | 1.375 | 1.375 | 1.375 | 11.00 | N/A | + +**Global cross-region inference (Ohio), Standard** uses different multimodal numbers on the same page — e.g. **Nova 2 Omni (Preview)** audio in **$1.00**, text out **$2.50**, image out **$40.00** (text/image/video in **$0.30** each). Use the Bedrock pricing page for the deployment you select. + +#### US East (Ohio) — Nova 2 Lite by tier (text + image + video) + +##### Global cross-region inference + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.30 | 2.50 | +| Priority | 0.525 | 4.375 | +| Flex | 0.15 | 1.25 | +| Batch | 0.15 | 1.25 | + +##### Geo cross-region inference and in-region + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.33 | 2.75 | +| Priority | 0.5775 | 4.8125 | +| Flex | 0.165 | 1.375 | +| Batch | 0.1595 | 1.342 | + +#### US East (Ohio) — Batch tier (text + image + video), selected models + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.1595 | 1.342 | +| Amazon Nova Micro | 0.0175 | 0.07 | +| Amazon Nova Lite | 0.03 | 0.12 | +| Amazon Nova Pro | 0.40 | 1.60 | +| Amazon Nova Premier | 1.25 | 6.25 | + +#### On-demand inference (listed rates, cache read column) + +| Model | Input $/1M | Cache read $/1M | Output $/1M | +| -------------------- | ---------- | --------------- | ----------- | +| Amazon Nova 2.0 Lite | N/A | N/A | N/A | +| Amazon Nova Micro | 0.035 | N/A | 0.14 | +| Amazon Nova Lite | 0.06 | N/A | 0.24 | +| Amazon Nova Pro | 0.80 | N/A | 3.20 | + +#### Built-in tools — Web grounding (US East Ohio) + +| Model | Price | +| ---------------------------- | ---------------------- | +| Amazon Nova 2 Omni (Preview) | $30.00 per 1K requests | +| Amazon Nova 2 Pro (Preview) | $30.00 per 1K requests | + +#### Creative — US East (N. Virginia) + +> **Lifecycle note:** Nova Canvas v1 and Nova Reel v1 are **excluded** — both are Legacy with EOL 2026-09-30, inside the 90-day exclusion zone. Do not list them in recommendation or comparison tables. Rates are retained for users already on them. See `vendored/ai/ai-model-lifecycle.md`. + +**Amazon Nova Canvas** (on-demand, per image): up to **1024×1024** — Standard **$0.04**, Premium **$0.06**; up to **2048×2048** — Standard **$0.06**, Premium **$0.08**. + +**Model customization (Nova Canvas):** **$0.005** per image seen; **$1.95**/month per custom model stored; provisioned inference per model unit per hour (no commit / 1-mo / 6-mo) **$60.50 / $55.00 / $30.25**. + +**Amazon Nova Reel** (video): **720p, 24 fps** — **$0.08** per second of video generated. + +#### Speech — US East (N. Virginia) + +> **Lifecycle note:** Nova Sonic v1 reached **EOL on 2026-09-14** and its requests now fail. Rates are retained only to explain the migration to **Nova 2 Sonic**. See `vendored/ai/ai-model-lifecycle.md`. + +Per 1M tokens. + +| Model | Modality | Input $/1M | Output $/1M | Status | +| ------------------- | -------- | ---------- | ----------- | ---------------- | +| Amazon Nova Sonic | Speech | 3.40 | 13.60 | EOL (2026-09-14) | +| Amazon Nova Sonic | Text | 0.06 | 0.24 | EOL (2026-09-14) | +| Amazon Nova 2 Sonic | Speech | 3.00 | 12.00 | active | +| Amazon Nova 2 Sonic | Text | 0.33 | 2.75 | active | + +#### Multimodal embeddings — US East (N. Virginia) + +| Offering | Text $/1M | Std image / doc image / video sec / audio sec | +| --------------------------------------------- | --------- | ------------------------------------------------ | +| Amazon Nova Multimodal Embeddings (On-demand) | 0.135 | $0.00006 / $0.0006 / $0.0007 / $0.00014 per unit | +| Amazon Nova Multimodal Embeddings (Batch) | 0.0675 | $0.00003 / $0.00048 / $0.00056 / $0.000112 | + +_On-demand inference for **custom Nova** models matches **base Nova** inference pricing per AWS._ + +--- diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index a5e154143..5a01723c3 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -34,6 +34,7 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/ai-migration-guardrails.md` | `skills/shared/ai/ai-migration-guardrails.md` | | `ai/ai-model-lifecycle.md` | `skills/shared/ai/ai-model-lifecycle.md` | | `ai/ai-openai-to-bedrock.md` | `skills/shared/ai/ai-openai-to-bedrock.md` | +| `ai/bedrock-pricing-cache.md` | `skills/shared/ai/bedrock-pricing-cache.md` | | `ai/bedrock-quotas.md` | `skills/shared/ai/bedrock-quotas.md` | | `ai/design-ref-agentic-to-agentcore.md` | `skills/shared/ai/design-ref-agentic-to-agentcore.md` | | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md index 87bc66b98..aeb78d0b1 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md @@ -116,7 +116,7 @@ Before recommending any Bedrock model in an agentic design: For agentic workload cost estimation: -1. **Primary:** `references/shared/pricing-cache.md` (±5-10% accuracy) +1. **Primary:** `references/vendored/ai/bedrock-pricing-cache.md` (±15-25% accuracy) 2. **Fallback:** `references/shared/pricing-fallback.md` (cache-miss → `estimated`/`unavailable`) AgentCore Runtime and Harness pricing: consumption-based, no upfront cost. Include in estimate only if the user selects Harness or Strands path. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-model-lifecycle.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-model-lifecycle.md index c3facc7bc..701d7e84e 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-model-lifecycle.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-model-lifecycle.md @@ -133,7 +133,7 @@ When building the model comparison table: - **Active**: no restrictions. - **Restricted** (`restricted (…)` in the pricing-cache Status column): never `recommended_model` or `backup_model`, never a default in a mapping guide. Include in `model_comparison` only when the user explicitly asks about frontier / Covered Models, annotated with the access requirement. -### Pricing Cache (`pricing-cache.md`) +### Pricing Cache (`bedrock-pricing-cache.md`) The multi-provider quick reference table includes a `Status` column: @@ -161,6 +161,6 @@ When refreshing the cache, recompute `days_to_eol` and refresh the `active` / `l **Newer models are not covered by the table.** A model launched on or after 2026-09-07 will never appear in the Legacy/EOL table above, and its absence is not evidence that it is Active. When a candidate is not in the table, verify it by calling `GetFoundationModel` (or `ListFoundationModels`) and reading `modelLifecycle.status`: `LEGACY` and `EOL` are never valid targets for a new migration. If the model is Legacy, read its model card for the actual EOL date and whether the Legacy period is 6 months or 45 days. If neither the API nor the card is reachable, treat the model's lifecycle as **unverified** and say so in the output rather than inferring `active` from a `Status` column in a pricing cache. -**Periodic table refresh:** When the table itself needs updating (new models added, EOL dates changed by AWS, or past-EOL rows to remove), update this file and `pricing-cache.md` together. Edit the canonical `skills/shared/ai/ai-model-lifecycle.md`, then copy it over every vendored copy in the same change so they stay byte-identical. +**Periodic table refresh:** When the table itself needs updating (new models added, EOL dates changed by AWS, or past-EOL rows to remove), update this file and `bedrock-pricing-cache.md` together. Edit the canonical files under `skills/shared/ai/`, then copy them over every vendored copy in the same change so they stay byte-identical. **Past-EOL rows:** Once `days_to_eol ≤ 0`, move the model out of the live table into **Removed**, and grep this plugin's skill trees for the model ID to catch any remaining reference to it as a target. Keep the Removed entry long enough that users already on the model still get a warning. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index cf41cb343..da1c5fcdf 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -131,7 +131,7 @@ See "Cost-optimized cross-family alternatives" below for the numbers. All figures per 1M tokens. Percentages are blended savings at a 2:1 input-to-output ratio. **Both sides of these comparisons are now Bedrock prices** where the OpenAI model is on Bedrock — this is a Bedrock-vs-Bedrock model -choice, not a provider comparison. Verify rates via `shared/pricing-cache.md` — pricing is cache-only, so a +choice, not a provider comparison. Verify rates via `references/vendored/ai/bedrock-pricing-cache.md` — pricing is cache-only, so a GPT-5.x rate that is absent or `_unverified_` in the cache has no live lookup to fall back to and must be reported as unavailable rather than priced (see `shared/openai-on-bedrock.md`). diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md new file mode 100644 index 000000000..b1f94f147 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -0,0 +1,449 @@ +# Amazon Bedrock Pricing Cache + +**Last updated:** 2026-09-03 +**Region:** us-east-1 +**Currency:** USD +**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) + +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> +> **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. + +--- +## Bedrock Models (On-Demand) + +**Anthropic Claude (Standard on-demand)** figures below match **US East (N. Virginia)** on [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) as of cache refresh. **Recommend defaults (new migrations):** Claude Sonnet 5 (flagship), Claude Opus 4.8 (hardest reasoning), Claude Haiku 4.5 (cost/speed). Do not default to any Claude Fable / Mythos frontier model (Fable 5, Fable 5.1, and successors). **Claude Fable 5** and **Claude Fable 5.1** (GA Sep 1, 2026) are the most expensive Anthropic models at $10/$50 per 1M tokens (Mythos-class); 5.1's on-demand rate difference is cache read at $0.25 (0.025x) vs $1.00; Bedrock lists no batch rate for either. Both are Anthropic **Covered Models**: using them requires opting the account into the `aws_review` data-retention mode, and Fable 5.1 is served commercially only via `us.`/`global.` CRIS profiles (no in-region — the bare `anthropic.claude-fable-5-1` id is not invokable on `bedrock-runtime`, hence the `us.` form in the table), with in-region access in AWS GovCloud (US) only. Claude Mythos 5.1 is a gated Preview for vetted cyber/bio research organizations and is not a migration target. **Claude Opus 4.8** keeps the same $5/$25 rate as Opus 4.6/4.7. **Claude Sonnet 5** launched June 30 at $2/$10; that launch rate became the standard price on Sep 1, 2026 (the scheduled increase to $3/$15 was cancelled), so Sonnet 5 is now both newer and cheaper than Sonnet 4.6 ($3/$15). **Claude Opus 4.7** lists the same headline on-demand input/output as **Opus 4.6** on that page; confirm **batch** availability per model (Opus 4.7 batch was **not** listed on the global cross-region table when this row was added). **Claude Opus 4.1** entered **Legacy** on Jul 8, 2026 (EOL Jan 8, 2027). **Batch**, **prompt cache** (5m / 1h write + cache read), and **geo / in-region cross-region** rows on that page can differ; e.g. **US East (Ohio)** cross-region inference for Claude Sonnet 4.6 is listed at **$3.30 / $16.50** per 1M input/output (≈10% above N. Virginia). Long-context SKUs **do not** all use the same multiplier: **Sonnet 4.6** and **Opus 4.6** long-context modes share the same on-demand rates as the non–long-context rows on the standard table; **Sonnet 4.5** and **Sonnet 4** long-context rows are priced higher on that same table. + +### Multi-provider quick reference (per 1M tokens) + +See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommend Legacy models for new migrations.** Rows whose Status is `restricted (…)` are never `recommended_model` / `backup_model` and never a default (Covered Models, gated previews). + +| Model | Model ID | Provider | Input $/1M | Output $/1M | Context | Tier | Status | +| -------------------------------- | ---------------------------------------- | --------- | ---------- | ----------- | ------- | --------- | ------------------------------------------------------------ | +| Claude Fable 5 | anthropic.claude-fable-5 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in) | +| Claude Fable 5.1 | us.anthropic.claude-fable-5-1 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in; CRIS only) | +| Claude Sonnet 5 | anthropic.claude-sonnet-5 | Anthropic | 2.00 | 10.00 | 1M | flagship | active ($2/$10 made the standard price Sep 1, 2026) | +| Claude Opus 4.8 | anthropic.claude-opus-4-8 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.6 | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.6 — Long Context | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K+ | flagship | active | +| Claude Opus 4.6 | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Opus 4.6 — Long Context | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K+ | premium | active | +| Claude Opus 4.5 | — | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.5 | — | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.5 — Long Context | — | Anthropic | 6.00 | 22.50 | 200K+ | flagship | active | +| Claude Haiku 4.5 | anthropic.claude-haiku-4-5-20251001-v1:0 | Anthropic | 1.00 | 5.00 | 200K | fast | active | +| Claude Opus 4.1 | anthropic.claude-opus-4-1-20250805-v1:0 | Anthropic | 15.00 | 75.00 | 200K | premium | legacy (EOL Jan 8, 2027) | +| Claude Sonnet 4 | anthropic.claude-sonnet-4-20250514-v1:0 | Anthropic | 3.00 | 15.00 | 200K | flagship | excluded (EOL Oct 14, 2026) | +| Llama 4 Maverick | meta.llama4-maverick-17b-instruct-v1:0 | Meta | 0.24 | 0.97 | 1M | mid | active | +| Llama 4 Scout | meta.llama4-scout-17b-instruct-v1:0 | Meta | 0.17 | 0.66 | 10M | efficient | active | +| Llama 3.3 70B | meta.llama3-3-70b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | active | +| Llama 3.2 90B | meta.llama3-2-90b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | EOL (Jul 7, 2026) | +| Nova 2 Lite | amazon.nova-2-lite-v1:0 | Amazon | 0.33 | 2.75 | 1M | mid | active | +| Nova 2 Pro (Preview) | amazon.nova-2-pro-v1:0 | Amazon | 1.375 | 11.00 | 1M | flagship | active | +| Nova Pro | amazon.nova-pro-v1:0 | Amazon | 0.80 | 3.20 | 300K | mid | active | +| Nova Pro (latency optimized) | — | Amazon | 1.00 | 4.00 | 300K | mid | active | +| Nova Lite | amazon.nova-lite-v1:0 | Amazon | 0.06 | 0.24 | 300K | fast | active | +| Nova Micro | amazon.nova-micro-v1:0 | Amazon | 0.035 | 0.14 | 128K | budget | active | +| Nova Premier | amazon.nova-premier-v1:0 | Amazon | 2.50 | 12.50 | 1M | reasoning | EOL (2026-09-14) | +| Mistral Large 3 | mistral.mistral-large-3-675b-instruct | Mistral | 0.50 | 1.50 | 256K | flagship | active | +| DeepSeek-R1 | deepseek.r1-v1:0 | DeepSeek | 1.35 | 5.40 | 128K | reasoning | active | +| DeepSeek-V3.1 | — | DeepSeek | 0.58 | 1.68 | — | mid | active (Sydney only) | +| gpt-oss-20b | openai.gpt-oss-20b-1:0 | OpenAI | 0.07 | 0.30 | 128K | budget | active | +| gpt-oss-120b | openai.gpt-oss-120b-1:0 | OpenAI | 0.15 | 0.60 | 128K | efficient | active | +| GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | +| GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | +| GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | +| Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | +| Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | +| Gemma 3 4B IT | google.gemma-3-4b-it | Google | 0.04 | 0.08 | 128K | budget | active | +| Gemma 3 12B IT | google.gemma-3-12b-it | Google | 0.09 | 0.29 | 128K | budget | active | +| Gemma 3 27B IT | google.gemma-3-27b-it | Google | 0.23 | 0.38 | 128K | efficient | active | +| MiniMax M2 | minimax.minimax-m2 | MiniMax | 0.30 | 1.20 | 1M | mid | active | +| MiniMax M2.1 | minimax.minimax-m2.1 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| MiniMax M2.5 | minimax.minimax-m2.5 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| Jamba 1.5 Large | ai21.jamba-1-5-large-v1:0 | AI21 Labs | 2.00 | 8.00 | — | mid | excluded (EOL 2026-11-26) | +| Jamba 1.5 Mini | ai21.jamba-1-5-mini-v1:0 | AI21 Labs | 0.20 | 0.40 | — | efficient | excluded (EOL 2026-11-26) | +| Jurassic-2 Mid | — | AI21 Labs | 12.50 | 12.50 | — | legacy | legacy | +| Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | +| Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | + +### Stability AI — Image Generation (per image, US East) + +Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. + +| Model | Model ID | Price/image | Resolution | Tier | Status | +| -------------------------- | --------------------------------- | ----------- | ---------- | -------- | ------ | +| Stable Image Ultra | stability.stable-image-ultra-v1:0 | $0.08 | up to 4MP | premium | active | +| Stable Diffusion 3.5 Large | stability.sd3-5-large-v1:0 | $0.065 | up to 1MP | flagship | active | +| Stable Image Core | stability.stable-image-core-v1:0 | $0.04 | up to 1MP | fast | active | + +Image editing services (inpaint, erase, upscale, etc.) are priced at $0.03–$0.60/operation. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full list. + +> **Cost comparison note:** DALL-E 3 (OpenAI) charges $0.04–$0.12/image depending on resolution. Google Imagen charges per character of prompt. When comparing, use per-image cost directly — do not convert Stability AI prices to per-token equivalents. + +### Anthropic Claude — batch & prompt cache (Standard, US East N. Virginia) + +Per 1M tokens unless noted. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full regional and tier tables. + +> **Batch support verified 2026-09-02** against the +> [supported Regions and models for batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) +> table: **Opus 4.6, Sonnet 4.6, Opus 4.5, Sonnet 4.5, Sonnet 4.5 — LC, and Haiku 4.5 are listed** (via cross-region +> inference profiles; batch is 50% of on-demand per the pricing page, matching the rows below). **Fable 5, Sonnet 5, +> and Opus 4.8 are NOT on that table** — Sonnet 5 and Opus 4.8's batch cells below are **unverified (‡)**: do not build a batch-discount +> TCO on them; price on-demand and note batch as a possible future saving pending the docs table listing them. +> Fable 5 / 5.1 instead carry **N/A** per the separate pricing-page verification below. +> Prompt-cache columns are unaffected (caching support is independent of batch). +> **Batch rate vs. cross-region inference (CRIS):** the confirmed rows show 50% of the **N. Virginia** on-demand +> price. Where batch is only reachable through a CRIS profile that lists above N. Virginia (e.g. Sonnet 4.6 in US East +> (Ohio) at ≈10% higher — see the on-demand section), the effective batch rate is 50% of that CRIS price, i.e. ≈10% +> above the cells here. Re-derive from the profile's on-demand rate for a region-specific quote. +> **Do not cross-check these rows against the AWS Pricing Calculator** — as of Sep 2026 the calculator carries no +> batch metered IDs for Opus 4.6 / Sonnet 4.6 and collapses other batch/cache SKUs (known calculator defect, not a +> pricing-page signal). The pricing page and the docs table above are the sources of truth. + +| Model | Batch in | Batch out | 5m cache write | 1h cache write | Cache read | +| ------------------------ | -------- | --------- | -------------- | -------------- | ---------- | +| Claude Fable 5 | N/A | N/A | 12.50 | 20.00 | 1.00 | +| Claude Fable 5.1 | N/A | N/A | 12.50 | 20.00 | 0.25 | +| Claude Sonnet 5 | 1.00 ‡ | 5.00 ‡ | 2.50 | 4.00 | 0.20 | +| Claude Opus 4.8 | 2.50 ‡ | 12.50 ‡ | 6.25 | 10.00 | 0.50 | +| Claude Sonnet 4.6 (+ LC) | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Opus 4.6 (+ LC) | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Opus 4.5 | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Haiku 4.5 | 0.50 | 2.50 | 1.25 | 2.00 | 0.10 | +| Claude Sonnet 4.5 | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Sonnet 4.5 — LC | 3.00 | 11.25 | 7.50 | 12.00 | 0.60 | + +_Batch: the Bedrock pricing page lists **N/A** for Claude Fable 5 and Fable 5.1 in both the Global and the Geo / In-region Anthropic tables (read 2026-09-03). Anthropic's first-party API offers a 50% batch discount on these models; Bedrock does not list one — do not assume a batch rate for Fable-class models._ + +‡ Not listed on the batch-supported models table as of 2026-09-02 — rate is the standard 50%-of-on-demand +projection, not a confirmed SKU. Mark these batch cells `_unverified_` in estimate output; the `unverified` gate in +`estimate-ai.md` (§ Pricing source caveat) treats any `_unverified_` row as blocking for a quoted figure — resolve +from the Bedrock pricing page first. + +### AI21 Labs + +**On-demand**, **US East (N. Virginia)** per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/). Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ---------------- | ---------- | ----------- | +| Jamba 1.5 Large | 2.00 | 8.00 | +| Jamba 1.5 Mini | 0.20 | 0.40 | +| Jurassic-2 Mid | 12.50 | 12.50 | +| Jurassic-2 Ultra | 18.80 | 18.80 | +| Jamba-Instruct | 0.50 | 0.70 | + +_Quick-reference rows use **—** for **model ID** and **context**; resolve in the Bedrock console or AWS model documentation._ + +### Mistral AI + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Mistral AI). **Priority** tier ≈ **+75%** vs Standard; **Flex** ≈ **−50%** vs Standard on the same page. + +#### US East (Ohio) — Pixtral (on-demand + batch header from page) + +| Model | Input $/1M | Output $/1M | Batch in | Batch out | +| --------------------- | ---------- | ----------- | -------- | --------- | +| Pixtral Large (25.02) | 2.00 | 6.00 | N/A | N/A | + +#### On-demand — US East (N. Virginia), US East (Ohio), US West (Oregon) + +Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Devstral 2 123B | 0.40 | 2.00 | +| Magistral Small 1.2 | 0.50 | 1.50 | +| Voxtral Mini 1.0 | 0.04 | 0.04 | +| Voxtral Small 1.0 | 0.10 | 0.30 | +| Ministral 3B 3.0 | 0.10 | 0.10 | +| Ministral 8B 3.0 | 0.15 | 0.15 | +| Ministral 14B 3.0 | 0.20 | 0.20 | +| Mistral Large 3 | 0.50 | 1.50 | + +_Rates differ in **Asia Pacific (Mumbai)**, **São Paulo**, **Tokyo**, **Europe (Ireland/Milan/London)**, **Sydney**, **Jakarta**, **Frankfurt**, **Stockholm** (e.g. Mistral Large 3 Mumbai **$0.59 / $1.76**). **Devstral 2 123B** appears in Jakarta / Frankfurt / Stockholm at **$0.48 / $2.40** on the same page._ + +### DeepSeek (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (DeepSeek). + +#### US East (Ohio) — Standard tier + +> **Note:** DeepSeek-V3.1 is no longer listed for US East regions on the Bedrock pricing page as of May 2026. Use DeepSeek-V3.2 for US East deployments. DeepSeek-V3.1 remains available in Asia Pacific (Sydney). + +| Model | Input $/1M | Output $/1M | +| ----------- | ---------- | ----------- | +| DeepSeek-R1 | 1.35 | 5.40 | + +#### US East (N. Virginia), US East (Ohio), US West (Oregon) + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| DeepSeek v3.2 | 0.62 | 1.85 | + +#### Other regions (examples on same page) + +- **Asia Pacific (Mumbai)**, **São Paulo**, **Jakarta**, **Tokyo**, **Stockholm**: DeepSeek v3.2 **$0.74 / $2.22** per 1M. +- **Asia Pacific (Sydney)**: DeepSeek v3.1 **$0.5974 / $1.7304**; v3.2 **$0.6386 / $1.9055** (and **Priority** / **Flex** tiers listed separately on the page). + +### OpenAI on Bedrock (GPT-5.6 and gpt-oss) + +See `shared/openai-on-bedrock.md` for model IDs, endpoint paths, regions, quotas, and caching rules. **Bedrock +in-region is priced at parity with OpenAI's _data residency_ tier — exactly 1.10x OpenAI's standard list price** — so +a same-model migration costs about 10% MORE, not the same. Do not describe it as cost-neutral. + +#### Proprietary GPT models — short context (272K), in-region + +Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally carries Terra, Luna and GPT-5.4; AWS GovCloud us-gov-west-1 / us-gov-east-1 carry Terra and Luna, us-gov-west-1 also GPT-5.4 (at its own rate below). Rates reflect the July 30, 2026 reduction (Luna −80%, Terra −20%) and the Aug 21, 2026 Sol reduction (−20% input / −33% output, promotional through at least Nov 21, 2026). + +| Model | Input $/1M | Output $/1M | Cache write $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | ---------------- | --------------- | +| GPT-5.6 Sol | 4.40 | 22.00 | 5.50 | 0.44 | +| GPT-5.6 Terra | 2.20 | 13.20 | 2.75 | 0.22 | +| GPT-5.6 Luna | 0.22 | 1.32 | 0.275 | 0.022 | +| GPT-5.5 | 5.50 | 33.00 | — | 0.55 | +| GPT-5.4 | 2.75 | 16.50 | — | 0.275 | + +#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| GPT-5.6 Sol | 8.80 | 33.00 | +| GPT-5.6 Terra | 4.40 | 19.80 | +| GPT-5.6 Luna | 0.44 | 1.98 | + +A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so +their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — +see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. + +> **One source conflict.** These models are absent +> from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and +> filtering on `GPT-5` or a `gpt-5` usage type returns zero rows (price-list publication 2026-08-04). An empty +> price-list result is **not** evidence the model is unavailable. (An earlier revision also +> flagged the AWS News Blog's Luna 0.20 / 1.20 as conflicting; it is the Global CRIS rate, not an error.) + +**Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum +1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. +GPT-5.5 and GPT-5.4 do not list prompt caching support. + +#### Open-weight gpt-oss + +| Model | Region | Input $/1M | Output $/1M | +| ------------ | --------------------- | ---------- | ----------- | +| gpt-oss-20b | US East (Ohio) | 0.07 | 0.30 | +| gpt-oss-120b | US East (Ohio) | 0.15 | 0.60 | +| gpt-oss-20b | Asia Pacific (Sydney) | 0.0721 | 0.3090 | +| gpt-oss-120b | Asia Pacific (Sydney) | 0.1545 | 0.6180 | + +_The Bedrock page also lists **Flex**, **Priority**, **Batch**, **GPT OSS Safeguard**, and **model customization** pricing for other regions._ + +### Meta Llama (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Meta). Prices per 1M input / output tokens unless noted. + +#### Llama 4 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------- | ------------ | ------------- | -------- | --------- | +| Llama 4 Maverick 17B | 0.24 | 0.97 | 0.12 | 0.485 | +| Llama 4 Scout 17B | 0.17 | 0.66 | 0.085 | 0.33 | + +#### Llama 3.3 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.3 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | + +#### Llama 3.2 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** All Llama 3.2 Instruct models reached **EOL on Jul 7, 2026** and are no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Scout / Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.2 Instruct (1B) | 0.10 | 0.10 | N/A | N/A | +| Llama 3.2 Instruct (3B) | 0.15 | 0.15 | N/A | N/A | +| Llama 3.2 Instruct (11B) | 0.16 | 0.16 | N/A | N/A | +| Llama 3.2 Instruct (90B) | 0.72 | 0.72 | N/A | N/A | + +#### Llama 3.2 — customization & provisioned throughput (US West Oregon) + +**Fine-tuning** (per 1M tokens trained / month storage / no-commit PT per model unit-hour): + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.2 Instruct (1B) | $0.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (3B) | $1.10 | $1.95 | $23.00 | +| Llama 3.2 Instruct (11B) | $3.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (90B) | $7.90 | $1.95 | $30.00 | + +**Provisioned throughput** ($/hour per model unit): + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.2 Instruct (1B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (3B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (11B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (90B) | $30.00 | $26.48 | $16.35 | + +#### Llama 3.1 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** Llama 3.1 405B Instruct reached **EOL on Jul 7, 2026** and is no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------------------------------------- | ------------ | ------------- | -------- | --------- | +| Llama 3.1 Instruct (8B) | 0.22 | 0.22 | 0.11 | 0.11 | +| Llama 3.1 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | +| Llama 3.1 Instruct (405B) | 2.40 | 2.40 | 1.20 | 1.20 | +| Llama 3.1 Instruct (70B) (latency optimized inf.) | 0.90 | 0.90 | N/A | N/A | +| Llama 3.1 Instruct (405B) (latency optimized inf.) | 3.00 | 3.00 | N/A | N/A | + +#### Llama 3.1 — customization & provisioned throughput (US West Oregon) + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.1 Instruct (8B) | $1.49 | $1.95 | $24.00 | +| Llama 3.1 Instruct (70B) | $7.99 | $1.95 | $24.00 | + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.1 Instruct (8B) | $24.00 | $21.18 | $13.08 | +| Llama 3.1 Instruct (70B) | $24.00 | $21.18 | $13.08 | + +#### Llama 3 — US East (N. Virginia), on-demand + +| Model | Input $/1M | Output $/1M | +| ---------------------- | ---------- | ----------- | +| Llama 3 Instruct (8B) | 0.30 | 0.60 | +| Llama 3 Instruct (70B) | 2.65 | 3.50 | + +#### Llama 2 — US East (N. Virginia) and US West (Oregon), on-demand + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Llama 2 Chat (13B) | 0.75 | 1.00 | +| Llama 2 Chat (70B) | 1.95 | 2.56 | + +#### Llama 2 — customization & provisioned throughput + +**Fine-tuning:** Llama 2 Pretrained (13B) **$1.49** per 1M tokens trained, **$1.95**/month storage, **$23.50**/unit-hr no-commit infer; (70B) **$7.99** / **$1.95** / **$23.50**. + +**Provisioned throughput** (per model unit-hour): 13B and 70B — **$21.18** (1-mo commit), **$13.08** (6-mo commit). Pretrained Llama 2 is **provisioned throughput only after customization** per AWS. + +### Amazon Nova + +All figures from [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Amazon Nova sections). Nova pricing depends on **inference deployment** (e.g. **Global cross-region** vs **Geo cross-region and in-region**), **tier** (Standard, Priority, Flex, Batch), **modality** (text / image / video / audio), and **AWS Region**. The quick-reference table above uses **Geo cross-region inference and in-region**, **Standard tier**, **US East (Ohio)**, **text + image + video** rates where a single input price applies to all three. Confirm the page before estimates. + +**Cache read (Nova):** AWS states cache read input tokens are **75% below** the on-demand input price for the applicable tier/modality. + +#### US East (Ohio) — Standard tier, text + image + video (Geo cross-region inference and in-region) + +Per 1M input / output tokens unless noted. + +| Model | Input $/1M | Output $/1M | +| ---------------------------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.33 | 2.75 | +| Amazon Nova Micro | 0.035 | 0.14 | +| Amazon Nova Lite | 0.06 | 0.24 | +| Amazon Nova Pro | 0.80 | 3.20 | +| Amazon Nova Pro (latency optimized inf.) | 1.00 | 4.00 | +| Amazon Nova Premier | 2.50 | 12.50 | + +#### US East (Ohio) — Standard tier, Global cross-region inference (text + image + video) + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.30 | 2.50 | + +#### US East (Ohio) — Standard tier, text / image / video / audio (Geo cross-region and in-region) + +Per 1M tokens. **Nova 2 Omni** and **Nova 2 Pro** are **Preview**. Image column is **output** image pricing where listed. + +| Model | Text in | Image in | Video in | Audio in | Text out | Image out | +| ---------------------------- | ------- | -------- | -------- | -------- | -------- | --------- | +| Amazon Nova 2 Omni (Preview) | 0.30 | 0.30 | 0.30 | 1.10 | 2.80 | 44.00 | +| Amazon Nova 2 Pro (Preview) | 1.375 | 1.375 | 1.375 | 1.375 | 11.00 | N/A | + +**Global cross-region inference (Ohio), Standard** uses different multimodal numbers on the same page — e.g. **Nova 2 Omni (Preview)** audio in **$1.00**, text out **$2.50**, image out **$40.00** (text/image/video in **$0.30** each). Use the Bedrock pricing page for the deployment you select. + +#### US East (Ohio) — Nova 2 Lite by tier (text + image + video) + +##### Global cross-region inference + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.30 | 2.50 | +| Priority | 0.525 | 4.375 | +| Flex | 0.15 | 1.25 | +| Batch | 0.15 | 1.25 | + +##### Geo cross-region inference and in-region + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.33 | 2.75 | +| Priority | 0.5775 | 4.8125 | +| Flex | 0.165 | 1.375 | +| Batch | 0.1595 | 1.342 | + +#### US East (Ohio) — Batch tier (text + image + video), selected models + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.1595 | 1.342 | +| Amazon Nova Micro | 0.0175 | 0.07 | +| Amazon Nova Lite | 0.03 | 0.12 | +| Amazon Nova Pro | 0.40 | 1.60 | +| Amazon Nova Premier | 1.25 | 6.25 | + +#### On-demand inference (listed rates, cache read column) + +| Model | Input $/1M | Cache read $/1M | Output $/1M | +| -------------------- | ---------- | --------------- | ----------- | +| Amazon Nova 2.0 Lite | N/A | N/A | N/A | +| Amazon Nova Micro | 0.035 | N/A | 0.14 | +| Amazon Nova Lite | 0.06 | N/A | 0.24 | +| Amazon Nova Pro | 0.80 | N/A | 3.20 | + +#### Built-in tools — Web grounding (US East Ohio) + +| Model | Price | +| ---------------------------- | ---------------------- | +| Amazon Nova 2 Omni (Preview) | $30.00 per 1K requests | +| Amazon Nova 2 Pro (Preview) | $30.00 per 1K requests | + +#### Creative — US East (N. Virginia) + +> **Lifecycle note:** Nova Canvas v1 and Nova Reel v1 are **excluded** — both are Legacy with EOL 2026-09-30, inside the 90-day exclusion zone. Do not list them in recommendation or comparison tables. Rates are retained for users already on them. See `vendored/ai/ai-model-lifecycle.md`. + +**Amazon Nova Canvas** (on-demand, per image): up to **1024×1024** — Standard **$0.04**, Premium **$0.06**; up to **2048×2048** — Standard **$0.06**, Premium **$0.08**. + +**Model customization (Nova Canvas):** **$0.005** per image seen; **$1.95**/month per custom model stored; provisioned inference per model unit per hour (no commit / 1-mo / 6-mo) **$60.50 / $55.00 / $30.25**. + +**Amazon Nova Reel** (video): **720p, 24 fps** — **$0.08** per second of video generated. + +#### Speech — US East (N. Virginia) + +> **Lifecycle note:** Nova Sonic v1 reached **EOL on 2026-09-14** and its requests now fail. Rates are retained only to explain the migration to **Nova 2 Sonic**. See `vendored/ai/ai-model-lifecycle.md`. + +Per 1M tokens. + +| Model | Modality | Input $/1M | Output $/1M | Status | +| ------------------- | -------- | ---------- | ----------- | ---------------- | +| Amazon Nova Sonic | Speech | 3.40 | 13.60 | EOL (2026-09-14) | +| Amazon Nova Sonic | Text | 0.06 | 0.24 | EOL (2026-09-14) | +| Amazon Nova 2 Sonic | Speech | 3.00 | 12.00 | active | +| Amazon Nova 2 Sonic | Text | 0.33 | 2.75 | active | + +#### Multimodal embeddings — US East (N. Virginia) + +| Offering | Text $/1M | Std image / doc image / video sec / audio sec | +| --------------------------------------------- | --------- | ------------------------------------------------ | +| Amazon Nova Multimodal Embeddings (On-demand) | 0.135 | $0.00006 / $0.0006 / $0.0007 / $0.00014 per unit | +| Amazon Nova Multimodal Embeddings (Batch) | 0.0675 | $0.00003 / $0.00048 / $0.00056 / $0.000112 | + +_On-demand inference for **custom Nova** models matches **base Nova** inference pricing per AWS._ + +--- diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md index 3d35f8b82..9e8d58990 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md @@ -255,7 +255,7 @@ Replace `MMDD-HHMM` with the actual migration ID, generate the `last_updated` IS - Provides `aws___search_documentation`, `aws___read_documentation`, `aws___list_regions`, `aws___get_regional_availability`, `aws___retrieve_skill` tools - Used during Design for regional availability checks and documentation lookups. -- Primary pricing source: `references/shared/pricing-cache.md` (cached 2026 rates, ±5-10% for infrastructure, ±15-25% for AI models). Pricing is cache-only — no live pricing MCP. +- Primary pricing sources: `references/shared/pricing-cache.md` (cached 2026 infrastructure and source-provider rates, ±5-25%) and `references/vendored/ai/bedrock-pricing-cache.md` (provider-neutral Bedrock rates, ±15-25%). Pricing is cache-only — no live pricing MCP. --- @@ -340,13 +340,15 @@ gcp-to-aws/ │ ├── validate-artifacts.md # Pre-report validation (Generate Step 0; read-only) │ ├── validate-migration-report.md # Post-write HTML completeness (Generate Step 4; also decision-report.html via --mode decision) │ ├── migration-complexity.md # Complexity tier definitions (small/medium/large) for timeline scaling -│ ├── pricing-cache.md # Cached AWS + source provider pricing (±5-25%, primary source) +│ ├── pricing-cache.md # Cached AWS infrastructure + source provider pricing +│ ├── gcp-infra-pricing-cache.md # Cached source-side GCP infrastructure pricing │ ├── graviton.md # Graviton/ARM64 tiers, mapping, per-phase rules (conditional load) │ └── schema-graviton.md # graviton_profile + cpu_architecture + architecture_comparison schemas │ └── references/vendored/ # byte-synced copies of skills/shared/ — DO NOT EDIT ├── workshop/workshop-invariants.md # cross-skill what-if workshop contract └── ai/ # plugin-neutral, source-cloud-agnostic AI content + ├── bedrock-pricing-cache.md # Canonical provider-neutral Bedrock model + service rates ├── ai-model-lifecycle.md # Bedrock Active/Legacy/EOL registry + 90-day exclusion rule ├── ai-migration-guardrails.md # shared constraints for every agentic migration path ├── bedrock-quotas.md # Bedrock TPM/RPM quota awareness, burndown rates, capacity planning @@ -361,7 +363,7 @@ gcp-to-aws/ | ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | No GCP sources found (no `.tf`, no app code, no billing data) | Offer live gcloud discovery per `discover.md` Step 1d. Only if declined or unavailable: Stop. Output: "No GCP sources detected. Provide at least one source type (Terraform files, application code, or billing exports), or re-run and accept live discovery." | | `.phase-status.json` missing phase gate | Stop. Output: "Cannot enter Phase X: Phase Y-1 not completed. Start from Phase Y or resume Phase Y-1." | -| Service not in pricing cache | Display user warning about ±5-25% accuracy. Use `pricing-cache.md`. Set `pricing_source: "unavailable"` for that service in the applicable `estimation-*.json` file. | +| Service not in applicable pricing cache | Display user warning about ±5-25% accuracy. Use the infrastructure/source-provider cache or Bedrock cache as applicable. Set `pricing_source: "unavailable"` for that service in the applicable `estimation-*.json` file. | | User skips questions or says "use defaults for the rest" | Apply documented defaults for all remaining questions (essential questions and any unconfirmed sheet rows in wizard mode; current and subsequent batches in full mode). Q2/Q3 defaults add a report caveat. Phase 2 completes either way. | | `aws-design.json` missing required clusters | Stop Phase 4. Output: "Re-run Phase 3 to generate missing cluster designs." | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/design-refs/ai-gemini-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/design-refs/ai-gemini-to-bedrock.md index 8de6891b9..abef0d9be 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/design-refs/ai-gemini-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/design-refs/ai-gemini-to-bedrock.md @@ -4,7 +4,7 @@ This file is loaded by `design-ai.md` when `ai-workload-profile.json` has `summary.ai_source` = `"gemini"` or `"both"`. It provides model mapping tables with pricing and honest competitive analysis for Gemini → Bedrock migration decisions. -Verify all pricing against `references/shared/pricing-cache.md`. +Verify source-provider pricing against `references/shared/pricing-cache.md` and Bedrock pricing against `references/vendored/ai/bedrock-pricing-cache.md`. **Model lifecycle:** Before recommending any Bedrock model, check `references/vendored/ai/ai-model-lifecycle.md`. Do not recommend Legacy models as primary selections for new migrations. Legacy models are annotated below where they appear. @@ -95,7 +95,7 @@ Gemini 3.5 Flash is now GA (May 2026) — the current flagship Flash model. Gemi | Gemini 1.5 Pro | **Legacy — EOL Sep 24, 2025. Migrate to Gemini 2.5 Pro or 3.x Pro.** | Claude Sonnet 5 | $2.00 / $10.00 | If still in use, migrate source model first | | text-bison / chat-bison | Legacy | Llama 4 Scout | $0.17 / $0.66 | Bedrock (better quality + cheaper) | | text-embedding-004 | $0.025 / N/A | Titan Embeddings V2 | $0.02 / N/A | Bedrock 20% cheaper | -| Gemini Embedding 2 (multimodal — text/image/video/audio/PDF) | text $0.20/1M, image $0.45/1M ($0.00012/img), audio $6.50/1M, video $12.00/1M | Amazon Nova Multimodal Embeddings | text $0.135/1M, image $0.00006/unit (see `shared/pricing-cache.md` § Multimodal embeddings) | Bedrock ~32% cheaper on text, ~50% cheaper on image | +| Gemini Embedding 2 (multimodal — text/image/video/audio/PDF) | text $0.20/1M, image $0.45/1M ($0.00012/img), audio $6.50/1M, video $12.00/1M | Amazon Nova Multimodal Embeddings | text $0.135/1M, image $0.00006/unit (see `references/vendored/ai/bedrock-pricing-cache.md` § Multimodal embeddings) | Bedrock ~32% cheaper on text, ~50% cheaper on image | | imagen-* | Varies | Stable Image Core | $0.04/img | Nova Canvas excluded (EOL Sep 30, 2026); Ultra $0.08/img if quality-first | _Percentages are blended savings using a 2:1 input-to-output token ratio. Actual savings depend on your input/output ratio._ diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index a82b2490b..8160794ba 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -94,7 +94,7 @@ Record what was applied: every constraint that changed a model choice or region ## Part 1: Bedrock Model Selection -**Restricted models — applies whichever guide Step 0 loaded and whatever the compliance value:** a row whose `Status` in `shared/pricing-cache.md` is `restricted (…)` (Covered Models such as Claude Fable 5 / 5.1, gated previews such as Claude Mythos) is never `recommended_model` or `backup_model` and never a default in any mapping guide. Offer one only when the user explicitly asks for a frontier model, and state its data-retention opt-in and any endpoint-specific regional or CRIS-only restrictions in the same sentence. +**Restricted models — applies whichever guide Step 0 loaded and whatever the compliance value:** a row whose `Status` in `references/vendored/ai/bedrock-pricing-cache.md` is `restricted (…)` (Covered Models such as Claude Fable 5 / 5.1, gated previews such as Claude Mythos) is never `recommended_model` or `backup_model` and never a default in any mapping guide. Offer one only when the user explicitly asks for a frontier model, and state its data-retention opt-in and any endpoint-specific regional or CRIS-only restrictions in the same sentence. **Multi-workload iteration (when `workloads[]` is present):** @@ -188,7 +188,7 @@ Treat model mapping as compatibility-guided, not 1:1 parity. Before cutover, req | `ai_priority = "quality"` | Prefer Claude Sonnet/Opus regardless of cost | | `ai_priority = "speed"` | Prefer Claude Sonnet (fastest integration) | | `ai_latency = "critical"` | Prefer smaller/faster models (Haiku, Nova Lite) | -| `ai_latency = "flexible"` | Any model; flag Batch API for 50% savings — but only for confirmed-batch models (skip the discount for models whose batch cells are `_unverified_` in `shared/pricing-cache.md`) | +| `ai_latency = "flexible"` | Any model; flag Batch API for 50% savings — but only for confirmed-batch models (skip the discount for models whose batch cells are `_unverified_` in `references/vendored/ai/bedrock-pricing-cache.md`) | **Stay-or-migrate assessment per model:** diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md index 49d0a4cc5..370d87e1b 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md @@ -80,7 +80,7 @@ for spend, AI-only Q7 for usage volume). For each model in `models[]` of `ai-workload-profile.json`, map to the closest Bedrock equivalent using the table below, then look up both source and Bedrock per-token prices -from `references/shared/pricing-cache.md` (Source Provider Pricing + Bedrock Models sections). +from `references/shared/pricing-cache.md` (Source Provider Pricing) and `references/vendored/ai/bedrock-pricing-cache.md` (Bedrock Models). **Source model → Bedrock equivalent mapping:** diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md index 1c624aad3..ed85b4884 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md @@ -10,17 +10,18 @@ The parent `estimate.md` selects the pricing mode before loading this file. **Price lookup order:** -1. **`shared/pricing-cache.md` (primary)** — Look up Bedrock model pricing and source provider pricing by table. Set `pricing_source: "cached"`. -2. **Unavailable** — If a model is NOT in the cache, set `pricing_source: "unavailable"` and warn the user. An `_unverified_` cache cell is blocking — do not use it as if it were a confirmed price. +1. **`references/vendored/ai/bedrock-pricing-cache.md` (primary for Bedrock)** — Look up Bedrock model and service pricing by table. Set `pricing_source: "cached"`. +2. **`shared/pricing-cache.md` (primary for source providers)** — Look up Gemini and OpenAI source-provider comparison pricing by table. Set `pricing_source: "cached"`. +3. **Unavailable** — If a model is NOT in the applicable cache, set `pricing_source: "unavailable"` and warn the user. An `_unverified_` cache cell is blocking — do not use it as if it were a confirmed price. -For typical migrations (Claude, Llama, Nova, Mistral, DeepSeek, Gemma, OpenAI gpt-oss, Gemini source pricing), ALL prices are in `pricing-cache.md`. Zero MCP calls needed. +For typical migrations (Claude, Llama, Nova, Mistral, DeepSeek, Gemma, OpenAI gpt-oss, Gemini source pricing), all prices are in the two caches. Zero MCP calls needed. **Model lifecycle:** When building the model comparison table, check `references/vendored/ai/ai-model-lifecycle.md` and apply the 90-day exclusion rule: - **Excluded** (≤90 days to EOL): omit entirely from `model_comparison`, `recommended_model`, and `backup_model`. - **Legacy** (>90 days to EOL): include in `model_comparison` with `(Legacy — EOL YYYY-MM-DD)` annotation. Do not select as `recommended_model` unless no Active alternative exists. - **Active**: no restrictions. -- **Restricted** (`restricted (…)` in the `pricing-cache.md` Status column — Covered Models such as Claude Fable 5 / 5.1, gated previews such as Claude Mythos): never `recommended_model` or `backup_model`. Include in `model_comparison` only when the user explicitly asked for a frontier model, annotated with the data-retention opt-in it requires. +- **Restricted** (`restricted (…)` in the `references/vendored/ai/bedrock-pricing-cache.md` Status column — Covered Models such as Claude Fable 5 / 5.1, gated previews such as Claude Mythos): never `recommended_model` or `backup_model`. Include in `model_comparison` only when the user explicitly asked for a frontier model, annotated with the data-retention opt-in it requires. ## Prerequisites @@ -125,7 +126,7 @@ Reference `aws-design-ai.json` → `honest_assessment`. If `"recommend_stay"`, p **Pricing source caveat (all providers):** a `pricing-cache.md` cell marked `_unverified_` is **blocking for any quoted figure, whatever the provider** — resolve it from the Bedrock pricing page or the model card before the row enters `model_comparison` or the ROI table; never substitute a guess or a same-tier sibling's rate. An `_unverified_` cache cell is not evidence the model is unavailable or free — it means the rate was not confirmed when the cache was last updated. See `shared/openai-on-bedrock.md`. -**Unverified gate (all providers, not just OpenAI):** any cell marked `_unverified_` in `shared/pricing-cache.md` — including Anthropic batch cells for models not yet on the [batch-supported models table](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) (Sonnet 5, Opus 4.8 as of 2026-09-02) — is blocking for any quoted figure that depends on it. Do not apply a batch discount to an `_unverified_` batch cell; price on-demand and note batch as a possible future saving, or resolve the rate from the Bedrock pricing page first. +**Unverified gate (all providers, not just OpenAI):** any cell marked `_unverified_` in `references/vendored/ai/bedrock-pricing-cache.md` — including Anthropic batch cells for models not yet on the [batch-supported models table](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) (Sonnet 5, Opus 4.8 as of 2026-09-02) — is blocking for any quoted figure that depends on it. Do not apply a batch discount to an `_unverified_` batch cell; price on-demand and note batch as a possible future saving, or resolve the rate from the Bedrock pricing page first. **Note:** Human/professional-services one-time migration costs are intentionally out of scope for this advisor and excluded from ROI calculations. @@ -143,7 +144,7 @@ Present applicable optimizations with estimated savings. **Every entry requires | -------------------------- | ---------------------------- | -------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Model downsizing / tiering | `model_tiering` | 60-87% | `"none"` | High volume, premium model selected. Available: Day 1. | | Prompt caching | `prompt_caching` | up to 90% on cached input tokens (workload- and cache-hit-rate dependent — see caveat below) | `"none"` | Repeated system prompts, long cacheable context. Available: Day 1. | -| Batch API | `batch_api` | 50% of on-demand | `"none"` | Non-real-time workloads (`ai_latency = "flexible"`) — **confirmed-batch models only**; models whose batch cells are `_unverified_` in `shared/pricing-cache.md` must not be quoted with a batch discount. Available: Day 1. | +| Batch API | `batch_api` | 50% of on-demand | `"none"` | Non-real-time workloads (`ai_latency = "flexible"`) — **confirmed-batch models only**; models whose batch cells are `_unverified_` in `references/vendored/ai/bedrock-pricing-cache.md` must not be quoted with a batch discount. Available: Day 1. | | Input token reduction | `input_token_reduction` | 10-30% | `"none"` | Prompt optimization, shorter context. Available: Day 1. | | Multi-model tiered routing | `multi_model_routing` | 60-87% | `"none"` | High/very-high volume, `tiered_strategy` in design. Available: Day 1 (design-time choice). | | Intelligent Prompt Routing | `intelligent_prompt_routing` | up to 30% (AWS-published ceiling) | `"none"` | Same model family (Anthropic or Meta Llama by default; additional models via configurable routers) available in 2+ tiers, latency-tolerant routing. **Does not apply to a Gemini-only or OpenAI-only Bedrock stack** — check the design's model family has a router available. Available: after 2+ weeks production traffic (this plugin's caution, not an AWS-imposed minimum -- AWS documents no waiting period). | @@ -283,7 +284,7 @@ All cost values are numbers, not strings. Output must be valid JSON. - [ ] `model_comparison` includes ALL viable Bedrock models, not just recommended - [ ] Legacy models in `model_comparison` are annotated with EOL dates (per `vendored/ai/ai-model-lifecycle.md`) - [ ] `recommended_model` is an Active model (not Legacy) unless no Active alternative exists -- [ ] Neither `recommended_model` nor `backup_model` carries a `restricted (…)` Status in `shared/pricing-cache.md` +- [ ] Neither `recommended_model` nor `backup_model` carries a `restricted (…)` Status in `references/vendored/ai/bedrock-pricing-cache.md` - [ ] Every model has `capabilities_match` checked against `ai_capabilities_required` - [ ] `recommended_model.rationale` references user's priority, preference, and volume - [ ] `roi_analysis` is honest — if migration increases cost, says so diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md index 616d62259..15371312f 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md @@ -6,12 +6,12 @@ Before running any sub-estimate file, determine the pricing source. -### Step 0a: Load Pricing Cache +### Step 0a: Load Pricing Caches -Read `shared/pricing-cache.md`. Check the `Last updated` date in the header: +Read `shared/pricing-cache.md` for AWS infrastructure and source-provider rates. When the AI route is active, also read `references/vendored/ai/bedrock-pricing-cache.md` for Bedrock model and service rates. Check each cache's `Last updated` date in its header: -- If <= 90 days old: **Cached prices are the primary source.** No MCP calls needed for services listed in the cache. Proceed to Step 1. -- If > 90 days old: Cache is stale. There is no live pricing lookup to fall back to — keep using the cached rates, widen the stated accuracy band, and surface the staleness warning in Step 0b. +- If every applicable cache is <= 90 days old: **Cached prices are the primary source.** No MCP calls needed for services listed in the caches. Proceed to Step 1. +- If an applicable cache is > 90 days old: That cache is stale. There is no live pricing lookup to fall back to — keep using the cached rates, widen the stated accuracy band, and surface the staleness warning in Step 0b. ### Step 0b: Surface Status to User (ALWAYS run) @@ -27,15 +27,16 @@ This prevents silent failures — the user sees the pricing constraint upfront, Each sub-estimate file uses this lookup order per service: -1. **`shared/pricing-cache.md`** (primary) — Cached prices (±5-25% accuracy). Set `pricing_source: "cached"`. Used first because it requires zero API calls and covers most common services. -2. **Unavailable** — If a service is NOT in the cache, set `pricing_source: "unavailable"` for that service. Add the service to `services_with_missing_fallback` and display a warning to the user: "Pricing unavailable for [service] — not in cache. Exclude from totals or provide a manual estimate." +1. **`shared/pricing-cache.md`** (primary for infrastructure and source providers) — Cached prices (±5-25% accuracy). Set `pricing_source: "cached"`. +2. **`references/vendored/ai/bedrock-pricing-cache.md`** (primary for Bedrock models and services) — Cached prices (±15-25% accuracy). Set `pricing_source: "cached"`. +3. **Unavailable** — If a service is NOT in its applicable cache, set `pricing_source: "unavailable"` for that service. Add the service to `services_with_missing_fallback` and display a warning to the user: "Pricing unavailable for [service] — not in cache. Exclude from totals or provide a manual estimate." **`pricing_source` values summary:** | Value | Meaning | | --------------- | ------------------------------------------ | -| `"cached"` | Found in pricing-cache.md (normal path) | -| `"unavailable"` | Not in cache; service excluded from totals | +| `"cached"` | Found in the applicable pricing cache (normal path) | +| `"unavailable"` | Not in cache; service excluded from totals | If cache is > 90 days old: @@ -264,7 +265,8 @@ _Breadcrumbs are emitted only after outer-run `HANDOFF_OK` — never on `GATE_FA ## Reference Files -- `shared/pricing-cache.md` — Cached AWS + source provider pricing (±5-25%, primary source) +- `shared/pricing-cache.md` — Cached AWS infrastructure + source-provider pricing (±5-25%) +- `references/vendored/ai/bedrock-pricing-cache.md` — Cached provider-neutral Bedrock pricing (±15-25%) ## Scope Boundary diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md index 5ab82810a..8fe56f258 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md @@ -244,7 +244,7 @@ customer's IAM policies, inside their VPC, logged to CloudTrail, and in-region i **Codex on Bedrock: unverified — do not price it.** An earlier revision of this file stated Codex is GA on Bedrock with pay-per-token pricing. As of 2026-09-02, Codex does not appear on the [OpenAI model card index](https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html), and no -Codex rate exists on the Bedrock pricing page, in `pricing-cache.md`, or in `bedrock_pricing.py`'s static table. +Codex rate exists on the Bedrock pricing page, in `references/vendored/ai/bedrock-pricing-cache.md`, or in `bedrock_pricing.py`'s static table. When the source workload is a coding agent, re-check the model card index first; if Codex is still absent, treat it as "a model not in the catalog above" (see the table in this section): plan a cross-family target (GPT-5.6 Sol / Terra are the coding-agent tier fits) and price that target — or report `pricing_source: "unverified"` with no @@ -264,7 +264,7 @@ This model family is moving fast (two GA waves and a repricing inside 10 weeks). 4. Recheck whether the Price List API has gained GPT-5.x coverage; if it has, drop the caveat above and let `estimate-ai.md` price these models from the MCP. 5. Recheck whether Chat Completions and `bedrock-runtime` support have been added or clarified. -6. Feed any lifecycle change into `ai-model-lifecycle.md` and any rate change into `pricing-cache.md`. +6. Feed any lifecycle change into `ai-model-lifecycle.md` and any rate change into `references/vendored/ai/bedrock-pricing-cache.md`. 7. **Re-verify within 14 days of any merge touching this file.** Item 5's prediction fired on 2026-08-21: between 2026-08-10 and 2026-08-21 the GPT-5.6 family gained a `bedrock-runtime`/CRIS path, published Global CRIS pricing at standard-price parity, and listed Chat Completions/Converse as supported — invalidating three of this file's diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md index eb2deb492..574feb68c 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md @@ -383,442 +383,11 @@ Serverless inference: $0.0000200 per second per GB memory. --- -## Bedrock Models (On-Demand) - -**Anthropic Claude (Standard on-demand)** figures below match **US East (N. Virginia)** on [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) as of cache refresh. **Recommend defaults (new migrations):** Claude Sonnet 5 (flagship), Claude Opus 4.8 (hardest reasoning), Claude Haiku 4.5 (cost/speed). Do not default to any Claude Fable / Mythos frontier model (Fable 5, Fable 5.1, and successors). **Claude Fable 5** and **Claude Fable 5.1** (GA Sep 1, 2026) are the most expensive Anthropic models at $10/$50 per 1M tokens (Mythos-class); 5.1's on-demand rate difference is cache read at $0.25 (0.025x) vs $1.00; Bedrock lists no batch rate for either. Both are Anthropic **Covered Models**: using them requires opting the account into the `aws_review` data-retention mode, and Fable 5.1 is served commercially only via `us.`/`global.` CRIS profiles (no in-region — the bare `anthropic.claude-fable-5-1` id is not invokable on `bedrock-runtime`, hence the `us.` form in the table), with in-region access in AWS GovCloud (US) only. Claude Mythos 5.1 is a gated Preview for vetted cyber/bio research organizations and is not a migration target. **Claude Opus 4.8** keeps the same $5/$25 rate as Opus 4.6/4.7. **Claude Sonnet 5** launched June 30 at $2/$10; that launch rate became the standard price on Sep 1, 2026 (the scheduled increase to $3/$15 was cancelled), so Sonnet 5 is now both newer and cheaper than Sonnet 4.6 ($3/$15). **Claude Opus 4.7** lists the same headline on-demand input/output as **Opus 4.6** on that page; confirm **batch** availability per model (Opus 4.7 batch was **not** listed on the global cross-region table when this row was added). **Claude Opus 4.1** entered **Legacy** on Jul 8, 2026 (EOL Jan 8, 2027). **Batch**, **prompt cache** (5m / 1h write + cache read), and **geo / in-region cross-region** rows on that page can differ; e.g. **US East (Ohio)** cross-region inference for Claude Sonnet 4.6 is listed at **$3.30 / $16.50** per 1M input/output (≈10% above N. Virginia). Long-context SKUs **do not** all use the same multiplier: **Sonnet 4.6** and **Opus 4.6** long-context modes share the same on-demand rates as the non–long-context rows on the standard table; **Sonnet 4.5** and **Sonnet 4** long-context rows are priced higher on that same table. - -### Multi-provider quick reference (per 1M tokens) - -See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommend Legacy models for new migrations.** Rows whose Status is `restricted (…)` are never `recommended_model` / `backup_model` and never a default (Covered Models, gated previews). - -| Model | Model ID | Provider | Input $/1M | Output $/1M | Context | Tier | Status | -| -------------------------------- | ---------------------------------------- | --------- | ---------- | ----------- | ------- | --------- | ------------------------------------------------------------ | -| Claude Fable 5 | anthropic.claude-fable-5 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in) | -| Claude Fable 5.1 | us.anthropic.claude-fable-5-1 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in; CRIS only) | -| Claude Sonnet 5 | anthropic.claude-sonnet-5 | Anthropic | 2.00 | 10.00 | 1M | flagship | active ($2/$10 made the standard price Sep 1, 2026) | -| Claude Opus 4.8 | anthropic.claude-opus-4-8 | Anthropic | 5.00 | 25.00 | 200K | premium | active | -| Claude Sonnet 4.6 | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K | flagship | active | -| Claude Sonnet 4.6 — Long Context | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K+ | flagship | active | -| Claude Opus 4.6 | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K | premium | active | -| Claude Opus 4.6 — Long Context | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K+ | premium | active | -| Claude Opus 4.5 | — | Anthropic | 5.00 | 25.00 | 200K | premium | active | -| Claude Sonnet 4.5 | — | Anthropic | 3.00 | 15.00 | 200K | flagship | active | -| Claude Sonnet 4.5 — Long Context | — | Anthropic | 6.00 | 22.50 | 200K+ | flagship | active | -| Claude Haiku 4.5 | anthropic.claude-haiku-4-5-20251001-v1:0 | Anthropic | 1.00 | 5.00 | 200K | fast | active | -| Claude Opus 4.1 | anthropic.claude-opus-4-1-20250805-v1:0 | Anthropic | 15.00 | 75.00 | 200K | premium | legacy (EOL Jan 8, 2027) | -| Claude Sonnet 4 | anthropic.claude-sonnet-4-20250514-v1:0 | Anthropic | 3.00 | 15.00 | 200K | flagship | excluded (EOL Oct 14, 2026) | -| Llama 4 Maverick | meta.llama4-maverick-17b-instruct-v1:0 | Meta | 0.24 | 0.97 | 1M | mid | active | -| Llama 4 Scout | meta.llama4-scout-17b-instruct-v1:0 | Meta | 0.17 | 0.66 | 10M | efficient | active | -| Llama 3.3 70B | meta.llama3-3-70b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | active | -| Llama 3.2 90B | meta.llama3-2-90b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | EOL (Jul 7, 2026) | -| Nova 2 Lite | amazon.nova-2-lite-v1:0 | Amazon | 0.33 | 2.75 | 1M | mid | active | -| Nova 2 Pro (Preview) | amazon.nova-2-pro-v1:0 | Amazon | 1.375 | 11.00 | 1M | flagship | active | -| Nova Pro | amazon.nova-pro-v1:0 | Amazon | 0.80 | 3.20 | 300K | mid | active | -| Nova Pro (latency optimized) | — | Amazon | 1.00 | 4.00 | 300K | mid | active | -| Nova Lite | amazon.nova-lite-v1:0 | Amazon | 0.06 | 0.24 | 300K | fast | active | -| Nova Micro | amazon.nova-micro-v1:0 | Amazon | 0.035 | 0.14 | 128K | budget | active | -| Nova Premier | amazon.nova-premier-v1:0 | Amazon | 2.50 | 12.50 | 1M | reasoning | EOL (2026-09-14) | -| Mistral Large 3 | mistral.mistral-large-3-675b-instruct | Mistral | 0.50 | 1.50 | 256K | flagship | active | -| DeepSeek-R1 | deepseek.r1-v1:0 | DeepSeek | 1.35 | 5.40 | 128K | reasoning | active | -| DeepSeek-V3.1 | — | DeepSeek | 0.58 | 1.68 | — | mid | active (Sydney only) | -| gpt-oss-20b | openai.gpt-oss-20b-1:0 | OpenAI | 0.07 | 0.30 | 128K | budget | active | -| gpt-oss-120b | openai.gpt-oss-120b-1:0 | OpenAI | 0.15 | 0.60 | 128K | efficient | active | -| GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | -| GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | -| GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | -| Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | -| Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | -| Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | -| Gemma 3 4B IT | google.gemma-3-4b-it | Google | 0.04 | 0.08 | 128K | budget | active | -| Gemma 3 12B IT | google.gemma-3-12b-it | Google | 0.09 | 0.29 | 128K | budget | active | -| Gemma 3 27B IT | google.gemma-3-27b-it | Google | 0.23 | 0.38 | 128K | efficient | active | -| MiniMax M2 | minimax.minimax-m2 | MiniMax | 0.30 | 1.20 | 1M | mid | active | -| MiniMax M2.1 | minimax.minimax-m2.1 | MiniMax | 0.30 | 1.20 | 196K | mid | active | -| MiniMax M2.5 | minimax.minimax-m2.5 | MiniMax | 0.30 | 1.20 | 196K | mid | active | -| Jamba 1.5 Large | ai21.jamba-1-5-large-v1:0 | AI21 Labs | 2.00 | 8.00 | — | mid | excluded (EOL 2026-11-26) | -| Jamba 1.5 Mini | ai21.jamba-1-5-mini-v1:0 | AI21 Labs | 0.20 | 0.40 | — | efficient | excluded (EOL 2026-11-26) | -| Jurassic-2 Mid | — | AI21 Labs | 12.50 | 12.50 | — | legacy | legacy | -| Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | -| Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | - -### Stability AI — Image Generation (per image, US East) - -Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. - -| Model | Model ID | Price/image | Resolution | Tier | Status | -| -------------------------- | --------------------------------- | ----------- | ---------- | -------- | ------ | -| Stable Image Ultra | stability.stable-image-ultra-v1:0 | $0.08 | up to 4MP | premium | active | -| Stable Diffusion 3.5 Large | stability.sd3-5-large-v1:0 | $0.065 | up to 1MP | flagship | active | -| Stable Image Core | stability.stable-image-core-v1:0 | $0.04 | up to 1MP | fast | active | - -Image editing services (inpaint, erase, upscale, etc.) are priced at $0.03–$0.60/operation. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full list. - -> **Cost comparison note:** DALL-E 3 (OpenAI) charges $0.04–$0.12/image depending on resolution. Google Imagen charges per character of prompt. When comparing, use per-image cost directly — do not convert Stability AI prices to per-token equivalents. - -### Anthropic Claude — batch & prompt cache (Standard, US East N. Virginia) - -Per 1M tokens unless noted. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full regional and tier tables. - -> **Batch support verified 2026-09-02** against the -> [supported Regions and models for batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) -> table: **Opus 4.6, Sonnet 4.6, Opus 4.5, Sonnet 4.5, Sonnet 4.5 — LC, and Haiku 4.5 are listed** (via cross-region -> inference profiles; batch is 50% of on-demand per the pricing page, matching the rows below). **Fable 5, Sonnet 5, -> and Opus 4.8 are NOT on that table** — Sonnet 5 and Opus 4.8's batch cells below are **unverified (‡)**: do not build a batch-discount -> TCO on them; price on-demand and note batch as a possible future saving pending the docs table listing them. -> Fable 5 / 5.1 instead carry **N/A** per the separate pricing-page verification below. -> Prompt-cache columns are unaffected (caching support is independent of batch). -> **Batch rate vs. cross-region inference (CRIS):** the confirmed rows show 50% of the **N. Virginia** on-demand -> price. Where batch is only reachable through a CRIS profile that lists above N. Virginia (e.g. Sonnet 4.6 in US East -> (Ohio) at ≈10% higher — see the on-demand section), the effective batch rate is 50% of that CRIS price, i.e. ≈10% -> above the cells here. Re-derive from the profile's on-demand rate for a region-specific quote. -> **Do not cross-check these rows against the AWS Pricing Calculator** — as of Sep 2026 the calculator carries no -> batch metered IDs for Opus 4.6 / Sonnet 4.6 and collapses other batch/cache SKUs (known calculator defect, not a -> pricing-page signal). The pricing page and the docs table above are the sources of truth. - -| Model | Batch in | Batch out | 5m cache write | 1h cache write | Cache read | -| ------------------------ | -------- | --------- | -------------- | -------------- | ---------- | -| Claude Fable 5 | N/A | N/A | 12.50 | 20.00 | 1.00 | -| Claude Fable 5.1 | N/A | N/A | 12.50 | 20.00 | 0.25 | -| Claude Sonnet 5 | 1.00 ‡ | 5.00 ‡ | 2.50 | 4.00 | 0.20 | -| Claude Opus 4.8 | 2.50 ‡ | 12.50 ‡ | 6.25 | 10.00 | 0.50 | -| Claude Sonnet 4.6 (+ LC) | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | -| Claude Opus 4.6 (+ LC) | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | -| Claude Opus 4.5 | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | -| Claude Haiku 4.5 | 0.50 | 2.50 | 1.25 | 2.00 | 0.10 | -| Claude Sonnet 4.5 | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | -| Claude Sonnet 4.5 — LC | 3.00 | 11.25 | 7.50 | 12.00 | 0.60 | - -_Batch: the Bedrock pricing page lists **N/A** for Claude Fable 5 and Fable 5.1 in both the Global and the Geo / In-region Anthropic tables (read 2026-09-03). Anthropic's first-party API offers a 50% batch discount on these models; Bedrock does not list one — do not assume a batch rate for Fable-class models._ - -‡ Not listed on the batch-supported models table as of 2026-09-02 — rate is the standard 50%-of-on-demand -projection, not a confirmed SKU. Mark these batch cells `_unverified_` in estimate output; the `unverified` gate in -`estimate-ai.md` (§ Pricing source caveat) treats any `_unverified_` row as blocking for a quoted figure — resolve -from the Bedrock pricing page first. - -### AI21 Labs - -**On-demand**, **US East (N. Virginia)** per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/). Prices per 1M input / output tokens. - -| Model | Input $/1M | Output $/1M | -| ---------------- | ---------- | ----------- | -| Jamba 1.5 Large | 2.00 | 8.00 | -| Jamba 1.5 Mini | 0.20 | 0.40 | -| Jurassic-2 Mid | 12.50 | 12.50 | -| Jurassic-2 Ultra | 18.80 | 18.80 | -| Jamba-Instruct | 0.50 | 0.70 | - -_Quick-reference rows use **—** for **model ID** and **context**; resolve in the Bedrock console or AWS model documentation._ - -### Mistral AI - -Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Mistral AI). **Priority** tier ≈ **+75%** vs Standard; **Flex** ≈ **−50%** vs Standard on the same page. - -#### US East (Ohio) — Pixtral (on-demand + batch header from page) - -| Model | Input $/1M | Output $/1M | Batch in | Batch out | -| --------------------- | ---------- | ----------- | -------- | --------- | -| Pixtral Large (25.02) | 2.00 | 6.00 | N/A | N/A | - -#### On-demand — US East (N. Virginia), US East (Ohio), US West (Oregon) +## Amazon Bedrock model and service pricing -Prices per 1M input / output tokens. - -| Model | Input $/1M | Output $/1M | -| ------------------- | ---------- | ----------- | -| Devstral 2 123B | 0.40 | 2.00 | -| Magistral Small 1.2 | 0.50 | 1.50 | -| Voxtral Mini 1.0 | 0.04 | 0.04 | -| Voxtral Small 1.0 | 0.10 | 0.30 | -| Ministral 3B 3.0 | 0.10 | 0.10 | -| Ministral 8B 3.0 | 0.15 | 0.15 | -| Ministral 14B 3.0 | 0.20 | 0.20 | -| Mistral Large 3 | 0.50 | 1.50 | - -_Rates differ in **Asia Pacific (Mumbai)**, **São Paulo**, **Tokyo**, **Europe (Ireland/Milan/London)**, **Sydney**, **Jakarta**, **Frankfurt**, **Stockholm** (e.g. Mistral Large 3 Mumbai **$0.59 / $1.76**). **Devstral 2 123B** appears in Jakarta / Frankfurt / Stockholm at **$0.48 / $2.40** on the same page._ - -### DeepSeek (Amazon Bedrock) - -Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (DeepSeek). - -#### US East (Ohio) — Standard tier - -> **Note:** DeepSeek-V3.1 is no longer listed for US East regions on the Bedrock pricing page as of May 2026. Use DeepSeek-V3.2 for US East deployments. DeepSeek-V3.1 remains available in Asia Pacific (Sydney). - -| Model | Input $/1M | Output $/1M | -| ----------- | ---------- | ----------- | -| DeepSeek-R1 | 1.35 | 5.40 | - -#### US East (N. Virginia), US East (Ohio), US West (Oregon) - -| Model | Input $/1M | Output $/1M | -| ------------- | ---------- | ----------- | -| DeepSeek v3.2 | 0.62 | 1.85 | - -#### Other regions (examples on same page) - -- **Asia Pacific (Mumbai)**, **São Paulo**, **Jakarta**, **Tokyo**, **Stockholm**: DeepSeek v3.2 **$0.74 / $2.22** per 1M. -- **Asia Pacific (Sydney)**: DeepSeek v3.1 **$0.5974 / $1.7304**; v3.2 **$0.6386 / $1.9055** (and **Priority** / **Flex** tiers listed separately on the page). - -### OpenAI on Bedrock (GPT-5.6 and gpt-oss) - -See `shared/openai-on-bedrock.md` for model IDs, endpoint paths, regions, quotas, and caching rules. **Bedrock -in-region is priced at parity with OpenAI's _data residency_ tier — exactly 1.10x OpenAI's standard list price** — so -a same-model migration costs about 10% MORE, not the same. Do not describe it as cost-neutral. - -#### Proprietary GPT models — short context (272K), in-region - -Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally carries Terra, Luna and GPT-5.4; AWS GovCloud us-gov-west-1 / us-gov-east-1 carry Terra and Luna, us-gov-west-1 also GPT-5.4 (at its own rate below). Rates reflect the July 30, 2026 reduction (Luna −80%, Terra −20%) and the Aug 21, 2026 Sol reduction (−20% input / −33% output, promotional through at least Nov 21, 2026). - -| Model | Input $/1M | Output $/1M | Cache write $/1M | Cache read $/1M | -| ------------- | ---------- | ----------- | ---------------- | --------------- | -| GPT-5.6 Sol | 4.40 | 22.00 | 5.50 | 0.44 | -| GPT-5.6 Terra | 2.20 | 13.20 | 2.75 | 0.22 | -| GPT-5.6 Luna | 0.22 | 1.32 | 0.275 | 0.022 | -| GPT-5.5 | 5.50 | 33.00 | — | 0.55 | -| GPT-5.4 | 2.75 | 16.50 | — | 0.275 | - -#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output - -| Model | Input $/1M | Output $/1M | -| ------------- | ---------- | ----------- | -| GPT-5.6 Sol | 8.80 | 33.00 | -| GPT-5.6 Terra | 4.40 | 19.80 | -| GPT-5.6 Luna | 0.44 | 1.98 | - -A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so -their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — -see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. - -> **One source conflict.** These models are absent -> from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and -> filtering on `GPT-5` or a `gpt-5` usage type returns zero rows (price-list publication 2026-08-04). An empty -> price-list result is **not** evidence the model is unavailable. (An earlier revision also -> flagged the AWS News Blog's Luna 0.20 / 1.20 as conflicting; it is the Global CRIS rate, not an error.) - -**Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum -1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. -GPT-5.5 and GPT-5.4 do not list prompt caching support. - -#### Open-weight gpt-oss - -| Model | Region | Input $/1M | Output $/1M | -| ------------ | --------------------- | ---------- | ----------- | -| gpt-oss-20b | US East (Ohio) | 0.07 | 0.30 | -| gpt-oss-120b | US East (Ohio) | 0.15 | 0.60 | -| gpt-oss-20b | Asia Pacific (Sydney) | 0.0721 | 0.3090 | -| gpt-oss-120b | Asia Pacific (Sydney) | 0.1545 | 0.6180 | - -_The Bedrock page also lists **Flex**, **Priority**, **Batch**, **GPT OSS Safeguard**, and **model customization** pricing for other regions._ - -### Meta Llama (Amazon Bedrock) - -Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Meta). Prices per 1M input / output tokens unless noted. - -#### Llama 4 — US East (Ohio), on-demand and batch - -| Model | On-demand in | On-demand out | Batch in | Batch out | -| -------------------- | ------------ | ------------- | -------- | --------- | -| Llama 4 Maverick 17B | 0.24 | 0.97 | 0.12 | 0.485 | -| Llama 4 Scout 17B | 0.17 | 0.66 | 0.085 | 0.33 | - -#### Llama 3.3 — US East (Ohio), on-demand and batch - -| Model | On-demand in | On-demand out | Batch in | Batch out | -| ------------------------ | ------------ | ------------- | -------- | --------- | -| Llama 3.3 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | - -#### Llama 3.2 — US East (Ohio), on-demand and batch - -> **Lifecycle note:** All Llama 3.2 Instruct models reached **EOL on Jul 7, 2026** and are no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Scout / Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. - -| Model | On-demand in | On-demand out | Batch in | Batch out | -| ------------------------ | ------------ | ------------- | -------- | --------- | -| Llama 3.2 Instruct (1B) | 0.10 | 0.10 | N/A | N/A | -| Llama 3.2 Instruct (3B) | 0.15 | 0.15 | N/A | N/A | -| Llama 3.2 Instruct (11B) | 0.16 | 0.16 | N/A | N/A | -| Llama 3.2 Instruct (90B) | 0.72 | 0.72 | N/A | N/A | - -#### Llama 3.2 — customization & provisioned throughput (US West Oregon) - -**Fine-tuning** (per 1M tokens trained / month storage / no-commit PT per model unit-hour): - -| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | -| ------------------------ | --------------- | ------------- | --------------------------- | -| Llama 3.2 Instruct (1B) | $0.50 | $1.95 | $23.00 | -| Llama 3.2 Instruct (3B) | $1.10 | $1.95 | $23.00 | -| Llama 3.2 Instruct (11B) | $3.50 | $1.95 | $23.00 | -| Llama 3.2 Instruct (90B) | $7.90 | $1.95 | $30.00 | - -**Provisioned throughput** ($/hour per model unit): - -| Model | No commit | 1-mo commit | 6-mo commit | -| ------------------------ | --------- | ----------- | ----------- | -| Llama 3.2 Instruct (1B) | $23.00 | $20.30 | $12.54 | -| Llama 3.2 Instruct (3B) | $23.00 | $20.30 | $12.54 | -| Llama 3.2 Instruct (11B) | $23.00 | $20.30 | $12.54 | -| Llama 3.2 Instruct (90B) | $30.00 | $26.48 | $16.35 | - -#### Llama 3.1 — US East (Ohio), on-demand and batch - -> **Lifecycle note:** Llama 3.1 405B Instruct reached **EOL on Jul 7, 2026** and is no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. - -| Model | On-demand in | On-demand out | Batch in | Batch out | -| -------------------------------------------------- | ------------ | ------------- | -------- | --------- | -| Llama 3.1 Instruct (8B) | 0.22 | 0.22 | 0.11 | 0.11 | -| Llama 3.1 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | -| Llama 3.1 Instruct (405B) | 2.40 | 2.40 | 1.20 | 1.20 | -| Llama 3.1 Instruct (70B) (latency optimized inf.) | 0.90 | 0.90 | N/A | N/A | -| Llama 3.1 Instruct (405B) (latency optimized inf.) | 3.00 | 3.00 | N/A | N/A | - -#### Llama 3.1 — customization & provisioned throughput (US West Oregon) - -| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | -| ------------------------ | --------------- | ------------- | --------------------------- | -| Llama 3.1 Instruct (8B) | $1.49 | $1.95 | $24.00 | -| Llama 3.1 Instruct (70B) | $7.99 | $1.95 | $24.00 | - -| Model | No commit | 1-mo commit | 6-mo commit | -| ------------------------ | --------- | ----------- | ----------- | -| Llama 3.1 Instruct (8B) | $24.00 | $21.18 | $13.08 | -| Llama 3.1 Instruct (70B) | $24.00 | $21.18 | $13.08 | - -#### Llama 3 — US East (N. Virginia), on-demand - -| Model | Input $/1M | Output $/1M | -| ---------------------- | ---------- | ----------- | -| Llama 3 Instruct (8B) | 0.30 | 0.60 | -| Llama 3 Instruct (70B) | 2.65 | 3.50 | - -#### Llama 2 — US East (N. Virginia) and US West (Oregon), on-demand - -| Model | Input $/1M | Output $/1M | -| ------------------ | ---------- | ----------- | -| Llama 2 Chat (13B) | 0.75 | 1.00 | -| Llama 2 Chat (70B) | 1.95 | 2.56 | - -#### Llama 2 — customization & provisioned throughput - -**Fine-tuning:** Llama 2 Pretrained (13B) **$1.49** per 1M tokens trained, **$1.95**/month storage, **$23.50**/unit-hr no-commit infer; (70B) **$7.99** / **$1.95** / **$23.50**. - -**Provisioned throughput** (per model unit-hour): 13B and 70B — **$21.18** (1-mo commit), **$13.08** (6-mo commit). Pretrained Llama 2 is **provisioned throughput only after customization** per AWS. - -### Amazon Nova - -All figures from [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Amazon Nova sections). Nova pricing depends on **inference deployment** (e.g. **Global cross-region** vs **Geo cross-region and in-region**), **tier** (Standard, Priority, Flex, Batch), **modality** (text / image / video / audio), and **AWS Region**. The quick-reference table above uses **Geo cross-region inference and in-region**, **Standard tier**, **US East (Ohio)**, **text + image + video** rates where a single input price applies to all three. Confirm the page before estimates. - -**Cache read (Nova):** AWS states cache read input tokens are **75% below** the on-demand input price for the applicable tier/modality. - -#### US East (Ohio) — Standard tier, text + image + video (Geo cross-region inference and in-region) - -Per 1M input / output tokens unless noted. - -| Model | Input $/1M | Output $/1M | -| ---------------------------------------- | ---------- | ----------- | -| Amazon Nova 2 Lite | 0.33 | 2.75 | -| Amazon Nova Micro | 0.035 | 0.14 | -| Amazon Nova Lite | 0.06 | 0.24 | -| Amazon Nova Pro | 0.80 | 3.20 | -| Amazon Nova Pro (latency optimized inf.) | 1.00 | 4.00 | -| Amazon Nova Premier | 2.50 | 12.50 | - -#### US East (Ohio) — Standard tier, Global cross-region inference (text + image + video) - -| Model | Input $/1M | Output $/1M | -| ------------------ | ---------- | ----------- | -| Amazon Nova 2 Lite | 0.30 | 2.50 | - -#### US East (Ohio) — Standard tier, text / image / video / audio (Geo cross-region and in-region) - -Per 1M tokens. **Nova 2 Omni** and **Nova 2 Pro** are **Preview**. Image column is **output** image pricing where listed. - -| Model | Text in | Image in | Video in | Audio in | Text out | Image out | -| ---------------------------- | ------- | -------- | -------- | -------- | -------- | --------- | -| Amazon Nova 2 Omni (Preview) | 0.30 | 0.30 | 0.30 | 1.10 | 2.80 | 44.00 | -| Amazon Nova 2 Pro (Preview) | 1.375 | 1.375 | 1.375 | 1.375 | 11.00 | N/A | - -**Global cross-region inference (Ohio), Standard** uses different multimodal numbers on the same page — e.g. **Nova 2 Omni (Preview)** audio in **$1.00**, text out **$2.50**, image out **$40.00** (text/image/video in **$0.30** each). Use the Bedrock pricing page for the deployment you select. - -#### US East (Ohio) — Nova 2 Lite by tier (text + image + video) - -##### Global cross-region inference - -| Tier | Input $/1M | Output $/1M | -| -------- | ---------- | ----------- | -| Standard | 0.30 | 2.50 | -| Priority | 0.525 | 4.375 | -| Flex | 0.15 | 1.25 | -| Batch | 0.15 | 1.25 | - -##### Geo cross-region inference and in-region - -| Tier | Input $/1M | Output $/1M | -| -------- | ---------- | ----------- | -| Standard | 0.33 | 2.75 | -| Priority | 0.5775 | 4.8125 | -| Flex | 0.165 | 1.375 | -| Batch | 0.1595 | 1.342 | - -#### US East (Ohio) — Batch tier (text + image + video), selected models - -| Model | Input $/1M | Output $/1M | -| ------------------- | ---------- | ----------- | -| Amazon Nova 2 Lite | 0.1595 | 1.342 | -| Amazon Nova Micro | 0.0175 | 0.07 | -| Amazon Nova Lite | 0.03 | 0.12 | -| Amazon Nova Pro | 0.40 | 1.60 | -| Amazon Nova Premier | 1.25 | 6.25 | - -#### On-demand inference (listed rates, cache read column) - -| Model | Input $/1M | Cache read $/1M | Output $/1M | -| -------------------- | ---------- | --------------- | ----------- | -| Amazon Nova 2.0 Lite | N/A | N/A | N/A | -| Amazon Nova Micro | 0.035 | N/A | 0.14 | -| Amazon Nova Lite | 0.06 | N/A | 0.24 | -| Amazon Nova Pro | 0.80 | N/A | 3.20 | - -#### Built-in tools — Web grounding (US East Ohio) - -| Model | Price | -| ---------------------------- | ---------------------- | -| Amazon Nova 2 Omni (Preview) | $30.00 per 1K requests | -| Amazon Nova 2 Pro (Preview) | $30.00 per 1K requests | - -#### Creative — US East (N. Virginia) - -> **Lifecycle note:** Nova Canvas v1 and Nova Reel v1 are **excluded** — both are Legacy with EOL 2026-09-30, inside the 90-day exclusion zone. Do not list them in recommendation or comparison tables. Rates are retained for users already on them. See `vendored/ai/ai-model-lifecycle.md`. - -**Amazon Nova Canvas** (on-demand, per image): up to **1024×1024** — Standard **$0.04**, Premium **$0.06**; up to **2048×2048** — Standard **$0.06**, Premium **$0.08**. - -**Model customization (Nova Canvas):** **$0.005** per image seen; **$1.95**/month per custom model stored; provisioned inference per model unit per hour (no commit / 1-mo / 6-mo) **$60.50 / $55.00 / $30.25**. - -**Amazon Nova Reel** (video): **720p, 24 fps** — **$0.08** per second of video generated. - -#### Speech — US East (N. Virginia) - -> **Lifecycle note:** Nova Sonic v1 reached **EOL on 2026-09-14** and its requests now fail. Rates are retained only to explain the migration to **Nova 2 Sonic**. See `vendored/ai/ai-model-lifecycle.md`. - -Per 1M tokens. - -| Model | Modality | Input $/1M | Output $/1M | Status | -| ------------------- | -------- | ---------- | ----------- | ---------------- | -| Amazon Nova Sonic | Speech | 3.40 | 13.60 | EOL (2026-09-14) | -| Amazon Nova Sonic | Text | 0.06 | 0.24 | EOL (2026-09-14) | -| Amazon Nova 2 Sonic | Speech | 3.00 | 12.00 | active | -| Amazon Nova 2 Sonic | Text | 0.33 | 2.75 | active | - -#### Multimodal embeddings — US East (N. Virginia) - -| Offering | Text $/1M | Std image / doc image / video sec / audio sec | -| --------------------------------------------- | --------- | ------------------------------------------------ | -| Amazon Nova Multimodal Embeddings (On-demand) | 0.135 | $0.00006 / $0.0006 / $0.0007 / $0.00014 per unit | -| Amazon Nova Multimodal Embeddings (Batch) | 0.0675 | $0.00003 / $0.00048 / $0.00056 / $0.000112 | - -_On-demand inference for **custom Nova** models matches **base Nova** inference pricing per AWS._ - ---- +Bedrock pricing is provider-neutral and canonical in +`references/vendored/ai/bedrock-pricing-cache.md`. Load that cache for every Bedrock +model and service rate; this file intentionally does not duplicate those values. ## Source Provider Pricing (for Migration Comparison) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-fallback.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-fallback.md index 3a82f3dfc..704fcc84d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-fallback.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-fallback.md @@ -5,7 +5,7 @@ ## Policy -When a service is NOT in `references/shared/pricing-cache.md`: +When a service is NOT in `references/shared/pricing-cache.md`, or a Bedrock model or service is NOT in `references/vendored/ai/bedrock-pricing-cache.md`: 1. If the cost engine's own formulas carry a well-known published rate (stated verbatim), set `pricing_source: "estimated"` and include a warning naming the rate and its source. 2. Otherwise set `pricing_source: "unavailable"`, add the service to `services_with_missing_fallback`, and warn the user: "Pricing unavailable for [service] — not in cache and no formula constant. Exclude from totals or provide a manual estimate." diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 01040332d..4dd7b9d24 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -27,6 +27,7 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/ai-migration-guardrails.md` | `skills/shared/ai/ai-migration-guardrails.md` | | `ai/ai-model-lifecycle.md` | `skills/shared/ai/ai-model-lifecycle.md` | | `ai/ai-openai-to-bedrock.md` | `skills/shared/ai/ai-openai-to-bedrock.md` | +| `ai/bedrock-pricing-cache.md` | `skills/shared/ai/bedrock-pricing-cache.md` | | `ai/bedrock-quotas.md` | `skills/shared/ai/bedrock-quotas.md` | | `ai/design-ref-agentic-to-agentcore.md` | `skills/shared/ai/design-ref-agentic-to-agentcore.md` | | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md index 87bc66b98..aeb78d0b1 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md @@ -116,7 +116,7 @@ Before recommending any Bedrock model in an agentic design: For agentic workload cost estimation: -1. **Primary:** `references/shared/pricing-cache.md` (±5-10% accuracy) +1. **Primary:** `references/vendored/ai/bedrock-pricing-cache.md` (±15-25% accuracy) 2. **Fallback:** `references/shared/pricing-fallback.md` (cache-miss → `estimated`/`unavailable`) AgentCore Runtime and Harness pricing: consumption-based, no upfront cost. Include in estimate only if the user selects Harness or Strands path. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-model-lifecycle.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-model-lifecycle.md index c3facc7bc..701d7e84e 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-model-lifecycle.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-model-lifecycle.md @@ -133,7 +133,7 @@ When building the model comparison table: - **Active**: no restrictions. - **Restricted** (`restricted (…)` in the pricing-cache Status column): never `recommended_model` or `backup_model`, never a default in a mapping guide. Include in `model_comparison` only when the user explicitly asks about frontier / Covered Models, annotated with the access requirement. -### Pricing Cache (`pricing-cache.md`) +### Pricing Cache (`bedrock-pricing-cache.md`) The multi-provider quick reference table includes a `Status` column: @@ -161,6 +161,6 @@ When refreshing the cache, recompute `days_to_eol` and refresh the `active` / `l **Newer models are not covered by the table.** A model launched on or after 2026-09-07 will never appear in the Legacy/EOL table above, and its absence is not evidence that it is Active. When a candidate is not in the table, verify it by calling `GetFoundationModel` (or `ListFoundationModels`) and reading `modelLifecycle.status`: `LEGACY` and `EOL` are never valid targets for a new migration. If the model is Legacy, read its model card for the actual EOL date and whether the Legacy period is 6 months or 45 days. If neither the API nor the card is reachable, treat the model's lifecycle as **unverified** and say so in the output rather than inferring `active` from a `Status` column in a pricing cache. -**Periodic table refresh:** When the table itself needs updating (new models added, EOL dates changed by AWS, or past-EOL rows to remove), update this file and `pricing-cache.md` together. Edit the canonical `skills/shared/ai/ai-model-lifecycle.md`, then copy it over every vendored copy in the same change so they stay byte-identical. +**Periodic table refresh:** When the table itself needs updating (new models added, EOL dates changed by AWS, or past-EOL rows to remove), update this file and `bedrock-pricing-cache.md` together. Edit the canonical files under `skills/shared/ai/`, then copy them over every vendored copy in the same change so they stay byte-identical. **Past-EOL rows:** Once `days_to_eol ≤ 0`, move the model out of the live table into **Removed**, and grep this plugin's skill trees for the model ID to catch any remaining reference to it as a target. Keep the Removed entry long enough that users already on the model still get a warning. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index cf41cb343..da1c5fcdf 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -131,7 +131,7 @@ See "Cost-optimized cross-family alternatives" below for the numbers. All figures per 1M tokens. Percentages are blended savings at a 2:1 input-to-output ratio. **Both sides of these comparisons are now Bedrock prices** where the OpenAI model is on Bedrock — this is a Bedrock-vs-Bedrock model -choice, not a provider comparison. Verify rates via `shared/pricing-cache.md` — pricing is cache-only, so a +choice, not a provider comparison. Verify rates via `references/vendored/ai/bedrock-pricing-cache.md` — pricing is cache-only, so a GPT-5.x rate that is absent or `_unverified_` in the cache has no live lookup to fall back to and must be reported as unavailable rather than priced (see `shared/openai-on-bedrock.md`). diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md new file mode 100644 index 000000000..b1f94f147 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -0,0 +1,449 @@ +# Amazon Bedrock Pricing Cache + +**Last updated:** 2026-09-03 +**Region:** us-east-1 +**Currency:** USD +**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) + +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> +> **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. + +--- +## Bedrock Models (On-Demand) + +**Anthropic Claude (Standard on-demand)** figures below match **US East (N. Virginia)** on [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) as of cache refresh. **Recommend defaults (new migrations):** Claude Sonnet 5 (flagship), Claude Opus 4.8 (hardest reasoning), Claude Haiku 4.5 (cost/speed). Do not default to any Claude Fable / Mythos frontier model (Fable 5, Fable 5.1, and successors). **Claude Fable 5** and **Claude Fable 5.1** (GA Sep 1, 2026) are the most expensive Anthropic models at $10/$50 per 1M tokens (Mythos-class); 5.1's on-demand rate difference is cache read at $0.25 (0.025x) vs $1.00; Bedrock lists no batch rate for either. Both are Anthropic **Covered Models**: using them requires opting the account into the `aws_review` data-retention mode, and Fable 5.1 is served commercially only via `us.`/`global.` CRIS profiles (no in-region — the bare `anthropic.claude-fable-5-1` id is not invokable on `bedrock-runtime`, hence the `us.` form in the table), with in-region access in AWS GovCloud (US) only. Claude Mythos 5.1 is a gated Preview for vetted cyber/bio research organizations and is not a migration target. **Claude Opus 4.8** keeps the same $5/$25 rate as Opus 4.6/4.7. **Claude Sonnet 5** launched June 30 at $2/$10; that launch rate became the standard price on Sep 1, 2026 (the scheduled increase to $3/$15 was cancelled), so Sonnet 5 is now both newer and cheaper than Sonnet 4.6 ($3/$15). **Claude Opus 4.7** lists the same headline on-demand input/output as **Opus 4.6** on that page; confirm **batch** availability per model (Opus 4.7 batch was **not** listed on the global cross-region table when this row was added). **Claude Opus 4.1** entered **Legacy** on Jul 8, 2026 (EOL Jan 8, 2027). **Batch**, **prompt cache** (5m / 1h write + cache read), and **geo / in-region cross-region** rows on that page can differ; e.g. **US East (Ohio)** cross-region inference for Claude Sonnet 4.6 is listed at **$3.30 / $16.50** per 1M input/output (≈10% above N. Virginia). Long-context SKUs **do not** all use the same multiplier: **Sonnet 4.6** and **Opus 4.6** long-context modes share the same on-demand rates as the non–long-context rows on the standard table; **Sonnet 4.5** and **Sonnet 4** long-context rows are priced higher on that same table. + +### Multi-provider quick reference (per 1M tokens) + +See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommend Legacy models for new migrations.** Rows whose Status is `restricted (…)` are never `recommended_model` / `backup_model` and never a default (Covered Models, gated previews). + +| Model | Model ID | Provider | Input $/1M | Output $/1M | Context | Tier | Status | +| -------------------------------- | ---------------------------------------- | --------- | ---------- | ----------- | ------- | --------- | ------------------------------------------------------------ | +| Claude Fable 5 | anthropic.claude-fable-5 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in) | +| Claude Fable 5.1 | us.anthropic.claude-fable-5-1 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in; CRIS only) | +| Claude Sonnet 5 | anthropic.claude-sonnet-5 | Anthropic | 2.00 | 10.00 | 1M | flagship | active ($2/$10 made the standard price Sep 1, 2026) | +| Claude Opus 4.8 | anthropic.claude-opus-4-8 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.6 | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.6 — Long Context | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K+ | flagship | active | +| Claude Opus 4.6 | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Opus 4.6 — Long Context | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K+ | premium | active | +| Claude Opus 4.5 | — | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.5 | — | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.5 — Long Context | — | Anthropic | 6.00 | 22.50 | 200K+ | flagship | active | +| Claude Haiku 4.5 | anthropic.claude-haiku-4-5-20251001-v1:0 | Anthropic | 1.00 | 5.00 | 200K | fast | active | +| Claude Opus 4.1 | anthropic.claude-opus-4-1-20250805-v1:0 | Anthropic | 15.00 | 75.00 | 200K | premium | legacy (EOL Jan 8, 2027) | +| Claude Sonnet 4 | anthropic.claude-sonnet-4-20250514-v1:0 | Anthropic | 3.00 | 15.00 | 200K | flagship | excluded (EOL Oct 14, 2026) | +| Llama 4 Maverick | meta.llama4-maverick-17b-instruct-v1:0 | Meta | 0.24 | 0.97 | 1M | mid | active | +| Llama 4 Scout | meta.llama4-scout-17b-instruct-v1:0 | Meta | 0.17 | 0.66 | 10M | efficient | active | +| Llama 3.3 70B | meta.llama3-3-70b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | active | +| Llama 3.2 90B | meta.llama3-2-90b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | EOL (Jul 7, 2026) | +| Nova 2 Lite | amazon.nova-2-lite-v1:0 | Amazon | 0.33 | 2.75 | 1M | mid | active | +| Nova 2 Pro (Preview) | amazon.nova-2-pro-v1:0 | Amazon | 1.375 | 11.00 | 1M | flagship | active | +| Nova Pro | amazon.nova-pro-v1:0 | Amazon | 0.80 | 3.20 | 300K | mid | active | +| Nova Pro (latency optimized) | — | Amazon | 1.00 | 4.00 | 300K | mid | active | +| Nova Lite | amazon.nova-lite-v1:0 | Amazon | 0.06 | 0.24 | 300K | fast | active | +| Nova Micro | amazon.nova-micro-v1:0 | Amazon | 0.035 | 0.14 | 128K | budget | active | +| Nova Premier | amazon.nova-premier-v1:0 | Amazon | 2.50 | 12.50 | 1M | reasoning | EOL (2026-09-14) | +| Mistral Large 3 | mistral.mistral-large-3-675b-instruct | Mistral | 0.50 | 1.50 | 256K | flagship | active | +| DeepSeek-R1 | deepseek.r1-v1:0 | DeepSeek | 1.35 | 5.40 | 128K | reasoning | active | +| DeepSeek-V3.1 | — | DeepSeek | 0.58 | 1.68 | — | mid | active (Sydney only) | +| gpt-oss-20b | openai.gpt-oss-20b-1:0 | OpenAI | 0.07 | 0.30 | 128K | budget | active | +| gpt-oss-120b | openai.gpt-oss-120b-1:0 | OpenAI | 0.15 | 0.60 | 128K | efficient | active | +| GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | +| GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | +| GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | +| Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | +| Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | +| Gemma 3 4B IT | google.gemma-3-4b-it | Google | 0.04 | 0.08 | 128K | budget | active | +| Gemma 3 12B IT | google.gemma-3-12b-it | Google | 0.09 | 0.29 | 128K | budget | active | +| Gemma 3 27B IT | google.gemma-3-27b-it | Google | 0.23 | 0.38 | 128K | efficient | active | +| MiniMax M2 | minimax.minimax-m2 | MiniMax | 0.30 | 1.20 | 1M | mid | active | +| MiniMax M2.1 | minimax.minimax-m2.1 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| MiniMax M2.5 | minimax.minimax-m2.5 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| Jamba 1.5 Large | ai21.jamba-1-5-large-v1:0 | AI21 Labs | 2.00 | 8.00 | — | mid | excluded (EOL 2026-11-26) | +| Jamba 1.5 Mini | ai21.jamba-1-5-mini-v1:0 | AI21 Labs | 0.20 | 0.40 | — | efficient | excluded (EOL 2026-11-26) | +| Jurassic-2 Mid | — | AI21 Labs | 12.50 | 12.50 | — | legacy | legacy | +| Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | +| Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | + +### Stability AI — Image Generation (per image, US East) + +Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. + +| Model | Model ID | Price/image | Resolution | Tier | Status | +| -------------------------- | --------------------------------- | ----------- | ---------- | -------- | ------ | +| Stable Image Ultra | stability.stable-image-ultra-v1:0 | $0.08 | up to 4MP | premium | active | +| Stable Diffusion 3.5 Large | stability.sd3-5-large-v1:0 | $0.065 | up to 1MP | flagship | active | +| Stable Image Core | stability.stable-image-core-v1:0 | $0.04 | up to 1MP | fast | active | + +Image editing services (inpaint, erase, upscale, etc.) are priced at $0.03–$0.60/operation. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full list. + +> **Cost comparison note:** DALL-E 3 (OpenAI) charges $0.04–$0.12/image depending on resolution. Google Imagen charges per character of prompt. When comparing, use per-image cost directly — do not convert Stability AI prices to per-token equivalents. + +### Anthropic Claude — batch & prompt cache (Standard, US East N. Virginia) + +Per 1M tokens unless noted. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full regional and tier tables. + +> **Batch support verified 2026-09-02** against the +> [supported Regions and models for batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) +> table: **Opus 4.6, Sonnet 4.6, Opus 4.5, Sonnet 4.5, Sonnet 4.5 — LC, and Haiku 4.5 are listed** (via cross-region +> inference profiles; batch is 50% of on-demand per the pricing page, matching the rows below). **Fable 5, Sonnet 5, +> and Opus 4.8 are NOT on that table** — Sonnet 5 and Opus 4.8's batch cells below are **unverified (‡)**: do not build a batch-discount +> TCO on them; price on-demand and note batch as a possible future saving pending the docs table listing them. +> Fable 5 / 5.1 instead carry **N/A** per the separate pricing-page verification below. +> Prompt-cache columns are unaffected (caching support is independent of batch). +> **Batch rate vs. cross-region inference (CRIS):** the confirmed rows show 50% of the **N. Virginia** on-demand +> price. Where batch is only reachable through a CRIS profile that lists above N. Virginia (e.g. Sonnet 4.6 in US East +> (Ohio) at ≈10% higher — see the on-demand section), the effective batch rate is 50% of that CRIS price, i.e. ≈10% +> above the cells here. Re-derive from the profile's on-demand rate for a region-specific quote. +> **Do not cross-check these rows against the AWS Pricing Calculator** — as of Sep 2026 the calculator carries no +> batch metered IDs for Opus 4.6 / Sonnet 4.6 and collapses other batch/cache SKUs (known calculator defect, not a +> pricing-page signal). The pricing page and the docs table above are the sources of truth. + +| Model | Batch in | Batch out | 5m cache write | 1h cache write | Cache read | +| ------------------------ | -------- | --------- | -------------- | -------------- | ---------- | +| Claude Fable 5 | N/A | N/A | 12.50 | 20.00 | 1.00 | +| Claude Fable 5.1 | N/A | N/A | 12.50 | 20.00 | 0.25 | +| Claude Sonnet 5 | 1.00 ‡ | 5.00 ‡ | 2.50 | 4.00 | 0.20 | +| Claude Opus 4.8 | 2.50 ‡ | 12.50 ‡ | 6.25 | 10.00 | 0.50 | +| Claude Sonnet 4.6 (+ LC) | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Opus 4.6 (+ LC) | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Opus 4.5 | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Haiku 4.5 | 0.50 | 2.50 | 1.25 | 2.00 | 0.10 | +| Claude Sonnet 4.5 | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Sonnet 4.5 — LC | 3.00 | 11.25 | 7.50 | 12.00 | 0.60 | + +_Batch: the Bedrock pricing page lists **N/A** for Claude Fable 5 and Fable 5.1 in both the Global and the Geo / In-region Anthropic tables (read 2026-09-03). Anthropic's first-party API offers a 50% batch discount on these models; Bedrock does not list one — do not assume a batch rate for Fable-class models._ + +‡ Not listed on the batch-supported models table as of 2026-09-02 — rate is the standard 50%-of-on-demand +projection, not a confirmed SKU. Mark these batch cells `_unverified_` in estimate output; the `unverified` gate in +`estimate-ai.md` (§ Pricing source caveat) treats any `_unverified_` row as blocking for a quoted figure — resolve +from the Bedrock pricing page first. + +### AI21 Labs + +**On-demand**, **US East (N. Virginia)** per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/). Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ---------------- | ---------- | ----------- | +| Jamba 1.5 Large | 2.00 | 8.00 | +| Jamba 1.5 Mini | 0.20 | 0.40 | +| Jurassic-2 Mid | 12.50 | 12.50 | +| Jurassic-2 Ultra | 18.80 | 18.80 | +| Jamba-Instruct | 0.50 | 0.70 | + +_Quick-reference rows use **—** for **model ID** and **context**; resolve in the Bedrock console or AWS model documentation._ + +### Mistral AI + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Mistral AI). **Priority** tier ≈ **+75%** vs Standard; **Flex** ≈ **−50%** vs Standard on the same page. + +#### US East (Ohio) — Pixtral (on-demand + batch header from page) + +| Model | Input $/1M | Output $/1M | Batch in | Batch out | +| --------------------- | ---------- | ----------- | -------- | --------- | +| Pixtral Large (25.02) | 2.00 | 6.00 | N/A | N/A | + +#### On-demand — US East (N. Virginia), US East (Ohio), US West (Oregon) + +Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Devstral 2 123B | 0.40 | 2.00 | +| Magistral Small 1.2 | 0.50 | 1.50 | +| Voxtral Mini 1.0 | 0.04 | 0.04 | +| Voxtral Small 1.0 | 0.10 | 0.30 | +| Ministral 3B 3.0 | 0.10 | 0.10 | +| Ministral 8B 3.0 | 0.15 | 0.15 | +| Ministral 14B 3.0 | 0.20 | 0.20 | +| Mistral Large 3 | 0.50 | 1.50 | + +_Rates differ in **Asia Pacific (Mumbai)**, **São Paulo**, **Tokyo**, **Europe (Ireland/Milan/London)**, **Sydney**, **Jakarta**, **Frankfurt**, **Stockholm** (e.g. Mistral Large 3 Mumbai **$0.59 / $1.76**). **Devstral 2 123B** appears in Jakarta / Frankfurt / Stockholm at **$0.48 / $2.40** on the same page._ + +### DeepSeek (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (DeepSeek). + +#### US East (Ohio) — Standard tier + +> **Note:** DeepSeek-V3.1 is no longer listed for US East regions on the Bedrock pricing page as of May 2026. Use DeepSeek-V3.2 for US East deployments. DeepSeek-V3.1 remains available in Asia Pacific (Sydney). + +| Model | Input $/1M | Output $/1M | +| ----------- | ---------- | ----------- | +| DeepSeek-R1 | 1.35 | 5.40 | + +#### US East (N. Virginia), US East (Ohio), US West (Oregon) + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| DeepSeek v3.2 | 0.62 | 1.85 | + +#### Other regions (examples on same page) + +- **Asia Pacific (Mumbai)**, **São Paulo**, **Jakarta**, **Tokyo**, **Stockholm**: DeepSeek v3.2 **$0.74 / $2.22** per 1M. +- **Asia Pacific (Sydney)**: DeepSeek v3.1 **$0.5974 / $1.7304**; v3.2 **$0.6386 / $1.9055** (and **Priority** / **Flex** tiers listed separately on the page). + +### OpenAI on Bedrock (GPT-5.6 and gpt-oss) + +See `shared/openai-on-bedrock.md` for model IDs, endpoint paths, regions, quotas, and caching rules. **Bedrock +in-region is priced at parity with OpenAI's _data residency_ tier — exactly 1.10x OpenAI's standard list price** — so +a same-model migration costs about 10% MORE, not the same. Do not describe it as cost-neutral. + +#### Proprietary GPT models — short context (272K), in-region + +Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally carries Terra, Luna and GPT-5.4; AWS GovCloud us-gov-west-1 / us-gov-east-1 carry Terra and Luna, us-gov-west-1 also GPT-5.4 (at its own rate below). Rates reflect the July 30, 2026 reduction (Luna −80%, Terra −20%) and the Aug 21, 2026 Sol reduction (−20% input / −33% output, promotional through at least Nov 21, 2026). + +| Model | Input $/1M | Output $/1M | Cache write $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | ---------------- | --------------- | +| GPT-5.6 Sol | 4.40 | 22.00 | 5.50 | 0.44 | +| GPT-5.6 Terra | 2.20 | 13.20 | 2.75 | 0.22 | +| GPT-5.6 Luna | 0.22 | 1.32 | 0.275 | 0.022 | +| GPT-5.5 | 5.50 | 33.00 | — | 0.55 | +| GPT-5.4 | 2.75 | 16.50 | — | 0.275 | + +#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| GPT-5.6 Sol | 8.80 | 33.00 | +| GPT-5.6 Terra | 4.40 | 19.80 | +| GPT-5.6 Luna | 0.44 | 1.98 | + +A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so +their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — +see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. + +> **One source conflict.** These models are absent +> from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and +> filtering on `GPT-5` or a `gpt-5` usage type returns zero rows (price-list publication 2026-08-04). An empty +> price-list result is **not** evidence the model is unavailable. (An earlier revision also +> flagged the AWS News Blog's Luna 0.20 / 1.20 as conflicting; it is the Global CRIS rate, not an error.) + +**Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum +1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. +GPT-5.5 and GPT-5.4 do not list prompt caching support. + +#### Open-weight gpt-oss + +| Model | Region | Input $/1M | Output $/1M | +| ------------ | --------------------- | ---------- | ----------- | +| gpt-oss-20b | US East (Ohio) | 0.07 | 0.30 | +| gpt-oss-120b | US East (Ohio) | 0.15 | 0.60 | +| gpt-oss-20b | Asia Pacific (Sydney) | 0.0721 | 0.3090 | +| gpt-oss-120b | Asia Pacific (Sydney) | 0.1545 | 0.6180 | + +_The Bedrock page also lists **Flex**, **Priority**, **Batch**, **GPT OSS Safeguard**, and **model customization** pricing for other regions._ + +### Meta Llama (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Meta). Prices per 1M input / output tokens unless noted. + +#### Llama 4 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------- | ------------ | ------------- | -------- | --------- | +| Llama 4 Maverick 17B | 0.24 | 0.97 | 0.12 | 0.485 | +| Llama 4 Scout 17B | 0.17 | 0.66 | 0.085 | 0.33 | + +#### Llama 3.3 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.3 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | + +#### Llama 3.2 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** All Llama 3.2 Instruct models reached **EOL on Jul 7, 2026** and are no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Scout / Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.2 Instruct (1B) | 0.10 | 0.10 | N/A | N/A | +| Llama 3.2 Instruct (3B) | 0.15 | 0.15 | N/A | N/A | +| Llama 3.2 Instruct (11B) | 0.16 | 0.16 | N/A | N/A | +| Llama 3.2 Instruct (90B) | 0.72 | 0.72 | N/A | N/A | + +#### Llama 3.2 — customization & provisioned throughput (US West Oregon) + +**Fine-tuning** (per 1M tokens trained / month storage / no-commit PT per model unit-hour): + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.2 Instruct (1B) | $0.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (3B) | $1.10 | $1.95 | $23.00 | +| Llama 3.2 Instruct (11B) | $3.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (90B) | $7.90 | $1.95 | $30.00 | + +**Provisioned throughput** ($/hour per model unit): + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.2 Instruct (1B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (3B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (11B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (90B) | $30.00 | $26.48 | $16.35 | + +#### Llama 3.1 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** Llama 3.1 405B Instruct reached **EOL on Jul 7, 2026** and is no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------------------------------------- | ------------ | ------------- | -------- | --------- | +| Llama 3.1 Instruct (8B) | 0.22 | 0.22 | 0.11 | 0.11 | +| Llama 3.1 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | +| Llama 3.1 Instruct (405B) | 2.40 | 2.40 | 1.20 | 1.20 | +| Llama 3.1 Instruct (70B) (latency optimized inf.) | 0.90 | 0.90 | N/A | N/A | +| Llama 3.1 Instruct (405B) (latency optimized inf.) | 3.00 | 3.00 | N/A | N/A | + +#### Llama 3.1 — customization & provisioned throughput (US West Oregon) + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.1 Instruct (8B) | $1.49 | $1.95 | $24.00 | +| Llama 3.1 Instruct (70B) | $7.99 | $1.95 | $24.00 | + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.1 Instruct (8B) | $24.00 | $21.18 | $13.08 | +| Llama 3.1 Instruct (70B) | $24.00 | $21.18 | $13.08 | + +#### Llama 3 — US East (N. Virginia), on-demand + +| Model | Input $/1M | Output $/1M | +| ---------------------- | ---------- | ----------- | +| Llama 3 Instruct (8B) | 0.30 | 0.60 | +| Llama 3 Instruct (70B) | 2.65 | 3.50 | + +#### Llama 2 — US East (N. Virginia) and US West (Oregon), on-demand + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Llama 2 Chat (13B) | 0.75 | 1.00 | +| Llama 2 Chat (70B) | 1.95 | 2.56 | + +#### Llama 2 — customization & provisioned throughput + +**Fine-tuning:** Llama 2 Pretrained (13B) **$1.49** per 1M tokens trained, **$1.95**/month storage, **$23.50**/unit-hr no-commit infer; (70B) **$7.99** / **$1.95** / **$23.50**. + +**Provisioned throughput** (per model unit-hour): 13B and 70B — **$21.18** (1-mo commit), **$13.08** (6-mo commit). Pretrained Llama 2 is **provisioned throughput only after customization** per AWS. + +### Amazon Nova + +All figures from [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Amazon Nova sections). Nova pricing depends on **inference deployment** (e.g. **Global cross-region** vs **Geo cross-region and in-region**), **tier** (Standard, Priority, Flex, Batch), **modality** (text / image / video / audio), and **AWS Region**. The quick-reference table above uses **Geo cross-region inference and in-region**, **Standard tier**, **US East (Ohio)**, **text + image + video** rates where a single input price applies to all three. Confirm the page before estimates. + +**Cache read (Nova):** AWS states cache read input tokens are **75% below** the on-demand input price for the applicable tier/modality. + +#### US East (Ohio) — Standard tier, text + image + video (Geo cross-region inference and in-region) + +Per 1M input / output tokens unless noted. + +| Model | Input $/1M | Output $/1M | +| ---------------------------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.33 | 2.75 | +| Amazon Nova Micro | 0.035 | 0.14 | +| Amazon Nova Lite | 0.06 | 0.24 | +| Amazon Nova Pro | 0.80 | 3.20 | +| Amazon Nova Pro (latency optimized inf.) | 1.00 | 4.00 | +| Amazon Nova Premier | 2.50 | 12.50 | + +#### US East (Ohio) — Standard tier, Global cross-region inference (text + image + video) + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.30 | 2.50 | + +#### US East (Ohio) — Standard tier, text / image / video / audio (Geo cross-region and in-region) + +Per 1M tokens. **Nova 2 Omni** and **Nova 2 Pro** are **Preview**. Image column is **output** image pricing where listed. + +| Model | Text in | Image in | Video in | Audio in | Text out | Image out | +| ---------------------------- | ------- | -------- | -------- | -------- | -------- | --------- | +| Amazon Nova 2 Omni (Preview) | 0.30 | 0.30 | 0.30 | 1.10 | 2.80 | 44.00 | +| Amazon Nova 2 Pro (Preview) | 1.375 | 1.375 | 1.375 | 1.375 | 11.00 | N/A | + +**Global cross-region inference (Ohio), Standard** uses different multimodal numbers on the same page — e.g. **Nova 2 Omni (Preview)** audio in **$1.00**, text out **$2.50**, image out **$40.00** (text/image/video in **$0.30** each). Use the Bedrock pricing page for the deployment you select. + +#### US East (Ohio) — Nova 2 Lite by tier (text + image + video) + +##### Global cross-region inference + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.30 | 2.50 | +| Priority | 0.525 | 4.375 | +| Flex | 0.15 | 1.25 | +| Batch | 0.15 | 1.25 | + +##### Geo cross-region inference and in-region + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.33 | 2.75 | +| Priority | 0.5775 | 4.8125 | +| Flex | 0.165 | 1.375 | +| Batch | 0.1595 | 1.342 | + +#### US East (Ohio) — Batch tier (text + image + video), selected models + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.1595 | 1.342 | +| Amazon Nova Micro | 0.0175 | 0.07 | +| Amazon Nova Lite | 0.03 | 0.12 | +| Amazon Nova Pro | 0.40 | 1.60 | +| Amazon Nova Premier | 1.25 | 6.25 | + +#### On-demand inference (listed rates, cache read column) + +| Model | Input $/1M | Cache read $/1M | Output $/1M | +| -------------------- | ---------- | --------------- | ----------- | +| Amazon Nova 2.0 Lite | N/A | N/A | N/A | +| Amazon Nova Micro | 0.035 | N/A | 0.14 | +| Amazon Nova Lite | 0.06 | N/A | 0.24 | +| Amazon Nova Pro | 0.80 | N/A | 3.20 | + +#### Built-in tools — Web grounding (US East Ohio) + +| Model | Price | +| ---------------------------- | ---------------------- | +| Amazon Nova 2 Omni (Preview) | $30.00 per 1K requests | +| Amazon Nova 2 Pro (Preview) | $30.00 per 1K requests | + +#### Creative — US East (N. Virginia) + +> **Lifecycle note:** Nova Canvas v1 and Nova Reel v1 are **excluded** — both are Legacy with EOL 2026-09-30, inside the 90-day exclusion zone. Do not list them in recommendation or comparison tables. Rates are retained for users already on them. See `vendored/ai/ai-model-lifecycle.md`. + +**Amazon Nova Canvas** (on-demand, per image): up to **1024×1024** — Standard **$0.04**, Premium **$0.06**; up to **2048×2048** — Standard **$0.06**, Premium **$0.08**. + +**Model customization (Nova Canvas):** **$0.005** per image seen; **$1.95**/month per custom model stored; provisioned inference per model unit per hour (no commit / 1-mo / 6-mo) **$60.50 / $55.00 / $30.25**. + +**Amazon Nova Reel** (video): **720p, 24 fps** — **$0.08** per second of video generated. + +#### Speech — US East (N. Virginia) + +> **Lifecycle note:** Nova Sonic v1 reached **EOL on 2026-09-14** and its requests now fail. Rates are retained only to explain the migration to **Nova 2 Sonic**. See `vendored/ai/ai-model-lifecycle.md`. + +Per 1M tokens. + +| Model | Modality | Input $/1M | Output $/1M | Status | +| ------------------- | -------- | ---------- | ----------- | ---------------- | +| Amazon Nova Sonic | Speech | 3.40 | 13.60 | EOL (2026-09-14) | +| Amazon Nova Sonic | Text | 0.06 | 0.24 | EOL (2026-09-14) | +| Amazon Nova 2 Sonic | Speech | 3.00 | 12.00 | active | +| Amazon Nova 2 Sonic | Text | 0.33 | 2.75 | active | + +#### Multimodal embeddings — US East (N. Virginia) + +| Offering | Text $/1M | Std image / doc image / video sec / audio sec | +| --------------------------------------------- | --------- | ------------------------------------------------ | +| Amazon Nova Multimodal Embeddings (On-demand) | 0.135 | $0.00006 / $0.0006 / $0.0007 / $0.00014 per unit | +| Amazon Nova Multimodal Embeddings (Batch) | 0.0675 | $0.00003 / $0.00048 / $0.00056 / $0.000112 | + +_On-demand inference for **custom Nova** models matches **base Nova** inference pricing per AWS._ + +--- diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py index 9115ed959..4505b72ba 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py @@ -36,7 +36,7 @@ def parse_price_dimensions(price_item: dict) -> dict: # Static fallback table: per-1K-token USD rates from public pricing pages. # Used when the PriceList API doesn't return data (e.g. new cross-region inference profile IDs). # Source: https://aws.amazon.com/bedrock/pricing/, cross-checked row-by-row against -# skills/gcp-to-aws/references/shared/pricing-cache.md (its per-1M rates / 1000). +# skills/shared/ai/bedrock-pricing-cache.md (its per-1M rates / 1000). # Every row below was re-verified against that cache on 2026-08-04; the Opus 4.8 row # had been copied from Opus 4.1's legacy $15/$75 and was corrected to $5/$25. # Re-check this table against that cache (and the public pricing page) whenever either moves. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py index a59a3a55f..bdeffd327 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py @@ -28,7 +28,7 @@ def test_static_fallback_returns_known_model(): def test_static_fallback_opus_4_8_rate_is_5_and_25_per_1m(): """Opus 4.8 is $5/$25 per 1M tokens (0.005/0.025 per 1K), NOT Opus 4.1's legacy - $15/$75 — see skills/gcp-to-aws/references/shared/pricing-cache.md for the + $15/$75 — see skills/shared/ai/bedrock-pricing-cache.md for the source rates. The table now keys by dateless family id, so the raw entries are asserted on those keys and ALL four id shapes (bare/us., dateless/date-pinned) must resolve behaviorally — diff --git a/plugins/aws-startup-advisor/skills/shared/ai/README.md b/plugins/aws-startup-advisor/skills/shared/ai/README.md index 43e4de142..692b5ed07 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/README.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/README.md @@ -18,14 +18,13 @@ before opening a pull request. | `design-ref-harness.md` | AgentCore Harness design reference | | `design-ref-agentic-to-agentcore.md` | Strands Agents + AgentCore Runtime design reference | | `sdk-capability-map.json` | SDK method → capability lookup used by app-code discovery | +| `bedrock-pricing-cache.md` | Provider-neutral Bedrock model and service pricing | Deliberately NOT here: a source-cloud's own mapping guide (e.g. gcp-to-aws's `ai-gemini-to-bedrock.md`) and its `schema-discover-ai.md`, both of which are written against one provider's SDK surface. -**Consumer contract.** A skill that vendors `references/vendored/ai/` must also ship -`references/shared/pricing-cache.md` and `references/shared/pricing-fallback.md` — -the files below reference them by that path for Bedrock model rates. They are -deliberately NOT part of this canonical set: `pricing-cache.md` doubles as some -skills' AWS-infrastructure rate card and carries source-provider rates, so it is -per-skill by design. +**Consumer contract.** A skill that consumes Bedrock pricing must vendor +`references/vendored/ai/bedrock-pricing-cache.md` from this canonical set. Source-cloud +and AWS infrastructure caches remain skill-owned; pricing fallback policy remains at +`references/shared/pricing-fallback.md` in each consuming skill. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md index 87bc66b98..aeb78d0b1 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md @@ -116,7 +116,7 @@ Before recommending any Bedrock model in an agentic design: For agentic workload cost estimation: -1. **Primary:** `references/shared/pricing-cache.md` (±5-10% accuracy) +1. **Primary:** `references/vendored/ai/bedrock-pricing-cache.md` (±15-25% accuracy) 2. **Fallback:** `references/shared/pricing-fallback.md` (cache-miss → `estimated`/`unavailable`) AgentCore Runtime and Harness pricing: consumption-based, no upfront cost. Include in estimate only if the user selects Harness or Strands path. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-model-lifecycle.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-model-lifecycle.md index c3facc7bc..701d7e84e 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-model-lifecycle.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-model-lifecycle.md @@ -133,7 +133,7 @@ When building the model comparison table: - **Active**: no restrictions. - **Restricted** (`restricted (…)` in the pricing-cache Status column): never `recommended_model` or `backup_model`, never a default in a mapping guide. Include in `model_comparison` only when the user explicitly asks about frontier / Covered Models, annotated with the access requirement. -### Pricing Cache (`pricing-cache.md`) +### Pricing Cache (`bedrock-pricing-cache.md`) The multi-provider quick reference table includes a `Status` column: @@ -161,6 +161,6 @@ When refreshing the cache, recompute `days_to_eol` and refresh the `active` / `l **Newer models are not covered by the table.** A model launched on or after 2026-09-07 will never appear in the Legacy/EOL table above, and its absence is not evidence that it is Active. When a candidate is not in the table, verify it by calling `GetFoundationModel` (or `ListFoundationModels`) and reading `modelLifecycle.status`: `LEGACY` and `EOL` are never valid targets for a new migration. If the model is Legacy, read its model card for the actual EOL date and whether the Legacy period is 6 months or 45 days. If neither the API nor the card is reachable, treat the model's lifecycle as **unverified** and say so in the output rather than inferring `active` from a `Status` column in a pricing cache. -**Periodic table refresh:** When the table itself needs updating (new models added, EOL dates changed by AWS, or past-EOL rows to remove), update this file and `pricing-cache.md` together. Edit the canonical `skills/shared/ai/ai-model-lifecycle.md`, then copy it over every vendored copy in the same change so they stay byte-identical. +**Periodic table refresh:** When the table itself needs updating (new models added, EOL dates changed by AWS, or past-EOL rows to remove), update this file and `bedrock-pricing-cache.md` together. Edit the canonical files under `skills/shared/ai/`, then copy them over every vendored copy in the same change so they stay byte-identical. **Past-EOL rows:** Once `days_to_eol ≤ 0`, move the model out of the live table into **Removed**, and grep this plugin's skill trees for the model ID to catch any remaining reference to it as a target. Keep the Removed entry long enough that users already on the model still get a warning. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index cf41cb343..da1c5fcdf 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -131,7 +131,7 @@ See "Cost-optimized cross-family alternatives" below for the numbers. All figures per 1M tokens. Percentages are blended savings at a 2:1 input-to-output ratio. **Both sides of these comparisons are now Bedrock prices** where the OpenAI model is on Bedrock — this is a Bedrock-vs-Bedrock model -choice, not a provider comparison. Verify rates via `shared/pricing-cache.md` — pricing is cache-only, so a +choice, not a provider comparison. Verify rates via `references/vendored/ai/bedrock-pricing-cache.md` — pricing is cache-only, so a GPT-5.x rate that is absent or `_unverified_` in the cache has no live lookup to fall back to and must be reported as unavailable rather than priced (see `shared/openai-on-bedrock.md`). diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md new file mode 100644 index 000000000..b1f94f147 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md @@ -0,0 +1,449 @@ +# Amazon Bedrock Pricing Cache + +**Last updated:** 2026-09-03 +**Region:** us-east-1 +**Currency:** USD +**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) + +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> +> **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. + +--- +## Bedrock Models (On-Demand) + +**Anthropic Claude (Standard on-demand)** figures below match **US East (N. Virginia)** on [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) as of cache refresh. **Recommend defaults (new migrations):** Claude Sonnet 5 (flagship), Claude Opus 4.8 (hardest reasoning), Claude Haiku 4.5 (cost/speed). Do not default to any Claude Fable / Mythos frontier model (Fable 5, Fable 5.1, and successors). **Claude Fable 5** and **Claude Fable 5.1** (GA Sep 1, 2026) are the most expensive Anthropic models at $10/$50 per 1M tokens (Mythos-class); 5.1's on-demand rate difference is cache read at $0.25 (0.025x) vs $1.00; Bedrock lists no batch rate for either. Both are Anthropic **Covered Models**: using them requires opting the account into the `aws_review` data-retention mode, and Fable 5.1 is served commercially only via `us.`/`global.` CRIS profiles (no in-region — the bare `anthropic.claude-fable-5-1` id is not invokable on `bedrock-runtime`, hence the `us.` form in the table), with in-region access in AWS GovCloud (US) only. Claude Mythos 5.1 is a gated Preview for vetted cyber/bio research organizations and is not a migration target. **Claude Opus 4.8** keeps the same $5/$25 rate as Opus 4.6/4.7. **Claude Sonnet 5** launched June 30 at $2/$10; that launch rate became the standard price on Sep 1, 2026 (the scheduled increase to $3/$15 was cancelled), so Sonnet 5 is now both newer and cheaper than Sonnet 4.6 ($3/$15). **Claude Opus 4.7** lists the same headline on-demand input/output as **Opus 4.6** on that page; confirm **batch** availability per model (Opus 4.7 batch was **not** listed on the global cross-region table when this row was added). **Claude Opus 4.1** entered **Legacy** on Jul 8, 2026 (EOL Jan 8, 2027). **Batch**, **prompt cache** (5m / 1h write + cache read), and **geo / in-region cross-region** rows on that page can differ; e.g. **US East (Ohio)** cross-region inference for Claude Sonnet 4.6 is listed at **$3.30 / $16.50** per 1M input/output (≈10% above N. Virginia). Long-context SKUs **do not** all use the same multiplier: **Sonnet 4.6** and **Opus 4.6** long-context modes share the same on-demand rates as the non–long-context rows on the standard table; **Sonnet 4.5** and **Sonnet 4** long-context rows are priced higher on that same table. + +### Multi-provider quick reference (per 1M tokens) + +See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommend Legacy models for new migrations.** Rows whose Status is `restricted (…)` are never `recommended_model` / `backup_model` and never a default (Covered Models, gated previews). + +| Model | Model ID | Provider | Input $/1M | Output $/1M | Context | Tier | Status | +| -------------------------------- | ---------------------------------------- | --------- | ---------- | ----------- | ------- | --------- | ------------------------------------------------------------ | +| Claude Fable 5 | anthropic.claude-fable-5 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in) | +| Claude Fable 5.1 | us.anthropic.claude-fable-5-1 | Anthropic | 10.00 | 50.00 | 1M | frontier | restricted (Covered Model; `aws_review` opt-in; CRIS only) | +| Claude Sonnet 5 | anthropic.claude-sonnet-5 | Anthropic | 2.00 | 10.00 | 1M | flagship | active ($2/$10 made the standard price Sep 1, 2026) | +| Claude Opus 4.8 | anthropic.claude-opus-4-8 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.6 | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.6 — Long Context | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K+ | flagship | active | +| Claude Opus 4.6 | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Opus 4.6 — Long Context | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K+ | premium | active | +| Claude Opus 4.5 | — | Anthropic | 5.00 | 25.00 | 200K | premium | active | +| Claude Sonnet 4.5 | — | Anthropic | 3.00 | 15.00 | 200K | flagship | active | +| Claude Sonnet 4.5 — Long Context | — | Anthropic | 6.00 | 22.50 | 200K+ | flagship | active | +| Claude Haiku 4.5 | anthropic.claude-haiku-4-5-20251001-v1:0 | Anthropic | 1.00 | 5.00 | 200K | fast | active | +| Claude Opus 4.1 | anthropic.claude-opus-4-1-20250805-v1:0 | Anthropic | 15.00 | 75.00 | 200K | premium | legacy (EOL Jan 8, 2027) | +| Claude Sonnet 4 | anthropic.claude-sonnet-4-20250514-v1:0 | Anthropic | 3.00 | 15.00 | 200K | flagship | excluded (EOL Oct 14, 2026) | +| Llama 4 Maverick | meta.llama4-maverick-17b-instruct-v1:0 | Meta | 0.24 | 0.97 | 1M | mid | active | +| Llama 4 Scout | meta.llama4-scout-17b-instruct-v1:0 | Meta | 0.17 | 0.66 | 10M | efficient | active | +| Llama 3.3 70B | meta.llama3-3-70b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | active | +| Llama 3.2 90B | meta.llama3-2-90b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | EOL (Jul 7, 2026) | +| Nova 2 Lite | amazon.nova-2-lite-v1:0 | Amazon | 0.33 | 2.75 | 1M | mid | active | +| Nova 2 Pro (Preview) | amazon.nova-2-pro-v1:0 | Amazon | 1.375 | 11.00 | 1M | flagship | active | +| Nova Pro | amazon.nova-pro-v1:0 | Amazon | 0.80 | 3.20 | 300K | mid | active | +| Nova Pro (latency optimized) | — | Amazon | 1.00 | 4.00 | 300K | mid | active | +| Nova Lite | amazon.nova-lite-v1:0 | Amazon | 0.06 | 0.24 | 300K | fast | active | +| Nova Micro | amazon.nova-micro-v1:0 | Amazon | 0.035 | 0.14 | 128K | budget | active | +| Nova Premier | amazon.nova-premier-v1:0 | Amazon | 2.50 | 12.50 | 1M | reasoning | EOL (2026-09-14) | +| Mistral Large 3 | mistral.mistral-large-3-675b-instruct | Mistral | 0.50 | 1.50 | 256K | flagship | active | +| DeepSeek-R1 | deepseek.r1-v1:0 | DeepSeek | 1.35 | 5.40 | 128K | reasoning | active | +| DeepSeek-V3.1 | — | DeepSeek | 0.58 | 1.68 | — | mid | active (Sydney only) | +| gpt-oss-20b | openai.gpt-oss-20b-1:0 | OpenAI | 0.07 | 0.30 | 128K | budget | active | +| gpt-oss-120b | openai.gpt-oss-120b-1:0 | OpenAI | 0.15 | 0.60 | 128K | efficient | active | +| GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | +| GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | +| GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | +| Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | +| Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | +| Gemma 3 4B IT | google.gemma-3-4b-it | Google | 0.04 | 0.08 | 128K | budget | active | +| Gemma 3 12B IT | google.gemma-3-12b-it | Google | 0.09 | 0.29 | 128K | budget | active | +| Gemma 3 27B IT | google.gemma-3-27b-it | Google | 0.23 | 0.38 | 128K | efficient | active | +| MiniMax M2 | minimax.minimax-m2 | MiniMax | 0.30 | 1.20 | 1M | mid | active | +| MiniMax M2.1 | minimax.minimax-m2.1 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| MiniMax M2.5 | minimax.minimax-m2.5 | MiniMax | 0.30 | 1.20 | 196K | mid | active | +| Jamba 1.5 Large | ai21.jamba-1-5-large-v1:0 | AI21 Labs | 2.00 | 8.00 | — | mid | excluded (EOL 2026-11-26) | +| Jamba 1.5 Mini | ai21.jamba-1-5-mini-v1:0 | AI21 Labs | 0.20 | 0.40 | — | efficient | excluded (EOL 2026-11-26) | +| Jurassic-2 Mid | — | AI21 Labs | 12.50 | 12.50 | — | legacy | legacy | +| Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | +| Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | + +### Stability AI — Image Generation (per image, US East) + +Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. + +| Model | Model ID | Price/image | Resolution | Tier | Status | +| -------------------------- | --------------------------------- | ----------- | ---------- | -------- | ------ | +| Stable Image Ultra | stability.stable-image-ultra-v1:0 | $0.08 | up to 4MP | premium | active | +| Stable Diffusion 3.5 Large | stability.sd3-5-large-v1:0 | $0.065 | up to 1MP | flagship | active | +| Stable Image Core | stability.stable-image-core-v1:0 | $0.04 | up to 1MP | fast | active | + +Image editing services (inpaint, erase, upscale, etc.) are priced at $0.03–$0.60/operation. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full list. + +> **Cost comparison note:** DALL-E 3 (OpenAI) charges $0.04–$0.12/image depending on resolution. Google Imagen charges per character of prompt. When comparing, use per-image cost directly — do not convert Stability AI prices to per-token equivalents. + +### Anthropic Claude — batch & prompt cache (Standard, US East N. Virginia) + +Per 1M tokens unless noted. See [Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) for full regional and tier tables. + +> **Batch support verified 2026-09-02** against the +> [supported Regions and models for batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) +> table: **Opus 4.6, Sonnet 4.6, Opus 4.5, Sonnet 4.5, Sonnet 4.5 — LC, and Haiku 4.5 are listed** (via cross-region +> inference profiles; batch is 50% of on-demand per the pricing page, matching the rows below). **Fable 5, Sonnet 5, +> and Opus 4.8 are NOT on that table** — Sonnet 5 and Opus 4.8's batch cells below are **unverified (‡)**: do not build a batch-discount +> TCO on them; price on-demand and note batch as a possible future saving pending the docs table listing them. +> Fable 5 / 5.1 instead carry **N/A** per the separate pricing-page verification below. +> Prompt-cache columns are unaffected (caching support is independent of batch). +> **Batch rate vs. cross-region inference (CRIS):** the confirmed rows show 50% of the **N. Virginia** on-demand +> price. Where batch is only reachable through a CRIS profile that lists above N. Virginia (e.g. Sonnet 4.6 in US East +> (Ohio) at ≈10% higher — see the on-demand section), the effective batch rate is 50% of that CRIS price, i.e. ≈10% +> above the cells here. Re-derive from the profile's on-demand rate for a region-specific quote. +> **Do not cross-check these rows against the AWS Pricing Calculator** — as of Sep 2026 the calculator carries no +> batch metered IDs for Opus 4.6 / Sonnet 4.6 and collapses other batch/cache SKUs (known calculator defect, not a +> pricing-page signal). The pricing page and the docs table above are the sources of truth. + +| Model | Batch in | Batch out | 5m cache write | 1h cache write | Cache read | +| ------------------------ | -------- | --------- | -------------- | -------------- | ---------- | +| Claude Fable 5 | N/A | N/A | 12.50 | 20.00 | 1.00 | +| Claude Fable 5.1 | N/A | N/A | 12.50 | 20.00 | 0.25 | +| Claude Sonnet 5 | 1.00 ‡ | 5.00 ‡ | 2.50 | 4.00 | 0.20 | +| Claude Opus 4.8 | 2.50 ‡ | 12.50 ‡ | 6.25 | 10.00 | 0.50 | +| Claude Sonnet 4.6 (+ LC) | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Opus 4.6 (+ LC) | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Opus 4.5 | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | +| Claude Haiku 4.5 | 0.50 | 2.50 | 1.25 | 2.00 | 0.10 | +| Claude Sonnet 4.5 | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | +| Claude Sonnet 4.5 — LC | 3.00 | 11.25 | 7.50 | 12.00 | 0.60 | + +_Batch: the Bedrock pricing page lists **N/A** for Claude Fable 5 and Fable 5.1 in both the Global and the Geo / In-region Anthropic tables (read 2026-09-03). Anthropic's first-party API offers a 50% batch discount on these models; Bedrock does not list one — do not assume a batch rate for Fable-class models._ + +‡ Not listed on the batch-supported models table as of 2026-09-02 — rate is the standard 50%-of-on-demand +projection, not a confirmed SKU. Mark these batch cells `_unverified_` in estimate output; the `unverified` gate in +`estimate-ai.md` (§ Pricing source caveat) treats any `_unverified_` row as blocking for a quoted figure — resolve +from the Bedrock pricing page first. + +### AI21 Labs + +**On-demand**, **US East (N. Virginia)** per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/). Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ---------------- | ---------- | ----------- | +| Jamba 1.5 Large | 2.00 | 8.00 | +| Jamba 1.5 Mini | 0.20 | 0.40 | +| Jurassic-2 Mid | 12.50 | 12.50 | +| Jurassic-2 Ultra | 18.80 | 18.80 | +| Jamba-Instruct | 0.50 | 0.70 | + +_Quick-reference rows use **—** for **model ID** and **context**; resolve in the Bedrock console or AWS model documentation._ + +### Mistral AI + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Mistral AI). **Priority** tier ≈ **+75%** vs Standard; **Flex** ≈ **−50%** vs Standard on the same page. + +#### US East (Ohio) — Pixtral (on-demand + batch header from page) + +| Model | Input $/1M | Output $/1M | Batch in | Batch out | +| --------------------- | ---------- | ----------- | -------- | --------- | +| Pixtral Large (25.02) | 2.00 | 6.00 | N/A | N/A | + +#### On-demand — US East (N. Virginia), US East (Ohio), US West (Oregon) + +Prices per 1M input / output tokens. + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Devstral 2 123B | 0.40 | 2.00 | +| Magistral Small 1.2 | 0.50 | 1.50 | +| Voxtral Mini 1.0 | 0.04 | 0.04 | +| Voxtral Small 1.0 | 0.10 | 0.30 | +| Ministral 3B 3.0 | 0.10 | 0.10 | +| Ministral 8B 3.0 | 0.15 | 0.15 | +| Ministral 14B 3.0 | 0.20 | 0.20 | +| Mistral Large 3 | 0.50 | 1.50 | + +_Rates differ in **Asia Pacific (Mumbai)**, **São Paulo**, **Tokyo**, **Europe (Ireland/Milan/London)**, **Sydney**, **Jakarta**, **Frankfurt**, **Stockholm** (e.g. Mistral Large 3 Mumbai **$0.59 / $1.76**). **Devstral 2 123B** appears in Jakarta / Frankfurt / Stockholm at **$0.48 / $2.40** on the same page._ + +### DeepSeek (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (DeepSeek). + +#### US East (Ohio) — Standard tier + +> **Note:** DeepSeek-V3.1 is no longer listed for US East regions on the Bedrock pricing page as of May 2026. Use DeepSeek-V3.2 for US East deployments. DeepSeek-V3.1 remains available in Asia Pacific (Sydney). + +| Model | Input $/1M | Output $/1M | +| ----------- | ---------- | ----------- | +| DeepSeek-R1 | 1.35 | 5.40 | + +#### US East (N. Virginia), US East (Ohio), US West (Oregon) + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| DeepSeek v3.2 | 0.62 | 1.85 | + +#### Other regions (examples on same page) + +- **Asia Pacific (Mumbai)**, **São Paulo**, **Jakarta**, **Tokyo**, **Stockholm**: DeepSeek v3.2 **$0.74 / $2.22** per 1M. +- **Asia Pacific (Sydney)**: DeepSeek v3.1 **$0.5974 / $1.7304**; v3.2 **$0.6386 / $1.9055** (and **Priority** / **Flex** tiers listed separately on the page). + +### OpenAI on Bedrock (GPT-5.6 and gpt-oss) + +See `shared/openai-on-bedrock.md` for model IDs, endpoint paths, regions, quotas, and caching rules. **Bedrock +in-region is priced at parity with OpenAI's _data residency_ tier — exactly 1.10x OpenAI's standard list price** — so +a same-model migration costs about 10% MORE, not the same. Do not describe it as cost-neutral. + +#### Proprietary GPT models — short context (272K), in-region + +Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally carries Terra, Luna and GPT-5.4; AWS GovCloud us-gov-west-1 / us-gov-east-1 carry Terra and Luna, us-gov-west-1 also GPT-5.4 (at its own rate below). Rates reflect the July 30, 2026 reduction (Luna −80%, Terra −20%) and the Aug 21, 2026 Sol reduction (−20% input / −33% output, promotional through at least Nov 21, 2026). + +| Model | Input $/1M | Output $/1M | Cache write $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | ---------------- | --------------- | +| GPT-5.6 Sol | 4.40 | 22.00 | 5.50 | 0.44 | +| GPT-5.6 Terra | 2.20 | 13.20 | 2.75 | 0.22 | +| GPT-5.6 Luna | 0.22 | 1.32 | 0.275 | 0.022 | +| GPT-5.5 | 5.50 | 33.00 | — | 0.55 | +| GPT-5.4 | 2.75 | 16.50 | — | 0.275 | + +#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output + +| Model | Input $/1M | Output $/1M | +| ------------- | ---------- | ----------- | +| GPT-5.6 Sol | 8.80 | 33.00 | +| GPT-5.6 Terra | 4.40 | 19.80 | +| GPT-5.6 Luna | 0.44 | 1.98 | + +A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so +their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — +see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. + +> **One source conflict.** These models are absent +> from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and +> filtering on `GPT-5` or a `gpt-5` usage type returns zero rows (price-list publication 2026-08-04). An empty +> price-list result is **not** evidence the model is unavailable. (An earlier revision also +> flagged the AWS News Blog's Luna 0.20 / 1.20 as conflicting; it is the Global CRIS rate, not an error.) + +**Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum +1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. +GPT-5.5 and GPT-5.4 do not list prompt caching support. + +#### Open-weight gpt-oss + +| Model | Region | Input $/1M | Output $/1M | +| ------------ | --------------------- | ---------- | ----------- | +| gpt-oss-20b | US East (Ohio) | 0.07 | 0.30 | +| gpt-oss-120b | US East (Ohio) | 0.15 | 0.60 | +| gpt-oss-20b | Asia Pacific (Sydney) | 0.0721 | 0.3090 | +| gpt-oss-120b | Asia Pacific (Sydney) | 0.1545 | 0.6180 | + +_The Bedrock page also lists **Flex**, **Priority**, **Batch**, **GPT OSS Safeguard**, and **model customization** pricing for other regions._ + +### Meta Llama (Amazon Bedrock) + +Per [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Meta). Prices per 1M input / output tokens unless noted. + +#### Llama 4 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------- | ------------ | ------------- | -------- | --------- | +| Llama 4 Maverick 17B | 0.24 | 0.97 | 0.12 | 0.485 | +| Llama 4 Scout 17B | 0.17 | 0.66 | 0.085 | 0.33 | + +#### Llama 3.3 — US East (Ohio), on-demand and batch + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.3 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | + +#### Llama 3.2 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** All Llama 3.2 Instruct models reached **EOL on Jul 7, 2026** and are no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Scout / Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| ------------------------ | ------------ | ------------- | -------- | --------- | +| Llama 3.2 Instruct (1B) | 0.10 | 0.10 | N/A | N/A | +| Llama 3.2 Instruct (3B) | 0.15 | 0.15 | N/A | N/A | +| Llama 3.2 Instruct (11B) | 0.16 | 0.16 | N/A | N/A | +| Llama 3.2 Instruct (90B) | 0.72 | 0.72 | N/A | N/A | + +#### Llama 3.2 — customization & provisioned throughput (US West Oregon) + +**Fine-tuning** (per 1M tokens trained / month storage / no-commit PT per model unit-hour): + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.2 Instruct (1B) | $0.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (3B) | $1.10 | $1.95 | $23.00 | +| Llama 3.2 Instruct (11B) | $3.50 | $1.95 | $23.00 | +| Llama 3.2 Instruct (90B) | $7.90 | $1.95 | $30.00 | + +**Provisioned throughput** ($/hour per model unit): + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.2 Instruct (1B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (3B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (11B) | $23.00 | $20.30 | $12.54 | +| Llama 3.2 Instruct (90B) | $30.00 | $26.48 | $16.35 | + +#### Llama 3.1 — US East (Ohio), on-demand and batch + +> **Lifecycle note:** Llama 3.1 405B Instruct reached **EOL on Jul 7, 2026** and is no longer available for inference on Bedrock. Do not select for any migrations. Use **Llama 4 Maverick** instead. See `vendored/ai/ai-model-lifecycle.md`. + +| Model | On-demand in | On-demand out | Batch in | Batch out | +| -------------------------------------------------- | ------------ | ------------- | -------- | --------- | +| Llama 3.1 Instruct (8B) | 0.22 | 0.22 | 0.11 | 0.11 | +| Llama 3.1 Instruct (70B) | 0.72 | 0.72 | 0.36 | 0.36 | +| Llama 3.1 Instruct (405B) | 2.40 | 2.40 | 1.20 | 1.20 | +| Llama 3.1 Instruct (70B) (latency optimized inf.) | 0.90 | 0.90 | N/A | N/A | +| Llama 3.1 Instruct (405B) (latency optimized inf.) | 3.00 | 3.00 | N/A | N/A | + +#### Llama 3.1 — customization & provisioned throughput (US West Oregon) + +| Model | Train 1M tokens | Store / month | Infer / unit-hr (no commit) | +| ------------------------ | --------------- | ------------- | --------------------------- | +| Llama 3.1 Instruct (8B) | $1.49 | $1.95 | $24.00 | +| Llama 3.1 Instruct (70B) | $7.99 | $1.95 | $24.00 | + +| Model | No commit | 1-mo commit | 6-mo commit | +| ------------------------ | --------- | ----------- | ----------- | +| Llama 3.1 Instruct (8B) | $24.00 | $21.18 | $13.08 | +| Llama 3.1 Instruct (70B) | $24.00 | $21.18 | $13.08 | + +#### Llama 3 — US East (N. Virginia), on-demand + +| Model | Input $/1M | Output $/1M | +| ---------------------- | ---------- | ----------- | +| Llama 3 Instruct (8B) | 0.30 | 0.60 | +| Llama 3 Instruct (70B) | 2.65 | 3.50 | + +#### Llama 2 — US East (N. Virginia) and US West (Oregon), on-demand + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Llama 2 Chat (13B) | 0.75 | 1.00 | +| Llama 2 Chat (70B) | 1.95 | 2.56 | + +#### Llama 2 — customization & provisioned throughput + +**Fine-tuning:** Llama 2 Pretrained (13B) **$1.49** per 1M tokens trained, **$1.95**/month storage, **$23.50**/unit-hr no-commit infer; (70B) **$7.99** / **$1.95** / **$23.50**. + +**Provisioned throughput** (per model unit-hour): 13B and 70B — **$21.18** (1-mo commit), **$13.08** (6-mo commit). Pretrained Llama 2 is **provisioned throughput only after customization** per AWS. + +### Amazon Nova + +All figures from [Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) (Amazon Nova sections). Nova pricing depends on **inference deployment** (e.g. **Global cross-region** vs **Geo cross-region and in-region**), **tier** (Standard, Priority, Flex, Batch), **modality** (text / image / video / audio), and **AWS Region**. The quick-reference table above uses **Geo cross-region inference and in-region**, **Standard tier**, **US East (Ohio)**, **text + image + video** rates where a single input price applies to all three. Confirm the page before estimates. + +**Cache read (Nova):** AWS states cache read input tokens are **75% below** the on-demand input price for the applicable tier/modality. + +#### US East (Ohio) — Standard tier, text + image + video (Geo cross-region inference and in-region) + +Per 1M input / output tokens unless noted. + +| Model | Input $/1M | Output $/1M | +| ---------------------------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.33 | 2.75 | +| Amazon Nova Micro | 0.035 | 0.14 | +| Amazon Nova Lite | 0.06 | 0.24 | +| Amazon Nova Pro | 0.80 | 3.20 | +| Amazon Nova Pro (latency optimized inf.) | 1.00 | 4.00 | +| Amazon Nova Premier | 2.50 | 12.50 | + +#### US East (Ohio) — Standard tier, Global cross-region inference (text + image + video) + +| Model | Input $/1M | Output $/1M | +| ------------------ | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.30 | 2.50 | + +#### US East (Ohio) — Standard tier, text / image / video / audio (Geo cross-region and in-region) + +Per 1M tokens. **Nova 2 Omni** and **Nova 2 Pro** are **Preview**. Image column is **output** image pricing where listed. + +| Model | Text in | Image in | Video in | Audio in | Text out | Image out | +| ---------------------------- | ------- | -------- | -------- | -------- | -------- | --------- | +| Amazon Nova 2 Omni (Preview) | 0.30 | 0.30 | 0.30 | 1.10 | 2.80 | 44.00 | +| Amazon Nova 2 Pro (Preview) | 1.375 | 1.375 | 1.375 | 1.375 | 11.00 | N/A | + +**Global cross-region inference (Ohio), Standard** uses different multimodal numbers on the same page — e.g. **Nova 2 Omni (Preview)** audio in **$1.00**, text out **$2.50**, image out **$40.00** (text/image/video in **$0.30** each). Use the Bedrock pricing page for the deployment you select. + +#### US East (Ohio) — Nova 2 Lite by tier (text + image + video) + +##### Global cross-region inference + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.30 | 2.50 | +| Priority | 0.525 | 4.375 | +| Flex | 0.15 | 1.25 | +| Batch | 0.15 | 1.25 | + +##### Geo cross-region inference and in-region + +| Tier | Input $/1M | Output $/1M | +| -------- | ---------- | ----------- | +| Standard | 0.33 | 2.75 | +| Priority | 0.5775 | 4.8125 | +| Flex | 0.165 | 1.375 | +| Batch | 0.1595 | 1.342 | + +#### US East (Ohio) — Batch tier (text + image + video), selected models + +| Model | Input $/1M | Output $/1M | +| ------------------- | ---------- | ----------- | +| Amazon Nova 2 Lite | 0.1595 | 1.342 | +| Amazon Nova Micro | 0.0175 | 0.07 | +| Amazon Nova Lite | 0.03 | 0.12 | +| Amazon Nova Pro | 0.40 | 1.60 | +| Amazon Nova Premier | 1.25 | 6.25 | + +#### On-demand inference (listed rates, cache read column) + +| Model | Input $/1M | Cache read $/1M | Output $/1M | +| -------------------- | ---------- | --------------- | ----------- | +| Amazon Nova 2.0 Lite | N/A | N/A | N/A | +| Amazon Nova Micro | 0.035 | N/A | 0.14 | +| Amazon Nova Lite | 0.06 | N/A | 0.24 | +| Amazon Nova Pro | 0.80 | N/A | 3.20 | + +#### Built-in tools — Web grounding (US East Ohio) + +| Model | Price | +| ---------------------------- | ---------------------- | +| Amazon Nova 2 Omni (Preview) | $30.00 per 1K requests | +| Amazon Nova 2 Pro (Preview) | $30.00 per 1K requests | + +#### Creative — US East (N. Virginia) + +> **Lifecycle note:** Nova Canvas v1 and Nova Reel v1 are **excluded** — both are Legacy with EOL 2026-09-30, inside the 90-day exclusion zone. Do not list them in recommendation or comparison tables. Rates are retained for users already on them. See `vendored/ai/ai-model-lifecycle.md`. + +**Amazon Nova Canvas** (on-demand, per image): up to **1024×1024** — Standard **$0.04**, Premium **$0.06**; up to **2048×2048** — Standard **$0.06**, Premium **$0.08**. + +**Model customization (Nova Canvas):** **$0.005** per image seen; **$1.95**/month per custom model stored; provisioned inference per model unit per hour (no commit / 1-mo / 6-mo) **$60.50 / $55.00 / $30.25**. + +**Amazon Nova Reel** (video): **720p, 24 fps** — **$0.08** per second of video generated. + +#### Speech — US East (N. Virginia) + +> **Lifecycle note:** Nova Sonic v1 reached **EOL on 2026-09-14** and its requests now fail. Rates are retained only to explain the migration to **Nova 2 Sonic**. See `vendored/ai/ai-model-lifecycle.md`. + +Per 1M tokens. + +| Model | Modality | Input $/1M | Output $/1M | Status | +| ------------------- | -------- | ---------- | ----------- | ---------------- | +| Amazon Nova Sonic | Speech | 3.40 | 13.60 | EOL (2026-09-14) | +| Amazon Nova Sonic | Text | 0.06 | 0.24 | EOL (2026-09-14) | +| Amazon Nova 2 Sonic | Speech | 3.00 | 12.00 | active | +| Amazon Nova 2 Sonic | Text | 0.33 | 2.75 | active | + +#### Multimodal embeddings — US East (N. Virginia) + +| Offering | Text $/1M | Std image / doc image / video sec / audio sec | +| --------------------------------------------- | --------- | ------------------------------------------------ | +| Amazon Nova Multimodal Embeddings (On-demand) | 0.135 | $0.00006 / $0.0006 / $0.0007 / $0.00014 per unit | +| Amazon Nova Multimodal Embeddings (Batch) | 0.0675 | $0.00003 / $0.00048 / $0.00056 / $0.000112 | + +_On-demand inference for **custom Nova** models matches **base Nova** inference pricing per AWS._ + +--- diff --git a/plugins/aws-startup-advisor/tests/test_model_id_lint.py b/plugins/aws-startup-advisor/tests/test_model_id_lint.py index 596c1623d..b17a79c4a 100644 --- a/plugins/aws-startup-advisor/tests/test_model_id_lint.py +++ b/plugins/aws-startup-advisor/tests/test_model_id_lint.py @@ -135,7 +135,7 @@ def test_retired_id_allowed_in_both_pricing_caches(tmp_path: Path) -> None: model_id = _banned(PLUGIN_ROOT)[0] root = _seed(tmp_path) for rel in ( - "skills/gcp-to-aws/references/shared/pricing-cache.md", + "skills/shared/ai/bedrock-pricing-cache.md", "skills/azure-to-aws/references/shared/pricing-cache.md", ): _write(root, rel, f"existing users may still see `{model_id}`\n") From d3ab4ab3c79905c1c6a124c1dec48f0abf1f35b0 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 22:00:44 -0700 Subject: [PATCH 20/53] fix(aws-startup-advisor): preserve pricing accuracy metadata --- .../scripts/check-bedrock-pricing-parity.py | 21 +++++++++++++++++++ .../skills/gcp-to-aws/SKILL.md | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py index 1f2c87a58..e803f23c8 100755 --- a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py +++ b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py @@ -22,8 +22,10 @@ REPO = PLUGIN.parents[1] OLD_CACHE = "plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md" OLD_SCRIPT = "plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py" +OLD_SKILL = "plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md" CANONICAL = PLUGIN / "skills/shared/ai/bedrock-pricing-cache.md" GCP_CACHE = PLUGIN / "skills/gcp-to-aws/references/shared/pricing-cache.md" +GCP_SKILL = PLUGIN / "skills/gcp-to-aws/SKILL.md" VENDORED = ( PLUGIN / "skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md", PLUGIN / "skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md", @@ -71,6 +73,17 @@ def model_rows(text: str) -> dict[str, tuple[str, str]]: return rows +def pricing_accuracy_bands(text: str) -> tuple[str, str]: + line = next( + (line for line in text.splitlines() if line.startswith("- Primary pricing source")), + None, + ) + assert line, "missing primary pricing source contract" + bands = re.findall(r"±\d+-\d+%", line) + assert len(bands) == 2, "expected infrastructure and AI pricing accuracy bands" + return bands[0], bands[1] + + def load_module(path: Path, name: str): module = types.ModuleType(name) code = compile( @@ -110,6 +123,10 @@ def main() -> int: old = git_show(OLD_CACHE) canonical = CANONICAL.read_text() remaining = GCP_CACHE.read_text() + old_accuracy = pricing_accuracy_bands(git_show(OLD_SKILL)) + new_accuracy = pricing_accuracy_bands(GCP_SKILL.read_text()) + + assert new_accuracy == old_accuracy, "estimate accuracy metadata changed" old_bedrock = section(old, "Bedrock Models (On-Demand)") new_bedrock = section(canonical, "Bedrock Models (On-Demand)") @@ -170,6 +187,10 @@ def main() -> int: "bedrock_model_lookup_keys": len(new_rows), "static_fallback_keys": len(pricing_behavior(after)["table"]), "representative_monthly_estimates": new_estimates, + "pricing_accuracy_bands": { + "infrastructure": new_accuracy[0], + "ai_models": new_accuracy[1], + }, "vendored_copies": len(VENDORED), "baseline": BASE_SHA, }, indent=2, sort_keys=True)) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md index 9e8d58990..1db6e0c0d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md @@ -255,7 +255,7 @@ Replace `MMDD-HHMM` with the actual migration ID, generate the `last_updated` IS - Provides `aws___search_documentation`, `aws___read_documentation`, `aws___list_regions`, `aws___get_regional_availability`, `aws___retrieve_skill` tools - Used during Design for regional availability checks and documentation lookups. -- Primary pricing sources: `references/shared/pricing-cache.md` (cached 2026 infrastructure and source-provider rates, ±5-25%) and `references/vendored/ai/bedrock-pricing-cache.md` (provider-neutral Bedrock rates, ±15-25%). Pricing is cache-only — no live pricing MCP. +- Primary pricing sources: `references/shared/pricing-cache.md` (cached 2026 infrastructure and source-provider rates, ±5-10%) and `references/vendored/ai/bedrock-pricing-cache.md` (provider-neutral Bedrock rates, ±15-25%). Pricing is cache-only — no live pricing MCP. --- From edbca3520b3da941492f8ee6c747c70cbaa75130 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 22:15:22 -0700 Subject: [PATCH 21/53] fix: align integration validation run IDs Accept uppercase UUID hex in integration validation to match the phase-status contract, cover uppercase/lowercase/malformed IDs through the artifact validator, and document the distinct OpenRouter fixture pair. Validated: targeted matrix suite (8 passed); uppercase artifact probe; empty-reference rejection; failed infrastructure/AI/IAM children; phase-status schema probe; mise run build; git diff --check; clean merge-tree against upstream/main. --- .../aws-startup-advisor/fixtures/README.md | 2 +- .../integration-validation.schema.json | 2 +- .../test_ai_migration_matrix_contracts.py | 41 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 8250ad48e..9636d2610 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -2,7 +2,7 @@ Also see: -- `ai-migration-matrix/` — target-only contract snapshots for infrastructure-only, AI-only, and combined migration modes. These fixtures do not claim current runtime support. They keep infrastructure source, AI provider/model family, and router/gateway independent; combined cases use one `.phase-status.json` and a separate `integration-validation.json` cross-track verdict. +- `ai-migration-matrix/` — target-only contract snapshots for infrastructure-only, AI-only, and combined migration modes. These fixtures do not claim current runtime support. They keep infrastructure source, AI provider/model family, and router/gateway independent; combined cases use one `.phase-status.json` and a separate `integration-validation.json` cross-track verdict. The two combined OpenRouter cases deliberately pair different infrastructure sources with different underlying providers/models (GCP + OpenAI/GPT and Azure + Anthropic/Claude), proving that OpenRouter is an infrastructure-independent gateway rather than duplicating one provider case. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json index 492b6fa17..6ab37456a 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json @@ -8,7 +8,7 @@ "additionalProperties": false, "properties": { "schema_version": {"const": "1.0"}, - "run_id": {"type": "string", "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"}, + "run_id": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"}, "verdict": {"enum": ["passed", "failed"]}, "track_results": { "type": "object", diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py index 1955ab0da..1e1662771 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -181,6 +181,47 @@ def test_combined_validation_references_both_tracks_and_all_categories() -> None assert result["references"] +def test_integration_validation_accepts_uppercase_and_lowercase_run_ids(tmp_path: Path) -> None: + source = FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json" + run_ids = { + "uppercase": ("ABCDEF12-ABCD-4ABC-8DEF-ABCDEF123456", 0), + "lowercase": ("abcdef12-abcd-4abc-8def-abcdef123456", 0), + "malformed": ("not-a-uuid", 1), + } + + for label, (run_id, expected_returncode) in run_ids.items(): + validation = _load(source) + validation["run_id"] = run_id + case_dir = tmp_path / label + case_dir.mkdir() + (case_dir / "integration-validation.json").write_text(json.dumps(validation)) + + result = subprocess.run( + [ + sys.executable, + str(ARTIFACT_VALIDATOR), + "--run-dir", + str(case_dir), + "--skill", + "gcp-to-aws", + "--manifest", + str(ARTIFACT_MANIFEST), + "--no-baseline", + "--json", + ], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == expected_returncode + findings = json.loads(result.stdout)["errors"] + if expected_returncode == 0: + assert findings == [] + else: + assert any(finding["code"] == "SCHEMA_VIOLATION" for finding in findings) + + def test_success_verdict_rejects_any_failed_child(tmp_path: Path) -> None: source = FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json" failed_children = [ From b7141feeeabed6c4ba1177109b3831ec9f19452d Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 22:29:06 -0700 Subject: [PATCH 22/53] fix(aws-startup-advisor): correct Bedrock cache ownership Route Azure Bedrock estimates through the canonical vendored cache, remove duplicate Azure Bedrock rows, and align infrastructure and AI accuracy contracts. Validated: pricing tests 21 passed; agent-advisor 323 passed; model/vendor tests 19 passed; Azure/GCP fixtures passed; parity checker and four mutation probes passed; artifact, vendored, skill-ref, Markdown, aggregate validation, full build, and gitleaks passed. --- .../scripts/check-bedrock-pricing-parity.py | 51 ++++++- .../vendored/ai/bedrock-pricing-cache.md | 6 +- .../references/phases/estimate/estimate-ai.md | 61 ++++---- .../references/phases/estimate/estimate.md | 1 + .../references/shared/openai-on-bedrock.md | 3 +- .../references/shared/pricing-cache.md | 130 +++--------------- .../references/shared/pricing-fallback.md | 3 +- .../vendored/ai/bedrock-pricing-cache.md | 6 +- .../skills/gcp-to-aws/SKILL.md | 2 +- .../references/phases/estimate/estimate-ai.md | 2 +- .../references/phases/estimate/estimate.md | 12 +- .../references/shared/pricing-cache.md | 6 +- .../vendored/ai/bedrock-pricing-cache.md | 6 +- .../skills/shared/ai/bedrock-pricing-cache.md | 6 +- 14 files changed, 131 insertions(+), 164 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py index e803f23c8..62e5e8da6 100755 --- a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py +++ b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py @@ -23,9 +23,16 @@ OLD_CACHE = "plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md" OLD_SCRIPT = "plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py" OLD_SKILL = "plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md" +OLD_AZURE_CACHE = "plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-cache.md" CANONICAL = PLUGIN / "skills/shared/ai/bedrock-pricing-cache.md" GCP_CACHE = PLUGIN / "skills/gcp-to-aws/references/shared/pricing-cache.md" GCP_SKILL = PLUGIN / "skills/gcp-to-aws/SKILL.md" +GCP_ESTIMATE = PLUGIN / "skills/gcp-to-aws/references/phases/estimate/estimate.md" +GCP_ESTIMATE_AI = PLUGIN / "skills/gcp-to-aws/references/phases/estimate/estimate-ai.md" +AZURE_CACHE = PLUGIN / "skills/azure-to-aws/references/shared/pricing-cache.md" +AZURE_ESTIMATE = PLUGIN / "skills/azure-to-aws/references/phases/estimate/estimate.md" +AZURE_ESTIMATE_AI = PLUGIN / "skills/azure-to-aws/references/phases/estimate/estimate-ai.md" +AZURE_OPENAI = PLUGIN / "skills/azure-to-aws/references/shared/openai-on-bedrock.md" VENDORED = ( PLUGIN / "skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md", PLUGIN / "skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md", @@ -73,6 +80,17 @@ def model_rows(text: str) -> dict[str, tuple[str, str]]: return rows +def markdown_data_rows(text: str) -> list[str]: + """Return table data rows while excluding separators and headers.""" + return [ + line + for line in text.splitlines() + if line.startswith("|") + and not re.match(r"^\|[\s:-]+\|", line) + and not line.startswith("| Model ") + ] + + def pricing_accuracy_bands(text: str) -> tuple[str, str]: line = next( (line for line in text.splitlines() if line.startswith("- Primary pricing source")), @@ -123,6 +141,13 @@ def main() -> int: old = git_show(OLD_CACHE) canonical = CANONICAL.read_text() remaining = GCP_CACHE.read_text() + gcp_estimate = GCP_ESTIMATE.read_text() + gcp_estimate_ai = GCP_ESTIMATE_AI.read_text() + azure_old = git_show(OLD_AZURE_CACHE) + azure_cache = AZURE_CACHE.read_text() + azure_estimate = AZURE_ESTIMATE.read_text() + azure_estimate_ai = AZURE_ESTIMATE_AI.read_text() + azure_openai = AZURE_OPENAI.read_text() old_accuracy = pricing_accuracy_bands(git_show(OLD_SKILL)) new_accuracy = pricing_accuracy_bands(GCP_SKILL.read_text()) @@ -131,7 +156,14 @@ def main() -> int: old_bedrock = section(old, "Bedrock Models (On-Demand)") new_bedrock = section(canonical, "Bedrock Models (On-Demand)") assert new_bedrock.rstrip() == old_bedrock.rstrip(), "Bedrock section changed during extraction" - assert metadata(canonical) == metadata(old), "freshness/source metadata changed" + old_metadata = metadata(old) + canonical_metadata = metadata(canonical) + for field in ("Last updated", "Region", "Currency"): + assert canonical_metadata[field] == old_metadata[field], f"Bedrock {field} metadata changed" + assert canonical_metadata["Accuracy"].startswith("±15-25% for AI/Bedrock") + assert "±5-10%" not in canonical_metadata["Accuracy"] + assert "other services in this file" not in canonical + assert "infrastructure services" not in canonical_metadata["Accuracy"] for heading in ( "Compute", "Database", "Storage", "Networking", "Supporting Services", @@ -140,6 +172,23 @@ def main() -> int: assert section(remaining, heading) == section(old, heading), f"non-Bedrock section changed: {heading}" assert "## Bedrock Models (On-Demand)" not in remaining assert "references/vendored/ai/bedrock-pricing-cache.md" in remaining + assert metadata(remaining)["Accuracy"].startswith("±5-10%") + assert "±15-25%" not in metadata(remaining)["Accuracy"] + assert "Bedrock subsection" not in remaining + assert "Amazon Nova" not in remaining[: remaining.index("## Compute")] + assert "±5-25%" not in gcp_estimate + assert "infrastructure/source-provider cache (updated [date], ±5-10%)" in gcp_estimate + assert "Bedrock cache (updated [date], ±15-25%)" in gcp_estimate + assert "a `references/vendored/ai/bedrock-pricing-cache.md` cell marked `_unverified_`" in gcp_estimate_ai + + assert "references/vendored/ai/bedrock-pricing-cache.md" in azure_estimate + assert "references/vendored/ai/bedrock-pricing-cache.md` (primary for Bedrock)" in azure_estimate_ai + assert "`shared/pricing-cache.md` (primary for source providers)" in azure_estimate_ai + assert "## Bedrock Models (On-Demand)" not in azure_cache + assert "Claude Fable" not in azure_cache + assert metadata(azure_cache)["Last updated"] == metadata(azure_old)["Last updated"] + assert markdown_data_rows(section(azure_cache, "Source Provider Pricing (for Migration Comparison)")) == markdown_data_rows(section(azure_old, "Source Provider Pricing (for Migration Comparison)")), "Azure source-provider rate rows changed" + assert "any Bedrock rate change into\n `references/vendored/ai/bedrock-pricing-cache.md`" in azure_openai canonical_bytes = CANONICAL.read_bytes() for copy in VENDORED: diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md index b1f94f147..19bfad8d5 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md @@ -3,10 +3,10 @@ **Last updated:** 2026-09-03 **Region:** us-east-1 **Currency:** USD -**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) +**Accuracy:** ±15-25% for AI/Bedrock models and services (sourced from public pricing pages) -> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. -> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures often reference **US East (Ohio)** and **inference mode** (global vs geo); retain those qualifications when estimating. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. > > **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-ai.md index 8fdb51212..91da814f8 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-ai.md @@ -24,28 +24,35 @@ The parent `estimate.md` selects the pricing mode before loading this file. **Price lookup order:** -1. **`shared/pricing-cache.md` (primary)** — Look up Bedrock model pricing and source - provider pricing by table. Set `pricing_source: "cached"`. -2. **Unavailable** — If a model is NOT in the cache, consult `shared/pricing-fallback.md`, - set `pricing_source: "unavailable"`, emit `null` for the affected per-token fields, and warn - the user. Never fabricate a token price. An `_unverified_` cache cell is blocking — do not - use it as if it were a confirmed price. There is no live pricing lookup to top the cache up. - -For typical migrations (Claude, Llama, Nova, Mistral, DeepSeek, Gemma, OpenAI gpt-oss, and the -proprietary GPT-5.x source rows), ALL prices are in `pricing-cache.md`. - -**Staleness:** if `pricing-cache.md` is more than 30 days past its **Last updated** date, treat -AI prices as potentially stale, set `pricing_source: "cached_stale"`, and note it — per that -file's staleness warning. - -**Bedrock pricing is per-1M-tokens.** Every generative Bedrock figure in `pricing-cache.md` is -stated per 1M input tokens and per 1M output tokens. Divide token counts by 1,000,000 before -multiplying by the rate. **Embedding models are input-only** — the § Embeddings — Bedrock table -gives a single per-1M-input rate and no output column; price an embedding workload as -input_tokens × rate (no output term), and compare against the § Embeddings — OpenAI / Azure OpenAI -source rate for the "$X today" baseline. A source `text-embedding-3-*` → Titan v2 move is a -re-embedding task (dimension change), not a free swap — carry the note into the estimate. There is no per-hour or baked-in-Multi-AZ dimension here — that RDS-style rule belongs -to `estimate-infra.md` and does NOT apply to token pricing. +1. **`references/vendored/ai/bedrock-pricing-cache.md` (primary for Bedrock)** — Look up + Bedrock model and service pricing, lifecycle-sensitive statuses, and per-token units by table. + Set `pricing_source: "cached"`. +2. **`shared/pricing-cache.md` (primary for source providers)** — Look up OpenAI / Azure OpenAI + source-provider comparison pricing by table. Set `pricing_source: "cached"`. +3. **Unavailable** — If a model is NOT in its applicable cache, consult + `shared/pricing-fallback.md`, set `pricing_source: "unavailable"`, emit `null` for the affected + per-token fields, and warn the user. Never fabricate a token price. An `_unverified_` cache + cell is blocking — do not use it as if it were a confirmed price. There is no live pricing + lookup to top either cache up. + +For typical migrations, Bedrock target prices are in the vendored Bedrock cache. Proprietary +OpenAI / Azure OpenAI source comparison rows are in `shared/pricing-cache.md`. + +**Staleness:** check each applicable cache's **Last updated** date. If the vendored Bedrock cache +is more than 30 days old, treat Bedrock prices as potentially stale; if the source-provider cache +is more than 30 days old, treat source comparison prices as potentially stale. Set +`pricing_source: "cached_stale"` and note the applicable warning. + +**Bedrock pricing is per-1M-tokens.** Every generative Bedrock figure in +`references/vendored/ai/bedrock-pricing-cache.md` is stated per 1M input tokens and per 1M output +tokens. Divide token counts by 1,000,000 before multiplying by the rate. **Embedding models are +input-only** — the vendored cache's § Embeddings — Bedrock table gives a single per-1M-input rate +and no output column; price an embedding workload as input_tokens × rate (no output term), and +compare against the § Embeddings — OpenAI / Azure OpenAI source rate in `shared/pricing-cache.md` +for the "$X today" baseline. A source `text-embedding-3-*` → Titan v2 move is a re-embedding task +(dimension change), not a free swap — carry the note into the estimate. There is no per-hour or +baked-in-Multi-AZ dimension here — that RDS-style rule belongs to `estimate-infra.md` and does NOT +apply to token pricing. **Model lifecycle:** When building the model comparison table, check `references/vendored/ai/ai-model-lifecycle.md` and apply the 90-day exclusion rule: @@ -107,7 +114,8 @@ match wins:** too short to be a monthly baseline — do NOT rank it above levels 3–4; fall back and present the partial actuals as a reference figure only, labeled with `active_days`. 3. **Estimated from token volume** — use `ai_constraints.ai_token_volume.value` from - `preferences.json` with **OpenAI / Azure OpenAI source list prices** from `pricing-cache.md` + `preferences.json` with **OpenAI / Azure OpenAI source list prices** from + `shared/pricing-cache.md` (under "Source Provider Pricing → OpenAI / Azure OpenAI"). Azure OpenAI serves the same GPT models and reads those same OpenAI rows — there is no separate Azure-OpenAI table. Apply the 60/40 input/output ratio if the actual ratio is unknown. @@ -225,9 +233,10 @@ and — for a same-model move — the elimination of behavior-delta and prompt-r **Pricing source caveat for OpenAI models:** the AWS Price List API does not carry the proprietary GPT-5.x models. A missing or empty price-list result is **not** evidence the model is -unavailable or free. Use `shared/pricing-cache.md`, and treat rows -marked `unverified` there as blocking for any quoted figure — resolve them from the Bedrock -pricing page first. See `shared/openai-on-bedrock.md`. +unavailable or free. Use `references/vendored/ai/bedrock-pricing-cache.md` for Bedrock rates and +statuses, and treat rows marked `unverified` there as blocking for any quoted figure — resolve +them from the Bedrock pricing page first. Use `shared/pricing-cache.md` only for OpenAI / Azure +OpenAI source-provider comparison rates. See `shared/openai-on-bedrock.md`. **Note:** Human / professional-services one-time migration costs are intentionally out of scope for this advisor and excluded from ROI calculations. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate.md index 53efd59a8..840ff9082 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate.md @@ -16,6 +16,7 @@ _knowledge: - { file: knowledge/estimate/rightsizing-thresholds.json } - { file: knowledge/estimate/estimate-defaults.json } - { file: references/shared/pricing-cache.md, _when: "ai-workload-profile.json exists in $MIGRATION_DIR" } + - { file: references/vendored/ai/bedrock-pricing-cache.md, _when: "ai-workload-profile.json exists in $MIGRATION_DIR" } - { file: references/shared/pricing-fallback.md, _when: "ai-workload-profile.json exists in $MIGRATION_DIR" } - { file: references/vendored/ai/ai-model-lifecycle.md, _when: "ai-workload-profile.json exists in $MIGRATION_DIR" } - { file: references/vendored/ai/bedrock-quotas.md, _when: "ai-workload-profile.json exists in $MIGRATION_DIR" } diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md index c3ffc7654..46754a69b 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md @@ -257,7 +257,8 @@ This model family is moving fast (two GA waves and a repricing inside 10 weeks). 4. Recheck whether the Price List API has gained GPT-5.x coverage; if it has, drop the caveat above and let `estimate-ai.md` price these models from the MCP. 5. Recheck whether Chat Completions and `bedrock-runtime` support have been added or clarified. -6. Feed any lifecycle change into `ai-model-lifecycle.md` and any rate change into `pricing-cache.md`. +6. Feed any lifecycle change into `ai-model-lifecycle.md` and any Bedrock rate change into + `references/vendored/ai/bedrock-pricing-cache.md` through its canonical shared source. 7. **Re-verify within 14 days of any merge touching this file.** Item 5's prediction fired on 2026-08-21: between 2026-08-10 and 2026-08-21 the GPT-5.6 family gained a `bedrock-runtime`/CRIS path, published Global CRIS pricing at standard-price parity, and listed Chat Completions/Converse as supported — invalidating three of this file's diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-cache.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-cache.md index 83b67ab83..ce22a978f 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-cache.md @@ -1,122 +1,25 @@ -# AI Pricing Cache (Bedrock + source-provider) +# Source-Provider AI Pricing Cache **Last updated:** 2026-08-24 **Region:** us-east-1 **Currency:** USD -**Accuracy:** ±15-25% for AI models (sourced from public pricing pages) - -> **Scope: AI/Bedrock only.** This file prices Bedrock model inference and the source-provider -> baselines used for migration ROI. It does NOT price infrastructure — azure-to-aws prices -> EC2/RDS/S3/ElastiCache/etc. from `references/vendored/pricing/aws-infra-pricing.json` (the -> file the `pricing-coverage` gate enforces). Two infra pricing sources would be a drift trap, -> so this cache deliberately carries no infra rows. `estimate-ai.md` reads this file for AI -> cost; `estimate-infra.md` reads the infra JSON. +**Accuracy:** ±15-25% for source-provider AI models (sourced from public pricing pages) + +> **Scope: source-provider AI only.** This file contains the OpenAI / Azure OpenAI +> baselines used for migration ROI. Bedrock model and service rates, lifecycle-sensitive +> statuses, units, and staleness guidance are canonical in +> `references/vendored/ai/bedrock-pricing-cache.md`. Infrastructure is priced from +> `references/vendored/pricing/aws-infra-pricing.json` (the file the `pricing-coverage` +> gate enforces). > > Prices vary by region and change over time; use for estimation only. There is no live pricing > lookup — on a cache miss apply `pricing-fallback.md` (`estimated` or `unavailable`). -> **Staleness warning:** if today is more than 30 days after **Last updated**, treat AI model +> **Staleness warning:** if today is more than 30 days after **Last updated**, treat source-provider > prices as potentially stale; set `pricing_source: "cached_stale"` in `estimation-ai.json` and -> note it. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/). +> note it. --- -## Bedrock Models (On-Demand) - -**Anthropic Claude (Standard on-demand)** figures match **US East (N. Virginia)** on -[Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/) as of cache refresh. -**Recommend defaults (new migrations):** Claude Sonnet 5 (flagship), Claude Opus 4.8 (hardest -reasoning), Claude Haiku 4.5 (cost/speed). Do not default to Claude Fable 5 (frontier). Long-context -SKUs do not all use the same multiplier; confirm batch/cache and cross-region rows per model on -that page. See `references/vendored/ai/ai-model-lifecycle.md` for lifecycle detail — **do not -recommend Legacy/excluded models for new migrations.** - -### Multi-provider quick reference (per 1M tokens) - -| Model | Model ID | Provider | Input $/1M | Output $/1M | Context | Tier | Status | -| ----------------- | ---------------------------------------- | --------- | ---------- | ----------- | ------- | --------- | ---------------------------------------------------------------------------------------------- | -| Claude Fable 5 | anthropic.claude-fable-5 | Anthropic | 10.00 | 50.00 | 1M | frontier | active | -| Claude Sonnet 5 | anthropic.claude-sonnet-5 | Anthropic | 2.00 | 10.00 | 1M | flagship | active ($2/$10 — launch rate became standard on Sep 1, 2026; the $3/$15 step-up was cancelled) | -| Claude Opus 4.8 | anthropic.claude-opus-4-8 | Anthropic | 5.00 | 25.00 | 200K | premium | active | -| Claude Sonnet 4.6 | anthropic.claude-sonnet-4-6 | Anthropic | 3.00 | 15.00 | 200K | flagship | active | -| Claude Opus 4.6 | anthropic.claude-opus-4-6-v1 | Anthropic | 5.00 | 25.00 | 200K | premium | active | -| Claude Haiku 4.5 | anthropic.claude-haiku-4-5-20251001-v1:0 | Anthropic | 1.00 | 5.00 | 200K | fast | active | -| Claude Opus 4.1 | anthropic.claude-opus-4-1-20250805-v1:0 | Anthropic | 15.00 | 75.00 | 200K | premium | legacy (EOL Jan 8, 2027) | -| Llama 4 Maverick | meta.llama4-maverick-17b-instruct-v1:0 | Meta | 0.24 | 0.97 | 1M | mid | active | -| Llama 4 Scout | meta.llama4-scout-17b-instruct-v1:0 | Meta | 0.17 | 0.66 | 10M | efficient | active | -| Llama 3.3 70B | meta.llama3-3-70b-instruct-v1:0 | Meta | 0.72 | 0.72 | 128K | mid | active | -| Nova 2 Lite | amazon.nova-2-lite-v1:0 | Amazon | 0.33 | 2.75 | 1M | mid | active | -| Nova Pro | amazon.nova-pro-v1:0 | Amazon | 0.80 | 3.20 | 300K | mid | active | -| Nova Lite | amazon.nova-lite-v1:0 | Amazon | 0.06 | 0.24 | 300K | fast | active | -| Nova Micro | amazon.nova-micro-v1:0 | Amazon | 0.035 | 0.14 | 128K | budget | active | -| Mistral Large 3 | mistral.mistral-large-3-675b-instruct | Mistral | 0.50 | 1.50 | 256K | flagship | active | -| DeepSeek-R1 | deepseek.r1-v1:0 | DeepSeek | 1.35 | 5.40 | 128K | reasoning | active | -| gpt-oss-20b | openai.gpt-oss-20b-1:0 | OpenAI | 0.07 | 0.30 | 128K | budget | active | -| gpt-oss-120b | openai.gpt-oss-120b-1:0 | OpenAI | 0.15 | 0.60 | 128K | efficient | active | -| GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | -| GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | -| GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | - -_Quick-reference rows use **—** for **model ID** and **context**; resolve in the Bedrock console -or AWS model documentation. This is a curated subset for migration selection — see the Bedrock -pricing page for the full catalog._ - -### Embeddings — Bedrock (per 1M input tokens, US East) - -Embedding models are **input-only** — priced per 1M input tokens, no output charge. Use for -`embedding` capability workloads (RAG corpora, semantic search, FAQ retrieval). A migration from -an OpenAI/Azure embedding deployment lands on one of these; note the **dimension** must match (or -the corpus must be re-embedded and any similarity threshold recalibrated). - -| Model | Model ID | Provider | Input $/1M | Dimensions | Tier | Status | -| ---------------------------- | ---------------------------- | -------- | ---------- | ----------------- | -------- | ------ | -| Titan Text Embeddings v2 | amazon.titan-embed-text-v2:0 | Amazon | 0.02 | 1024/512/256 | default | active | -| Titan Text Embeddings v1 | amazon.titan-embed-text-v1 | Amazon | 0.10 | 1536 | legacy | active | -| Cohere Embed v4 | cohere.embed-v4:0 | Cohere | 0.12 | 1536/1024/512/256 | flagship | active | -| Cohere Embed English v3 | cohere.embed-english-v3 | Cohere | 0.10 | 1024 | mid | active | -| Cohere Embed Multilingual v3 | cohere.embed-multilingual-v3 | Cohere | 0.10 | 1024 | mid | active | - -**Default target for a migrating OpenAI/Azure embedding workload:** Titan Text Embeddings v2 -(`amazon.titan-embed-text-v2:0`) — cheapest, configurable dimensions (1024 default; 512/256 for -cost/latency). Choose Cohere Embed v4 when the source used a large-dimension model and matrix -compatibility or multilingual quality matters. Titan v2 output dimension is configurable, so map -`text-embedding-3-large` (3072-dim) or `-small` (1536-dim) to Titan 1024 and **re-embed** — there -is no dimension-preserving swap. See `references/vendored/ai/ai-openai-to-bedrock.md`. - -### Stability AI — Image Generation (per image, US East) - -Priced **per image** (not per token). Use for `image_generation` capability workloads. - -| Model | Model ID | Price/image | Resolution | Tier | Status | -| -------------------------- | --------------------------------- | ----------- | ---------- | -------- | ------ | -| Stable Image Ultra | stability.stable-image-ultra-v1:0 | $0.08 | up to 4MP | premium | active | -| Stable Diffusion 3.5 Large | stability.sd3-5-large-v1:0 | $0.065 | up to 1MP | flagship | active | -| Stable Image Core | stability.stable-image-core-v1:0 | $0.04 | up to 1MP | fast | active | - -> **Cost comparison note:** DALL-E 3 (OpenAI / Azure OpenAI) charges $0.04–$0.12/image by -> resolution. When comparing, use per-image cost directly — do not convert Stability AI prices -> to per-token equivalents. - -### Anthropic Claude — batch & prompt cache (Standard, US East N. Virginia) - -Per 1M tokens unless noted. - -| Model | Batch in | Batch out | 5m cache write | 1h cache write | Cache read | -| ----------------- | -------- | --------- | -------------- | -------------- | ---------- | -| Claude Sonnet 5 | 1.00 | 5.00 | 2.50 | 4.00 | 0.20 | -| Claude Opus 4.8 | 2.50 | 12.50 | 6.25 | 10.00 | 0.50 | -| Claude Sonnet 4.6 | 1.50 | 7.50 | 3.75 | 6.00 | 0.30 | -| Claude Haiku 4.5 | 0.50 | 2.50 | 1.25 | 2.00 | 0.10 | - -### OpenAI on Bedrock — the same-model path - -The proprietary GPT-5.x models on Bedrock are priced at OpenAI's **data-residency tier ≈ 1.10x** -the OpenAI standard rates below, and run **`bedrock-mantle` only, in-region only** (no -`bedrock-runtime`, no cross-region inference profile for GPT-5.5/5.4). `gpt-oss` DOES support -`bedrock-runtime`. For endpoint paths, region matrix, quotas, and prompt-caching detail see -`references/shared/openai-on-bedrock.md` (the fact base the mapping guide reads). - ## Source Provider Pricing (for Migration Comparison) Use alongside the Bedrock rates to compute migration ROI. This is the **source-side baseline** — @@ -134,11 +37,12 @@ fallback when no measured spend exists, and the source side of the per-model "vs Prices per 1M tokens. > **Tier note — these are OpenAI STANDARD-tier rates.** Bedrock in-region for the same models is -> the OpenAI _data-residency_ tier, exactly 1.10x these figures (see the OpenAI-on-Bedrock note -> above). So a same-model move for GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 is a ~10% increase, -> not parity — compute the target from the Bedrock table, not by carrying these over. These rows -> are the right source-side baseline, and the only figures available for models with **no** -> Bedrock equivalent (GPT-5.x Pro, GPT-5.2/5.1, GPT-4.x, o-series). +> the OpenAI _data-residency_ tier, exactly 1.10x these figures (see +> `references/shared/openai-on-bedrock.md`). So a same-model move for GPT-5.6 Sol/Terra/Luna, +> GPT-5.5, GPT-5.4 is a ~10% increase, not parity — compute the target from the vendored Bedrock +> cache, not by carrying these over. These rows are the right source-side baseline, and the only +> figures available for models with **no** Bedrock equivalent (GPT-5.x Pro, GPT-5.2/5.1, GPT-4.x, +> o-series). | Model | Input $/1M | Output $/1M | Context | Tier | | ------------- | ------------ | ------------ | ------- | --------- | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-fallback.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-fallback.md index 5de015c4a..e0080bf74 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-fallback.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/pricing-fallback.md @@ -5,7 +5,8 @@ ## Policy -When a service is NOT in `references/shared/pricing-cache.md` (AI models) or +When a service is NOT in `references/vendored/ai/bedrock-pricing-cache.md` (Bedrock), +`references/shared/pricing-cache.md` (OpenAI / Azure OpenAI source-provider models), or `references/vendored/pricing/aws-infra-pricing.json` (infrastructure): 1. If the cost engine's own formulas carry a well-known published rate (stated verbatim), set `pricing_source: "estimated"` and include a warning naming the rate and its source. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md index b1f94f147..19bfad8d5 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -3,10 +3,10 @@ **Last updated:** 2026-09-03 **Region:** us-east-1 **Currency:** USD -**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) +**Accuracy:** ±15-25% for AI/Bedrock models and services (sourced from public pricing pages) -> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. -> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures often reference **US East (Ohio)** and **inference mode** (global vs geo); retain those qualifications when estimating. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. > > **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md index 1db6e0c0d..11b7f1ef1 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md @@ -363,7 +363,7 @@ gcp-to-aws/ | ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | No GCP sources found (no `.tf`, no app code, no billing data) | Offer live gcloud discovery per `discover.md` Step 1d. Only if declined or unavailable: Stop. Output: "No GCP sources detected. Provide at least one source type (Terraform files, application code, or billing exports), or re-run and accept live discovery." | | `.phase-status.json` missing phase gate | Stop. Output: "Cannot enter Phase X: Phase Y-1 not completed. Start from Phase Y or resume Phase Y-1." | -| Service not in applicable pricing cache | Display user warning about ±5-25% accuracy. Use the infrastructure/source-provider cache or Bedrock cache as applicable. Set `pricing_source: "unavailable"` for that service in the applicable `estimation-*.json` file. | +| Service not in applicable pricing cache | Display the applicable accuracy warning: ±5-10% for infrastructure/source-provider rates or ±15-25% for Bedrock rates. Use the corresponding cache and set `pricing_source: "unavailable"` for that service in the applicable `estimation-*.json` file. | | User skips questions or says "use defaults for the rest" | Apply documented defaults for all remaining questions (essential questions and any unconfirmed sheet rows in wizard mode; current and subsequent batches in full mode). Q2/Q3 defaults add a report caveat. Phase 2 completes either way. | | `aws-design.json` missing required clusters | Stop Phase 4. Output: "Re-run Phase 3 to generate missing cluster designs." | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md index ed85b4884..7aa9c1f58 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate-ai.md @@ -124,7 +124,7 @@ Reference `aws-design-ai.json` → `honest_assessment`. If `"recommend_stay"`, p **Non-cost benefits to present:** usage counting toward existing AWS commitments, IAM/VPC/PrivateLink/KMS/CloudTrail governance, in-region processing for data residency, prompt caching (Claude, and GPT-5.6 at 90% off cached input with cached tokens exempt from the input-TPM quota), model flexibility (100+ models), AWS ecosystem (Guardrails, Knowledge Bases, AgentCore), and — for a same-model move — the elimination of behavior-delta and prompt-regression risk. -**Pricing source caveat (all providers):** a `pricing-cache.md` cell marked `_unverified_` is **blocking for any quoted figure, whatever the provider** — resolve it from the Bedrock pricing page or the model card before the row enters `model_comparison` or the ROI table; never substitute a guess or a same-tier sibling's rate. An `_unverified_` cache cell is not evidence the model is unavailable or free — it means the rate was not confirmed when the cache was last updated. See `shared/openai-on-bedrock.md`. +**Pricing source caveat (all providers):** a `references/vendored/ai/bedrock-pricing-cache.md` cell marked `_unverified_` is **blocking for any quoted Bedrock figure, whatever the provider** — resolve it from the Bedrock pricing page or the model card before the row enters `model_comparison` or the ROI table; never substitute a guess or a same-tier sibling's rate. An `_unverified_` Bedrock cache cell is not evidence the model is unavailable or free — it means the rate was not confirmed when the cache was last updated. See `shared/openai-on-bedrock.md`. **Unverified gate (all providers, not just OpenAI):** any cell marked `_unverified_` in `references/vendored/ai/bedrock-pricing-cache.md` — including Anthropic batch cells for models not yet on the [batch-supported models table](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference-supported.html) (Sonnet 5, Opus 4.8 as of 2026-09-02) — is blocking for any quoted figure that depends on it. Do not apply a batch discount to an `_unverified_` batch cell; price on-demand and note batch as a possible future saving, or resolve the rate from the Bedrock pricing page first. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md index 15371312f..ad6d6f19a 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md @@ -17,9 +17,11 @@ Read `shared/pricing-cache.md` for AWS infrastructure and source-provider rates. **Before any sub-estimate file runs**, display the pricing mode to the user so they know what to expect: -- **If cache ≤ 90 days**: "Pricing source: cached (updated [date], ±5-25% accuracy). No live pricing API required." -- **If cache > 90 days**: "⚠️ Pricing source: stale cache only (updated [date]). Proceeding with cached pricing; accuracy may be ±15-25% for AI models." -- **If a required service is NOT in cache**: "⚠️ Some services not in pricing cache. Those services will show `pricing_source: unavailable` in the estimate." +- **Infrastructure/source-provider cache only, ≤ 90 days**: "Pricing source: cached (updated [date], ±5-10% accuracy). No live pricing API required." +- **Bedrock cache only, ≤ 90 days**: "Pricing source: cached (updated [date], ±15-25% accuracy). No live pricing API required." +- **Both caches, ≤ 90 days**: "Pricing sources: infrastructure/source-provider cache (updated [date], ±5-10%) and Bedrock cache (updated [date], ±15-25%). No live pricing API required." +- **If an applicable cache > 90 days**: "⚠️ Pricing source: stale cache only (updated [date]). Proceeding with cached pricing; the applicable accuracy band may widen." +- **If a required service is NOT in cache**: "⚠️ Some services not in the applicable pricing cache. Those services will show `pricing_source: unavailable` in the estimate." This prevents silent failures — the user sees the pricing constraint upfront, not after 5 minutes of estimation work. @@ -27,7 +29,7 @@ This prevents silent failures — the user sees the pricing constraint upfront, Each sub-estimate file uses this lookup order per service: -1. **`shared/pricing-cache.md`** (primary for infrastructure and source providers) — Cached prices (±5-25% accuracy). Set `pricing_source: "cached"`. +1. **`shared/pricing-cache.md`** (primary for infrastructure and source providers) — Cached prices (±5-10% accuracy). Set `pricing_source: "cached"`. 2. **`references/vendored/ai/bedrock-pricing-cache.md`** (primary for Bedrock models and services) — Cached prices (±15-25% accuracy). Set `pricing_source: "cached"`. 3. **Unavailable** — If a service is NOT in its applicable cache, set `pricing_source: "unavailable"` for that service. Add the service to `services_with_missing_fallback` and display a warning to the user: "Pricing unavailable for [service] — not in cache. Exclude from totals or provide a manual estimate." @@ -265,7 +267,7 @@ _Breadcrumbs are emitted only after outer-run `HANDOFF_OK` — never on `GATE_FA ## Reference Files -- `shared/pricing-cache.md` — Cached AWS infrastructure + source-provider pricing (±5-25%) +- `shared/pricing-cache.md` — Cached AWS infrastructure + source-provider pricing (±5-10%) - `references/vendored/ai/bedrock-pricing-cache.md` — Cached provider-neutral Bedrock pricing (±15-25%) ## Scope Boundary diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md index 574feb68c..5bb24ee9c 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/pricing-cache.md @@ -3,10 +3,10 @@ **Last updated:** 2026-09-03 **Region:** us-east-1 **Currency:** USD -**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) +**Accuracy:** ±5-10% for infrastructure and source-provider services (sourced from public pricing pages and the AWS Price List API) -> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. -> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the applicable public provider pricing page. Rates default to **us-east-1** unless noted. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat cached rates as potentially stale. Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object. Surface the same warning to the user in the estimate output. > > **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md index b1f94f147..19bfad8d5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -3,10 +3,10 @@ **Last updated:** 2026-09-03 **Region:** us-east-1 **Currency:** USD -**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) +**Accuracy:** ±15-25% for AI/Bedrock models and services (sourced from public pricing pages) -> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. -> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures often reference **US East (Ohio)** and **inference mode** (global vs geo); retain those qualifications when estimating. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. > > **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md index b1f94f147..19bfad8d5 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md @@ -3,10 +3,10 @@ **Last updated:** 2026-09-03 **Region:** us-east-1 **Currency:** USD -**Accuracy:** ±5-10% for infrastructure services (sourced from AWS Price List API), ±15-25% for AI models (sourced from public pricing pages) +**Accuracy:** ±15-25% for AI/Bedrock models and services (sourced from public pricing pages) -> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures in the Bedrock subsection often reference **US East (Ohio)** and **inference mode** (global vs geo); other services in this file default to **us-east-1** unless noted. -> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). Infrastructure prices (Fargate, RDS, S3, etc.) change rarely and remain reliable longer. When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. +> Prices may vary by region and change over time. Use for estimation only; there is no live pricing lookup — for the latest rates, check the public AWS pricing pages (e.g. https://aws.amazon.com/bedrock/pricing). **Amazon Nova** figures often reference **US East (Ohio)** and **inference mode** (global vs geo); retain those qualifications when estimating. +> **Staleness warning:** If today's date is more than 30 days after the **Last updated** date above, treat AI model prices as potentially stale (±15-25% accuracy may widen). When staleness is detected, keep `pricing_source.status: "cached"` (the schema enum is `cached | cached_fallback | unavailable` — there is no `cached_stale` status) and record the staleness in the dedicated `pricing_source.fallback_staleness` object: set `is_stale: true` and `staleness_warning: "Pricing cache is more than 30 days old — AI model prices may have changed. Verify against [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/)."` Surface that same warning to the user in the estimate output. > > **Lifecycle is not a cached price field.** The `Status` column below is a dated snapshot, not permission to skip the lifecycle check. Before selecting any model, call `GetFoundationModel` or `ListFoundationModels` and read `modelLifecycle.status`. For a model launched on or after 2026-09-07, also read its model card: its Legacy period may be 45 days rather than 6 months, and it will never appear in the Legacy/EOL table. See `vendored/ai/ai-model-lifecycle.md`. From 83843da68273f6d2837ffe60262220325cf48625 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 22:48:16 -0700 Subject: [PATCH 23/53] test(aws-startup-advisor): make AI matrix composable Validated 31 contract tests and all 14 scenario snapshots. Full build, artifact lint, vendored parity, skill refs, markdown lint, and gitleaks pass. --- .../aws-startup-advisor/fixtures/README.md | 2 +- .../ai-only-anthropic/.phase-status.json | 1 + .../ai-only-anthropic/expected-artifacts.json | 17 + .../ai-only-gemini/expected-artifacts.json | 6 +- .../ai-only-openai/expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../expected-artifacts.json | 6 +- .../.phase-status.json | 1 + .../expected-artifacts.json | 17 + .../integration-validation.json | 1 + .../ai-migration-matrix/expected-matrix.json | 14 +- .../infra-only-azure/expected-artifacts.json | 8 +- .../infra-only-gcp/expected-artifacts.json | 8 +- .../infra-only-heroku/.phase-status.json | 1 + .../infra-only-heroku/expected-artifacts.json | 13 + .../ai-migration-scenario.schema.json | 169 +++++++ .../test_ai_migration_matrix_contracts.py | 418 +++++++++++++----- 22 files changed, 553 insertions(+), 171 deletions(-) create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/integration-validation.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/.phase-status.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/expected-artifacts.json create mode 100644 plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 9636d2610..91a356a08 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -2,7 +2,7 @@ Also see: -- `ai-migration-matrix/` — target-only contract snapshots for infrastructure-only, AI-only, and combined migration modes. These fixtures do not claim current runtime support. They keep infrastructure source, AI provider/model family, and router/gateway independent; combined cases use one `.phase-status.json` and a separate `integration-validation.json` cross-track verdict. The two combined OpenRouter cases deliberately pair different infrastructure sources with different underlying providers/models (GCP + OpenAI/GPT and Azure + Anthropic/Claude), proving that OpenRouter is an infrastructure-independent gateway rather than duplicating one provider case. +- `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. These fixtures do not claim current runtime support or enumerate the Cartesian product. Infrastructure source, direct AI sources/model families, and optional gateways are independent dimensions validated by `scripts/contracts/ai-migration-scenario.schema.json`; synthetic composition tests cover legal combinations without duplicate fixture directories. GCP, Azure, and Heroku are represented as infrastructure sources; Google/Gemini, OpenAI (with direct versus Azure endpoint provenance), and direct Anthropic/Claude are represented as AI sources. OpenRouter remains a separate gateway retaining its known upstream provider/model signals. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/.phase-status.json new file mode 100644 index 000000000..cb9218b92 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"ai-only-anthropic","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000013","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json new file mode 100644 index 000000000..bd46baa38 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json @@ -0,0 +1,17 @@ +{ + "id": "ai-only-anthropic", + "mode": "ai_only", + "migration_id": "ai-only-anthropic", + "run_id": "00000000-0000-4000-8000-000000000013", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": null, + "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"]}], + "gateways": [] + }, + "outcomes": { + "infrastructure": null, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json index 43c796d13..f9155f68a 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_provider": "gcp_ai", - "model_families": ["gemini"], - "gateway_type": null, - "upstream_provider_signals": ["gcp_ai"] + "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"]}], + "gateways": [] }, "outcomes": { "infrastructure": null, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json index 60a4b8abc..b0966b8cd 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_provider": "openai", - "model_families": ["gpt"], - "gateway_type": null, - "upstream_provider_signals": ["openai", "azure_openai"] + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], + "gateways": [] }, "outcomes": { "infrastructure": null, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json index 51a5df9cb..ae02ea871 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_provider": "openai", - "model_families": ["gpt"], - "gateway_type": "llm_router", - "upstream_provider_signals": ["openai"] + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], + "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_provider_signals": ["openai"], "upstream_model_family_signals": ["gpt"]}] }, "outcomes": { "infrastructure": null, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json index 60c37b691..3f03644ae 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "azure", - "ai_provider": "gcp_ai", - "model_families": ["gemini"], - "gateway_type": null, - "upstream_provider_signals": ["gcp_ai"] + "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"]}], + "gateways": [] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json index 38e1ad6c1..fe4bfd804 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "azure", - "ai_provider": "azure_openai", - "model_families": ["gpt"], - "gateway_type": null, - "upstream_provider_signals": ["azure_openai"] + "ai_sources": [{"provider": "openai", "source_service": "azure_openai", "model_families": ["gpt"]}], + "gateways": [] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json index 691798b5a..0cfafe27a 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "azure", - "ai_provider": "anthropic", - "model_families": ["claude"], - "gateway_type": "llm_router", - "upstream_provider_signals": ["anthropic"] + "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"]}], + "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_provider_signals": ["anthropic"], "upstream_model_family_signals": ["claude"]}] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json index e4f8febb5..ecc62b60b 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "gcp", - "ai_provider": "gcp_ai", - "model_families": ["gemini"], - "gateway_type": null, - "upstream_provider_signals": ["gcp_ai"] + "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"]}], + "gateways": [] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json index acffaa389..c15453f66 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "gcp", - "ai_provider": "openai", - "model_families": ["gpt"], - "gateway_type": null, - "upstream_provider_signals": ["openai", "azure_openai"] + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], + "gateways": [] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json index c89e9c313..7bc7796b1 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json @@ -6,10 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "gcp", - "ai_provider": "openai", - "model_families": ["gpt"], - "gateway_type": "llm_router", - "upstream_provider_signals": ["openai"] + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], + "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_provider_signals": ["openai"], "upstream_model_family_signals": ["gpt"]}] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/.phase-status.json new file mode 100644 index 000000000..6de31c8e8 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"combined-heroku-anthropic","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000014","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json new file mode 100644 index 000000000..1ad8bc215 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json @@ -0,0 +1,17 @@ +{ + "id": "combined-heroku-anthropic", + "mode": "combined", + "migration_id": "combined-heroku-anthropic", + "run_id": "00000000-0000-4000-8000-000000000014", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": "heroku", + "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"]}], + "gateways": [] + }, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": {"verdict": "passed", "artifacts": ["ai-workload-profile.json", "aws-design-ai.json", "estimation-ai.json", "generation-ai.json"]}, + "integration_validation": {"verdict": "passed", "artifact": "integration-validation.json"} + } +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/integration-validation.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/integration-validation.json new file mode 100644 index 000000000..e24ea08a6 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/integration-validation.json @@ -0,0 +1 @@ +{"schema_version":"1.0","run_id":"00000000-0000-4000-8000-000000000014","verdict":"passed","track_results":{"infrastructure":{"verdict":"passed","references":["aws-design.json","estimation-infra.json","generation-infra.json","validation-report.json"]},"ai":{"verdict":"passed","references":["ai-workload-profile.json","aws-design-ai.json","estimation-ai.json","generation-ai.json"]}},"checks":{"networking":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"iam_and_secrets":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"data_flow":{"verdict":"passed","references":["aws-design.json","aws-design-ai.json"]},"observability":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"deployment_order":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cutover":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"rollback":{"verdict":"passed","references":["generation-infra.json","generation-ai.json"]},"cost_estimate":{"verdict":"passed","references":["estimation-infra.json","estimation-ai.json"]},"validation_dependencies":{"verdict":"passed","references":["validation-report.json","generation-ai.json"]}}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json index 024f87cc5..a4d275815 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json @@ -1,17 +1,25 @@ { - "contract_version": "1.0", - "description": "Target migration-mode contracts only; these fixtures do not assert current runtime support.", + "contract_version": "2.0", + "description": "Target-only representative contracts for composable migration dimensions; these fixtures do not assert current runtime support or enumerate a Cartesian matrix.", + "supported": { + "infrastructure_sources": ["gcp", "azure", "heroku"], + "direct_ai_providers": ["google", "openai", "anthropic"], + "gateway_products": ["openrouter"] + }, "cases": [ {"id": "infra-only-gcp", "snapshot": "infra-only-gcp/expected-artifacts.json"}, {"id": "infra-only-azure", "snapshot": "infra-only-azure/expected-artifacts.json"}, + {"id": "infra-only-heroku", "snapshot": "infra-only-heroku/expected-artifacts.json"}, {"id": "ai-only-gemini", "snapshot": "ai-only-gemini/expected-artifacts.json"}, {"id": "ai-only-openai", "snapshot": "ai-only-openai/expected-artifacts.json"}, + {"id": "ai-only-anthropic", "snapshot": "ai-only-anthropic/expected-artifacts.json"}, {"id": "ai-only-openrouter", "snapshot": "ai-only-openrouter/expected-artifacts.json"}, {"id": "combined-gcp-gemini", "snapshot": "combined-gcp-gemini/expected-artifacts.json"}, {"id": "combined-azure-openai", "snapshot": "combined-azure-openai/expected-artifacts.json"}, {"id": "combined-gcp-openai", "snapshot": "combined-gcp-openai/expected-artifacts.json"}, {"id": "combined-azure-gemini", "snapshot": "combined-azure-gemini/expected-artifacts.json"}, {"id": "combined-gcp-openrouter", "snapshot": "combined-gcp-openrouter/expected-artifacts.json"}, - {"id": "combined-azure-openrouter", "snapshot": "combined-azure-openrouter/expected-artifacts.json"} + {"id": "combined-azure-openrouter", "snapshot": "combined-azure-openrouter/expected-artifacts.json"}, + {"id": "combined-heroku-anthropic", "snapshot": "combined-heroku-anthropic/expected-artifacts.json"} ] } diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json index 5835e00d6..5fbf51b5f 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-azure/expected-artifacts.json @@ -4,13 +4,7 @@ "migration_id": "infra-only-azure", "run_id": "00000000-0000-4000-8000-000000000002", "state": ".phase-status.json", - "dimensions": { - "infrastructure_source": "azure", - "ai_provider": null, - "model_families": [], - "gateway_type": null, - "upstream_provider_signals": [] - }, + "dimensions": {"infrastructure_source": "azure", "ai_sources": [], "gateways": []}, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, "ai": null, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json index cb105b2df..92c6beb65 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-gcp/expected-artifacts.json @@ -4,13 +4,7 @@ "migration_id": "infra-only-gcp", "run_id": "00000000-0000-4000-8000-000000000001", "state": ".phase-status.json", - "dimensions": { - "infrastructure_source": "gcp", - "ai_provider": null, - "model_families": [], - "gateway_type": null, - "upstream_provider_signals": [] - }, + "dimensions": {"infrastructure_source": "gcp", "ai_sources": [], "gateways": []}, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, "ai": null, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/.phase-status.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/.phase-status.json new file mode 100644 index 000000000..51c150a8e --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/.phase-status.json @@ -0,0 +1 @@ +{"migration_id":"infra-only-heroku","last_updated":"2026-01-01T00:00:00Z","run_id":"00000000-0000-4000-8000-000000000012","phases":{"discover":"completed","clarify":"completed","design":"completed","estimate":"completed","generate":"completed","validate":"completed"}} diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/expected-artifacts.json new file mode 100644 index 000000000..43de4ca3e --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/infra-only-heroku/expected-artifacts.json @@ -0,0 +1,13 @@ +{ + "id": "infra-only-heroku", + "mode": "infrastructure_only", + "migration_id": "infra-only-heroku", + "run_id": "00000000-0000-4000-8000-000000000012", + "state": ".phase-status.json", + "dimensions": {"infrastructure_source": "heroku", "ai_sources": [], "gateways": []}, + "outcomes": { + "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, + "ai": null, + "integration_validation": null + } +} diff --git a/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json b/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json new file mode 100644 index 000000000..5f0794b38 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json @@ -0,0 +1,169 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:ai-migration-scenario", + "title": "AI migration scenario snapshot", + "description": "Target-only compositional contract for representative infrastructure and AI migration scenarios.", + "type": "object", + "required": ["id", "mode", "migration_id", "run_id", "state", "dimensions", "outcomes"], + "additionalProperties": false, + "properties": { + "id": {"$ref": "#/definitions/identifier"}, + "mode": {"enum": ["infrastructure_only", "ai_only", "combined"]}, + "migration_id": {"$ref": "#/definitions/identifier"}, + "run_id": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"}, + "state": {"const": ".phase-status.json"}, + "dimensions": { + "type": "object", + "required": ["infrastructure_source", "ai_sources", "gateways"], + "additionalProperties": false, + "properties": { + "infrastructure_source": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/definitions/extensibleIdentity"} + ] + }, + "ai_sources": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/aiSource"} + }, + "gateways": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/gateway"} + } + } + }, + "outcomes": { + "type": "object", + "required": ["infrastructure", "ai", "integration_validation"], + "additionalProperties": false, + "properties": { + "infrastructure": {"oneOf": [{"type": "null"}, {"$ref": "#/definitions/trackOutcome"}]}, + "ai": {"oneOf": [{"type": "null"}, {"$ref": "#/definitions/trackOutcome"}]}, + "integration_validation": {"oneOf": [{"type": "null"}, {"$ref": "#/definitions/integrationOutcome"}]} + } + } + }, + "oneOf": [ + { + "properties": { + "mode": {"const": "infrastructure_only"}, + "dimensions": { + "properties": { + "infrastructure_source": {"$ref": "#/definitions/extensibleIdentity"}, + "ai_sources": {"maxItems": 0}, + "gateways": {"maxItems": 0} + } + }, + "outcomes": { + "properties": { + "infrastructure": {"$ref": "#/definitions/trackOutcome"}, + "ai": {"type": "null"}, + "integration_validation": {"type": "null"} + } + } + } + }, + { + "properties": { + "mode": {"const": "ai_only"}, + "dimensions": { + "properties": { + "infrastructure_source": {"type": "null"}, + "ai_sources": {"minItems": 1} + } + }, + "outcomes": { + "properties": { + "infrastructure": {"type": "null"}, + "ai": {"$ref": "#/definitions/trackOutcome"}, + "integration_validation": {"type": "null"} + } + } + } + }, + { + "properties": { + "mode": {"const": "combined"}, + "dimensions": { + "properties": { + "infrastructure_source": {"$ref": "#/definitions/extensibleIdentity"}, + "ai_sources": {"minItems": 1} + } + }, + "outcomes": { + "properties": { + "infrastructure": {"$ref": "#/definitions/trackOutcome"}, + "ai": {"$ref": "#/definitions/trackOutcome"}, + "integration_validation": {"$ref": "#/definitions/integrationOutcome"} + } + } + } + } + ], + "definitions": { + "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$"}, + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter)$)[a-z][a-z0-9_]*$"}, + "modelFamily": {"type": "string", "pattern": "^(?!both$)[a-z][a-z0-9_.-]*$"}, + "gatewayIdentity": {"type": "string", "pattern": "^(?!both$)[a-z][a-z0-9_]*$"}, + "aiSource": { + "type": "object", + "required": ["provider", "source_service", "model_families"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/modelFamily"} + } + } + }, + "gateway": { + "type": "object", + "required": ["product", "type", "upstream_provider_signals", "upstream_model_family_signals"], + "additionalProperties": false, + "properties": { + "product": {"$ref": "#/definitions/gatewayIdentity"}, + "type": {"$ref": "#/definitions/extensibleIdentity"}, + "upstream_provider_signals": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/extensibleIdentity"} + }, + "upstream_model_family_signals": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/modelFamily"} + } + } + }, + "trackOutcome": { + "type": "object", + "required": ["verdict", "artifacts"], + "additionalProperties": false, + "properties": { + "verdict": {"const": "passed"}, + "artifacts": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + }, + "integrationOutcome": { + "type": "object", + "required": ["verdict", "artifact"], + "additionalProperties": false, + "properties": { + "verdict": {"const": "passed"}, + "artifact": {"const": "integration-validation.json"} + } + } + } +} diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py index 1e1662771..1810121d3 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -1,20 +1,30 @@ -"""Contract tests for target AI-only and combined migration modes. +"""Contract tests for composable target migration scenarios. These tests intentionally validate deterministic fixtures rather than runtime routing. -They pin the contract that later implementation PRs must satisfy without claiming that -all scenarios are supported by the current skills. +They pin contracts that later implementation PRs must satisfy without claiming current +skill support or treating the representative fixtures as a Cartesian support boundary. """ from __future__ import annotations +import copy +import importlib.util import json import subprocess import sys from pathlib import Path +import pytest + FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "ai-migration-matrix" PLUGIN_ROOT = FIXTURE_ROOT.parent.parent ARTIFACT_VALIDATOR = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" ARTIFACT_MANIFEST = PLUGIN_ROOT / "scripts" / "artifact-contracts.json" +SCENARIO_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "ai-migration-scenario.schema.json" +_VALIDATOR_SPEC = importlib.util.spec_from_file_location("artifact_validator", ARTIFACT_VALIDATOR) +assert _VALIDATOR_SPEC and _VALIDATOR_SPEC.loader +ARTIFACT_VALIDATOR_MODULE = importlib.util.module_from_spec(_VALIDATOR_SPEC) +sys.modules[_VALIDATOR_SPEC.name] = ARTIFACT_VALIDATOR_MODULE +_VALIDATOR_SPEC.loader.exec_module(ARTIFACT_VALIDATOR_MODULE) INFRA_ARTIFACTS = { "aws-design.json", "estimation-infra.json", @@ -38,26 +48,10 @@ "cost_estimate", "validation_dependencies", } -EXPECTED_CASES = { - "infra-only-gcp", - "infra-only-azure", - "ai-only-gemini", - "ai-only-openai", - "ai-only-openrouter", - "combined-gcp-gemini", - "combined-azure-openai", - "combined-gcp-openai", - "combined-azure-gemini", - "combined-gcp-openrouter", - "combined-azure-openrouter", -} -EXPECTED_COMBINATIONS = { - ("gcp", "gcp_ai", None), - ("azure", "azure_openai", None), - ("gcp", "openai", None), - ("azure", "gcp_ai", None), - ("gcp", "openai", "llm_router"), - ("azure", "anthropic", "llm_router"), +REQUIRED_ADDITIONS = { + "infra-only-heroku", + "ai-only-anthropic", + "combined-heroku-anthropic", } @@ -65,22 +59,113 @@ def _load(path: Path) -> dict: return json.loads(path.read_text()) -def _cases() -> dict[str, tuple[Path, dict]]: +def _manifest() -> dict: manifest = _load(FIXTURE_ROOT / "expected-matrix.json") - assert manifest["contract_version"] == "1.0" + assert manifest["contract_version"] == "2.0" + return manifest + + +def _cases() -> dict[str, tuple[Path, dict]]: loaded = {} - for entry in manifest["cases"]: + for entry in _manifest()["cases"]: path = FIXTURE_ROOT / entry["snapshot"] snapshot = _load(path) assert snapshot["id"] == entry["id"] + assert entry["id"] not in loaded loaded[entry["id"]] = (path.parent, snapshot) return loaded -def test_manifest_has_complete_scenario_coverage() -> None: +def _scenario_validation_findings(snapshot: dict) -> list: + findings = [] + ARTIFACT_VALIDATOR_MODULE.validate_json_schema( + _load(SCENARIO_SCHEMA), + snapshot, + "", + "expected-artifacts.json", + str(SCENARIO_SCHEMA), + findings, + ) + return findings + + +def _validate_scenario(snapshot: dict) -> None: + assert not _scenario_validation_findings(snapshot) + + +def _assert_gateway_correspondence(snapshot: dict) -> None: + dimensions = snapshot["dimensions"] + providers = {source["provider"] for source in dimensions["ai_sources"]} + model_families = { + family + for source in dimensions["ai_sources"] + for family in source["model_families"] + } + assert dimensions["infrastructure_source"] != "openrouter" + assert "openrouter" not in providers + for gateway in dimensions["gateways"]: + assert set(gateway["upstream_provider_signals"]) <= providers + assert set(gateway["upstream_model_family_signals"]) <= model_families + + +def _run_artifact_validator(case_dir: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(ARTIFACT_VALIDATOR), + "--run-dir", + str(case_dir), + "--skill", + "gcp-to-aws", + "--manifest", + str(ARTIFACT_MANIFEST), + "--no-baseline", + "--json", + ], + capture_output=True, + check=False, + text=True, + ) + + +def _synthetic( + infrastructure_source: str, + provider: str, + source_service: str, + family: str, + *, + openrouter: bool = False, +) -> dict: + scenario = copy.deepcopy(_cases()["combined-heroku-anthropic"][1]) + scenario["id"] = "synthetic-composition" + scenario["migration_id"] = "synthetic-composition" + scenario["dimensions"] = { + "infrastructure_source": infrastructure_source, + "ai_sources": [ + { + "provider": provider, + "source_service": source_service, + "model_families": [family], + } + ], + "gateways": [], + } + if openrouter: + scenario["dimensions"]["gateways"] = [ + { + "product": "openrouter", + "type": "llm_router", + "upstream_provider_signals": [provider], + "upstream_model_family_signals": [family], + } + ] + return scenario + + +def test_manifest_has_fourteen_representative_scenarios_and_all_modes() -> None: cases = _cases() - assert set(cases) == EXPECTED_CASES - assert len(cases) == len(EXPECTED_CASES) + assert len(cases) == 14 + assert REQUIRED_ADDITIONS <= set(cases) assert {case["mode"] for _, case in cases.values()} == { "infrastructure_only", "ai_only", @@ -88,55 +173,86 @@ def test_manifest_has_complete_scenario_coverage() -> None: } -def test_infrastructure_ai_and_gateway_are_independent_dimensions() -> None: - cases = _cases() - for _, snapshot in cases.values(): - dimensions = snapshot["dimensions"] - assert set(dimensions) == {"infrastructure_source", "ai_provider", "model_families", "gateway_type", "upstream_provider_signals"} - assert dimensions["infrastructure_source"] in {None, "gcp", "azure"} - assert dimensions["ai_provider"] in {None, "gcp_ai", "openai", "azure_openai", "anthropic"} - assert dimensions["gateway_type"] in {None, "llm_router"} - assert "openrouter" not in {dimensions["infrastructure_source"], dimensions["ai_provider"]} - if dimensions["gateway_type"] == "llm_router": - assert dimensions["ai_provider"] is not None - assert dimensions["model_families"] - assert dimensions["upstream_provider_signals"] - - combined = { - ( - snapshot["dimensions"]["infrastructure_source"], - snapshot["dimensions"]["ai_provider"], - snapshot["dimensions"]["gateway_type"], +def test_every_snapshot_validates_against_scenario_schema() -> None: + schema = _load(SCENARIO_SCHEMA) + assert ARTIFACT_VALIDATOR_MODULE.unsupported_keywords(schema) == [] + for _, snapshot in _cases().values(): + _validate_scenario(snapshot) + _assert_gateway_correspondence(snapshot) + + +def test_supported_axes_have_independent_representatives() -> None: + manifest = _manifest() + snapshots = [snapshot for _, snapshot in _cases().values()] + supported = manifest["supported"] + + for infrastructure_source in supported["infrastructure_sources"]: + assert any( + snapshot["mode"] == "infrastructure_only" + and snapshot["dimensions"]["infrastructure_source"] == infrastructure_source + for snapshot in snapshots + ) + assert any( + snapshot["mode"] == "combined" + and snapshot["dimensions"]["infrastructure_source"] == infrastructure_source + for snapshot in snapshots ) - for _, snapshot in cases.values() - if snapshot["mode"] == "combined" - } - assert combined == EXPECTED_COMBINATIONS + for provider in supported["direct_ai_providers"]: + assert any( + snapshot["mode"] == "ai_only" + and not snapshot["dimensions"]["gateways"] + and provider + in {source["provider"] for source in snapshot["dimensions"]["ai_sources"]} + for snapshot in snapshots + ) + assert any( + snapshot["mode"] == "combined" + and provider + in {source["provider"] for source in snapshot["dimensions"]["ai_sources"]} + for snapshot in snapshots + ) + + for product in supported["gateway_products"]: + assert any( + snapshot["mode"] == "ai_only" + and product + in {gateway["product"] for gateway in snapshot["dimensions"]["gateways"]} + for snapshot in snapshots + ) + assert any( + snapshot["mode"] == "combined" + and product + in {gateway["product"] for gateway in snapshot["dimensions"]["gateways"]} + for snapshot in snapshots + ) -def test_ai_only_origins_require_no_infrastructure() -> None: + +def test_openai_provider_preserves_direct_and_azure_endpoint_provenance() -> None: cases = _cases() - ai_only = [snapshot for _, snapshot in cases.values() if snapshot["mode"] == "ai_only"] - assert {(case["dimensions"]["ai_provider"], case["dimensions"]["gateway_type"]) for case in ai_only} == { - ("gcp_ai", None), - ("openai", None), - ("openai", "llm_router"), + direct = cases["combined-gcp-openai"][1]["dimensions"]["ai_sources"][0] + azure = cases["combined-azure-openai"][1]["dimensions"]["ai_sources"][0] + assert direct == { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + } + assert azure == { + "provider": "openai", + "source_service": "azure_openai", + "model_families": ["gpt"], } - openai = cases["ai-only-openai"][1] - assert openai["dimensions"]["upstream_provider_signals"] == ["openai", "azure_openai"] - for snapshot in ai_only: - assert snapshot["dimensions"]["infrastructure_source"] is None - assert snapshot["outcomes"]["infrastructure"] is None - assert snapshot["outcomes"]["integration_validation"] is None -def test_each_case_has_one_shared_run_state() -> None: +def test_each_case_has_one_shared_run_state_validated_by_repository_tooling() -> None: for case_dir, snapshot in _cases().values(): assert snapshot["state"] == ".phase-status.json" state = _load(case_dir / snapshot["state"]) assert state["migration_id"] == snapshot["migration_id"] assert state["run_id"] == snapshot["run_id"] assert set(state["phases"].values()) == {"completed"} + result = _run_artifact_validator(case_dir) + assert result.returncode == 0, result.stdout + result.stderr def test_required_artifacts_and_verdicts_match_each_mode() -> None: @@ -157,7 +273,10 @@ def test_required_artifacts_and_verdicts_match_each_mode() -> None: integration_path = case_dir / "integration-validation.json" if snapshot["mode"] == "combined": - assert integration == {"verdict": "passed", "artifact": "integration-validation.json"} + assert integration == { + "verdict": "passed", + "artifact": "integration-validation.json", + } assert integration_path.is_file() else: assert integration is None @@ -181,6 +300,90 @@ def test_combined_validation_references_both_tracks_and_all_categories() -> None assert result["references"] +@pytest.mark.parametrize( + ("infrastructure_source", "provider", "source_service", "family", "openrouter"), + [ + ("heroku", "google", "vertex_ai", "gemini", False), + ("heroku", "openai", "openai_api", "gpt", False), + ("heroku", "openai", "openai_api", "gpt", True), + ("gcp", "anthropic", "anthropic_api", "claude", False), + ("azure", "anthropic", "anthropic_api", "claude", False), + ], +) +def test_non_materialized_compositions_are_schema_valid( + infrastructure_source: str, + provider: str, + source_service: str, + family: str, + openrouter: bool, +) -> None: + scenario = _synthetic( + infrastructure_source, + provider, + source_service, + family, + openrouter=openrouter, + ) + _validate_scenario(scenario) + _assert_gateway_correspondence(scenario) + + +def test_future_sources_and_multiple_direct_providers_need_no_schema_enum_change() -> None: + scenario = _synthetic("future_platform", "future_provider", "future_api", "future_model") + scenario["dimensions"]["ai_sources"].append( + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + } + ) + _validate_scenario(scenario) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda scenario: scenario.update(mode="ai_only"), + lambda scenario: scenario["dimensions"].update(infrastructure_source=None), + lambda scenario: scenario["dimensions"].update(ai_sources=[]), + lambda scenario: scenario["outcomes"].update(infrastructure=None), + lambda scenario: scenario["outcomes"].update(ai=None), + lambda scenario: scenario["outcomes"].update(integration_validation=None), + lambda scenario: scenario["dimensions"].update(infrastructure_source="both"), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(provider="both"), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(provider="openrouter"), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(source_service="both"), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(model_families=[]), + lambda scenario: scenario["dimensions"].update( + gateways=[ + { + "product": "both", + "type": "llm_router", + "upstream_provider_signals": ["anthropic"], + "upstream_model_family_signals": ["claude"], + } + ] + ), + ], +) +def test_schema_rejects_invalid_mode_dimensions_tracks_and_sentinels(mutation) -> None: + scenario = copy.deepcopy(_cases()["combined-heroku-anthropic"][1]) + mutation(scenario) + assert _scenario_validation_findings(scenario) + + +def test_gateway_signals_must_match_retained_ai_sources() -> None: + scenario = _synthetic("heroku", "openai", "openai_api", "gpt", openrouter=True) + scenario["dimensions"]["gateways"][0]["upstream_provider_signals"] = ["anthropic"] + with pytest.raises(AssertionError): + _assert_gateway_correspondence(scenario) + + scenario = _synthetic("heroku", "openai", "openai_api", "gpt", openrouter=True) + scenario["dimensions"]["gateways"][0]["upstream_model_family_signals"] = ["claude"] + with pytest.raises(AssertionError): + _assert_gateway_correspondence(scenario) + + def test_integration_validation_accepts_uppercase_and_lowercase_run_ids(tmp_path: Path) -> None: source = FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json" run_ids = { @@ -196,24 +399,7 @@ def test_integration_validation_accepts_uppercase_and_lowercase_run_ids(tmp_path case_dir.mkdir() (case_dir / "integration-validation.json").write_text(json.dumps(validation)) - result = subprocess.run( - [ - sys.executable, - str(ARTIFACT_VALIDATOR), - "--run-dir", - str(case_dir), - "--skill", - "gcp-to-aws", - "--manifest", - str(ARTIFACT_MANIFEST), - "--no-baseline", - "--json", - ], - capture_output=True, - check=False, - text=True, - ) - + result = _run_artifact_validator(case_dir) assert result.returncode == expected_returncode findings = json.loads(result.stdout)["errors"] if expected_returncode == 0: @@ -222,39 +408,37 @@ def test_integration_validation_accepts_uppercase_and_lowercase_run_ids(tmp_path assert any(finding["code"] == "SCHEMA_VIOLATION" for finding in findings) -def test_success_verdict_rejects_any_failed_child(tmp_path: Path) -> None: - source = FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json" - failed_children = [ +@pytest.mark.parametrize( + ("section", "child"), + [ ("track_results", "infrastructure"), ("track_results", "ai"), ("checks", "iam_and_secrets"), - ] - - for section, child in failed_children: - invalid = _load(source) - invalid[section][child]["verdict"] = "failed" - case_dir = tmp_path / f"{section}-{child}" - case_dir.mkdir() - (case_dir / "integration-validation.json").write_text(json.dumps(invalid)) - - result = subprocess.run( - [ - sys.executable, - str(ARTIFACT_VALIDATOR), - "--run-dir", - str(case_dir), - "--skill", - "gcp-to-aws", - "--manifest", - str(ARTIFACT_MANIFEST), - "--no-baseline", - "--json", - ], - capture_output=True, - check=False, - text=True, - ) - - assert result.returncode == 1 - findings = json.loads(result.stdout)["errors"] - assert any(finding["code"] == "SCHEMA_VIOLATION" for finding in findings) + ], +) +def test_success_verdict_rejects_any_failed_child( + tmp_path: Path, section: str, child: str +) -> None: + invalid = _load(FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json") + invalid[section][child]["verdict"] = "failed" + case_dir = tmp_path / f"{section}-{child}" + case_dir.mkdir() + (case_dir / "integration-validation.json").write_text(json.dumps(invalid)) + + result = _run_artifact_validator(case_dir) + assert result.returncode == 1 + findings = json.loads(result.stdout)["errors"] + assert any(finding["code"] == "SCHEMA_VIOLATION" for finding in findings) + + +def test_success_verdict_rejects_empty_references(tmp_path: Path) -> None: + invalid = _load(FIXTURE_ROOT / "combined-gcp-gemini" / "integration-validation.json") + invalid["checks"]["networking"]["references"] = [] + case_dir = tmp_path / "empty-references" + case_dir.mkdir() + (case_dir / "integration-validation.json").write_text(json.dumps(invalid)) + + result = _run_artifact_validator(case_dir) + assert result.returncode == 1 + findings = json.loads(result.stdout)["errors"] + assert any(finding["code"] == "SCHEMA_VIOLATION" for finding in findings) From 9ffba548ed2ac126431de05831642bd204f07f3d Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 23:19:26 -0700 Subject: [PATCH 24/53] test(aws-startup-advisor): expand AI migration matrix --- .../aws-startup-advisor/fixtures/README.md | 2 +- .../ai-only-anthropic/expected-artifacts.json | 2 +- .../ai-only-gemini/expected-artifacts.json | 2 +- .../ai-only-openai/expected-artifacts.json | 2 +- .../expected-artifacts.json | 4 +- .../expected-artifacts.json | 2 +- .../expected-artifacts.json | 2 +- .../expected-artifacts.json | 4 +- .../expected-artifacts.json | 2 +- .../expected-artifacts.json | 2 +- .../expected-artifacts.json | 4 +- .../expected-artifacts.json | 2 +- .../ai-migration-matrix/expected-matrix.json | 4 +- .../scripts/artifact-contracts.json | 3 +- .../ai-migration-scenario.schema.json | 106 +++- .../integration-validation.schema.json | 2 +- .../test_ai_migration_matrix_contracts.py | 458 ++++++++++++++---- 17 files changed, 488 insertions(+), 115 deletions(-) diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 91a356a08..12cd8ba16 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -2,7 +2,7 @@ Also see: -- `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. These fixtures do not claim current runtime support or enumerate the Cartesian product. Infrastructure source, direct AI sources/model families, and optional gateways are independent dimensions validated by `scripts/contracts/ai-migration-scenario.schema.json`; synthetic composition tests cover legal combinations without duplicate fixture directories. GCP, Azure, and Heroku are represented as infrastructure sources; Google/Gemini, OpenAI (with direct versus Azure endpoint provenance), and direct Anthropic/Claude are represented as AI sources. OpenRouter remains a separate gateway retaining its known upstream provider/model signals. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. +- `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json index bd46baa38..04a8f8014 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-anthropic/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"]}], + "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"], "model_evidence": "observed", "models": ["claude-3-5-sonnet-20240620"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json index f9155f68a..532715872 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-gemini/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"]}], + "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"], "model_evidence": "observed", "models": ["gemini-1.5-pro"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json index b0966b8cd..f5f642f29 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openai/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"], "model_evidence": "observed", "models": ["gpt-4o"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json index ae02ea871..98e054d3c 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/ai-only-openrouter/expected-artifacts.json @@ -6,8 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": null, - "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], - "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_provider_signals": ["openai"], "upstream_model_family_signals": ["gpt"]}] + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"], "model_evidence": "observed", "models": ["gpt-4o"]}], + "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_evidence": "observed", "upstreams": [{"provider": "openai", "source_service": "openai_api", "model_families": [{"family": "gpt", "model_evidence": "observed", "models": ["gpt-4o"]}]}]}] }, "outcomes": { "infrastructure": null, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json index 3f03644ae..a2aedb0c6 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-gemini/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "azure", - "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"]}], + "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"], "model_evidence": "observed", "models": ["gemini-1.5-pro"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json index fe4bfd804..8e2382e4d 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openai/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "azure", - "ai_sources": [{"provider": "openai", "source_service": "azure_openai", "model_families": ["gpt"]}], + "ai_sources": [{"provider": "openai", "source_service": "azure_openai", "model_families": ["gpt"], "model_evidence": "observed", "models": ["gpt-4o"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json index 0cfafe27a..59d56e823 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-azure-openrouter/expected-artifacts.json @@ -6,8 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "azure", - "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"]}], - "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_provider_signals": ["anthropic"], "upstream_model_family_signals": ["claude"]}] + "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"], "model_evidence": "observed", "models": ["claude-3-5-sonnet-20240620"]}], + "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_evidence": "observed", "upstreams": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": [{"family": "claude", "model_evidence": "observed", "models": ["claude-3-5-sonnet-20240620"]}]}]}] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json index ecc62b60b..82eea3042 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-gemini/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "gcp", - "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"]}], + "ai_sources": [{"provider": "google", "source_service": "vertex_ai", "model_families": ["gemini"], "model_evidence": "observed", "models": ["gemini-1.5-pro"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json index c15453f66..29b71f87d 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openai/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "gcp", - "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"], "model_evidence": "observed", "models": ["gpt-4o"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json index 7bc7796b1..746f68c84 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-gcp-openrouter/expected-artifacts.json @@ -6,8 +6,8 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "gcp", - "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"]}], - "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_provider_signals": ["openai"], "upstream_model_family_signals": ["gpt"]}] + "ai_sources": [{"provider": "openai", "source_service": "openai_api", "model_families": ["gpt"], "model_evidence": "observed", "models": ["gpt-4o"]}], + "gateways": [{"product": "openrouter", "type": "llm_router", "upstream_evidence": "observed", "upstreams": [{"provider": "openai", "source_service": "openai_api", "model_families": [{"family": "gpt", "model_evidence": "observed", "models": ["gpt-4o"]}]}]}] }, "outcomes": { "infrastructure": {"verdict": "passed", "artifacts": ["aws-design.json", "estimation-infra.json", "generation-infra.json", "validation-report.json"]}, diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json index 1ad8bc215..c70f73047 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/combined-heroku-anthropic/expected-artifacts.json @@ -6,7 +6,7 @@ "state": ".phase-status.json", "dimensions": { "infrastructure_source": "heroku", - "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"]}], + "ai_sources": [{"provider": "anthropic", "source_service": "anthropic_api", "model_families": ["claude"], "model_evidence": "observed", "models": ["claude-3-5-sonnet-20240620"]}], "gateways": [] }, "outcomes": { diff --git a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json index a4d275815..639fdd054 100644 --- a/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json +++ b/plugins/aws-startup-advisor/fixtures/ai-migration-matrix/expected-matrix.json @@ -1,6 +1,6 @@ { - "contract_version": "2.0", - "description": "Target-only representative contracts for composable migration dimensions; these fixtures do not assert current runtime support or enumerate a Cartesian matrix.", + "contract_version": "3.0", + "description": "Target-only represented contract coverage for composable migration dimensions; this is not a runtime capability catalog, does not assert current runtime support, and does not enumerate a Cartesian matrix.", "supported": { "infrastructure_sources": ["gcp", "azure", "heroku"], "direct_ai_providers": ["google", "openai", "anthropic"], diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index bf5fa1fd9..1bbe3608e 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -3,8 +3,7 @@ "fixture_prefixes": { "gcp-": "gcp-to-aws", "heroku-": "heroku-to-aws", - "azure-": "azure-to-aws", - "ai-migration-matrix": "gcp-to-aws" + "azure-": "azure-to-aws" }, "shared": { ".phase-status.json": { diff --git a/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json b/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json index 5f0794b38..b1e027f95 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/ai-migration-scenario.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "urn:awslabs:startups:migration:ai-migration-scenario", "title": "AI migration scenario snapshot", - "description": "Target-only compositional contract for representative infrastructure and AI migration scenarios.", + "description": "Target-only compositional contract for representative infrastructure and AI migration scenarios. Structural validity does not assert current runtime support; a versioned runtime capability catalog is a later deliverable.", "type": "object", "required": ["id", "mode", "migration_id", "run_id", "state", "dimensions", "outcomes"], "additionalProperties": false, @@ -18,12 +18,14 @@ "additionalProperties": false, "properties": { "infrastructure_source": { + "description": "Option A: zero or one infrastructure source per migration run. Polycloud or multiple simultaneous infrastructure sources are outside this contract and deferred.", "oneOf": [ {"type": "null"}, {"$ref": "#/definitions/extensibleIdentity"} ] }, "ai_sources": { + "description": "Retained AI sources. Semantic uniqueness is the provider and source_service pair; duplicate or overlapping pairs are invalid, while one provider through distinct services is valid.", "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/aiSource"} @@ -105,12 +107,14 @@ ], "definitions": { "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$"}, - "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter)$)[a-z][a-z0-9_]*$"}, - "modelFamily": {"type": "string", "pattern": "^(?!both$)[a-z][a-z0-9_.-]*$"}, - "gatewayIdentity": {"type": "string", "pattern": "^(?!both$)[a-z][a-z0-9_]*$"}, + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "modelFamily": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_.-]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "gatewayIdentity": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_]*$"}, "aiSource": { + "description": "A structurally extensible AI source keyed semantically by provider and source_service. Exact models are observed normalized IDs or an explicitly empty not-observed set; sentinel model IDs are invalid.", "type": "object", - "required": ["provider", "source_service", "model_families"], + "required": ["provider", "source_service", "model_families", "model_evidence", "models"], "additionalProperties": false, "properties": { "provider": {"$ref": "#/definitions/extensibleIdentity"}, @@ -120,27 +124,103 @@ "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/definitions/modelFamily"} + }, + "model_evidence": {"enum": ["observed", "not_observed"]}, + "models": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/normalizedModel"} + } + }, + "oneOf": [ + { + "properties": { + "model_evidence": {"const": "observed"}, + "models": {"minItems": 1} + } + }, + { + "properties": { + "model_evidence": {"const": "not_observed"}, + "models": {"maxItems": 0} + } + } + ] + }, + "upstreamModelFamily": { + "description": "One provider-associated model family and its exact observed models, or an explicit not-observed exact-model set.", + "type": "object", + "required": ["family", "model_evidence", "models"], + "additionalProperties": false, + "properties": { + "family": {"$ref": "#/definitions/modelFamily"}, + "model_evidence": {"enum": ["observed", "not_observed"]}, + "models": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/normalizedModel"} + } + }, + "oneOf": [ + { + "properties": { + "model_evidence": {"const": "observed"}, + "models": {"minItems": 1} + } + }, + { + "properties": { + "model_evidence": {"const": "not_observed"}, + "models": {"maxItems": 0} + } + } + ] + }, + "gatewayUpstream": { + "description": "Authoritative provider to optional source-service provenance to family to exact-model association. Semantic uniqueness is provider plus source_service when known; duplicate or overlapping entries are invalid.", + "type": "object", + "required": ["provider", "model_families"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/upstreamModelFamily"} } } }, "gateway": { + "description": "Optional gateway. Observed upstreams are structured associations; unresolved explicitly means that upstream identity is unknown and requires an empty upstreams array.", "type": "object", - "required": ["product", "type", "upstream_provider_signals", "upstream_model_family_signals"], + "required": ["product", "type", "upstream_evidence", "upstreams"], "additionalProperties": false, "properties": { "product": {"$ref": "#/definitions/gatewayIdentity"}, "type": {"$ref": "#/definitions/extensibleIdentity"}, - "upstream_provider_signals": { + "upstream_evidence": {"enum": ["observed", "unresolved"]}, + "upstreams": { "type": "array", "uniqueItems": true, - "items": {"$ref": "#/definitions/extensibleIdentity"} + "items": {"$ref": "#/definitions/gatewayUpstream"} + } + }, + "oneOf": [ + { + "properties": { + "upstream_evidence": {"const": "observed"}, + "upstreams": {"minItems": 1} + } }, - "upstream_model_family_signals": { - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/modelFamily"} + { + "properties": { + "upstream_evidence": {"const": "unresolved"}, + "upstreams": {"maxItems": 0} + } } - } + ] }, "trackOutcome": { "type": "object", diff --git a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json index 6ab37456a..23e3c0d9e 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/integration-validation.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "urn:awslabs:startups:migration:integration-validation", "title": "integration-validation.json", - "description": "Target snapshot contract for cross-track validation in a combined infrastructure and AI migration.", + "description": "Target snapshot contract for cross-track validation in a combined migration. Under Option A, track_results.infrastructure is the single zero-or-one infrastructure-source track; polycloud runs are deferred.", "type": "object", "required": ["schema_version", "run_id", "verdict", "track_results", "checks"], "additionalProperties": false, diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py index 1810121d3..a5c103463 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -20,6 +20,8 @@ ARTIFACT_VALIDATOR = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" ARTIFACT_MANIFEST = PLUGIN_ROOT / "scripts" / "artifact-contracts.json" SCENARIO_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "ai-migration-scenario.schema.json" +INTEGRATION_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "integration-validation.schema.json" +PHASE_STATUS_SCHEMA = PLUGIN_ROOT / "skills" / "shared" / "state" / "phase-status.schema.json" _VALIDATOR_SPEC = importlib.util.spec_from_file_location("artifact_validator", ARTIFACT_VALIDATOR) assert _VALIDATOR_SPEC and _VALIDATOR_SPEC.loader ARTIFACT_VALIDATOR_MODULE = importlib.util.module_from_spec(_VALIDATOR_SPEC) @@ -53,6 +55,12 @@ "ai-only-anthropic", "combined-heroku-anthropic", } +MODELS = { + "google": "gemini-1.5-pro", + "openai": "gpt-4o", + "anthropic": "claude-3-5-sonnet-20240620", + "future_provider": "future-model-v1", +} def _load(path: Path) -> dict: @@ -61,7 +69,7 @@ def _load(path: Path) -> dict: def _manifest() -> dict: manifest = _load(FIXTURE_ROOT / "expected-matrix.json") - assert manifest["contract_version"] == "2.0" + assert manifest["contract_version"] == "3.0" return manifest @@ -76,36 +84,75 @@ def _cases() -> dict[str, tuple[Path, dict]]: return loaded -def _scenario_validation_findings(snapshot: dict) -> list: +def _schema_findings(schema_path: Path, instance: dict, name: str) -> list: findings = [] ARTIFACT_VALIDATOR_MODULE.validate_json_schema( - _load(SCENARIO_SCHEMA), - snapshot, + _load(schema_path), + instance, "", - "expected-artifacts.json", - str(SCENARIO_SCHEMA), + name, + str(schema_path), findings, ) return findings +def _scenario_validation_findings(snapshot: dict) -> list: + return _schema_findings(SCENARIO_SCHEMA, snapshot, "expected-artifacts.json") + + def _validate_scenario(snapshot: dict) -> None: assert not _scenario_validation_findings(snapshot) -def _assert_gateway_correspondence(snapshot: dict) -> None: +def _assert_scenario_semantics(snapshot: dict) -> None: dimensions = snapshot["dimensions"] - providers = {source["provider"] for source in dimensions["ai_sources"]} - model_families = { - family - for source in dimensions["ai_sources"] - for family in source["model_families"] - } assert dimensions["infrastructure_source"] != "openrouter" - assert "openrouter" not in providers + + sources = {} + for source in dimensions["ai_sources"]: + key = (source["provider"], source["source_service"]) + assert key not in sources, f"duplicate or overlapping AI source: {key}" + assert source["provider"] != "openrouter" + sources[key] = source + for gateway in dimensions["gateways"]: - assert set(gateway["upstream_provider_signals"]) <= providers - assert set(gateway["upstream_model_family_signals"]) <= model_families + upstream_keys = set() + for upstream in gateway["upstreams"]: + key = (upstream["provider"], upstream.get("source_service")) + assert key not in upstream_keys, ( + f"duplicate or overlapping gateway upstream: {key}" + ) + upstream_keys.add(key) + + if "source_service" in upstream: + source = sources.get((upstream["provider"], upstream["source_service"])) + assert source is not None, f"unmatched gateway upstream: {key}" + else: + candidates = [ + source + for (provider, _), source in sources.items() + if provider == upstream["provider"] + ] + assert len(candidates) == 1, ( + f"service-less upstream must match one source: {upstream['provider']}" + ) + source = candidates[0] + + families = set(source["model_families"]) + models = set(source["models"]) + seen_families = set() + for family in upstream["model_families"]: + assert family["family"] not in seen_families, ( + f"duplicate or overlapping upstream family: {family['family']}" + ) + seen_families.add(family["family"]) + assert family["family"] in families, ( + f"wrongly paired upstream family: {family['family']}" + ) + assert set(family["models"]) <= models, ( + f"wrongly paired upstream models: {family['models']}" + ) def _run_artifact_validator(case_dir: Path) -> subprocess.CompletedProcess[str]: @@ -128,40 +175,98 @@ def _run_artifact_validator(case_dir: Path) -> subprocess.CompletedProcess[str]: ) +def _source(provider: str, source_service: str, family: str, *models: str) -> dict: + return { + "provider": provider, + "source_service": source_service, + "model_families": [family], + "model_evidence": "observed", + "models": list(models), + } + + +def _upstream(provider: str, source_service: str, family: str, *models: str) -> dict: + return { + "provider": provider, + "source_service": source_service, + "model_families": [ + { + "family": family, + "model_evidence": "observed", + "models": list(models), + } + ], + } + + +def _gateway(*upstreams: dict) -> dict: + return { + "product": "openrouter", + "type": "llm_router", + "upstream_evidence": "observed", + "upstreams": list(upstreams), + } + + def _synthetic( infrastructure_source: str, provider: str, source_service: str, family: str, + model: str, *, openrouter: bool = False, ) -> dict: scenario = copy.deepcopy(_cases()["combined-heroku-anthropic"][1]) scenario["id"] = "synthetic-composition" scenario["migration_id"] = "synthetic-composition" + source = _source(provider, source_service, family, model) scenario["dimensions"] = { "infrastructure_source": infrastructure_source, - "ai_sources": [ - { - "provider": provider, - "source_service": source_service, - "model_families": [family], - } - ], + "ai_sources": [source], "gateways": [], } if openrouter: scenario["dimensions"]["gateways"] = [ - { - "product": "openrouter", - "type": "llm_router", - "upstream_provider_signals": [provider], - "upstream_model_family_signals": [family], - } + _gateway(_upstream(provider, source_service, family, model)) ] return scenario +def _multi_openrouter_scenario(infrastructure_source: str = "heroku") -> dict: + scenario = copy.deepcopy(_cases()["combined-heroku-anthropic"][1]) + scenario["id"] = "multi-cardinality-openrouter" + scenario["migration_id"] = "multi-cardinality-openrouter" + associations = [ + ( + "anthropic", + "anthropic_api", + "claude", + ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307"], + ), + ("google", "vertex_ai", "gemini", ["gemini-1.5-pro"]), + ("openai", "openai_api", "gpt", ["gpt-4o"]), + ("openai", "azure_openai", "gpt", ["gpt-4o"]), + ("future_provider", "future_api", "future_family", ["future-model-v1"]), + ] + scenario["dimensions"] = { + "infrastructure_source": infrastructure_source, + "ai_sources": [ + _source(provider, service, family, *models) + for provider, service, family, models in associations + ], + "gateways": [ + _gateway( + *[ + _upstream(provider, service, family, *models) + for provider, service, family, models in associations + ] + ) + ], + } + return scenario + + def test_manifest_has_fourteen_representative_scenarios_and_all_modes() -> None: cases = _cases() assert len(cases) == 14 @@ -178,7 +283,7 @@ def test_every_snapshot_validates_against_scenario_schema() -> None: assert ARTIFACT_VALIDATOR_MODULE.unsupported_keywords(schema) == [] for _, snapshot in _cases().values(): _validate_scenario(snapshot) - _assert_gateway_correspondence(snapshot) + _assert_scenario_semantics(snapshot) def test_supported_axes_have_independent_representatives() -> None: @@ -189,12 +294,14 @@ def test_supported_axes_have_independent_representatives() -> None: for infrastructure_source in supported["infrastructure_sources"]: assert any( snapshot["mode"] == "infrastructure_only" - and snapshot["dimensions"]["infrastructure_source"] == infrastructure_source + and snapshot["dimensions"]["infrastructure_source"] + == infrastructure_source for snapshot in snapshots ) assert any( snapshot["mode"] == "combined" - and snapshot["dimensions"]["infrastructure_source"] == infrastructure_source + and snapshot["dimensions"]["infrastructure_source"] + == infrastructure_source for snapshot in snapshots ) @@ -232,27 +339,28 @@ def test_openai_provider_preserves_direct_and_azure_endpoint_provenance() -> Non cases = _cases() direct = cases["combined-gcp-openai"][1]["dimensions"]["ai_sources"][0] azure = cases["combined-azure-openai"][1]["dimensions"]["ai_sources"][0] - assert direct == { - "provider": "openai", - "source_service": "openai_api", - "model_families": ["gpt"], - } - assert azure == { - "provider": "openai", - "source_service": "azure_openai", - "model_families": ["gpt"], - } + assert direct == _source("openai", "openai_api", "gpt", "gpt-4o") + assert azure == _source("openai", "azure_openai", "gpt", "gpt-4o") -def test_each_case_has_one_shared_run_state_validated_by_repository_tooling() -> None: +def test_each_case_shared_artifacts_validate_directly() -> None: for case_dir, snapshot in _cases().values(): assert snapshot["state"] == ".phase-status.json" state = _load(case_dir / snapshot["state"]) assert state["migration_id"] == snapshot["migration_id"] assert state["run_id"] == snapshot["run_id"] assert set(state["phases"].values()) == {"completed"} - result = _run_artifact_validator(case_dir) - assert result.returncode == 0, result.stdout + result.stderr + assert not _schema_findings( + PHASE_STATUS_SCHEMA, state, ".phase-status.json" + ) + + integration_path = case_dir / "integration-validation.json" + if integration_path.is_file(): + assert not _schema_findings( + INTEGRATION_SCHEMA, + _load(integration_path), + "integration-validation.json", + ) def test_required_artifacts_and_verdicts_match_each_mode() -> None: @@ -301,43 +409,173 @@ def test_combined_validation_references_both_tracks_and_all_categories() -> None @pytest.mark.parametrize( - ("infrastructure_source", "provider", "source_service", "family", "openrouter"), + ("provider", "source_service", "family", "model"), [ - ("heroku", "google", "vertex_ai", "gemini", False), - ("heroku", "openai", "openai_api", "gpt", False), - ("heroku", "openai", "openai_api", "gpt", True), - ("gcp", "anthropic", "anthropic_api", "claude", False), - ("azure", "anthropic", "anthropic_api", "claude", False), + ("google", "vertex_ai", "gemini", "gemini-1.5-pro"), + ("openai", "openai_api", "gpt", "gpt-4o"), + ("anthropic", "anthropic_api", "claude", "claude-3-5-sonnet-20240620"), ], ) -def test_non_materialized_compositions_are_schema_valid( +def test_heroku_composes_with_each_direct_ai_provider( + provider: str, source_service: str, family: str, model: str +) -> None: + scenario = _synthetic("heroku", provider, source_service, family, model) + _validate_scenario(scenario) + _assert_scenario_semantics(scenario) + + +@pytest.mark.parametrize("infrastructure_source", ["gcp", "azure", "heroku"]) +def test_openrouter_dimensions_compose_with_each_infrastructure_source( infrastructure_source: str, - provider: str, - source_service: str, - family: str, - openrouter: bool, ) -> None: - scenario = _synthetic( - infrastructure_source, - provider, - source_service, - family, - openrouter=openrouter, + scenario = _multi_openrouter_scenario(infrastructure_source) + assert isinstance(scenario["dimensions"]["infrastructure_source"], str) + _validate_scenario(scenario) + _assert_scenario_semantics(scenario) + + +def test_multi_cardinality_openrouter_preserves_exact_associations() -> None: + scenario = _multi_openrouter_scenario() + dimensions = scenario["dimensions"] + sources = dimensions["ai_sources"] + upstreams = dimensions["gateways"][0]["upstreams"] + + assert len({source["provider"] for source in sources}) > 1 + assert len({family for source in sources for family in source["model_families"]}) > 1 + assert len({model for source in sources for model in source["models"]}) > 1 + claude = next( + family + for upstream in upstreams + if upstream["provider"] == "anthropic" + for family in upstream["model_families"] + if family["family"] == "claude" ) + assert claude["models"] == [ + "claude-3-5-sonnet-20240620", + "claude-3-haiku-20240307", + ] + assert { + (upstream["provider"], upstream["source_service"]): { + family["family"]: family["models"] + for family in upstream["model_families"] + } + for upstream in upstreams + } == { + (source["provider"], source["source_service"]): { + source["model_families"][0]: source["models"] + } + for source in sources + } + assert [ + source["source_service"] + for source in sources + if source["provider"] == "openai" + ] == ["openai_api", "azure_openai"] _validate_scenario(scenario) - _assert_gateway_correspondence(scenario) + _assert_scenario_semantics(scenario) -def test_future_sources_and_multiple_direct_providers_need_no_schema_enum_change() -> None: - scenario = _synthetic("future_platform", "future_provider", "future_api", "future_model") +def test_future_structural_identity_does_not_claim_runtime_support() -> None: + scenario = _synthetic( + "future_platform", + "future_provider", + "future_api", + "future_family", + "future-model-v1", + ) scenario["dimensions"]["ai_sources"].append( - { - "provider": "anthropic", - "source_service": "anthropic_api", - "model_families": ["claude"], - } + _source( + "anthropic", + "anthropic_api", + "claude", + "claude-3-5-sonnet-20240620", + ) ) _validate_scenario(scenario) + _assert_scenario_semantics(scenario) + assert "future_provider" not in _manifest()["supported"]["direct_ai_providers"] + + +@pytest.mark.parametrize( + "mutation", + [ + lambda scenario: scenario["dimensions"]["ai_sources"].append( + copy.deepcopy(scenario["dimensions"]["ai_sources"][0]) + ), + lambda scenario: scenario["dimensions"]["ai_sources"].append( + _source("anthropic", "anthropic_api", "claude", "claude-3-haiku-20240307") + ), + ], +) +def test_duplicate_or_overlapping_ai_sources_are_rejected(mutation) -> None: + scenario = _synthetic( + "heroku", + "anthropic", + "anthropic_api", + "claude", + "claude-3-5-sonnet-20240620", + ) + mutation(scenario) + if not _scenario_validation_findings(scenario): + with pytest.raises(AssertionError): + _assert_scenario_semantics(scenario) + else: + assert _scenario_validation_findings(scenario) + + +@pytest.mark.parametrize("overlap", ["exact", "split", "ambiguous-service"]) +def test_duplicate_or_overlapping_gateway_upstreams_are_rejected(overlap: str) -> None: + scenario = _multi_openrouter_scenario() + gateway = scenario["dimensions"]["gateways"][0] + if overlap == "exact": + gateway["upstreams"].append(copy.deepcopy(gateway["upstreams"][0])) + elif overlap == "split": + gateway["upstreams"].append( + _upstream( + "anthropic", + "anthropic_api", + "claude", + "claude-3-haiku-20240307", + ) + ) + else: + gateway["upstreams"].append( + { + "provider": "openai", + "model_families": [ + { + "family": "gpt", + "model_evidence": "observed", + "models": ["gpt-4o"], + } + ], + } + ) + if not _scenario_validation_findings(scenario): + with pytest.raises(AssertionError): + _assert_scenario_semantics(scenario) + else: + assert _scenario_validation_findings(scenario) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda upstream: upstream.update(provider="google"), + lambda upstream: upstream.update(source_service="azure_openai"), + lambda upstream: upstream["model_families"][0].update(family="claude"), + lambda upstream: upstream["model_families"][0].update( + models=["claude-3-5-sonnet-20240620"] + ), + ], +) +def test_gateway_upstreams_must_match_retained_ai_sources_exactly(mutation) -> None: + scenario = _synthetic( + "heroku", "openai", "openai_api", "gpt", "gpt-4o", openrouter=True + ) + mutation(scenario["dimensions"]["gateways"][0]["upstreams"][0]) + with pytest.raises(AssertionError): + _assert_scenario_semantics(scenario) @pytest.mark.parametrize( @@ -345,43 +583,99 @@ def test_future_sources_and_multiple_direct_providers_need_no_schema_enum_change [ lambda scenario: scenario.update(mode="ai_only"), lambda scenario: scenario["dimensions"].update(infrastructure_source=None), + lambda scenario: scenario["dimensions"].update(infrastructure_source=["heroku", "gcp"]), + lambda scenario: ( + scenario.update(mode="ai_only"), + scenario["dimensions"].update( + infrastructure_source=None, ai_sources=[], gateways=[] + ), + scenario["outcomes"].update( + infrastructure=None, ai=None, integration_validation=None + ), + ), lambda scenario: scenario["dimensions"].update(ai_sources=[]), lambda scenario: scenario["outcomes"].update(infrastructure=None), lambda scenario: scenario["outcomes"].update(ai=None), lambda scenario: scenario["outcomes"].update(integration_validation=None), lambda scenario: scenario["dimensions"].update(infrastructure_source="both"), + lambda scenario: scenario["dimensions"].update(infrastructure_source="openrouter"), lambda scenario: scenario["dimensions"]["ai_sources"][0].update(provider="both"), lambda scenario: scenario["dimensions"]["ai_sources"][0].update(provider="openrouter"), lambda scenario: scenario["dimensions"]["ai_sources"][0].update(source_service="both"), lambda scenario: scenario["dimensions"]["ai_sources"][0].update(model_families=[]), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(provider="OpenAI"), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(source_service="openai-api"), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(model_families=["GPT"]), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(models=["GPT-4o"]), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(models=["unknown"]), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update(models=[]), + lambda scenario: scenario["dimensions"]["ai_sources"][0].update( + model_evidence="not_observed" + ), lambda scenario: scenario["dimensions"].update( gateways=[ { "product": "both", "type": "llm_router", - "upstream_provider_signals": ["anthropic"], - "upstream_model_family_signals": ["claude"], + "upstream_evidence": "observed", + "upstreams": [ + _upstream("anthropic", "anthropic_api", "claude", MODELS["anthropic"]) + ], + } + ] + ), + lambda scenario: scenario["dimensions"].update( + gateways=[ + { + "product": "openrouter", + "type": "llm_router", + "upstream_evidence": "observed", + "upstreams": [], + } + ] + ), + lambda scenario: scenario["dimensions"].update( + gateways=[ + { + "product": "openrouter", + "type": "llm_router", + "upstream_evidence": "unresolved", + "upstreams": [ + _upstream("anthropic", "anthropic_api", "claude", MODELS["anthropic"]) + ], } ] ), ], ) -def test_schema_rejects_invalid_mode_dimensions_tracks_and_sentinels(mutation) -> None: +def test_schema_rejects_invalid_modes_ids_cardinality_and_sentinels(mutation) -> None: scenario = copy.deepcopy(_cases()["combined-heroku-anthropic"][1]) mutation(scenario) assert _scenario_validation_findings(scenario) -def test_gateway_signals_must_match_retained_ai_sources() -> None: - scenario = _synthetic("heroku", "openai", "openai_api", "gpt", openrouter=True) - scenario["dimensions"]["gateways"][0]["upstream_provider_signals"] = ["anthropic"] - with pytest.raises(AssertionError): - _assert_gateway_correspondence(scenario) +def test_unresolved_gateway_upstreams_are_explicit_and_unambiguous() -> None: + scenario = _synthetic( + "heroku", "openai", "openai_api", "gpt", "gpt-4o", openrouter=True + ) + scenario["dimensions"]["gateways"] = [ + { + "product": "openrouter", + "type": "llm_router", + "upstream_evidence": "unresolved", + "upstreams": [], + } + ] + _validate_scenario(scenario) + _assert_scenario_semantics(scenario) - scenario = _synthetic("heroku", "openai", "openai_api", "gpt", openrouter=True) - scenario["dimensions"]["gateways"][0]["upstream_model_family_signals"] = ["claude"] - with pytest.raises(AssertionError): - _assert_gateway_correspondence(scenario) + for evidence in ["observed", None]: + invalid = copy.deepcopy(scenario) + if evidence is None: + del invalid["dimensions"]["gateways"][0]["upstream_evidence"] + else: + invalid["dimensions"]["gateways"][0]["upstream_evidence"] = evidence + assert _scenario_validation_findings(invalid) def test_integration_validation_accepts_uppercase_and_lowercase_run_ids(tmp_path: Path) -> None: From ca88672648d06344c3e7810e16833dce283ad652 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 23:38:28 -0700 Subject: [PATCH 25/53] fix: enforce exact gateway source contracts --- .../test_ai_migration_matrix_contracts.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py index a5c103463..dde38f84c 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -142,6 +142,7 @@ def _assert_scenario_semantics(snapshot: dict) -> None: families = set(source["model_families"]) models = set(source["models"]) seen_families = set() + seen_models = set() for family in upstream["model_families"]: assert family["family"] not in seen_families, ( f"duplicate or overlapping upstream family: {family['family']}" @@ -150,9 +151,21 @@ def _assert_scenario_semantics(snapshot: dict) -> None: assert family["family"] in families, ( f"wrongly paired upstream family: {family['family']}" ) - assert set(family["models"]) <= models, ( + assert family["model_evidence"] == source["model_evidence"], ( + f"wrongly paired upstream model evidence: {family['model_evidence']}" + ) + family_models = set(family["models"]) + assert family_models <= models, ( f"wrongly paired upstream models: {family['models']}" ) + seen_models.update(family_models) + + assert seen_families == families, ( + f"gateway upstream families do not match source: {seen_families}" + ) + assert seen_models == models, ( + f"gateway upstream models do not match source: {seen_models}" + ) def _run_artifact_validator(case_dir: Path) -> subprocess.CompletedProcess[str]: @@ -578,6 +591,52 @@ def test_gateway_upstreams_must_match_retained_ai_sources_exactly(mutation) -> N _assert_scenario_semantics(scenario) +@pytest.mark.parametrize( + "mutation", + [ + lambda upstream: upstream["model_families"][1]["models"].remove( + "claude-3-haiku-20240307" + ), + lambda upstream: upstream["model_families"].pop(), + lambda upstream: upstream["model_families"][1].update( + model_evidence="not_observed", models=[] + ), + ], +) +def test_gateway_upstreams_cannot_omit_source_model_details(mutation) -> None: + scenario = _multi_openrouter_scenario() + source = next( + source + for source in scenario["dimensions"]["ai_sources"] + if source["provider"] == "anthropic" + ) + upstream = next( + upstream + for upstream in scenario["dimensions"]["gateways"][0]["upstreams"] + if upstream["provider"] == "anthropic" + ) + source["model_families"] = ["claude", "claude_haiku"] + upstream["model_families"] = [ + { + "family": "claude", + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620"], + }, + { + "family": "claude_haiku", + "model_evidence": "observed", + "models": ["claude-3-haiku-20240307"], + }, + ] + _validate_scenario(scenario) + _assert_scenario_semantics(scenario) + + mutation(upstream) + + with pytest.raises(AssertionError): + _assert_scenario_semantics(scenario) + + @pytest.mark.parametrize( "mutation", [ From 6ca33dadd700b34a12445c980093399507267b86 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 5 Oct 2026 23:40:58 -0700 Subject: [PATCH 26/53] fix: validate shared neutral fixtures --- .../scripts/validate-artifacts.py | 12 ++-- .../tests/test_validate_artifacts.py | 56 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 03ef685bb..de3817855 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -866,10 +866,11 @@ def load_manifest(path: Path) -> Dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) -def build_contracts(manifest: Dict[str, Any], skill: str, findings: List[Finding]) -> List[Contract]: +def build_contracts(manifest: Dict[str, Any], skill: Optional[str], findings: List[Finding]) -> List[Contract]: specs: Dict[str, Any] = {} specs.update(manifest.get("shared", {})) - specs.update(manifest.get("skills", {}).get(skill, {})) + if skill is not None: + specs.update(manifest.get("skills", {}).get(skill, {})) contracts: List[Contract] = [] for glob_, spec in specs.items(): c = Contract(artifact_glob=glob_) @@ -928,7 +929,7 @@ def find_artifacts(run_dir: Path) -> List[Path]: return out -def validate_run_dir(run_dir: Path, skill: str, manifest: Dict[str, Any], findings: List[Finding]) -> int: +def validate_run_dir(run_dir: Path, skill: Optional[str], manifest: Dict[str, Any], findings: List[Finding]) -> int: contracts = build_contracts(manifest, skill, findings) checked = 0 for p in find_artifacts(run_dir): @@ -936,7 +937,8 @@ def validate_run_dir(run_dir: Path, skill: str, manifest: Dict[str, Any], findin matched = [c for c in contracts if fnmatch.fnmatch(name, c.artifact_glob)] rel = _rel(p) if not matched: - findings.append(Finding("NO_CONTRACT", rel, "", f"no contract covers '{name}' for skill {skill}", "")) + if skill is not None: + findings.append(Finding("NO_CONTRACT", rel, "", f"no contract covers '{name}' for skill {skill}", "")) continue try: value = json.loads(p.read_text(encoding="utf-8")) @@ -1026,8 +1028,6 @@ def main(argv: Optional[List[str]] = None) -> int: fixtures_root = PLUGIN_ROOT / "fixtures" for d in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): skill = skill_for_fixture(d.name, manifest) - if skill is None: - continue checked += validate_run_dir(d, skill, manifest, findings) else: if not args.skill: diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index fb1a0bf17..b3b7a57da 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -320,6 +320,62 @@ def test_fixtures_pass_with_baseline_and_no_stale_entries(): assert report["stale_baseline_entries"] == [], "remove baseline entries that no longer match anything" +def test_neutral_matrix_shared_artifacts_extend_the_mapped_fixture_count(): + manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") + fixtures_root = PLUGIN_ROOT / "fixtures" + matrix = fixtures_root / "ai-migration-matrix" + shared_artifacts = [p for p in va.find_artifacts(matrix) if p.name in manifest["shared"]] + assert sum(p.name == ".phase-status.json" for p in shared_artifacts) == 14 + assert sum(p.name == "integration-validation.json" for p in shared_artifacts) == 7 + assert not any(p.name.startswith("expected-") for p in va.find_artifacts(matrix)) + + matrix_findings = [] + matrix_checked = va.validate_run_dir(matrix, None, manifest, matrix_findings) + assert matrix_checked == len(shared_artifacts) + assert matrix_findings == [] + + mapped_checked = 0 + mapped_findings = [] + for fixture_dir in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): + skill = va.skill_for_fixture(fixture_dir.name, manifest) + if skill is not None: + mapped_checked += va.validate_run_dir(fixture_dir, skill, manifest, mapped_findings) + + r = _run("--fixtures", "--json") + assert r.returncode == 0, r.stdout + r.stderr + report = json.loads(r.stdout) + assert report["checked_artifacts"] - mapped_checked == len(shared_artifacts) + + +def test_invalid_neutral_shared_artifacts_fail_their_schemas(tmp_path: Path): + matrix = PLUGIN_ROOT / "fixtures" / "ai-migration-matrix" + manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") + mutations = { + ".phase-status.json": lambda value: value.pop("migration_id"), + "integration-validation.json": lambda value: value.pop("run_id"), + } + for name, mutate in mutations.items(): + source = next(matrix.rglob(name)) + value = json.loads(source.read_text()) + mutate(value) + neutral = tmp_path / name.lstrip(".") + neutral.mkdir() + (neutral / name).write_text(json.dumps(value)) + findings = [] + assert va.validate_run_dir(neutral, None, manifest, findings) == 1 + assert any(f.code == "SCHEMA_VIOLATION" for f in findings), findings + + +def test_neutral_fixture_skips_provider_artifacts_and_expected_snapshots(tmp_path: Path): + manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") + (tmp_path / "gcp-resource-inventory.json").write_text("{}") + (tmp_path / "expected-artifacts.json").write_text("not json") + findings = [] + assert va.validate_run_dir(tmp_path, None, manifest, findings) == 0 + assert findings == [] + assert [p.name for p in va.find_artifacts(tmp_path)] == ["gcp-resource-inventory.json"] + + def test_injected_off_contract_key_fails(tmp_path: Path): """The regression this tool exists for: a producer emits a key its contract never documented. Copy a golden run, plant the key, expect exit 1 naming the path.""" From 5837b152592ab6db390ff9bda467739342c535c2 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 11:10:46 -0700 Subject: [PATCH 27/53] fix(aws-startup-advisor): keep mise.toml plugin-local; CI runs scripts directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root mise.toml now matches main exactly. lint:artifacts and lint:vendored-parity are defined in a new, fully self-contained plugins/aws-startup-advisor/mise.toml instead, for local dev convenience. CI enforcement bypasses mise entirely, same pattern as the existing model-staleness.yml precedent (PR #348) and artifact-contracts.yml (PR #385): two dedicated workflows invoke the plugin's scripts directly via python3, scoped to pull requests touching plugins/aws-startup-advisor/**. Net effect: root mise.toml has zero diff against main. The only files outside the plugin are the two GitHub Actions workflow files themselves, which GitHub requires under .github/workflows/ — each only invokes the plugin's own script and carries no shared logic. --- .github/workflows/artifact-contracts.yml | 27 ++++++++++++++++++++++++ .github/workflows/vendored-parity.yml | 27 ++++++++++++++++++++++++ mise.toml | 10 +-------- plugins/aws-startup-advisor/mise.toml | 7 ++++++ 4 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/artifact-contracts.yml create mode 100644 .github/workflows/vendored-parity.yml create mode 100644 plugins/aws-startup-advisor/mise.toml diff --git a/.github/workflows/artifact-contracts.yml b/.github/workflows/artifact-contracts.yml new file mode 100644 index 000000000..0c2a52f5f --- /dev/null +++ b/.github/workflows/artifact-contracts.yml @@ -0,0 +1,27 @@ +# Freshness/consistency gate for aws-startup-advisor's fixture artifact contracts. +# +# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every +# PR. A failing check is the notification: it shows on the pull request. +# Checks only the shipped fixture corpus against its published schema +# contracts (see plugins/aws-startup-advisor/scripts/validate-artifacts.py); +# does not touch anything outside the plugin. +name: Artifact contracts +on: + pull_request: + paths: + - "plugins/aws-startup-advisor/**" + - ".github/workflows/artifact-contracts.yml" + workflow_dispatch: {} +permissions: + actions: none + contents: none +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate fixture artifacts against their schema contracts + run: python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures diff --git a/.github/workflows/vendored-parity.yml b/.github/workflows/vendored-parity.yml new file mode 100644 index 000000000..cf8b62bff --- /dev/null +++ b/.github/workflows/vendored-parity.yml @@ -0,0 +1,27 @@ +# Parity gate for aws-startup-advisor's vendored/shared skill files. +# +# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every +# PR. A failing check is the notification: it shows on the pull request. +# Checks only that each skill's references/vendored/ copies are byte-identical +# to skills/shared/ (see plugins/aws-startup-advisor/tools/sync-vendored.py); +# does not touch anything outside the plugin. +name: Vendored parity +on: + pull_request: + paths: + - "plugins/aws-startup-advisor/**" + - ".github/workflows/vendored-parity.yml" + workflow_dispatch: {} +permissions: + actions: none + contents: none +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check vendored copies are in sync with skills/shared/ + run: python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check diff --git a/mise.toml b/mise.toml index c5b6d26aa..cad129e3b 100644 --- a/mise.toml +++ b/mise.toml @@ -27,17 +27,9 @@ run = "python3 tools/sync-plugin-skills.py --check" description = "Validate Agent Plugins v1.0.0 conformance (plugins/)" run = "uv run --with jsonschema --with pyyaml python tools/validate_spec.py" -[tasks."lint:artifacts"] -description = "Validate aws-startup-advisor fixture artifacts against their published schema contracts" -run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures" - -[tasks."lint:vendored-parity"] -description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" -run = "python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check" - [tasks.validate] description = "Run all validation" -depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts", "lint:vendored-parity"] +depends = ["lint:manifests", "lint:skill-parity", "lint:spec"] [tasks.security] description = "Run security scans" diff --git a/plugins/aws-startup-advisor/mise.toml b/plugins/aws-startup-advisor/mise.toml new file mode 100644 index 000000000..c0e90cc84 --- /dev/null +++ b/plugins/aws-startup-advisor/mise.toml @@ -0,0 +1,7 @@ +[tasks."lint:artifacts"] +description = "Validate aws-startup-advisor fixture artifacts against their published schema contracts" +run = "python3 scripts/validate-artifacts.py --fixtures" + +[tasks."lint:vendored-parity"] +description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" +run = "python3 tools/sync-vendored.py --check" From c42e7268fb426dd1bf552ae9958c20df2acedd88 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 11:13:13 -0700 Subject: [PATCH 28/53] fix(aws-startup-advisor): keep mise.toml plugin-local; CI runs scripts directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root mise.toml now matches main exactly. lint:artifacts, lint:vendored-parity, and lint:skill-refs are defined in a new, fully self-contained plugins/aws-startup-advisor/mise.toml instead, for local dev convenience. CI enforcement bypasses mise entirely, same pattern as the existing model-staleness.yml precedent (PR #348) and artifact-contracts.yml / vendored-parity.yml (PRs #385/#386): three dedicated workflows invoke the plugin's scripts directly via python3, scoped to pull requests touching plugins/aws-startup-advisor/**. Net effect: root mise.toml has zero diff against main. The only files outside the plugin are the three GitHub Actions workflow files themselves, which GitHub requires under .github/workflows/ — each only invokes the plugin's own script and carries no shared logic. --- .github/workflows/artifact-contracts.yml | 27 ++++++++++++++++++++++++ .github/workflows/skill-refs.yml | 27 ++++++++++++++++++++++++ .github/workflows/vendored-parity.yml | 27 ++++++++++++++++++++++++ mise.toml | 14 +----------- plugins/aws-startup-advisor/mise.toml | 11 ++++++++++ 5 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/artifact-contracts.yml create mode 100644 .github/workflows/skill-refs.yml create mode 100644 .github/workflows/vendored-parity.yml create mode 100644 plugins/aws-startup-advisor/mise.toml diff --git a/.github/workflows/artifact-contracts.yml b/.github/workflows/artifact-contracts.yml new file mode 100644 index 000000000..0c2a52f5f --- /dev/null +++ b/.github/workflows/artifact-contracts.yml @@ -0,0 +1,27 @@ +# Freshness/consistency gate for aws-startup-advisor's fixture artifact contracts. +# +# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every +# PR. A failing check is the notification: it shows on the pull request. +# Checks only the shipped fixture corpus against its published schema +# contracts (see plugins/aws-startup-advisor/scripts/validate-artifacts.py); +# does not touch anything outside the plugin. +name: Artifact contracts +on: + pull_request: + paths: + - "plugins/aws-startup-advisor/**" + - ".github/workflows/artifact-contracts.yml" + workflow_dispatch: {} +permissions: + actions: none + contents: none +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate fixture artifacts against their schema contracts + run: python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures diff --git a/.github/workflows/skill-refs.yml b/.github/workflows/skill-refs.yml new file mode 100644 index 000000000..e8b7b9e81 --- /dev/null +++ b/.github/workflows/skill-refs.yml @@ -0,0 +1,27 @@ +# Reference-path gate for aws-startup-advisor's skill instructions. +# +# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every +# PR. A failing check is the notification: it shows on the pull request. +# Checks only that every file path a skill's instructions mention actually +# exists (see plugins/aws-startup-advisor/tools/check-skill-refs.py); does +# not touch anything outside the plugin. +name: Skill reference paths +on: + pull_request: + paths: + - "plugins/aws-startup-advisor/**" + - ".github/workflows/skill-refs.yml" + workflow_dispatch: {} +permissions: + actions: none + contents: none +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check every referenced skill file path exists + run: python3 plugins/aws-startup-advisor/tools/check-skill-refs.py --check diff --git a/.github/workflows/vendored-parity.yml b/.github/workflows/vendored-parity.yml new file mode 100644 index 000000000..cf8b62bff --- /dev/null +++ b/.github/workflows/vendored-parity.yml @@ -0,0 +1,27 @@ +# Parity gate for aws-startup-advisor's vendored/shared skill files. +# +# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every +# PR. A failing check is the notification: it shows on the pull request. +# Checks only that each skill's references/vendored/ copies are byte-identical +# to skills/shared/ (see plugins/aws-startup-advisor/tools/sync-vendored.py); +# does not touch anything outside the plugin. +name: Vendored parity +on: + pull_request: + paths: + - "plugins/aws-startup-advisor/**" + - ".github/workflows/vendored-parity.yml" + workflow_dispatch: {} +permissions: + actions: none + contents: none +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check vendored copies are in sync with skills/shared/ + run: python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check diff --git a/mise.toml b/mise.toml index 1a8eec4d9..cad129e3b 100644 --- a/mise.toml +++ b/mise.toml @@ -27,21 +27,9 @@ run = "python3 tools/sync-plugin-skills.py --check" description = "Validate Agent Plugins v1.0.0 conformance (plugins/)" run = "uv run --with jsonschema --with pyyaml python tools/validate_spec.py" -[tasks."lint:artifacts"] -description = "Validate aws-startup-advisor fixture artifacts against their published schema contracts" -run = "python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures" - -[tasks."lint:vendored-parity"] -description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" -run = "python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check" - -[tasks."lint:skill-refs"] -description = "Check every file path the aws-startup-advisor skills reference exists" -run = "python3 plugins/aws-startup-advisor/tools/check-skill-refs.py --check" - [tasks.validate] description = "Run all validation" -depends = ["lint:manifests", "lint:skill-parity", "lint:spec", "lint:artifacts", "lint:vendored-parity", "lint:skill-refs"] +depends = ["lint:manifests", "lint:skill-parity", "lint:spec"] [tasks.security] description = "Run security scans" diff --git a/plugins/aws-startup-advisor/mise.toml b/plugins/aws-startup-advisor/mise.toml new file mode 100644 index 000000000..ab7ecd7f2 --- /dev/null +++ b/plugins/aws-startup-advisor/mise.toml @@ -0,0 +1,11 @@ +[tasks."lint:artifacts"] +description = "Validate aws-startup-advisor fixture artifacts against their published schema contracts" +run = "python3 scripts/validate-artifacts.py --fixtures" + +[tasks."lint:vendored-parity"] +description = "Check each skill's references/vendored/ copies are byte-identical to skills/shared/" +run = "python3 tools/sync-vendored.py --check" + +[tasks."lint:skill-refs"] +description = "Check every file path the aws-startup-advisor skills reference exists" +run = "python3 tools/check-skill-refs.py --check" From f690cbe3bb638c7f9dbe5729cdcf8dacff20d655 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 12:10:01 -0700 Subject: [PATCH 29/53] test(aws-startup-advisor): define neutral AI workload profile Add a target-only, provider-neutral ai-workload-profile.json contract plus representative fixtures and contract tests, so a future normalization output can preserve multiple AI providers/gateways instead of collapsing them into a lossy both/single-gateway-type shape. No producer or runtime behavior change; schema validity does not imply an implemented adapter or route. --- plugins/aws-startup-advisor/README.md | 2 +- .../aws-startup-advisor/fixtures/README.md | 1 + .../ai-only-multi-direct.json | 36 +++ .../openai-direct-and-azure-openai.json | 36 +++ .../openrouter-multi-upstream.json | 48 ++++ .../openrouter-unresolved.json | 18 ++ .../contracts/ai-workload-profile.schema.json | 155 +++++++++++ .../test_ai_workload_profile_contracts.py | 241 ++++++++++++++++++ 8 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream.json create mode 100644 plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved.json create mode 100644 plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json create mode 100644 plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py diff --git a/plugins/aws-startup-advisor/README.md b/plugins/aws-startup-advisor/README.md index cf62f3f6b..2891d5199 100644 --- a/plugins/aws-startup-advisor/README.md +++ b/plugins/aws-startup-advisor/README.md @@ -38,7 +38,7 @@ Alongside the skills, the plugin ships supporting material the skills load on de - [`skills/shared/`](skills/shared/) — Canonical reference material shared across skills rather than an invocable skill of its own: the phase-workflow interpreter contract, estimation schemas and complexity tiers, a snapshot of AWS infrastructure rates, phase-status schema, what-if workshop invariants, shared Clarify question fragments, and the AI-migration guardrails and Bedrock model references. Each consuming skill carries a byte-identical copy under its own `references/vendored/` so the skill folder stays self-contained. - [`agents/`](agents/) — Seven Claude Code subagent definitions: two generic, phase-agnostic migration phase workers (read/write and read/write/shell tiers) plus five specialists for the Bedrock rewrite flow (code analyzer, code rewriter, log ingestor, prompt evaluator, report generator). -- [`scripts/`](scripts/) — Five Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter, `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. +- [`scripts/`](scripts/) — Five Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter, `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. - [`fixtures/`](fixtures/) — Reference and regression fixtures for the migration report and estimation artifacts, used to keep generated output within the documented contract. `fixtures/azure-iac-terraform/` additionally carries deterministic asserters that pin the Azure Discover, Clarify, Design, and Estimate contracts against a committed synthetic estate. ## MCP Servers diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 12cd8ba16..a27b50e64 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -3,6 +3,7 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. +- `ai-workload-profiles/` — 4 representative target-only `ai-workload-profile.json` observation fixtures (not a Cartesian matrix): AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct.json new file mode 100644 index 000000000..37a7dbc91 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct.json @@ -0,0 +1,36 @@ +{ + "schema_version": "1.0", + "ai_sources": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620"], + "model_observations": [ + { + "normalized_model": "claude-3-5-sonnet-20240620", + "raw_model": "claude-3-5-sonnet-20240620", + "model_family": "claude", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/anthropic_client.py:18"}] + } + ] + }, + { + "provider": "google", + "source_service": "vertex_ai", + "model_families": ["gemini"], + "model_evidence": "observed", + "models": ["gemini-1.5-pro"], + "model_observations": [ + { + "normalized_model": "gemini-1.5-pro", + "raw_model": "models/gemini-1.5-pro-latest", + "model_family": "gemini", + "evidence": [{"provenance": "iac", "confidence": "high", "detail": "vertex.tf:google_vertex_ai_endpoint.main"}] + } + ] + } + ], + "gateways": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai.json new file mode 100644 index 000000000..64805f49f --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai.json @@ -0,0 +1,36 @@ +{ + "schema_version": "1.0", + "ai_sources": [ + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + { + "normalized_model": "gpt-4o", + "raw_model": "gpt-4o", + "model_family": "gpt", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/openai_client.py:9"}] + } + ] + }, + { + "provider": "openai", + "source_service": "azure_openai", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + { + "normalized_model": "gpt-4o", + "raw_model": "gpt-4o-2024-08-06", + "model_family": "gpt", + "evidence": [{"provenance": "iac", "confidence": "high", "detail": "azure.tf:azurerm_cognitive_deployment.gpt4o"}] + } + ] + } + ], + "gateways": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream.json new file mode 100644 index 000000000..adca587a8 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream.json @@ -0,0 +1,48 @@ +{ + "schema_version": "1.0", + "ai_sources": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307"], + "model_observations": [ + {"normalized_model": "claude-3-5-sonnet-20240620", "raw_model": "anthropic/claude-3.5-sonnet", "model_family": "claude", "evidence": [{"provenance": "usage_api", "confidence": "high"}]}, + {"normalized_model": "claude-3-haiku-20240307", "raw_model": "anthropic/claude-3-haiku", "model_family": "claude", "evidence": [{"provenance": "usage_api", "confidence": "medium"}]} + ] + }, + { + "provider": "google", + "source_service": "vertex_ai", + "model_families": ["gemini"], + "model_evidence": "observed", + "models": ["gemini-1.5-pro"], + "model_observations": [ + {"normalized_model": "gemini-1.5-pro", "raw_model": "google/gemini-pro-1.5", "model_family": "gemini", "evidence": [{"provenance": "usage_api", "confidence": "high"}]} + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + {"normalized_model": "gpt-4o", "raw_model": "openai/gpt-4o", "model_family": "gpt", "evidence": [{"provenance": "usage_api", "confidence": "high"}]} + ] + } + ], + "gateways": [ + { + "product": "openrouter", + "type": "llm_router", + "upstream_evidence": "observed", + "upstreams": [ + {"provider": "anthropic", "source_service": "anthropic_api", "model_families": [{"family": "claude", "model_evidence": "observed", "models": ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307"]}]}, + {"provider": "google", "source_service": "vertex_ai", "model_families": [{"family": "gemini", "model_evidence": "observed", "models": ["gemini-1.5-pro"]}]}, + {"provider": "openai", "source_service": "openai_api", "model_families": [{"family": "gpt", "model_evidence": "observed", "models": ["gpt-4o"]}]} + ] + } + ] +} diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved.json new file mode 100644 index 000000000..167c076d1 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved.json @@ -0,0 +1,18 @@ +{ + "schema_version": "1.0", + "ai_sources": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620"], + "model_observations": [ + {"normalized_model": "claude-3-5-sonnet-20240620", "raw_model": "claude-3-5-sonnet-20240620", "model_family": "claude", "evidence": [{"provenance": "code", "confidence": "high"}]} + ] + } + ], + "gateways": [ + {"product": "openrouter", "type": "llm_router", "upstream_evidence": "unresolved", "upstreams": []} + ] +} diff --git a/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json b/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json new file mode 100644 index 000000000..26ea0394b --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json @@ -0,0 +1,155 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:ai-workload-profile", + "title": "AI workload profile (target-only observation contract)", + "description": "Provider-neutral, target-only contract for a future ai-workload-profile.json observation/normalization output. Structural validity does NOT imply an implemented detector, adapter, route, model mapping, or runtime-supported identity. This document is a design-time target for a schema that does not yet have a producer.", + "type": "object", + "required": ["schema_version", "ai_sources", "gateways"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "1.0"}, + "ai_sources": { + "description": "Retained AI sources. Semantic uniqueness is the provider and source_service pair; duplicate or overlapping pairs are invalid, while one provider through distinct services is valid. Projects verbatim (provider, source_service, model_families, model_evidence, models) into ai-migration-scenario.schema.json's aiSource.", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/aiSource"} + }, + "gateways": { + "description": "Gateways independent of direct providers and infrastructure. OpenRouter is representable only here, never as a provider or infrastructure source. Projects verbatim into ai-migration-scenario.schema.json's gateway.", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/gateway"} + } + }, + "definitions": { + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "modelFamily": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_.-]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "gatewayIdentity": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_]*$"}, + "provenance": {"enum": ["code", "iac", "live_api", "usage_api", "user_confirmed"]}, + "confidence": {"enum": ["confirmed", "high", "medium", "low"]}, + "evidenceEntry": { + "type": "object", + "required": ["provenance", "confidence"], + "additionalProperties": false, + "properties": { + "provenance": {"$ref": "#/definitions/provenance"}, + "confidence": {"$ref": "#/definitions/confidence"}, + "detail": {"type": "string", "minLength": 1} + } + }, + "modelObservation": { + "description": "One exact normalized model's raw/observed alias and evidence trail. raw_model is the verbatim observed string and is never discarded even when it equals normalized_model.", + "type": "object", + "required": ["normalized_model", "raw_model", "model_family", "evidence"], + "additionalProperties": false, + "properties": { + "normalized_model": {"$ref": "#/definitions/normalizedModel"}, + "raw_model": {"type": "string", "minLength": 1}, + "model_family": {"$ref": "#/definitions/modelFamily"}, + "evidence": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/evidenceEntry"} + } + } + }, + "aiSource": { + "description": "A provider-neutral AI source keyed semantically by provider and source_service (e.g. direct openai vs azure_openai is a distinct source_service on the same provider). model_families/model_evidence/models mirror ai-migration-scenario.schema.json's aiSource exactly; model_observations carries the richer raw-alias/evidence trail and is cross-checked against models[] by the test suite, not by this schema (no $data support in the enforced subset).", + "type": "object", + "required": ["provider", "source_service", "model_families", "model_evidence", "models", "model_observations"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/modelFamily"} + }, + "model_evidence": {"enum": ["observed", "not_observed"]}, + "models": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/normalizedModel"} + }, + "model_observations": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/modelObservation"} + } + }, + "oneOf": [ + { + "properties": { + "model_evidence": {"const": "observed"}, + "models": {"minItems": 1} + } + }, + { + "properties": { + "model_evidence": {"const": "not_observed"}, + "models": {"maxItems": 0}, + "model_observations": {"maxItems": 0} + } + } + ] + }, + "upstreamModelFamily": { + "description": "Byte-identical in shape to ai-migration-scenario.schema.json's upstreamModelFamily.", + "type": "object", + "required": ["family", "model_evidence", "models"], + "additionalProperties": false, + "properties": { + "family": {"$ref": "#/definitions/modelFamily"}, + "model_evidence": {"enum": ["observed", "not_observed"]}, + "models": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/normalizedModel"} + } + }, + "oneOf": [ + {"properties": {"model_evidence": {"const": "observed"}, "models": {"minItems": 1}}}, + {"properties": {"model_evidence": {"const": "not_observed"}, "models": {"maxItems": 0}}} + ] + }, + "gatewayUpstream": { + "description": "Byte-identical in shape to ai-migration-scenario.schema.json's gatewayUpstream.", + "type": "object", + "required": ["provider", "model_families"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/upstreamModelFamily"} + } + } + }, + "gateway": { + "description": "Byte-identical in shape to ai-migration-scenario.schema.json's gateway. An observed gateway cannot have an empty upstreams array; an unresolved gateway cannot invent upstreams.", + "type": "object", + "required": ["product", "type", "upstream_evidence", "upstreams"], + "additionalProperties": false, + "properties": { + "product": {"$ref": "#/definitions/gatewayIdentity"}, + "type": {"$ref": "#/definitions/extensibleIdentity"}, + "upstream_evidence": {"enum": ["observed", "unresolved"]}, + "upstreams": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/gatewayUpstream"} + } + }, + "oneOf": [ + {"properties": {"upstream_evidence": {"const": "observed"}, "upstreams": {"minItems": 1}}}, + {"properties": {"upstream_evidence": {"const": "unresolved"}, "upstreams": {"maxItems": 0}}} + ] + } + } +} diff --git a/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py new file mode 100644 index 000000000..619cca1d5 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py @@ -0,0 +1,241 @@ +"""Contract tests for the target-only ai-workload-profile.json observation contract. + +Pins the schema's self-validity, the semantic invariants (duplicate/overlap rejection, +model_observations <-> models[] correspondence, no fabricated IDs), and a deterministic +projection into PR #410's ai-migration-scenario.schema.json so the two contracts cannot +silently diverge. Target-only: no producer exists yet, no runtime behavior is asserted. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "ai-workload-profiles" +PROFILE_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "ai-workload-profile.schema.json" +VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +MATRIX_TEST_PATH = Path(__file__).resolve().parent / "test_ai_migration_matrix_contracts.py" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +VALIDATOR = _load_module("ai_workload_profile__validator", VALIDATOR_PATH) +MATRIX = _load_module("ai_workload_profile__matrix_contracts", MATRIX_TEST_PATH) + +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _profile_schema() -> dict: + return _load(PROFILE_SCHEMA) + + +def _profile_findings(profile: dict) -> list: + findings = [] + VALIDATOR.validate_json_schema(_profile_schema(), profile, "", "profile", str(PROFILE_SCHEMA), findings) + return findings + + +def _assert_profile_semantics(profile: dict) -> None: + """Mirrors MATRIX._assert_scenario_semantics' duplicate/overlap style, plus the + model_observations <-> models[] correspondence this richer contract adds.""" + sources = {} + for source in profile["ai_sources"]: + key = (source["provider"], source["source_service"]) + assert key not in sources, f"duplicate or overlapping AI source: {key}" + assert source["provider"] != "openrouter" + sources[key] = source + + families = set(source["model_families"]) + models = set(source["models"]) + observations = {obs["normalized_model"]: obs for obs in source["model_observations"]} + assert set(observations) == models, ( + f"model_observations must correspond exactly to models[]: {set(observations)} != {models}" + ) + for obs in source["model_observations"]: + assert obs["model_family"] in families, ( + f"model_observations family not in model_families: {obs['model_family']}" + ) + + for gateway in profile["gateways"]: + upstream_keys = set() + for upstream in gateway["upstreams"]: + key = (upstream["provider"], upstream.get("source_service")) + assert key not in upstream_keys, f"duplicate or overlapping gateway upstream: {key}" + upstream_keys.add(key) + + +AI_ONLY_OUTCOMES = { + "infrastructure": None, + "ai": {"verdict": "passed", "artifacts": sorted(MATRIX.AI_ARTIFACTS)}, + "integration_validation": None, +} + + +def _project_to_scenario(profile: dict) -> dict: + """The deterministic projection: ai_sources[] copies its five #410-shaped fields + verbatim (dropping model_observations); gateways[] copies verbatim (already + byte-identical in shape to #410's gateway). Wrapped in a synthetic ai_only scenario + skeleton purely so #410's schema/semantics helpers can be reused unmodified.""" + ai_sources = [ + { + "provider": s["provider"], + "source_service": s["source_service"], + "model_families": s["model_families"], + "model_evidence": s["model_evidence"], + "models": s["models"], + } + for s in profile["ai_sources"] + ] + gateways = [copy.deepcopy(g) for g in profile["gateways"]] + return { + "id": "ai-workload-profile-projection", + "mode": "ai_only", + "migration_id": "ai-workload-profile-projection", + "run_id": "00000000-0000-4000-8000-000000000099", + "state": ".phase-status.json", + "dimensions": { + "infrastructure_source": None, + "ai_sources": ai_sources, + "gateways": gateways, + }, + "outcomes": AI_ONLY_OUTCOMES, + } + + +# --------------------------------------------------------------------------- schema self-validity + +def test_schema_has_no_unsupported_keywords() -> None: + assert VALIDATOR.unsupported_keywords(_profile_schema()) == [] + + +def test_schema_is_valid_json_and_draft07_shaped() -> None: + schema = _profile_schema() + assert schema["$schema"] == "http://json-schema.org/draft-07/schema#" + assert schema["type"] == "object" + + +# --------------------------------------------------------------------------- positive fixtures + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.stem) +def test_positive_fixtures_validate_against_profile_schema(fixture_path: Path) -> None: + profile = _load(fixture_path) + assert _profile_findings(profile) == [] + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.stem) +def test_positive_fixtures_satisfy_semantic_invariants(fixture_path: Path) -> None: + _assert_profile_semantics(_load(fixture_path)) + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.stem) +def test_positive_fixtures_project_exactly_into_pr410_semantics(fixture_path: Path) -> None: + profile = _load(fixture_path) + scenario = _project_to_scenario(profile) + assert MATRIX._schema_findings(MATRIX.SCENARIO_SCHEMA, scenario, fixture_path.name) == [] + MATRIX._assert_scenario_semantics(scenario) + + +def test_openrouter_multi_upstream_fixture_has_two_models_in_one_family() -> None: + profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream.json") + claude = next(s for s in profile["ai_sources"] if s["provider"] == "anthropic") + assert len(claude["models"]) >= 2 + + +def test_openai_direct_and_azure_fixture_preserves_distinct_provenance() -> None: + profile = _load(FIXTURE_ROOT / "openai-direct-and-azure-openai.json") + services = {s["source_service"] for s in profile["ai_sources"] if s["provider"] == "openai"} + assert services == {"openai_api", "azure_openai"} + # raw_model differs across services even though normalized_model is identical — + # proves the raw alias is retained independently of normalization. + raw_by_service = { + s["source_service"]: s["model_observations"][0]["raw_model"] + for s in profile["ai_sources"] if s["provider"] == "openai" + } + assert len(set(raw_by_service.values())) == 2 + + +def test_unresolved_gateway_fixture_has_empty_upstreams() -> None: + profile = _load(FIXTURE_ROOT / "openrouter-unresolved.json") + gateway = profile["gateways"][0] + assert gateway["upstream_evidence"] == "unresolved" + assert gateway["upstreams"] == [] + + +# --------------------------------------------------------------------------- negative mutations + +def _base_profile() -> dict: + return copy.deepcopy(_load(FIXTURE_ROOT / "ai-only-multi-direct.json")) + + +def test_duplicate_ai_source_identity_is_rejected() -> None: + """An exact-duplicate aiSource dict is already caught by the schema's uniqueItems; + confirmed by running validate_json_schema against the schema directly (a byte-identical + duplicate produces a uniqueItems SCHEMA_VIOLATION). Follow #410's own + test_duplicate_or_overlapping_ai_sources_are_rejected if/else pattern rather than + asserting one branch, since whether uniqueItems or the semantic check catches a given + mutation depends on whether the duplicate is byte-identical or merely overlapping.""" + profile = _base_profile() + profile["ai_sources"].append(copy.deepcopy(profile["ai_sources"][0])) + if not _profile_findings(profile): + with pytest.raises(AssertionError): + _assert_profile_semantics(profile) + else: + assert _profile_findings(profile) + + +def test_dropped_model_under_a_resolved_upstream_is_rejected() -> None: + profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream.json") + scenario = _project_to_scenario(profile) + # Drop one of Claude's two models from the gateway upstream while the source keeps both. + upstream = next( + u for u in scenario["dimensions"]["gateways"][0]["upstreams"] if u["provider"] == "anthropic" + ) + upstream["model_families"][0]["models"].remove("claude-3-haiku-20240307") + with pytest.raises(AssertionError): + MATRIX._assert_scenario_semantics(scenario) + + +def test_fabricated_unknown_model_id_is_rejected() -> None: + profile = _base_profile() + profile["ai_sources"][0]["models"] = ["unknown"] + profile["ai_sources"][0]["model_observations"][0]["normalized_model"] = "unknown" + assert any( + "unknown" in f.message or "pattern" in f.message for f in _profile_findings(profile) + ) + + +def test_openrouter_as_a_direct_provider_is_rejected() -> None: + profile = _base_profile() + profile["ai_sources"][0]["provider"] = "openrouter" + assert _profile_findings(profile) != [] + + +def test_empty_upstreams_on_an_observed_gateway_is_rejected() -> None: + profile = _base_profile() + profile["gateways"] = [ + {"product": "openrouter", "type": "llm_router", "upstream_evidence": "observed", "upstreams": []} + ] + assert _profile_findings(profile) != [] + + +def test_invented_upstream_on_an_unresolved_gateway_is_rejected() -> None: + profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream.json") + gateway = profile["gateways"][0] + gateway["upstream_evidence"] = "unresolved" + # upstreams[] still populated — invented upstream on an unresolved gateway. + assert _profile_findings(profile) != [] From 4958a2c446733d83b35353acb4347d14ab6855c2 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 12:49:57 -0700 Subject: [PATCH 30/53] test(aws-startup-advisor): lock observed-source evidence trail and cover ai-workload fixtures in lint:artifacts - Add regression test asserting _assert_profile_semantics rejects an observed ai_source whose model_observations has been emptied while models[] and model_evidence stay intact (the schema alone cannot express that correspondence). - Move the four ai-workload-profiles fixtures into /ai-workload-profile.json subdirectories and register ai-workload-profile.json in artifact-contracts.json's shared section so lint:artifacts (and mise run build) actually open these files. --- .../ai-workload-profile.json} | 0 .../ai-workload-profile.json} | 0 .../ai-workload-profile.json} | 0 .../ai-workload-profile.json} | 0 .../scripts/artifact-contracts.json | 3 ++ .../test_ai_workload_profile_contracts.py | 30 ++++++++++++------- 6 files changed, 23 insertions(+), 10 deletions(-) rename plugins/aws-startup-advisor/fixtures/ai-workload-profiles/{ai-only-multi-direct.json => ai-only-multi-direct/ai-workload-profile.json} (100%) rename plugins/aws-startup-advisor/fixtures/ai-workload-profiles/{openai-direct-and-azure-openai.json => openai-direct-and-azure-openai/ai-workload-profile.json} (100%) rename plugins/aws-startup-advisor/fixtures/ai-workload-profiles/{openrouter-multi-upstream.json => openrouter-multi-upstream/ai-workload-profile.json} (100%) rename plugins/aws-startup-advisor/fixtures/ai-workload-profiles/{openrouter-unresolved.json => openrouter-unresolved/ai-workload-profile.json} (100%) diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct/ai-workload-profile.json similarity index 100% rename from plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct.json rename to plugins/aws-startup-advisor/fixtures/ai-workload-profiles/ai-only-multi-direct/ai-workload-profile.json diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai/ai-workload-profile.json similarity index 100% rename from plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai.json rename to plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openai-direct-and-azure-openai/ai-workload-profile.json diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream/ai-workload-profile.json similarity index 100% rename from plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream.json rename to plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-multi-upstream/ai-workload-profile.json diff --git a/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved.json b/plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved/ai-workload-profile.json similarity index 100% rename from plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved.json rename to plugins/aws-startup-advisor/fixtures/ai-workload-profiles/openrouter-unresolved/ai-workload-profile.json diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 1bbe3608e..8cd686b9b 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -11,6 +11,9 @@ }, "integration-validation.json": { "json_schema": "scripts/contracts/integration-validation.schema.json" + }, + "ai-workload-profile.json": { + "json_schema": "scripts/contracts/ai-workload-profile.schema.json" } }, "skills": { diff --git a/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py index 619cca1d5..d147490c8 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py @@ -33,7 +33,7 @@ def _load_module(name: str, path: Path): VALIDATOR = _load_module("ai_workload_profile__validator", VALIDATOR_PATH) MATRIX = _load_module("ai_workload_profile__matrix_contracts", MATRIX_TEST_PATH) -POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*.json")) +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*/ai-workload-profile.json")) def _load(path: Path) -> dict: @@ -131,18 +131,18 @@ def test_schema_is_valid_json_and_draft07_shaped() -> None: # --------------------------------------------------------------------------- positive fixtures -@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.stem) +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) def test_positive_fixtures_validate_against_profile_schema(fixture_path: Path) -> None: profile = _load(fixture_path) assert _profile_findings(profile) == [] -@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.stem) +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) def test_positive_fixtures_satisfy_semantic_invariants(fixture_path: Path) -> None: _assert_profile_semantics(_load(fixture_path)) -@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.stem) +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) def test_positive_fixtures_project_exactly_into_pr410_semantics(fixture_path: Path) -> None: profile = _load(fixture_path) scenario = _project_to_scenario(profile) @@ -151,13 +151,13 @@ def test_positive_fixtures_project_exactly_into_pr410_semantics(fixture_path: Pa def test_openrouter_multi_upstream_fixture_has_two_models_in_one_family() -> None: - profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream.json") + profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream" / "ai-workload-profile.json") claude = next(s for s in profile["ai_sources"] if s["provider"] == "anthropic") assert len(claude["models"]) >= 2 def test_openai_direct_and_azure_fixture_preserves_distinct_provenance() -> None: - profile = _load(FIXTURE_ROOT / "openai-direct-and-azure-openai.json") + profile = _load(FIXTURE_ROOT / "openai-direct-and-azure-openai" / "ai-workload-profile.json") services = {s["source_service"] for s in profile["ai_sources"] if s["provider"] == "openai"} assert services == {"openai_api", "azure_openai"} # raw_model differs across services even though normalized_model is identical — @@ -170,7 +170,7 @@ def test_openai_direct_and_azure_fixture_preserves_distinct_provenance() -> None def test_unresolved_gateway_fixture_has_empty_upstreams() -> None: - profile = _load(FIXTURE_ROOT / "openrouter-unresolved.json") + profile = _load(FIXTURE_ROOT / "openrouter-unresolved" / "ai-workload-profile.json") gateway = profile["gateways"][0] assert gateway["upstream_evidence"] == "unresolved" assert gateway["upstreams"] == [] @@ -179,7 +179,17 @@ def test_unresolved_gateway_fixture_has_empty_upstreams() -> None: # --------------------------------------------------------------------------- negative mutations def _base_profile() -> dict: - return copy.deepcopy(_load(FIXTURE_ROOT / "ai-only-multi-direct.json")) + return copy.deepcopy(_load(FIXTURE_ROOT / "ai-only-multi-direct" / "ai-workload-profile.json")) + + +def test_empty_model_observations_on_an_observed_source_is_rejected() -> None: + """Lock in that _assert_profile_semantics rejects an observed source whose + model_observations has been emptied while models[] and model_evidence stay intact + (the JSON Schema alone cannot express this correspondence).""" + profile = _base_profile() + profile["ai_sources"][0]["model_observations"] = [] + with pytest.raises(AssertionError): + _assert_profile_semantics(profile) def test_duplicate_ai_source_identity_is_rejected() -> None: @@ -199,7 +209,7 @@ def test_duplicate_ai_source_identity_is_rejected() -> None: def test_dropped_model_under_a_resolved_upstream_is_rejected() -> None: - profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream.json") + profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream" / "ai-workload-profile.json") scenario = _project_to_scenario(profile) # Drop one of Claude's two models from the gateway upstream while the source keeps both. upstream = next( @@ -234,7 +244,7 @@ def test_empty_upstreams_on_an_observed_gateway_is_rejected() -> None: def test_invented_upstream_on_an_unresolved_gateway_is_rejected() -> None: - profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream.json") + profile = _load(FIXTURE_ROOT / "openrouter-multi-upstream" / "ai-workload-profile.json") gateway = profile["gateways"][0] gateway["upstream_evidence"] = "unresolved" # upstreams[] still populated — invented upstream on an unresolved gateway. From 183c6d925c1f495af23d79128d69b5db774b11cc Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 13:19:14 -0700 Subject: [PATCH 31/53] refactor(aws-startup-advisor): canonicalize aws-design-ai.json contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Canonicalizes Azure's prior schema-design-aws-ai.md into skills/shared/ai/schema-design-aws-ai.md, replacing the retired single-scalar metadata.ai_source (MUST equal summary.ai_source) with many-valued metadata.ai_sources[]/metadata.gateways[] mirroring #415's ai-workload-profile.schema.json exactly. Adds the matching target-only scripts/contracts/aws-design-ai.schema.json, vendors the canonical prose into both gcp-to-aws and azure-to-aws references/vendored/ai/, repoints artifact-contracts.json's aws-design-ai.json entry to the shared canonical shape (adding the GCP entry it previously lacked), and reduces Azure's old schema-design-aws-ai.md to a redirect pointer. Target-only / non-runtime, per #410 and #415's own framing: no producer is repointed in this change (design-ai.md is unchanged in both skills); structural validity does not imply runtime support. Adds plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py, a projection/equivalence suite that reuses #415's own semantic oracle (_assert_profile_semantics / MATRIX._assert_scenario_semantics) to prove metadata.ai_sources[]/gateways[] project exactly into ai-workload-profile.schema.json's shape, covering both a multi-provider direct case (Anthropic + OpenAI in one design) and a gateway-fronted multi-upstream case (one OpenRouter gateway fronting Anthropic + OpenAI simultaneously). Fixtures are nested as fixtures/aws-design-ai// aws-design-ai.json (not flat) and registered in artifact-contracts.json's shared section from the start, per #415's own Cursor-review finding about lint:artifacts fixture visibility. Validation (run in the worktree .worktrees/aws-design-ai-contract): - uv run --with pytest python -m pytest plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py -> 16 passed - uv run --with pytest python -m pytest plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py -> 57 passed (unchanged, regression-clean) - uv run --with pytest python -m pytest plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py -> 24 passed (unchanged, regression-clean) - mise run lint:artifacts -> PASS, 80 artifact(s) checked (up from 74 before this change), 0 errors, 50 baselined (pre-existing), 14 info (pre-existing NO_CONTRACT entries, unrelated to this change) - mise run lint:vendored-parity -> PASS, 46 vendored file(s) byte-identical to skills/shared/ and matching README tables - mise run lint:skill-refs -> PASS, 1181 references checked, 0 missing, 7 baselined (pre-existing), 0 stale - mise run lint:md -> 0 issues in 2478 files - mise run validate -> all sub-tasks (lint:manifests, lint:spec, lint:skill-parity, lint:artifacts, lint:vendored-parity, lint:skill-refs) pass - mise run build -> all validate tasks pass; security (gitleaks) scanned 503 commits, ~35.6MB, no leaks found - git diff --check -> clean (exit 0) - git merge-tree --write-tree origin/main HEAD -> clean tree SHA, no conflict markers - git diff against PR #415 head (4958a2c446733d83b35353acb4347d14ab6855c2) touches only: plugins/aws-startup-advisor/README.md, fixtures/README.md, scripts/artifact-contracts.json, scripts/contracts/aws-design-ai.schema.json, skills/azure-to-aws/references/shared/schema-design-aws-ai.md, skills/{azure,gcp}-to-aws/references/vendored/README.md, skills/{azure,gcp}-to-aws/references/vendored/ai/schema-design-aws-ai.md, skills/shared/ai/README.md, skills/shared/ai/schema-design-aws-ai.md, tests/test_aws_design_ai_contracts.py, and the two new fixtures/aws-design-ai//aws-design-ai.json files — contracts, fixtures, tests, docs, and vendored files only. No design-ai.md, estimate-ai.md, generate-artifacts-ai.md, or other phase/runtime file touched. --- plugins/aws-startup-advisor/README.md | 2 +- .../aws-startup-advisor/fixtures/README.md | 1 + .../multi-provider-direct/aws-design-ai.json | 114 ++++++++ .../aws-design-ai.json | 181 ++++++++++++ .../scripts/artifact-contracts.json | 7 +- .../contracts/aws-design-ai.schema.json | 198 +++++++++++++ .../references/shared/schema-design-aws-ai.md | 205 +------------ .../references/vendored/README.md | 1 + .../vendored/ai/schema-design-aws-ai.md | 275 ++++++++++++++++++ .../gcp-to-aws/references/vendored/README.md | 1 + .../vendored/ai/schema-design-aws-ai.md | 275 ++++++++++++++++++ .../skills/shared/ai/README.md | 1 + .../skills/shared/ai/schema-design-aws-ai.md | 275 ++++++++++++++++++ .../tests/test_aws_design_ai_contracts.py | 197 +++++++++++++ 14 files changed, 1539 insertions(+), 194 deletions(-) create mode 100644 plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json create mode 100644 plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json create mode 100644 plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json create mode 100644 plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md create mode 100644 plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md create mode 100644 plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md create mode 100644 plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py diff --git a/plugins/aws-startup-advisor/README.md b/plugins/aws-startup-advisor/README.md index 2891d5199..51192a259 100644 --- a/plugins/aws-startup-advisor/README.md +++ b/plugins/aws-startup-advisor/README.md @@ -38,7 +38,7 @@ Alongside the skills, the plugin ships supporting material the skills load on de - [`skills/shared/`](skills/shared/) — Canonical reference material shared across skills rather than an invocable skill of its own: the phase-workflow interpreter contract, estimation schemas and complexity tiers, a snapshot of AWS infrastructure rates, phase-status schema, what-if workshop invariants, shared Clarify question fragments, and the AI-migration guardrails and Bedrock model references. Each consuming skill carries a byte-identical copy under its own `references/vendored/` so the skill folder stays self-contained. - [`agents/`](agents/) — Seven Claude Code subagent definitions: two generic, phase-agnostic migration phase workers (read/write and read/write/shell tiers) plus five specialists for the Bedrock rewrite flow (code analyzer, code rewriter, log ingestor, prompt evaluator, report generator). -- [`scripts/`](scripts/) — Five Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter, `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. +- [`scripts/`](scripts/) — Five Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter, `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. `scripts/contracts/aws-design-ai.schema.json`, canonicalized from Azure's prior single-scalar `metadata.ai_source` contract into `skills/shared/ai/schema-design-aws-ai.md`, likewise defines a target-only `aws-design-ai.json` design contract whose `metadata.ai_sources[]`/`metadata.gateways[]` mirror the workload-profile contract exactly; `design-ai.md` in both skills still writes against the prior per-skill shape, so this is a target, not a claim about current output. - [`fixtures/`](fixtures/) — Reference and regression fixtures for the migration report and estimation artifacts, used to keep generated output within the documented contract. `fixtures/azure-iac-terraform/` additionally carries deterministic asserters that pin the Azure Discover, Clarify, Design, and Estimate contracts against a committed synthetic estate. ## MCP Servers diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index a27b50e64..c883ce295 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -4,6 +4,7 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `ai-workload-profiles/` — 4 representative target-only `ai-workload-profile.json` observation fixtures (not a Cartesian matrix): AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. +- `aws-design-ai/` — 2 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway) and `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json new file mode 100644 index 000000000..0330bfc88 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json @@ -0,0 +1,114 @@ +{ + "phase": "design", + "focus": "ai", + "timestamp": "2026-09-08T02:20:00Z", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620"], + "model_observations": [ + { + "normalized_model": "claude-3-5-sonnet-20240620", + "raw_model": "claude-3-5-sonnet-20240620", + "model_family": "claude", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/anthropic_client.py:18"}] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + { + "normalized_model": "gpt-4o", + "raw_model": "gpt-4o", + "model_family": "gpt", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/openai_client.py:9"}] + } + ] + } + ], + "gateways": [], + "bedrock_models_selected": ["anthropic.claude-sonnet-4-5-v1:0", "openai.gpt-5.6-terra"], + "regional_validation": "checked" + }, + "design_blocks": [ + { + "workload_id": "wl_anthropic_chat", + "model_id": "claude-3-5-sonnet-20240620", + "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "Anthropic-sourced chat workload maps to the current Claude Sonnet generation on Bedrock.", + "confidence_warning": null + }, + { + "workload_id": "wl_openai_support_bot", + "model_id": "gpt-4o", + "target_bedrock_model": "openai.gpt-5.6-terra", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "OpenAI-sourced support bot moves to GPT-5.6 Terra on Bedrock Mantle, same provider lineage.", + "confidence_warning": null + } + ], + "ai_architecture": { + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [ + { + "source_model_id": "claude-3-5-sonnet-20240620", + "aws_model_id": "anthropic.claude-sonnet-4-5-v1:0", + "capabilities_matched": ["text_generation"], + "capability_gaps": [], + "honest_assessment": "strong_migrate", + "source_provider_price": "3.00/15.00 per 1M tokens", + "bedrock_price": "3.00/15.00 per 1M tokens", + "price_comparison": "parity", + "migration_complexity": "low", + "model_change": true, + "migration_path": "direct", + "quota_risk": "low" + }, + { + "source_model_id": "gpt-4o", + "aws_model_id": "openai.gpt-5.6-terra", + "capabilities_matched": ["text_generation"], + "capability_gaps": [], + "honest_assessment": "strong_migrate", + "source_provider_price": "2.50/10.00 per 1M tokens", + "bedrock_price": "2.20/13.20 per 1M tokens", + "price_comparison": "mixed", + "migration_complexity": "low", + "model_change": true, + "migration_path": "mantle_openai_responses", + "quota_risk": "low" + } + ], + "capability_mapping": { + "text_generation": {"parity": "full", "notes": "Converse and Mantle Responses both cover the detected usage."} + }, + "code_migration": { + "primary_pattern": "direct_sdk", + "framework": "none", + "files_to_modify": ["src/ai/anthropic_client.py", "src/ai/openai_client.py"], + "dependency_changes": "boto3 for the Anthropic client; openai>=2.45.0 with aws_bedrock_token_generator for the OpenAI client." + }, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], + "multi_model_warnings": [], + "agentic_design": null +} diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json new file mode 100644 index 000000000..80da03388 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json @@ -0,0 +1,181 @@ +{ + "phase": "design", + "focus": "ai", + "timestamp": "2026-09-08T02:20:00Z", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307"], + "model_observations": [ + { + "normalized_model": "claude-3-5-sonnet-20240620", + "raw_model": "anthropic/claude-3.5-sonnet", + "model_family": "claude", + "evidence": [{"provenance": "usage_api", "confidence": "high"}] + }, + { + "normalized_model": "claude-3-haiku-20240307", + "raw_model": "anthropic/claude-3-haiku", + "model_family": "claude", + "evidence": [{"provenance": "usage_api", "confidence": "medium"}] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + { + "normalized_model": "gpt-4o", + "raw_model": "openai/gpt-4o", + "model_family": "gpt", + "evidence": [{"provenance": "usage_api", "confidence": "high"}] + } + ] + } + ], + "gateways": [ + { + "product": "openrouter", + "type": "llm_router", + "upstream_evidence": "observed", + "upstreams": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": [ + { + "family": "claude", + "model_evidence": "observed", + "models": ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307"] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "model_families": [ + { + "family": "gpt", + "model_evidence": "observed", + "models": ["gpt-4o"] + } + ] + } + ] + } + ], + "bedrock_models_selected": ["anthropic.claude-sonnet-4-5-v1:0", "anthropic.claude-haiku-4-5-v1:0", "openai.gpt-5.6-terra"], + "regional_validation": "checked" + }, + "design_blocks": [ + { + "workload_id": "wl_router_chat_primary", + "model_id": "claude-3-5-sonnet-20240620", + "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "Primary chat tier routed through OpenRouter to Claude Sonnet; same-family Bedrock target.", + "confidence_warning": null + }, + { + "workload_id": "wl_router_chat_cheap", + "model_id": "claude-3-haiku-20240307", + "target_bedrock_model": "anthropic.claude-haiku-4-5-v1:0", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "Low-cost tier routed through OpenRouter to Claude Haiku; same-family Bedrock target.", + "confidence_warning": null + }, + { + "workload_id": "wl_router_fallback", + "model_id": "gpt-4o", + "target_bedrock_model": "openai.gpt-5.6-terra", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "OpenRouter fallback upstream to OpenAI maps to GPT-5.6 Terra on Bedrock Mantle.", + "confidence_warning": null + } + ], + "ai_architecture": { + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": { + "tier_1": "anthropic.claude-sonnet-4-5-v1:0", + "tier_2": "anthropic.claude-haiku-4-5-v1:0", + "tier_3": "openai.gpt-5.6-terra" + }, + "bedrock_models": [ + { + "source_model_id": "claude-3-5-sonnet-20240620", + "aws_model_id": "anthropic.claude-sonnet-4-5-v1:0", + "capabilities_matched": ["text_generation"], + "capability_gaps": [], + "honest_assessment": "strong_migrate", + "source_provider_price": "via OpenRouter — see § Pricing", + "bedrock_price": "3.00/15.00 per 1M tokens", + "price_comparison": "favorable", + "migration_complexity": "medium", + "model_change": true, + "migration_path": "direct", + "quota_risk": "low" + }, + { + "source_model_id": "claude-3-haiku-20240307", + "aws_model_id": "anthropic.claude-haiku-4-5-v1:0", + "capabilities_matched": ["text_generation"], + "capability_gaps": [], + "honest_assessment": "strong_migrate", + "source_provider_price": "via OpenRouter — see § Pricing", + "bedrock_price": "0.25/1.25 per 1M tokens", + "price_comparison": "favorable", + "migration_complexity": "medium", + "model_change": true, + "migration_path": "direct", + "quota_risk": "low" + }, + { + "source_model_id": "gpt-4o", + "aws_model_id": "openai.gpt-5.6-terra", + "capabilities_matched": ["text_generation"], + "capability_gaps": [], + "honest_assessment": "strong_migrate", + "source_provider_price": "via OpenRouter — see § Pricing", + "bedrock_price": "2.20/13.20 per 1M tokens", + "price_comparison": "mixed", + "migration_complexity": "medium", + "model_change": true, + "migration_path": "mantle_openai_responses", + "quota_risk": "medium" + } + ], + "capability_mapping": { + "text_generation": {"parity": "full", "notes": "All three router upstreams land on Bedrock text-generation targets."} + }, + "code_migration": { + "primary_pattern": "framework", + "framework": "openrouter-sdk", + "files_to_modify": ["src/ai/router_client.py"], + "dependency_changes": "Replace the OpenRouter SDK call sites with per-provider Bedrock clients (boto3 Converse for Anthropic, Mantle Responses for OpenAI).", + "openrouter_path": "direct" + }, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], + "multi_model_warnings": [ + {"type": "multi_model_tiered", "message": "Three models detected behind one gateway; a 3-tier routing strategy is recorded above."} + ], + "agentic_design": null +} diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 8cd686b9b..c114eeade 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -14,6 +14,9 @@ }, "ai-workload-profile.json": { "json_schema": "scripts/contracts/ai-workload-profile.schema.json" + }, + "aws-design-ai.json": { + "json_schema": "scripts/contracts/aws-design-ai.schema.json" } }, "skills": { @@ -257,10 +260,6 @@ } ] }, - "aws-design-ai.json": { - "shape": "skills/azure-to-aws/references/shared/schema-design-aws-ai.md", - "root_heading": "Shape" - }, "estimation-infra.json": { "json_schema": "skills/shared/estimate/estimation-infra.schema.json" }, diff --git a/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json new file mode 100644 index 000000000..2f7a643dd --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json @@ -0,0 +1,198 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:aws-design-ai", + "title": "aws-design-ai.json (canonical, target-only design contract)", + "description": "Provider-neutral, target-only contract for the aws-design-ai.json AI design fragment. Structural validity does NOT imply an implemented detector, adapter, route, model mapping, or runtime-supported identity, and does NOT imply that any skill's current design-ai.md actually emits this shape — this is a design-time target, canonicalized from Azure's prior single-scalar metadata.ai_source contract. metadata.ai_sources[] and metadata.gateways[] are byte-identical in shape to ai-workload-profile.schema.json's ai_sources[]/gateways[] and project verbatim from the ai-workload-profile.json the design was built from.", + "type": "object", + "required": ["phase", "focus", "timestamp", "source_profile", "metadata", "design_blocks", "ai_architecture", "regional_warnings", "multi_model_warnings"], + "additionalProperties": false, + "properties": { + "phase": {"const": "design"}, + "focus": {"const": "ai"}, + "timestamp": {"type": "string", "minLength": 1}, + "source_profile": {"const": "ai-workload-profile.json"}, + "metadata": {"$ref": "#/definitions/metadata"}, + "design_blocks": { + "type": "array", + "items": {"$ref": "#/definitions/designBlock"} + }, + "ai_architecture": {"type": "object"}, + "regional_warnings": {"type": "array"}, + "multi_model_warnings": {"type": "array"}, + "agentic_design": {"oneOf": [{"type": "null"}, {"type": "object"}]}, + "halt": {"type": "object"} + }, + "definitions": { + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "modelFamily": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_.-]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "gatewayIdentity": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_]*$"}, + "provenance": {"enum": ["code", "iac", "live_api", "usage_api", "user_confirmed"]}, + "confidence": {"enum": ["confirmed", "high", "medium", "low"]}, + "evidenceEntry": { + "type": "object", + "required": ["provenance", "confidence"], + "additionalProperties": false, + "properties": { + "provenance": {"$ref": "#/definitions/provenance"}, + "confidence": {"$ref": "#/definitions/confidence"}, + "detail": {"type": "string", "minLength": 1} + } + }, + "modelObservation": { + "description": "Byte-identical in shape to ai-workload-profile.schema.json's modelObservation.", + "type": "object", + "required": ["normalized_model", "raw_model", "model_family", "evidence"], + "additionalProperties": false, + "properties": { + "normalized_model": {"$ref": "#/definitions/normalizedModel"}, + "raw_model": {"type": "string", "minLength": 1}, + "model_family": {"$ref": "#/definitions/modelFamily"}, + "evidence": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/evidenceEntry"} + } + } + }, + "aiSource": { + "description": "Byte-identical in shape to ai-workload-profile.schema.json's aiSource (provider, source_service, model_families, model_evidence, models, model_observations). metadata.ai_sources[] projects this verbatim from the profile.", + "type": "object", + "required": ["provider", "source_service", "model_families", "model_evidence", "models", "model_observations"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/modelFamily"} + }, + "model_evidence": {"enum": ["observed", "not_observed"]}, + "models": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/normalizedModel"} + }, + "model_observations": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/modelObservation"} + } + }, + "oneOf": [ + {"properties": {"model_evidence": {"const": "observed"}, "models": {"minItems": 1}}}, + {"properties": {"model_evidence": {"const": "not_observed"}, "models": {"maxItems": 0}, "model_observations": {"maxItems": 0}}} + ] + }, + "upstreamModelFamily": { + "description": "Byte-identical in shape to ai-workload-profile.schema.json's upstreamModelFamily.", + "type": "object", + "required": ["family", "model_evidence", "models"], + "additionalProperties": false, + "properties": { + "family": {"$ref": "#/definitions/modelFamily"}, + "model_evidence": {"enum": ["observed", "not_observed"]}, + "models": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/normalizedModel"} + } + }, + "oneOf": [ + {"properties": {"model_evidence": {"const": "observed"}, "models": {"minItems": 1}}}, + {"properties": {"model_evidence": {"const": "not_observed"}, "models": {"maxItems": 0}}} + ] + }, + "gatewayUpstream": { + "description": "Byte-identical in shape to ai-workload-profile.schema.json's gatewayUpstream.", + "type": "object", + "required": ["provider", "model_families"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/upstreamModelFamily"} + } + } + }, + "gateway": { + "description": "Byte-identical in shape to ai-workload-profile.schema.json's gateway. metadata.gateways[] projects this verbatim from the profile.", + "type": "object", + "required": ["product", "type", "upstream_evidence", "upstreams"], + "additionalProperties": false, + "properties": { + "product": {"$ref": "#/definitions/gatewayIdentity"}, + "type": {"$ref": "#/definitions/extensibleIdentity"}, + "upstream_evidence": {"enum": ["observed", "unresolved"]}, + "upstreams": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/gatewayUpstream"} + } + }, + "oneOf": [ + {"properties": {"upstream_evidence": {"const": "observed"}, "upstreams": {"minItems": 1}}}, + {"properties": {"upstream_evidence": {"const": "unresolved"}, "upstreams": {"maxItems": 0}}} + ] + }, + "metadata": { + "description": "Replaces the retired single-scalar ai_source (MUST equal summary.ai_source) with many-valued ai_sources[]/gateways[] mirroring ai-workload-profile.schema.json exactly. additionalProperties: false means a vestigial scalar ai_source field is rejected, not merely undocumented.", + "type": "object", + "required": ["ai_sources", "gateways", "bedrock_models_selected", "regional_validation"], + "additionalProperties": false, + "properties": { + "ai_sources": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/aiSource"} + }, + "gateways": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/gateway"} + }, + "bedrock_models_selected": { + "type": "array", + "items": {"type": "string", "minLength": 1} + }, + "regional_validation": {"enum": ["checked", "fallback_static"]} + } + }, + "designBlock": { + "type": "object", + "required": ["workload_id", "model_id", "target_bedrock_model", "target_aws_service", "capability", "capability_confidence", "rationale", "confidence_warning"], + "additionalProperties": false, + "properties": { + "workload_id": {"type": "string", "minLength": 1}, + "model_id": {"type": "string", "minLength": 1}, + "target_bedrock_model": {"oneOf": [{"type": "null"}, {"type": "string", "minLength": 1}]}, + "target_aws_service": { + "oneOf": [ + {"type": "null"}, + {"enum": ["textract", "rekognition", "transcribe", "comprehend", "translate", "polly", "sagemaker"]} + ] + }, + "capability": { + "enum": [ + "text_generation", "structured_output", "image_generation", "embedding", + "speech_to_text", "text_to_speech", "unknown", + "document_extraction", "image_analysis", "speech_transcription" + ] + }, + "capability_confidence": {"enum": ["high", "medium", "low"]}, + "rationale": {"type": "string", "minLength": 1}, + "confidence_warning": {"oneOf": [{"type": "null"}, {"type": "string", "minLength": 1}]} + }, + "oneOf": [ + {"properties": {"target_bedrock_model": {"type": "string"}, "target_aws_service": {"type": "null"}}}, + {"properties": {"target_bedrock_model": {"type": "null"}, "target_aws_service": {"type": "string"}}} + ] + } + } +} diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index c250c7a16..f011aa426 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -1,189 +1,16 @@ -# Schema — aws-design-ai.json - -> **Why this file exists.** `aws-design.json` had no schema once and a capability run -> reverse-engineered its shape from postconditions, inventing key names the golden did not -> share (§13.3a). The AI design artifact is the same risk one layer over: `design-ai.md` -> produces it, `estimate-ai.md` and `generate-artifacts-ai.md` consume it, and a prose-only -> contract drifts between the three by construction. This file is the single written contract -> for `aws-design-ai.json`. It is the AI analogue of `schema-design-aws.md`, and it is -> deliberately kept parallel to gcp-to-aws's `aws-design-ai.json` shape so the estimate and -> generate ports stay straight (§19.10). - -`aws-design-ai.json` is written by `phases/design/design-ai.md` **only when -`ai-workload-profile.json` exists**. It is a SEPARATE artifact from `aws-design.json` -(§19.10): the infra assembler still solely owns `aws-design.json`; the AI fragment owns this -file. When there is no AI workload, this file is not produced and nothing downstream looks -for it. - -## Shape - -```jsonc -{ - "phase": "design", - "focus": "ai", - "timestamp": "", - "source_profile": "ai-workload-profile.json", - "metadata": { - "ai_source": "azure_openai", // MUST equal summary.ai_source from the profile - "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) - "regional_validation": "checked" // checked | fallback_static (see § regional_validation) - }, - "design_blocks": [], // one per workloads[] entry — see § design_blocks - "ai_architecture": { // see § ai_architecture - "honest_assessment": "strong_migrate", - "honest_assessment_reason": null, - "tiered_strategy": null, - "bedrock_models": [], - "capability_mapping": {}, - "code_migration": {}, - "infrastructure": [], - "services_to_migrate": [] - }, - "regional_warnings": [], // ALWAYS present, [] when clean — see § regional_warnings - "multi_model_warnings": [], // ALWAYS present, [] when single model — see § multi_model_warnings - "agentic_design": null, // present ONLY when agentic_profile.is_agentic — see § agentic_design - "halt": {} // present ONLY when the AI design is failing its gate -} -``` - -**Accounting invariant.** Every entry in the profile's `workloads[]` (or `models[]` when -`workloads[]` is empty) appears in exactly one `design_blocks[]` row, in input order. - -## design_blocks[] - -One row per confirmed workload. This is the per-workload target decision. - -```jsonc -{ - "workload_id": "wl_3a1f2c", // from the profile's workloads[]; preserve verbatim - "model_id": "gpt-4o", // the source model / SDK method the workload used - "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", // XOR target_aws_service — see below - "target_aws_service": null, // XOR target_bedrock_model — see below - "capability": "text_generation", // the workload's capability (see § capability vocabulary) - "capability_confidence": "high", // high | medium | low - "rationale": "", - "confidence_warning": null // non-null string when capability_confidence == "low" -} -``` - -- **`target_bedrock_model` XOR `target_aws_service` — exactly one is non-null per row.** - Bedrock-model capabilities (`text_generation`, `structured_output`, `image_generation`, - `embedding`, `speech_to_text`, `text_to_speech`, `unknown`) set `target_bedrock_model` and - leave `target_aws_service` null. The three traditional-AI capabilities - (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` - (one of `"textract"`, `"rekognition"`, `"transcribe"`, `"comprehend"`, `"translate"`, - `"polly"`, `"sagemaker"`) and leave `target_bedrock_model` null. This is the same rule gcp - enforces; it exists because those three are AWS AI services, not Bedrock model swaps. -- **`confidence_warning`** is a non-null string (naming the workload and that manual review is - required) when `capability_confidence == "low"`; null for `high`/`medium`. -- **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. - -## capability vocabulary - -Bedrock-model capabilities: `text_generation`, `structured_output`, `image_generation`, -`embedding`, `speech_to_text`, `text_to_speech`, `unknown`. -Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, -`image_analysis`, `speech_transcription`. - -## ai_architecture - -- **`honest_assessment`** — `strong_migrate | moderate_migrate | weak_migrate | - recommend_stay | not_applicable`. Overall value = the weakest across all Bedrock models. - `not_applicable` is a PER-WORKLOAD value for the three traditional-AI capabilities only — - never the overall assessment (they are feature swaps, not a cost-driven model decision). -- **`honest_assessment_reason`** — REQUIRED (non-null) when `honest_assessment == - "recommend_stay"`; names the specific non-cost blocker (fixed region with no suitable model, - a Realtime/streaming dependency, an Assistants-style feature with no Bedrock equivalent, or a - confirmed unsupported API surface). Null otherwise. Cost parity alone is never sufficient to - recommend staying when the source models are on Bedrock. -- **`tiered_strategy`** — object when `ai_token_volume == "high"` (a 3-tier routing plan); - null for low/medium. -- **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, - `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, - `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), - `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities - do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. -- **`capability_mapping`** — per capability that is `true` in the profile's - `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. -- **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), - `framework`, `files_to_modify[]`, `dependency_changes`, and — when the source is Azure - OpenAI / OpenAI — `openrouter_path` when a router was detected - (`same_model_mantle|direct|litellm|keep_openrouter`). -- **`infrastructure[]`** — Azure AI infra resource → AWS equivalent, with confidence. -- **`services_to_migrate[]`** — Azure AI service → AWS service, effort, notes. - -## migration_path vocabulary - -`mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, region carries it — -keep the SDK, `model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; -set `model_change: true` when a proprietary GPT model must move off mantle for Guardrails / -Knowledge Bases / logging / an unsupported region) · `gpt-oss` (OpenAI-lineage model on the -Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). A proprietary -`openai.gpt-*` model ID is NEVER paired with a `converse`/`bedrock-runtime` path — those models -are mantle-only. - -## regional_validation - -`checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region -availability) · `fallback_static` (the call failed; the static table in -`vendored/ai/ai-migration-guardrails.md` was used). - -## regional_warnings[] - -ALWAYS present (`[]` when clean). Per unavailable service: -`{ service, target_region, nearest_available, impact }`. A compliance constraint that forced a -region (e.g. `fedramp` → GovCloud, `gdpr` → EU `eu.` profiles) records the models it excluded -here. Never blocks the design — it flags the constraint and proceeds. - -## multi_model_warnings[] - -ALWAYS present (`[]` when a single model / no coordination issue). Per warning: -`{ type, message }`, `type ∈ embeddings_reindex | cascade_pair | multi_model_tiered | -image_separate | speech_separate`. - -## agentic_design - -Present ONLY when the profile's `agentic_profile.is_agentic == true`; null/absent otherwise. -`{ migration_approach: "retarget|harness|strands|undecided", ...path-specific config }`. When -`migration_approach == "harness"` carry `harness_config` (with `source_model_provider`, which -is `"open_ai"` for an `azure_openai` source — see §19.9(b)); when `"strands"` carry the -AgentCore Runtime config from `vendored/ai/design-ref-agentic-to-agentcore.md`. - -## halt - -Present ONLY when the AI design is failing its gate. `{ reason, blocking[] }` where each -`blocking[]` entry is `{ kind, identifier, why, action }`. The AI design halts when a routed -rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) named by -`index.md` is not on disk — the same missing-rubric guard the infra design uses. - -## Validation Checklist - -- [ ] `metadata.ai_source` equals `summary.ai_source` from `ai-workload-profile.json`. -- [ ] Every `workloads[]` entry (or `models[]` when workloads is empty) has exactly one - `design_blocks[]` row, in input order. -- [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / - `target_aws_service` non-null (XOR). -- [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) - has `target_bedrock_model: null`, a non-null `target_aws_service`, and (in - `ai_architecture`) is not counted in `bedrock_models[]`. -- [ ] Every `bedrock_models[]` entry has `source_provider_price`, `bedrock_price`, - `price_comparison`. -- [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if - `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. -- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a - `converse`/`bedrock-runtime` migration path (mantle-only). -- [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on - Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not - to a Claude/Nova substitute. -- [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). -- [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with - `migration_approach` matching `preferences.json`; a `harness` source with an - `azure_openai` origin sets `source_model_provider: "open_ai"`. -- [ ] All Bedrock model IDs are Active per `vendored/ai/ai-model-lifecycle.md` (a Legacy ID is - used only if no Active alternative exists, with the EOL date noted). -- [ ] `"App Runner"` appears nowhere in the artifact. - -## Status — build step 2 (AI design route) - -Written alongside `design-ai.md`. `estimate-ai.md` and `generate-artifacts-ai.md` -read this contract. +# Schema — aws-design-ai.json (moved — see the canonical contract) + +> **This file has been canonicalized.** The `aws-design-ai.json` contract that used to live +> here — including its single-scalar `metadata.ai_source` MUST-equal-`summary.ai_source` +> rule — has moved to `plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md`, +> which replaces that scalar with many-valued `metadata.ai_sources[]` / `metadata.gateways[]` +> mirroring `ai-workload-profile.schema.json` (#415) exactly. This skill reads the canonical +> contract through its own byte-identical vendored copy at +> `references/vendored/ai/schema-design-aws-ai.md` (see that directory's README for the +> vendored-parity gate, `mise run lint:vendored-parity`). This file is kept only as a redirect +> so a reference to this path does not 404; do not restate the contract here — edit the +> canonical source instead. +> +> `scripts/artifact-contracts.json`'s `aws-design-ai.json` entry for `azure-to-aws` now points +> at the vendored copy. This change is target-only: `design-ai.md` has not been repointed to +> emit the new shape in this change (deferred runtime-adoption work). diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index a5e154143..9407b1e9d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -38,6 +38,7 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/design-ref-agentic-to-agentcore.md` | `skills/shared/ai/design-ref-agentic-to-agentcore.md` | | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | | `ai/sdk-capability-map.json` | `skills/shared/ai/sdk-capability-map.json` | +| `ai/schema-design-aws-ai.md` | `skills/shared/ai/schema-design-aws-ai.md` | | `clarify/clarify-availability.md` | `skills/shared/clarify/clarify-availability.md` | | `clarify/clarify-compliance.md` | `skills/shared/clarify/clarify-compliance.md` | | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md new file mode 100644 index 000000000..4c9f3a4fd --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -0,0 +1,275 @@ +# Schema — aws-design-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design.json` had no schema once and a capability run +> reverse-engineered its shape from postconditions, inventing key names the golden did not +> share (§13.3a). The AI design artifact is the same risk one layer over: `design-ai.md` +> produces it, `estimate-ai.md` and `generate-artifacts-ai.md` consume it, and a prose-only +> contract drifts between producers by construction. This file is the single, provider-neutral +> written contract for `aws-design-ai.json`, canonicalized from Azure's prior +> `references/shared/schema-design-aws-ai.md` so GCP and Azure share one definition instead of +> drifting independently (GCP had none). Each consuming skill carries a byte-identical vendored +> copy at `references/vendored/ai/schema-design-aws-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410) and +> `ai-workload-profile.schema.json` (#415).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/aws-design-ai.schema.json`) does NOT imply an +> implemented detector, adapter, route, model mapping, or runtime-supported identity, and does +> NOT imply that GCP's or Azure's current `design-ai.md` actually emits this shape yet. No +> producer has been repointed to this canonical contract in this change — that is deferred +> runtime-adoption work. +> +> **What changed from the prior Azure-only contract.** The prior contract's `metadata.ai_source` +> was a single scalar string (`azure_openai | openai | anthropic | both | other`) that MUST +> equal a single scalar `summary.ai_source` from the profile — the same lossy, single-identity +> `both`-sentinel pattern #410 and #415 retired in the observation layer. This canonical version +> replaces that scalar with many-valued `metadata.ai_sources[]` / `metadata.gateways[]`, mirroring +> `ai-workload-profile.schema.json`'s `ai_sources[]` / `gateways[]` shape EXACTLY (same field +> names, same types, same evidence-gating rules). A design can now carry simultaneous +> Anthropic-sourced and OpenAI-sourced workloads, or an OpenRouter gateway fronting several +> providers at once, without collapsing them into one scalar identity. + +`aws-design-ai.json` is written by `phases/design/design-ai.md` **only when +`ai-workload-profile.json` exists**. It is a SEPARATE artifact from `aws-design.json` +(§19.10): the infra assembler still solely owns `aws-design.json`; the AI fragment owns this +file. When there is no AI workload, this file is not produced and nothing downstream looks +for it. + +## Shape + +```jsonc +{ + "phase": "design", + "focus": "ai", + "timestamp": "", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [], // many-valued — see § metadata.ai_sources[]; mirrors the profile's ai_sources[] exactly + "gateways": [], // many-valued — see § metadata.gateways[]; mirrors the profile's gateways[] exactly + "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) + "regional_validation": "checked" // checked | fallback_static (see § regional_validation) + }, + "design_blocks": [], // one per workloads[] entry — see § design_blocks + "ai_architecture": { // see § ai_architecture + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [], + "capability_mapping": {}, + "code_migration": {}, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], // ALWAYS present, [] when clean — see § regional_warnings + "multi_model_warnings": [], // ALWAYS present, [] when single model — see § multi_model_warnings + "agentic_design": null, // present ONLY when agentic_profile.is_agentic — see § agentic_design + "halt": {} // present ONLY when the AI design is failing its gate +} +``` + +**Accounting invariant.** Every entry in the profile's `workloads[]` (or `models[]` when +`workloads[]` is empty) appears in exactly one `design_blocks[]` row, in input order. + +## metadata.ai_sources[] + +Byte-identical in shape to `ai-workload-profile.schema.json`'s `ai_sources[]` (`scripts/contracts/ai-workload-profile.schema.json`, +definition `aiSource`). Each entry: + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", // extensibleIdentity — e.g. "openai_api" vs "azure_openai" distinguishes direct vs Azure-fronted OpenAI + "model_families": ["claude"], // modelFamily[], non-empty + "model_evidence": "observed", // observed | not_observed + "models": ["claude-3-5-sonnet-20240620"], // normalizedModel[]; [] iff model_evidence == "not_observed" + "model_observations": [] // raw-alias/evidence trail; [] iff model_evidence == "not_observed" — see the profile schema's modelObservation +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair; a design can carry multiple +simultaneous sources (one Anthropic-sourced workload and one OpenAI-sourced workload in the +same `aws-design-ai.json` is valid and expected — this is the shape change this canonicalization +makes). `provider` is never `"openrouter"` — OpenRouter and other LLM routers are represented +only in `metadata.gateways[]`, never as a direct source. + +## metadata.gateways[] + +Byte-identical in shape to `ai-workload-profile.schema.json`'s `gateways[]` (definition +`gateway`). An observed gateway's `upstreams[]` must correspond exactly to the `ai_sources[]` +entries it fronts (same provider/source_service/model_families/model_evidence/models +association #410's gateway-correspondence rule enforces); an unresolved gateway's `upstreams[]` +is explicitly empty. + +```jsonc +{ + "product": "openrouter", // gatewayIdentity — never "both" | "unknown" + "type": "llm_router", + "upstream_evidence": "observed", // observed | unresolved + "upstreams": [] // gatewayUpstream[]; [] iff upstream_evidence == "unresolved" +} +``` + +Both `metadata.ai_sources[]` and `metadata.gateways[]` project **verbatim** from the +`ai-workload-profile.json` the design was built from — the test suite +(`test_aws_design_ai_contracts.py`) proves this projection against #415's own semantic +oracle (`_assert_profile_semantics`), the same way #415 proved its own projection into #410. +A design that drops a model, renames a provider, or collapses two sources from the profile +into one is a contract violation, not a presentation choice. + +## design_blocks[] + +One row per confirmed workload. This is the per-workload target decision. + +```jsonc +{ + "workload_id": "wl_3a1f2c", // from the profile's workloads[]; preserve verbatim + "model_id": "gpt-4o", // the source model / SDK method the workload used + "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", // XOR target_aws_service — see below + "target_aws_service": null, // XOR target_bedrock_model — see below + "capability": "text_generation", // the workload's capability (see § capability vocabulary) + "capability_confidence": "high", // high | medium | low + "rationale": "", + "confidence_warning": null // non-null string when capability_confidence == "low" +} +``` + +- **`target_bedrock_model` XOR `target_aws_service` — exactly one is non-null per row.** + Bedrock-model capabilities (`text_generation`, `structured_output`, `image_generation`, + `embedding`, `speech_to_text`, `text_to_speech`, `unknown`) set `target_bedrock_model` and + leave `target_aws_service` null. The three traditional-AI capabilities + (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` + (one of `"textract"`, `"rekognition"`, `"transcribe"`, `"comprehend"`, `"translate"`, + `"polly"`, `"sagemaker"`) and leave `target_bedrock_model` null. This is the same rule gcp + enforces; it exists because those three are AWS AI services, not Bedrock model swaps. +- **`confidence_warning`** is a non-null string (naming the workload and that manual review is + required) when `capability_confidence == "low"`; null for `high`/`medium`. +- **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. +- **`model_id` provenance:** every `design_blocks[]` row's source model MUST trace to a + `metadata.ai_sources[]` entry (directly, or through a `metadata.gateways[]` upstream) — a + `design_blocks[]` row naming a model no retained source or gateway upstream carries is a + contract violation. + +## capability vocabulary + +Bedrock-model capabilities: `text_generation`, `structured_output`, `image_generation`, +`embedding`, `speech_to_text`, `text_to_speech`, `unknown`. +Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, +`image_analysis`, `speech_transcription`. + +## ai_architecture + +- **`honest_assessment`** — `strong_migrate | moderate_migrate | weak_migrate | + recommend_stay | not_applicable`. Overall value = the weakest across all Bedrock models. + `not_applicable` is a PER-WORKLOAD value for the three traditional-AI capabilities only — + never the overall assessment (they are feature swaps, not a cost-driven model decision). +- **`honest_assessment_reason`** — REQUIRED (non-null) when `honest_assessment == + "recommend_stay"`; names the specific non-cost blocker (fixed region with no suitable model, + a Realtime/streaming dependency, an Assistants-style feature with no Bedrock equivalent, or a + confirmed unsupported API surface). Null otherwise. Cost parity alone is never sufficient to + recommend staying when the source models are on Bedrock. +- **`tiered_strategy`** — object when `ai_token_volume == "high"` (a 3-tier routing plan); + null for low/medium. +- **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, + `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, + `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), + `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities + do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. +- **`capability_mapping`** — per capability that is `true` in the profile's + `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. +- **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), + `framework`, `files_to_modify[]`, `dependency_changes`, and — when a retained source's + `source_service` is `azure_openai` or a `provider`/`source_service` pair resolves to OpenAI — + `openrouter_path` when a router was detected + (`same_model_mantle|direct|litellm|keep_openrouter`). +- **`infrastructure[]`** — source-cloud AI infra resource → AWS equivalent, with confidence. +- **`services_to_migrate[]`** — source-cloud AI service → AWS service, effort, notes. + +## migration_path vocabulary + +`mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model +on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `converse` +(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when a proprietary GPT +model must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported region) · +`gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic +Bedrock SDK swap). A proprietary `openai.gpt-*` model ID is NEVER paired with a +`converse`/`bedrock-runtime` path — those models are mantle-only. + +## regional_validation + +`checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region +availability) · `fallback_static` (the call failed; the static table in +`vendored/ai/ai-migration-guardrails.md` was used). + +## regional_warnings[] + +ALWAYS present (`[]` when clean). Per unavailable service: +`{ service, target_region, nearest_available, impact }`. A compliance constraint that forced a +region (e.g. `fedramp` → GovCloud, `gdpr` → EU `eu.` profiles) records the models it excluded +here. Never blocks the design — it flags the constraint and proceeds. + +## multi_model_warnings[] + +ALWAYS present (`[]` when a single model / no coordination issue). Per warning: +`{ type, message }`, `type ∈ embeddings_reindex | cascade_pair | multi_model_tiered | +image_separate | speech_separate`. + +## agentic_design + +Present ONLY when the profile's `agentic_profile.is_agentic == true`; null/absent otherwise. +`{ migration_approach: "retarget|harness|strands|undecided", ...path-specific config }`. When +`migration_approach == "harness"` carry `harness_config` (with `source_model_provider`, derived +per retained source rather than a single scalar — an OpenAI-protocol source with +`source_service: "azure_openai"` sets `source_model_provider: "open_ai"` — see §19.9(b)); when +`"strands"` carry the AgentCore Runtime config from `vendored/ai/design-ref-agentic-to-agentcore.md`. + +## halt + +Present ONLY when the AI design is failing its gate. `{ reason, blocking[] }` where each +`blocking[]` entry is `{ kind, identifier, why, action }`. The AI design halts when a routed +rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` ref) named by +`index.md` is not on disk — the same missing-rubric guard the infra design uses. + +## Validation Checklist + +- [ ] `metadata.ai_sources[]` is non-empty whenever the profile has at least one retained + source, and each entry's `(provider, source_service)` pair is unique — no duplicate or + overlapping source. +- [ ] `metadata.ai_sources[]` and `metadata.gateways[]` project verbatim from the + `ai-workload-profile.json` the design was built from (same provider, source_service, + model_families, model_evidence, models, model_observations, and gateway upstream + correspondence) — no dropped, renamed, or fabricated source or model. +- [ ] No entry in `metadata.ai_sources[]` has `provider: "openrouter"`; OpenRouter and other + routers appear only in `metadata.gateways[]`. +- [ ] Every `workloads[]` entry (or `models[]` when workloads is empty) has exactly one + `design_blocks[]` row, in input order. +- [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / + `target_aws_service` non-null (XOR), and its source model traces to a + `metadata.ai_sources[]` entry or a `metadata.gateways[]` upstream. +- [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) + has `target_bedrock_model: null`, a non-null `target_aws_service`, and (in + `ai_architecture`) is not counted in `bedrock_models[]`. +- [ ] Every `bedrock_models[]` entry has `source_provider_price`, `bedrock_price`, + `price_comparison`. +- [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if + `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. +- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves + to direct OpenAI: every source model that is available on Bedrock and carried by the + target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. +- [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). +- [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with + `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves + to Azure-fronted OpenAI sets `source_model_provider: "open_ai"`. +- [ ] All Bedrock model IDs are Active per `vendored/ai/ai-model-lifecycle.md` (a Legacy ID is + used only if no Active alternative exists, with the EOL date noted). +- [ ] `"App Runner"` appears nowhere in the artifact. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. `design-ai.md` (GCP and Azure) still +writes against the prior Azure-only `metadata.ai_source` scalar documented in each skill's own +`references/shared/schema-design-aws-ai.md` (Azure) / absence of a contract (GCP). Repointing +either skill's `design-ai.md` to emit this shape is deferred runtime-adoption work, named +explicitly so a future PR does not silently assume today's runtime output already satisfies +this contract. `estimate-ai.md` and `generate-artifacts-ai.md` are unchanged and still read +today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 01040332d..d10aa5cd5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -31,6 +31,7 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/design-ref-agentic-to-agentcore.md` | `skills/shared/ai/design-ref-agentic-to-agentcore.md` | | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | | `ai/sdk-capability-map.json` | `skills/shared/ai/sdk-capability-map.json` | +| `ai/schema-design-aws-ai.md` | `skills/shared/ai/schema-design-aws-ai.md` | | `clarify/clarify-availability.md` | `skills/shared/clarify/clarify-availability.md` | | `clarify/clarify-compliance.md` | `skills/shared/clarify/clarify-compliance.md` | | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md new file mode 100644 index 000000000..4c9f3a4fd --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -0,0 +1,275 @@ +# Schema — aws-design-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design.json` had no schema once and a capability run +> reverse-engineered its shape from postconditions, inventing key names the golden did not +> share (§13.3a). The AI design artifact is the same risk one layer over: `design-ai.md` +> produces it, `estimate-ai.md` and `generate-artifacts-ai.md` consume it, and a prose-only +> contract drifts between producers by construction. This file is the single, provider-neutral +> written contract for `aws-design-ai.json`, canonicalized from Azure's prior +> `references/shared/schema-design-aws-ai.md` so GCP and Azure share one definition instead of +> drifting independently (GCP had none). Each consuming skill carries a byte-identical vendored +> copy at `references/vendored/ai/schema-design-aws-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410) and +> `ai-workload-profile.schema.json` (#415).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/aws-design-ai.schema.json`) does NOT imply an +> implemented detector, adapter, route, model mapping, or runtime-supported identity, and does +> NOT imply that GCP's or Azure's current `design-ai.md` actually emits this shape yet. No +> producer has been repointed to this canonical contract in this change — that is deferred +> runtime-adoption work. +> +> **What changed from the prior Azure-only contract.** The prior contract's `metadata.ai_source` +> was a single scalar string (`azure_openai | openai | anthropic | both | other`) that MUST +> equal a single scalar `summary.ai_source` from the profile — the same lossy, single-identity +> `both`-sentinel pattern #410 and #415 retired in the observation layer. This canonical version +> replaces that scalar with many-valued `metadata.ai_sources[]` / `metadata.gateways[]`, mirroring +> `ai-workload-profile.schema.json`'s `ai_sources[]` / `gateways[]` shape EXACTLY (same field +> names, same types, same evidence-gating rules). A design can now carry simultaneous +> Anthropic-sourced and OpenAI-sourced workloads, or an OpenRouter gateway fronting several +> providers at once, without collapsing them into one scalar identity. + +`aws-design-ai.json` is written by `phases/design/design-ai.md` **only when +`ai-workload-profile.json` exists**. It is a SEPARATE artifact from `aws-design.json` +(§19.10): the infra assembler still solely owns `aws-design.json`; the AI fragment owns this +file. When there is no AI workload, this file is not produced and nothing downstream looks +for it. + +## Shape + +```jsonc +{ + "phase": "design", + "focus": "ai", + "timestamp": "", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [], // many-valued — see § metadata.ai_sources[]; mirrors the profile's ai_sources[] exactly + "gateways": [], // many-valued — see § metadata.gateways[]; mirrors the profile's gateways[] exactly + "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) + "regional_validation": "checked" // checked | fallback_static (see § regional_validation) + }, + "design_blocks": [], // one per workloads[] entry — see § design_blocks + "ai_architecture": { // see § ai_architecture + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [], + "capability_mapping": {}, + "code_migration": {}, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], // ALWAYS present, [] when clean — see § regional_warnings + "multi_model_warnings": [], // ALWAYS present, [] when single model — see § multi_model_warnings + "agentic_design": null, // present ONLY when agentic_profile.is_agentic — see § agentic_design + "halt": {} // present ONLY when the AI design is failing its gate +} +``` + +**Accounting invariant.** Every entry in the profile's `workloads[]` (or `models[]` when +`workloads[]` is empty) appears in exactly one `design_blocks[]` row, in input order. + +## metadata.ai_sources[] + +Byte-identical in shape to `ai-workload-profile.schema.json`'s `ai_sources[]` (`scripts/contracts/ai-workload-profile.schema.json`, +definition `aiSource`). Each entry: + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", // extensibleIdentity — e.g. "openai_api" vs "azure_openai" distinguishes direct vs Azure-fronted OpenAI + "model_families": ["claude"], // modelFamily[], non-empty + "model_evidence": "observed", // observed | not_observed + "models": ["claude-3-5-sonnet-20240620"], // normalizedModel[]; [] iff model_evidence == "not_observed" + "model_observations": [] // raw-alias/evidence trail; [] iff model_evidence == "not_observed" — see the profile schema's modelObservation +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair; a design can carry multiple +simultaneous sources (one Anthropic-sourced workload and one OpenAI-sourced workload in the +same `aws-design-ai.json` is valid and expected — this is the shape change this canonicalization +makes). `provider` is never `"openrouter"` — OpenRouter and other LLM routers are represented +only in `metadata.gateways[]`, never as a direct source. + +## metadata.gateways[] + +Byte-identical in shape to `ai-workload-profile.schema.json`'s `gateways[]` (definition +`gateway`). An observed gateway's `upstreams[]` must correspond exactly to the `ai_sources[]` +entries it fronts (same provider/source_service/model_families/model_evidence/models +association #410's gateway-correspondence rule enforces); an unresolved gateway's `upstreams[]` +is explicitly empty. + +```jsonc +{ + "product": "openrouter", // gatewayIdentity — never "both" | "unknown" + "type": "llm_router", + "upstream_evidence": "observed", // observed | unresolved + "upstreams": [] // gatewayUpstream[]; [] iff upstream_evidence == "unresolved" +} +``` + +Both `metadata.ai_sources[]` and `metadata.gateways[]` project **verbatim** from the +`ai-workload-profile.json` the design was built from — the test suite +(`test_aws_design_ai_contracts.py`) proves this projection against #415's own semantic +oracle (`_assert_profile_semantics`), the same way #415 proved its own projection into #410. +A design that drops a model, renames a provider, or collapses two sources from the profile +into one is a contract violation, not a presentation choice. + +## design_blocks[] + +One row per confirmed workload. This is the per-workload target decision. + +```jsonc +{ + "workload_id": "wl_3a1f2c", // from the profile's workloads[]; preserve verbatim + "model_id": "gpt-4o", // the source model / SDK method the workload used + "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", // XOR target_aws_service — see below + "target_aws_service": null, // XOR target_bedrock_model — see below + "capability": "text_generation", // the workload's capability (see § capability vocabulary) + "capability_confidence": "high", // high | medium | low + "rationale": "", + "confidence_warning": null // non-null string when capability_confidence == "low" +} +``` + +- **`target_bedrock_model` XOR `target_aws_service` — exactly one is non-null per row.** + Bedrock-model capabilities (`text_generation`, `structured_output`, `image_generation`, + `embedding`, `speech_to_text`, `text_to_speech`, `unknown`) set `target_bedrock_model` and + leave `target_aws_service` null. The three traditional-AI capabilities + (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` + (one of `"textract"`, `"rekognition"`, `"transcribe"`, `"comprehend"`, `"translate"`, + `"polly"`, `"sagemaker"`) and leave `target_bedrock_model` null. This is the same rule gcp + enforces; it exists because those three are AWS AI services, not Bedrock model swaps. +- **`confidence_warning`** is a non-null string (naming the workload and that manual review is + required) when `capability_confidence == "low"`; null for `high`/`medium`. +- **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. +- **`model_id` provenance:** every `design_blocks[]` row's source model MUST trace to a + `metadata.ai_sources[]` entry (directly, or through a `metadata.gateways[]` upstream) — a + `design_blocks[]` row naming a model no retained source or gateway upstream carries is a + contract violation. + +## capability vocabulary + +Bedrock-model capabilities: `text_generation`, `structured_output`, `image_generation`, +`embedding`, `speech_to_text`, `text_to_speech`, `unknown`. +Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, +`image_analysis`, `speech_transcription`. + +## ai_architecture + +- **`honest_assessment`** — `strong_migrate | moderate_migrate | weak_migrate | + recommend_stay | not_applicable`. Overall value = the weakest across all Bedrock models. + `not_applicable` is a PER-WORKLOAD value for the three traditional-AI capabilities only — + never the overall assessment (they are feature swaps, not a cost-driven model decision). +- **`honest_assessment_reason`** — REQUIRED (non-null) when `honest_assessment == + "recommend_stay"`; names the specific non-cost blocker (fixed region with no suitable model, + a Realtime/streaming dependency, an Assistants-style feature with no Bedrock equivalent, or a + confirmed unsupported API surface). Null otherwise. Cost parity alone is never sufficient to + recommend staying when the source models are on Bedrock. +- **`tiered_strategy`** — object when `ai_token_volume == "high"` (a 3-tier routing plan); + null for low/medium. +- **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, + `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, + `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), + `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities + do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. +- **`capability_mapping`** — per capability that is `true` in the profile's + `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. +- **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), + `framework`, `files_to_modify[]`, `dependency_changes`, and — when a retained source's + `source_service` is `azure_openai` or a `provider`/`source_service` pair resolves to OpenAI — + `openrouter_path` when a router was detected + (`same_model_mantle|direct|litellm|keep_openrouter`). +- **`infrastructure[]`** — source-cloud AI infra resource → AWS equivalent, with confidence. +- **`services_to_migrate[]`** — source-cloud AI service → AWS service, effort, notes. + +## migration_path vocabulary + +`mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model +on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `converse` +(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when a proprietary GPT +model must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported region) · +`gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic +Bedrock SDK swap). A proprietary `openai.gpt-*` model ID is NEVER paired with a +`converse`/`bedrock-runtime` path — those models are mantle-only. + +## regional_validation + +`checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region +availability) · `fallback_static` (the call failed; the static table in +`vendored/ai/ai-migration-guardrails.md` was used). + +## regional_warnings[] + +ALWAYS present (`[]` when clean). Per unavailable service: +`{ service, target_region, nearest_available, impact }`. A compliance constraint that forced a +region (e.g. `fedramp` → GovCloud, `gdpr` → EU `eu.` profiles) records the models it excluded +here. Never blocks the design — it flags the constraint and proceeds. + +## multi_model_warnings[] + +ALWAYS present (`[]` when a single model / no coordination issue). Per warning: +`{ type, message }`, `type ∈ embeddings_reindex | cascade_pair | multi_model_tiered | +image_separate | speech_separate`. + +## agentic_design + +Present ONLY when the profile's `agentic_profile.is_agentic == true`; null/absent otherwise. +`{ migration_approach: "retarget|harness|strands|undecided", ...path-specific config }`. When +`migration_approach == "harness"` carry `harness_config` (with `source_model_provider`, derived +per retained source rather than a single scalar — an OpenAI-protocol source with +`source_service: "azure_openai"` sets `source_model_provider: "open_ai"` — see §19.9(b)); when +`"strands"` carry the AgentCore Runtime config from `vendored/ai/design-ref-agentic-to-agentcore.md`. + +## halt + +Present ONLY when the AI design is failing its gate. `{ reason, blocking[] }` where each +`blocking[]` entry is `{ kind, identifier, why, action }`. The AI design halts when a routed +rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` ref) named by +`index.md` is not on disk — the same missing-rubric guard the infra design uses. + +## Validation Checklist + +- [ ] `metadata.ai_sources[]` is non-empty whenever the profile has at least one retained + source, and each entry's `(provider, source_service)` pair is unique — no duplicate or + overlapping source. +- [ ] `metadata.ai_sources[]` and `metadata.gateways[]` project verbatim from the + `ai-workload-profile.json` the design was built from (same provider, source_service, + model_families, model_evidence, models, model_observations, and gateway upstream + correspondence) — no dropped, renamed, or fabricated source or model. +- [ ] No entry in `metadata.ai_sources[]` has `provider: "openrouter"`; OpenRouter and other + routers appear only in `metadata.gateways[]`. +- [ ] Every `workloads[]` entry (or `models[]` when workloads is empty) has exactly one + `design_blocks[]` row, in input order. +- [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / + `target_aws_service` non-null (XOR), and its source model traces to a + `metadata.ai_sources[]` entry or a `metadata.gateways[]` upstream. +- [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) + has `target_bedrock_model: null`, a non-null `target_aws_service`, and (in + `ai_architecture`) is not counted in `bedrock_models[]`. +- [ ] Every `bedrock_models[]` entry has `source_provider_price`, `bedrock_price`, + `price_comparison`. +- [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if + `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. +- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves + to direct OpenAI: every source model that is available on Bedrock and carried by the + target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. +- [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). +- [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with + `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves + to Azure-fronted OpenAI sets `source_model_provider: "open_ai"`. +- [ ] All Bedrock model IDs are Active per `vendored/ai/ai-model-lifecycle.md` (a Legacy ID is + used only if no Active alternative exists, with the EOL date noted). +- [ ] `"App Runner"` appears nowhere in the artifact. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. `design-ai.md` (GCP and Azure) still +writes against the prior Azure-only `metadata.ai_source` scalar documented in each skill's own +`references/shared/schema-design-aws-ai.md` (Azure) / absence of a contract (GCP). Repointing +either skill's `design-ai.md` to emit this shape is deferred runtime-adoption work, named +explicitly so a future PR does not silently assume today's runtime output already satisfies +this contract. `estimate-ai.md` and `generate-artifacts-ai.md` are unchanged and still read +today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/README.md b/plugins/aws-startup-advisor/skills/shared/ai/README.md index 43e4de142..d6a3627f1 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/README.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/README.md @@ -18,6 +18,7 @@ before opening a pull request. | `design-ref-harness.md` | AgentCore Harness design reference | | `design-ref-agentic-to-agentcore.md` | Strands Agents + AgentCore Runtime design reference | | `sdk-capability-map.json` | SDK method → capability lookup used by app-code discovery | +| `schema-design-aws-ai.md` | Target-only `aws-design-ai.json` contract; `metadata.ai_sources[]`/`gateways[]` mirror `ai-workload-profile.schema.json` exactly | Deliberately NOT here: a source-cloud's own mapping guide (e.g. gcp-to-aws's `ai-gemini-to-bedrock.md`) and its `schema-discover-ai.md`, both of which are diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md new file mode 100644 index 000000000..4c9f3a4fd --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -0,0 +1,275 @@ +# Schema — aws-design-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design.json` had no schema once and a capability run +> reverse-engineered its shape from postconditions, inventing key names the golden did not +> share (§13.3a). The AI design artifact is the same risk one layer over: `design-ai.md` +> produces it, `estimate-ai.md` and `generate-artifacts-ai.md` consume it, and a prose-only +> contract drifts between producers by construction. This file is the single, provider-neutral +> written contract for `aws-design-ai.json`, canonicalized from Azure's prior +> `references/shared/schema-design-aws-ai.md` so GCP and Azure share one definition instead of +> drifting independently (GCP had none). Each consuming skill carries a byte-identical vendored +> copy at `references/vendored/ai/schema-design-aws-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410) and +> `ai-workload-profile.schema.json` (#415).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/aws-design-ai.schema.json`) does NOT imply an +> implemented detector, adapter, route, model mapping, or runtime-supported identity, and does +> NOT imply that GCP's or Azure's current `design-ai.md` actually emits this shape yet. No +> producer has been repointed to this canonical contract in this change — that is deferred +> runtime-adoption work. +> +> **What changed from the prior Azure-only contract.** The prior contract's `metadata.ai_source` +> was a single scalar string (`azure_openai | openai | anthropic | both | other`) that MUST +> equal a single scalar `summary.ai_source` from the profile — the same lossy, single-identity +> `both`-sentinel pattern #410 and #415 retired in the observation layer. This canonical version +> replaces that scalar with many-valued `metadata.ai_sources[]` / `metadata.gateways[]`, mirroring +> `ai-workload-profile.schema.json`'s `ai_sources[]` / `gateways[]` shape EXACTLY (same field +> names, same types, same evidence-gating rules). A design can now carry simultaneous +> Anthropic-sourced and OpenAI-sourced workloads, or an OpenRouter gateway fronting several +> providers at once, without collapsing them into one scalar identity. + +`aws-design-ai.json` is written by `phases/design/design-ai.md` **only when +`ai-workload-profile.json` exists**. It is a SEPARATE artifact from `aws-design.json` +(§19.10): the infra assembler still solely owns `aws-design.json`; the AI fragment owns this +file. When there is no AI workload, this file is not produced and nothing downstream looks +for it. + +## Shape + +```jsonc +{ + "phase": "design", + "focus": "ai", + "timestamp": "", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [], // many-valued — see § metadata.ai_sources[]; mirrors the profile's ai_sources[] exactly + "gateways": [], // many-valued — see § metadata.gateways[]; mirrors the profile's gateways[] exactly + "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) + "regional_validation": "checked" // checked | fallback_static (see § regional_validation) + }, + "design_blocks": [], // one per workloads[] entry — see § design_blocks + "ai_architecture": { // see § ai_architecture + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [], + "capability_mapping": {}, + "code_migration": {}, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], // ALWAYS present, [] when clean — see § regional_warnings + "multi_model_warnings": [], // ALWAYS present, [] when single model — see § multi_model_warnings + "agentic_design": null, // present ONLY when agentic_profile.is_agentic — see § agentic_design + "halt": {} // present ONLY when the AI design is failing its gate +} +``` + +**Accounting invariant.** Every entry in the profile's `workloads[]` (or `models[]` when +`workloads[]` is empty) appears in exactly one `design_blocks[]` row, in input order. + +## metadata.ai_sources[] + +Byte-identical in shape to `ai-workload-profile.schema.json`'s `ai_sources[]` (`scripts/contracts/ai-workload-profile.schema.json`, +definition `aiSource`). Each entry: + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", // extensibleIdentity — e.g. "openai_api" vs "azure_openai" distinguishes direct vs Azure-fronted OpenAI + "model_families": ["claude"], // modelFamily[], non-empty + "model_evidence": "observed", // observed | not_observed + "models": ["claude-3-5-sonnet-20240620"], // normalizedModel[]; [] iff model_evidence == "not_observed" + "model_observations": [] // raw-alias/evidence trail; [] iff model_evidence == "not_observed" — see the profile schema's modelObservation +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair; a design can carry multiple +simultaneous sources (one Anthropic-sourced workload and one OpenAI-sourced workload in the +same `aws-design-ai.json` is valid and expected — this is the shape change this canonicalization +makes). `provider` is never `"openrouter"` — OpenRouter and other LLM routers are represented +only in `metadata.gateways[]`, never as a direct source. + +## metadata.gateways[] + +Byte-identical in shape to `ai-workload-profile.schema.json`'s `gateways[]` (definition +`gateway`). An observed gateway's `upstreams[]` must correspond exactly to the `ai_sources[]` +entries it fronts (same provider/source_service/model_families/model_evidence/models +association #410's gateway-correspondence rule enforces); an unresolved gateway's `upstreams[]` +is explicitly empty. + +```jsonc +{ + "product": "openrouter", // gatewayIdentity — never "both" | "unknown" + "type": "llm_router", + "upstream_evidence": "observed", // observed | unresolved + "upstreams": [] // gatewayUpstream[]; [] iff upstream_evidence == "unresolved" +} +``` + +Both `metadata.ai_sources[]` and `metadata.gateways[]` project **verbatim** from the +`ai-workload-profile.json` the design was built from — the test suite +(`test_aws_design_ai_contracts.py`) proves this projection against #415's own semantic +oracle (`_assert_profile_semantics`), the same way #415 proved its own projection into #410. +A design that drops a model, renames a provider, or collapses two sources from the profile +into one is a contract violation, not a presentation choice. + +## design_blocks[] + +One row per confirmed workload. This is the per-workload target decision. + +```jsonc +{ + "workload_id": "wl_3a1f2c", // from the profile's workloads[]; preserve verbatim + "model_id": "gpt-4o", // the source model / SDK method the workload used + "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", // XOR target_aws_service — see below + "target_aws_service": null, // XOR target_bedrock_model — see below + "capability": "text_generation", // the workload's capability (see § capability vocabulary) + "capability_confidence": "high", // high | medium | low + "rationale": "", + "confidence_warning": null // non-null string when capability_confidence == "low" +} +``` + +- **`target_bedrock_model` XOR `target_aws_service` — exactly one is non-null per row.** + Bedrock-model capabilities (`text_generation`, `structured_output`, `image_generation`, + `embedding`, `speech_to_text`, `text_to_speech`, `unknown`) set `target_bedrock_model` and + leave `target_aws_service` null. The three traditional-AI capabilities + (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` + (one of `"textract"`, `"rekognition"`, `"transcribe"`, `"comprehend"`, `"translate"`, + `"polly"`, `"sagemaker"`) and leave `target_bedrock_model` null. This is the same rule gcp + enforces; it exists because those three are AWS AI services, not Bedrock model swaps. +- **`confidence_warning`** is a non-null string (naming the workload and that manual review is + required) when `capability_confidence == "low"`; null for `high`/`medium`. +- **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. +- **`model_id` provenance:** every `design_blocks[]` row's source model MUST trace to a + `metadata.ai_sources[]` entry (directly, or through a `metadata.gateways[]` upstream) — a + `design_blocks[]` row naming a model no retained source or gateway upstream carries is a + contract violation. + +## capability vocabulary + +Bedrock-model capabilities: `text_generation`, `structured_output`, `image_generation`, +`embedding`, `speech_to_text`, `text_to_speech`, `unknown`. +Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, +`image_analysis`, `speech_transcription`. + +## ai_architecture + +- **`honest_assessment`** — `strong_migrate | moderate_migrate | weak_migrate | + recommend_stay | not_applicable`. Overall value = the weakest across all Bedrock models. + `not_applicable` is a PER-WORKLOAD value for the three traditional-AI capabilities only — + never the overall assessment (they are feature swaps, not a cost-driven model decision). +- **`honest_assessment_reason`** — REQUIRED (non-null) when `honest_assessment == + "recommend_stay"`; names the specific non-cost blocker (fixed region with no suitable model, + a Realtime/streaming dependency, an Assistants-style feature with no Bedrock equivalent, or a + confirmed unsupported API surface). Null otherwise. Cost parity alone is never sufficient to + recommend staying when the source models are on Bedrock. +- **`tiered_strategy`** — object when `ai_token_volume == "high"` (a 3-tier routing plan); + null for low/medium. +- **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, + `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, + `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), + `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities + do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. +- **`capability_mapping`** — per capability that is `true` in the profile's + `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. +- **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), + `framework`, `files_to_modify[]`, `dependency_changes`, and — when a retained source's + `source_service` is `azure_openai` or a `provider`/`source_service` pair resolves to OpenAI — + `openrouter_path` when a router was detected + (`same_model_mantle|direct|litellm|keep_openrouter`). +- **`infrastructure[]`** — source-cloud AI infra resource → AWS equivalent, with confidence. +- **`services_to_migrate[]`** — source-cloud AI service → AWS service, effort, notes. + +## migration_path vocabulary + +`mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model +on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `converse` +(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when a proprietary GPT +model must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported region) · +`gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic +Bedrock SDK swap). A proprietary `openai.gpt-*` model ID is NEVER paired with a +`converse`/`bedrock-runtime` path — those models are mantle-only. + +## regional_validation + +`checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region +availability) · `fallback_static` (the call failed; the static table in +`vendored/ai/ai-migration-guardrails.md` was used). + +## regional_warnings[] + +ALWAYS present (`[]` when clean). Per unavailable service: +`{ service, target_region, nearest_available, impact }`. A compliance constraint that forced a +region (e.g. `fedramp` → GovCloud, `gdpr` → EU `eu.` profiles) records the models it excluded +here. Never blocks the design — it flags the constraint and proceeds. + +## multi_model_warnings[] + +ALWAYS present (`[]` when a single model / no coordination issue). Per warning: +`{ type, message }`, `type ∈ embeddings_reindex | cascade_pair | multi_model_tiered | +image_separate | speech_separate`. + +## agentic_design + +Present ONLY when the profile's `agentic_profile.is_agentic == true`; null/absent otherwise. +`{ migration_approach: "retarget|harness|strands|undecided", ...path-specific config }`. When +`migration_approach == "harness"` carry `harness_config` (with `source_model_provider`, derived +per retained source rather than a single scalar — an OpenAI-protocol source with +`source_service: "azure_openai"` sets `source_model_provider: "open_ai"` — see §19.9(b)); when +`"strands"` carry the AgentCore Runtime config from `vendored/ai/design-ref-agentic-to-agentcore.md`. + +## halt + +Present ONLY when the AI design is failing its gate. `{ reason, blocking[] }` where each +`blocking[]` entry is `{ kind, identifier, why, action }`. The AI design halts when a routed +rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` ref) named by +`index.md` is not on disk — the same missing-rubric guard the infra design uses. + +## Validation Checklist + +- [ ] `metadata.ai_sources[]` is non-empty whenever the profile has at least one retained + source, and each entry's `(provider, source_service)` pair is unique — no duplicate or + overlapping source. +- [ ] `metadata.ai_sources[]` and `metadata.gateways[]` project verbatim from the + `ai-workload-profile.json` the design was built from (same provider, source_service, + model_families, model_evidence, models, model_observations, and gateway upstream + correspondence) — no dropped, renamed, or fabricated source or model. +- [ ] No entry in `metadata.ai_sources[]` has `provider: "openrouter"`; OpenRouter and other + routers appear only in `metadata.gateways[]`. +- [ ] Every `workloads[]` entry (or `models[]` when workloads is empty) has exactly one + `design_blocks[]` row, in input order. +- [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / + `target_aws_service` non-null (XOR), and its source model traces to a + `metadata.ai_sources[]` entry or a `metadata.gateways[]` upstream. +- [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) + has `target_bedrock_model: null`, a non-null `target_aws_service`, and (in + `ai_architecture`) is not counted in `bedrock_models[]`. +- [ ] Every `bedrock_models[]` entry has `source_provider_price`, `bedrock_price`, + `price_comparison`. +- [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if + `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. +- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves + to direct OpenAI: every source model that is available on Bedrock and carried by the + target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. +- [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). +- [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with + `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves + to Azure-fronted OpenAI sets `source_model_provider: "open_ai"`. +- [ ] All Bedrock model IDs are Active per `vendored/ai/ai-model-lifecycle.md` (a Legacy ID is + used only if no Active alternative exists, with the EOL date noted). +- [ ] `"App Runner"` appears nowhere in the artifact. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. `design-ai.md` (GCP and Azure) still +writes against the prior Azure-only `metadata.ai_source` scalar documented in each skill's own +`references/shared/schema-design-aws-ai.md` (Azure) / absence of a contract (GCP). Repointing +either skill's `design-ai.md` to emit this shape is deferred runtime-adoption work, named +explicitly so a future PR does not silently assume today's runtime output already satisfies +this contract. `estimate-ai.md` and `generate-artifacts-ai.md` are unchanged and still read +today's artifact shape. diff --git a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py new file mode 100644 index 000000000..b1b13da02 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py @@ -0,0 +1,197 @@ +"""Contract tests for the target-only, canonical aws-design-ai.json contract. + +Pins the schema's self-validity and proves that `metadata.ai_sources[]` / `metadata.gateways[]` +project exactly into #415's `ai-workload-profile.schema.json` shape, reusing #415's own +semantic-assertion helper (`_assert_profile_semantics`) as the oracle — the same way #415 +reused #410's `test_ai_migration_matrix_contracts.py` helpers. Target-only: no producer emits +this shape yet (`design-ai.md` is unchanged in both skills); no runtime behavior is asserted. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "aws-design-ai" +DESIGN_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "aws-design-ai.schema.json" +VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +WORKLOAD_PROFILE_TEST_PATH = Path(__file__).resolve().parent / "test_ai_workload_profile_contracts.py" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +VALIDATOR = _load_module("aws_design_ai__validator", VALIDATOR_PATH) +# #415's own test module: imports/executes it to reuse _assert_profile_semantics as the oracle, +# exactly as #415 reused #410's test_ai_migration_matrix_contracts.py (MATRIX) module. +WORKLOAD_PROFILE = _load_module("aws_design_ai__workload_profile_contracts", WORKLOAD_PROFILE_TEST_PATH) + +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*/aws-design-ai.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _design_ai_schema() -> dict: + return _load(DESIGN_AI_SCHEMA) + + +def _design_ai_findings(design: dict) -> list: + findings = [] + VALIDATOR.validate_json_schema(_design_ai_schema(), design, "", "design", str(DESIGN_AI_SCHEMA), findings) + return findings + + +def _project_to_profile(design: dict) -> dict: + """The deterministic projection: metadata.ai_sources[]/gateways[] already carry the exact + #415-shaped fields (provider, source_service, model_families, model_evidence, models, + model_observations for sources; product/type/upstream_evidence/upstreams for gateways) — + this wraps them in a synthetic ai-workload-profile.json skeleton purely so #415's own + schema/semantics helpers can be reused unmodified.""" + return { + "schema_version": "1.0", + "ai_sources": copy.deepcopy(design["metadata"]["ai_sources"]), + "gateways": copy.deepcopy(design["metadata"]["gateways"]), + } + + +# --------------------------------------------------------------------------- schema self-validity + +def test_schema_has_no_unsupported_keywords() -> None: + assert VALIDATOR.unsupported_keywords(_design_ai_schema()) == [] + + +def test_schema_is_valid_json_and_draft07_shaped() -> None: + schema = _design_ai_schema() + assert schema["$schema"] == "http://json-schema.org/draft-07/schema#" + assert schema["type"] == "object" + + +def test_metadata_rejects_the_retired_single_scalar_ai_source() -> None: + """additionalProperties: false on the metadata object means a vestigial scalar ai_source + field (the retired #386-era shape) is rejected outright, not merely undocumented.""" + schema = _design_ai_schema() + assert schema["definitions"]["metadata"]["additionalProperties"] is False + assert "ai_source" not in schema["definitions"]["metadata"]["properties"] + + +# --------------------------------------------------------------------------- positive fixtures + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_validate_against_design_ai_schema(fixture_path: Path) -> None: + design = _load(fixture_path) + assert _design_ai_findings(design) == [] + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_project_exactly_into_pr415_semantics(fixture_path: Path) -> None: + design = _load(fixture_path) + profile = _project_to_profile(design) + assert WORKLOAD_PROFILE._profile_findings(profile) == [] + WORKLOAD_PROFILE._assert_profile_semantics(profile) + + +def test_multi_provider_fixture_carries_two_simultaneous_direct_sources() -> None: + """Exercises the multi-provider case: one Anthropic-sourced + one OpenAI-sourced + design_block in a single aws-design-ai.json, with no gateway involved.""" + design = _load(FIXTURE_ROOT / "multi-provider-direct" / "aws-design-ai.json") + providers = {s["provider"] for s in design["metadata"]["ai_sources"]} + assert providers == {"anthropic", "openai"} + assert not design["metadata"]["gateways"] + assert len(design["design_blocks"]) == 2 + + +def test_gateway_fixture_carries_multi_upstream_openrouter_fronting_two_providers() -> None: + """Exercises the gateway-fronted multi-upstream case: one OpenRouter gateway fronting + Anthropic (two models) and OpenAI simultaneously.""" + design = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "aws-design-ai.json") + gateways = design["metadata"]["gateways"] + assert len(gateways) == 1 + upstream_providers = {u["provider"] for u in gateways[0]["upstreams"]} + assert upstream_providers == {"anthropic", "openai"} + claude = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") + assert len(claude["models"]) == 2 + assert len(design["design_blocks"]) == 3 + + +# --------------------------------------------------------------------------- negative mutations + +def _base_design() -> dict: + return copy.deepcopy(_load(FIXTURE_ROOT / "multi-provider-direct" / "aws-design-ai.json")) + + +def test_old_single_scalar_ai_source_shape_is_rejected() -> None: + """The retired Azure-only shape: a single scalar metadata.ai_source string standing in + for the multi-provider reality, instead of metadata.ai_sources[]/gateways[]. Proves the + sentinel single-identity pattern is actually retired by this contract, not just + redocumented — directly answering #415's own Cursor-review precedent of locking a negative + case with an assertion rather than a fixture directory.""" + design = _base_design() + del design["metadata"]["ai_sources"] + del design["metadata"]["gateways"] + design["metadata"]["ai_source"] = "both" + assert _design_ai_findings(design) != [] + + +def test_old_single_scalar_ai_source_with_one_provider_is_also_rejected() -> None: + """Even a single-valued old-shape scalar (not just the 'both' sentinel) is rejected, + because additionalProperties: false on metadata means ai_sources[]/gateways[] are + required and ai_source is unknown — the scalar shape cannot coexist with the new one.""" + design = _base_design() + del design["metadata"]["ai_sources"] + del design["metadata"]["gateways"] + design["metadata"]["ai_source"] = "anthropic" + assert _design_ai_findings(design) != [] + + +def test_duplicate_ai_source_identity_is_rejected() -> None: + design = _base_design() + design["metadata"]["ai_sources"].append(copy.deepcopy(design["metadata"]["ai_sources"][0])) + assert _design_ai_findings(design) != [] + + +def test_openrouter_as_a_direct_provider_is_rejected() -> None: + design = _base_design() + design["metadata"]["ai_sources"][0]["provider"] = "openrouter" + assert _design_ai_findings(design) != [] + + +def test_dropped_model_under_a_resolved_gateway_upstream_is_rejected() -> None: + design = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "aws-design-ai.json") + profile = _project_to_profile(design) + upstream = next( + u for u in profile["gateways"][0]["upstreams"] if u["provider"] == "anthropic" + ) + upstream["model_families"][0]["models"].remove("claude-3-haiku-20240307") + with pytest.raises(AssertionError): + WORKLOAD_PROFILE.MATRIX._assert_scenario_semantics( + WORKLOAD_PROFILE._project_to_scenario(profile) + ) + + +def test_design_block_without_exactly_one_target_is_rejected() -> None: + design = _base_design() + design["design_blocks"][0]["target_bedrock_model"] = None + assert _design_ai_findings(design) != [] + + design = _base_design() + design["design_blocks"][0]["target_aws_service"] = "textract" + assert _design_ai_findings(design) != [] + + +def test_fabricated_unknown_model_in_metadata_is_rejected() -> None: + design = _base_design() + design["metadata"]["ai_sources"][0]["models"] = ["unknown"] + design["metadata"]["ai_sources"][0]["model_observations"][0]["normalized_model"] = "unknown" + assert _design_ai_findings(design) != [] From 04d0f79bc27627c6c2ef2f34e45da1cca71a6006 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 14:16:32 -0700 Subject: [PATCH 32/53] fix(aws-startup-advisor): address PR #416 Cursor findings on aws-design-ai contract Finding 1 (blocking): restore Azure's aws-design-ai.json emission contract. The manifest had dropped azure-to-aws's shape/root_heading entry outright instead of repointing it, so Azure's current scalar metadata.ai_source output silently failed the shared target-only schema. Restore skills/azure-to-aws/references/shared/schema-design-aws-ai.md to its PR #415 content and restore the azure-to-aws artifact-contracts.json entry so it shadows the shared schema until design-ai.md is repointed. Rewrite the canonical-status section in skills/shared/ai/schema-design-aws-ai.md to name the restored Azure file as the current emission contract and this file as the target. Vendored copies re-synced for lint:vendored-parity. Finding 2 (should-fix): add a regression test proving an observed source with empty model_observations is rejected by the reused #415 _assert_profile_semantics helper, locking the completeness gap the schema itself cannot express without $data. Finding 3 (should-fix): add _assert_model_traceability and tests proving every design_blocks[].model_id traces to a metadata.ai_sources[] entry or a gateway upstream, with a negative case for an untraceable model id. Verified: pytest tests/test_aws_design_ai_contracts.py (22 passed), mise run lint:artifacts (0 errors), mise run lint:vendored-parity, mise run build all green. --- .../scripts/artifact-contracts.json | 4 + .../references/shared/schema-design-aws-ai.md | 205 ++++++++++++++++-- .../vendored/ai/schema-design-aws-ai.md | 23 +- .../vendored/ai/schema-design-aws-ai.md | 23 +- .../skills/shared/ai/schema-design-aws-ai.md | 23 +- .../tests/test_aws_design_ai_contracts.py | 108 +++++++++ 6 files changed, 349 insertions(+), 37 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index c114eeade..af8823c4c 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -260,6 +260,10 @@ } ] }, + "aws-design-ai.json": { + "shape": "skills/azure-to-aws/references/shared/schema-design-aws-ai.md", + "root_heading": "Shape" + }, "estimation-infra.json": { "json_schema": "skills/shared/estimate/estimation-infra.schema.json" }, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index f011aa426..c250c7a16 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -1,16 +1,189 @@ -# Schema — aws-design-ai.json (moved — see the canonical contract) - -> **This file has been canonicalized.** The `aws-design-ai.json` contract that used to live -> here — including its single-scalar `metadata.ai_source` MUST-equal-`summary.ai_source` -> rule — has moved to `plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md`, -> which replaces that scalar with many-valued `metadata.ai_sources[]` / `metadata.gateways[]` -> mirroring `ai-workload-profile.schema.json` (#415) exactly. This skill reads the canonical -> contract through its own byte-identical vendored copy at -> `references/vendored/ai/schema-design-aws-ai.md` (see that directory's README for the -> vendored-parity gate, `mise run lint:vendored-parity`). This file is kept only as a redirect -> so a reference to this path does not 404; do not restate the contract here — edit the -> canonical source instead. -> -> `scripts/artifact-contracts.json`'s `aws-design-ai.json` entry for `azure-to-aws` now points -> at the vendored copy. This change is target-only: `design-ai.md` has not been repointed to -> emit the new shape in this change (deferred runtime-adoption work). +# Schema — aws-design-ai.json + +> **Why this file exists.** `aws-design.json` had no schema once and a capability run +> reverse-engineered its shape from postconditions, inventing key names the golden did not +> share (§13.3a). The AI design artifact is the same risk one layer over: `design-ai.md` +> produces it, `estimate-ai.md` and `generate-artifacts-ai.md` consume it, and a prose-only +> contract drifts between the three by construction. This file is the single written contract +> for `aws-design-ai.json`. It is the AI analogue of `schema-design-aws.md`, and it is +> deliberately kept parallel to gcp-to-aws's `aws-design-ai.json` shape so the estimate and +> generate ports stay straight (§19.10). + +`aws-design-ai.json` is written by `phases/design/design-ai.md` **only when +`ai-workload-profile.json` exists**. It is a SEPARATE artifact from `aws-design.json` +(§19.10): the infra assembler still solely owns `aws-design.json`; the AI fragment owns this +file. When there is no AI workload, this file is not produced and nothing downstream looks +for it. + +## Shape + +```jsonc +{ + "phase": "design", + "focus": "ai", + "timestamp": "", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_source": "azure_openai", // MUST equal summary.ai_source from the profile + "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) + "regional_validation": "checked" // checked | fallback_static (see § regional_validation) + }, + "design_blocks": [], // one per workloads[] entry — see § design_blocks + "ai_architecture": { // see § ai_architecture + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [], + "capability_mapping": {}, + "code_migration": {}, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], // ALWAYS present, [] when clean — see § regional_warnings + "multi_model_warnings": [], // ALWAYS present, [] when single model — see § multi_model_warnings + "agentic_design": null, // present ONLY when agentic_profile.is_agentic — see § agentic_design + "halt": {} // present ONLY when the AI design is failing its gate +} +``` + +**Accounting invariant.** Every entry in the profile's `workloads[]` (or `models[]` when +`workloads[]` is empty) appears in exactly one `design_blocks[]` row, in input order. + +## design_blocks[] + +One row per confirmed workload. This is the per-workload target decision. + +```jsonc +{ + "workload_id": "wl_3a1f2c", // from the profile's workloads[]; preserve verbatim + "model_id": "gpt-4o", // the source model / SDK method the workload used + "target_bedrock_model": "anthropic.claude-sonnet-4-5-v1:0", // XOR target_aws_service — see below + "target_aws_service": null, // XOR target_bedrock_model — see below + "capability": "text_generation", // the workload's capability (see § capability vocabulary) + "capability_confidence": "high", // high | medium | low + "rationale": "", + "confidence_warning": null // non-null string when capability_confidence == "low" +} +``` + +- **`target_bedrock_model` XOR `target_aws_service` — exactly one is non-null per row.** + Bedrock-model capabilities (`text_generation`, `structured_output`, `image_generation`, + `embedding`, `speech_to_text`, `text_to_speech`, `unknown`) set `target_bedrock_model` and + leave `target_aws_service` null. The three traditional-AI capabilities + (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` + (one of `"textract"`, `"rekognition"`, `"transcribe"`, `"comprehend"`, `"translate"`, + `"polly"`, `"sagemaker"`) and leave `target_bedrock_model` null. This is the same rule gcp + enforces; it exists because those three are AWS AI services, not Bedrock model swaps. +- **`confidence_warning`** is a non-null string (naming the workload and that manual review is + required) when `capability_confidence == "low"`; null for `high`/`medium`. +- **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. + +## capability vocabulary + +Bedrock-model capabilities: `text_generation`, `structured_output`, `image_generation`, +`embedding`, `speech_to_text`, `text_to_speech`, `unknown`. +Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, +`image_analysis`, `speech_transcription`. + +## ai_architecture + +- **`honest_assessment`** — `strong_migrate | moderate_migrate | weak_migrate | + recommend_stay | not_applicable`. Overall value = the weakest across all Bedrock models. + `not_applicable` is a PER-WORKLOAD value for the three traditional-AI capabilities only — + never the overall assessment (they are feature swaps, not a cost-driven model decision). +- **`honest_assessment_reason`** — REQUIRED (non-null) when `honest_assessment == + "recommend_stay"`; names the specific non-cost blocker (fixed region with no suitable model, + a Realtime/streaming dependency, an Assistants-style feature with no Bedrock equivalent, or a + confirmed unsupported API surface). Null otherwise. Cost parity alone is never sufficient to + recommend staying when the source models are on Bedrock. +- **`tiered_strategy`** — object when `ai_token_volume == "high"` (a 3-tier routing plan); + null for low/medium. +- **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, + `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, + `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), + `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities + do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. +- **`capability_mapping`** — per capability that is `true` in the profile's + `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. +- **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), + `framework`, `files_to_modify[]`, `dependency_changes`, and — when the source is Azure + OpenAI / OpenAI — `openrouter_path` when a router was detected + (`same_model_mantle|direct|litellm|keep_openrouter`). +- **`infrastructure[]`** — Azure AI infra resource → AWS equivalent, with confidence. +- **`services_to_migrate[]`** — Azure AI service → AWS service, effort, notes. + +## migration_path vocabulary + +`mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, region carries it — +keep the SDK, `model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; +set `model_change: true` when a proprietary GPT model must move off mantle for Guardrails / +Knowledge Bases / logging / an unsupported region) · `gpt-oss` (OpenAI-lineage model on the +Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). A proprietary +`openai.gpt-*` model ID is NEVER paired with a `converse`/`bedrock-runtime` path — those models +are mantle-only. + +## regional_validation + +`checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region +availability) · `fallback_static` (the call failed; the static table in +`vendored/ai/ai-migration-guardrails.md` was used). + +## regional_warnings[] + +ALWAYS present (`[]` when clean). Per unavailable service: +`{ service, target_region, nearest_available, impact }`. A compliance constraint that forced a +region (e.g. `fedramp` → GovCloud, `gdpr` → EU `eu.` profiles) records the models it excluded +here. Never blocks the design — it flags the constraint and proceeds. + +## multi_model_warnings[] + +ALWAYS present (`[]` when a single model / no coordination issue). Per warning: +`{ type, message }`, `type ∈ embeddings_reindex | cascade_pair | multi_model_tiered | +image_separate | speech_separate`. + +## agentic_design + +Present ONLY when the profile's `agentic_profile.is_agentic == true`; null/absent otherwise. +`{ migration_approach: "retarget|harness|strands|undecided", ...path-specific config }`. When +`migration_approach == "harness"` carry `harness_config` (with `source_model_provider`, which +is `"open_ai"` for an `azure_openai` source — see §19.9(b)); when `"strands"` carry the +AgentCore Runtime config from `vendored/ai/design-ref-agentic-to-agentcore.md`. + +## halt + +Present ONLY when the AI design is failing its gate. `{ reason, blocking[] }` where each +`blocking[]` entry is `{ kind, identifier, why, action }`. The AI design halts when a routed +rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) named by +`index.md` is not on disk — the same missing-rubric guard the infra design uses. + +## Validation Checklist + +- [ ] `metadata.ai_source` equals `summary.ai_source` from `ai-workload-profile.json`. +- [ ] Every `workloads[]` entry (or `models[]` when workloads is empty) has exactly one + `design_blocks[]` row, in input order. +- [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / + `target_aws_service` non-null (XOR). +- [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) + has `target_bedrock_model: null`, a non-null `target_aws_service`, and (in + `ai_architecture`) is not counted in `bedrock_models[]`. +- [ ] Every `bedrock_models[]` entry has `source_provider_price`, `bedrock_price`, + `price_comparison`. +- [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if + `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. +- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on + Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not + to a Claude/Nova substitute. +- [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). +- [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with + `migration_approach` matching `preferences.json`; a `harness` source with an + `azure_openai` origin sets `source_model_provider: "open_ai"`. +- [ ] All Bedrock model IDs are Active per `vendored/ai/ai-model-lifecycle.md` (a Legacy ID is + used only if no Active alternative exists, with the EOL date noted). +- [ ] `"App Runner"` appears nowhere in the artifact. + +## Status — build step 2 (AI design route) + +Written alongside `design-ai.md`. `estimate-ai.md` and `generate-artifacts-ai.md` +read this contract. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 4c9f3a4fd..f3f4005ce 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -266,10 +266,19 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` ## Status — target-only, no producer yet -This canonical contract has no producer in this change. `design-ai.md` (GCP and Azure) still -writes against the prior Azure-only `metadata.ai_source` scalar documented in each skill's own -`references/shared/schema-design-aws-ai.md` (Azure) / absence of a contract (GCP). Repointing -either skill's `design-ai.md` to emit this shape is deferred runtime-adoption work, named -explicitly so a future PR does not silently assume today's runtime output already satisfies -this contract. `estimate-ai.md` and `generate-artifacts-ai.md` are unchanged and still read -today's artifact shape. +This canonical contract has no producer in this change. The CURRENT emission contract for +Azure's `aws-design-ai.json` is still +`skills/azure-to-aws/references/shared/schema-design-aws-ai.md` — the single-scalar +`metadata.ai_source` shape (MUST equal `summary.ai_source` from the profile) that `design.md`, +`design-ai.md`, and `estimate-ai.md` read and assert today. `scripts/artifact-contracts.json`'s +`azure-to-aws` entry for `aws-design-ai.json` shadows the shared `json_schema` entry with that +file's `Shape` block for exactly this reason: Azure's actual runtime output has not changed. +GCP has no `aws-design-ai.json` contract today, so GCP's artifacts (if any) validate directly +against this canonical, target-only schema. + +This document (and its companion JSON Schema, `scripts/contracts/aws-design-ai.schema.json`) +is the TARGET contract — the many-valued `ai_sources[]`/`gateways[]` shape neither skill's +`design-ai.md` emits yet. Repointing either skill's `design-ai.md` to emit this shape is +deferred runtime-adoption work, named explicitly so a future PR does not silently assume +today's runtime output already satisfies this contract. `estimate-ai.md` and +`generate-artifacts-ai.md` are unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 4c9f3a4fd..f3f4005ce 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -266,10 +266,19 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` ## Status — target-only, no producer yet -This canonical contract has no producer in this change. `design-ai.md` (GCP and Azure) still -writes against the prior Azure-only `metadata.ai_source` scalar documented in each skill's own -`references/shared/schema-design-aws-ai.md` (Azure) / absence of a contract (GCP). Repointing -either skill's `design-ai.md` to emit this shape is deferred runtime-adoption work, named -explicitly so a future PR does not silently assume today's runtime output already satisfies -this contract. `estimate-ai.md` and `generate-artifacts-ai.md` are unchanged and still read -today's artifact shape. +This canonical contract has no producer in this change. The CURRENT emission contract for +Azure's `aws-design-ai.json` is still +`skills/azure-to-aws/references/shared/schema-design-aws-ai.md` — the single-scalar +`metadata.ai_source` shape (MUST equal `summary.ai_source` from the profile) that `design.md`, +`design-ai.md`, and `estimate-ai.md` read and assert today. `scripts/artifact-contracts.json`'s +`azure-to-aws` entry for `aws-design-ai.json` shadows the shared `json_schema` entry with that +file's `Shape` block for exactly this reason: Azure's actual runtime output has not changed. +GCP has no `aws-design-ai.json` contract today, so GCP's artifacts (if any) validate directly +against this canonical, target-only schema. + +This document (and its companion JSON Schema, `scripts/contracts/aws-design-ai.schema.json`) +is the TARGET contract — the many-valued `ai_sources[]`/`gateways[]` shape neither skill's +`design-ai.md` emits yet. Repointing either skill's `design-ai.md` to emit this shape is +deferred runtime-adoption work, named explicitly so a future PR does not silently assume +today's runtime output already satisfies this contract. `estimate-ai.md` and +`generate-artifacts-ai.md` are unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 4c9f3a4fd..f3f4005ce 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -266,10 +266,19 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` ## Status — target-only, no producer yet -This canonical contract has no producer in this change. `design-ai.md` (GCP and Azure) still -writes against the prior Azure-only `metadata.ai_source` scalar documented in each skill's own -`references/shared/schema-design-aws-ai.md` (Azure) / absence of a contract (GCP). Repointing -either skill's `design-ai.md` to emit this shape is deferred runtime-adoption work, named -explicitly so a future PR does not silently assume today's runtime output already satisfies -this contract. `estimate-ai.md` and `generate-artifacts-ai.md` are unchanged and still read -today's artifact shape. +This canonical contract has no producer in this change. The CURRENT emission contract for +Azure's `aws-design-ai.json` is still +`skills/azure-to-aws/references/shared/schema-design-aws-ai.md` — the single-scalar +`metadata.ai_source` shape (MUST equal `summary.ai_source` from the profile) that `design.md`, +`design-ai.md`, and `estimate-ai.md` read and assert today. `scripts/artifact-contracts.json`'s +`azure-to-aws` entry for `aws-design-ai.json` shadows the shared `json_schema` entry with that +file's `Shape` block for exactly this reason: Azure's actual runtime output has not changed. +GCP has no `aws-design-ai.json` contract today, so GCP's artifacts (if any) validate directly +against this canonical, target-only schema. + +This document (and its companion JSON Schema, `scripts/contracts/aws-design-ai.schema.json`) +is the TARGET contract — the many-valued `ai_sources[]`/`gateways[]` shape neither skill's +`design-ai.md` emits yet. Repointing either skill's `design-ai.md` to emit this shape is +deferred runtime-adoption work, named explicitly so a future PR does not silently assume +today's runtime output already satisfies this contract. `estimate-ai.md` and +`generate-artifacts-ai.md` are unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py index b1b13da02..1b86e9e0a 100644 --- a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py @@ -20,6 +20,7 @@ FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "aws-design-ai" DESIGN_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "aws-design-ai.schema.json" VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +MANIFEST_PATH = PLUGIN_ROOT / "scripts" / "artifact-contracts.json" WORKLOAD_PROFILE_TEST_PATH = Path(__file__).resolve().parent / "test_ai_workload_profile_contracts.py" @@ -195,3 +196,110 @@ def test_fabricated_unknown_model_in_metadata_is_rejected() -> None: design["metadata"]["ai_sources"][0]["models"] = ["unknown"] design["metadata"]["ai_sources"][0]["model_observations"][0]["normalized_model"] = "unknown" assert _design_ai_findings(design) != [] + + +def test_observed_source_with_empty_model_observations_is_rejected_by_profile_semantics() -> None: + """The schema's observed branch (model_evidence: 'observed') only requires models[] + non-empty; it does not require model_observations[] non-empty (Draft-07 cannot express + the models[]/model_observations[] correspondence without $data). The projected-profile + semantic oracle (#415's _assert_profile_semantics, reused unmodified) already closes + this gap, so this is the lock: clearing model_observations on an observed source must + still pass the schema but fail the semantic oracle.""" + design = _base_design() + anthropic = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") + anthropic["model_observations"] = [] + assert _design_ai_findings(design) == [] # schema alone accepts this — expected, not the lock + profile = _project_to_profile(design) + with pytest.raises(AssertionError): + WORKLOAD_PROFILE._assert_profile_semantics(profile) + + +# --------------------------------------------------------------------------- model traceability + + +def _assert_model_traceability(design: dict) -> None: + """Every design_blocks[].model_id MUST trace to a metadata.ai_sources[] entry (directly) + or a metadata.gateways[].upstreams[] entry (through a gateway upstream model_family) — + schema-design-aws-ai.md's model_id provenance rule. Draft-07 cannot express this + cross-array correspondence without $data, so this helper is the lock, mirroring the + pattern _assert_profile_semantics already established for ai_sources[]/gateways[] + internal consistency.""" + reachable_models: set[str] = set() + for source in design["metadata"]["ai_sources"]: + reachable_models.update(source["models"]) + for gateway in design["metadata"]["gateways"]: + for upstream in gateway["upstreams"]: + for family in upstream["model_families"]: + reachable_models.update(family["models"]) + for block in design["design_blocks"]: + assert block["model_id"] in reachable_models, ( + f"design_blocks[] model_id {block['model_id']!r} traces to no " + f"metadata.ai_sources[] entry or metadata.gateways[] upstream" + ) + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_have_traceable_design_block_models(fixture_path: Path) -> None: + design = _load(fixture_path) + _assert_model_traceability(design) + + +def test_design_block_naming_an_untraceable_model_is_rejected() -> None: + design = _base_design() + design["design_blocks"][0]["model_id"] = "made-up-model" + with pytest.raises(AssertionError): + _assert_model_traceability(design) + + +# --------------------------------------------------------------------------- manifest shadowing (Azure) + + +def _azure_scalar_design() -> dict: + """A document matching Azure's OLD single-scalar metadata.ai_source shape — the shape + Azure's design-ai.md actually emits today (schema-design-aws-ai.md, restored from #415's + head). Everything else matches the canonical top-level shape so only `metadata` differs.""" + design = _base_design() + design["metadata"] = { + "ai_source": "azure_openai", + "bedrock_models_selected": design["metadata"]["bedrock_models_selected"], + "regional_validation": design["metadata"]["regional_validation"], + } + return design + + +def test_azure_scalar_ai_source_validates_under_the_restored_azure_shape_contract() -> None: + """Validated AS skill=azure-to-aws, the scalar document follows the restored Shape + contract at skills/azure-to-aws/references/shared/schema-design-aws-ai.md and is + accepted — Azure's own manifest entry shadows the shared target-only schema.""" + manifest = VALIDATOR.load_manifest(MANIFEST_PATH) + findings: list = [] + contracts = VALIDATOR.build_contracts(manifest, "azure-to-aws", findings) + matched = [c for c in contracts if c.artifact_glob == "aws-design-ai.json"] + assert len(matched) == 1 + assert matched[0].shape is not None, ( + "azure-to-aws must shadow the shared aws-design-ai.json schema with its own Shape " + "contract, or Azure's actual scalar emission is validated against the target-only " + "schema and rejected" + ) + design = _azure_scalar_design() + design_findings: list = [] + VALIDATOR.validate_shape(matched[0].shape.root, design, "", "design", matched[0].shape.label, design_findings) + assert design_findings == [] + + +def test_azure_scalar_ai_source_fails_the_target_only_schema_with_no_skill() -> None: + """The SAME scalar document, validated with no skill (shared contracts only), fails the + target schema: additionalProperties:false on metadata plus missing + ai_sources/gateways/regional_validation-shaped required fields.""" + manifest = VALIDATOR.load_manifest(MANIFEST_PATH) + findings: list = [] + contracts = VALIDATOR.build_contracts(manifest, None, findings) + matched = [c for c in contracts if c.artifact_glob == "aws-design-ai.json"] + assert len(matched) == 1 + assert matched[0].json_schema is not None + design = _azure_scalar_design() + design_findings: list = [] + VALIDATOR.validate_json_schema( + matched[0].json_schema, design, "", "design", matched[0].json_schema_path, design_findings + ) + assert design_findings != [] From dbb2be6af05f3dc07e891ae0da1250cf9d5c90e1 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 14:56:32 -0700 Subject: [PATCH 33/53] fix(aws-startup-advisor): close GCP shadow gap, model_observations gap, gateway traceability test gap - Add gcp-to-aws manifest shadow entry for aws-design-ai.json mirroring azure-to-aws's pattern, with a new schema-design-aws-ai.md documenting GCP's current scalar emission; add the matching accept/reject test pair. - Require model_observations minItems:1 on the aiSource observed branch in aws-design-ai.schema.json, closing the schema-level gap the semantic oracle alone used to cover; update the existing test to assert the schema now rejects the mutation directly. - Add two tests exercising the gateway-upstream-walk branch of _assert_model_traceability (previously unexercised by any positive fixture). --- .../scripts/artifact-contracts.json | 4 + .../contracts/aws-design-ai.schema.json | 2 +- .../references/shared/schema-design-aws-ai.md | 88 +++++++++++++++ .../skills/shared/ai/schema-design-aws-ai.md | 12 +- .../tests/test_aws_design_ai_contracts.py | 106 ++++++++++++++++-- 5 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index af8823c4c..62239448a 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -60,6 +60,10 @@ "shape": "skills/gcp-to-aws/references/shared/schema-discover-ai.md", "root_heading": "ai-workload-profile.json" }, + "aws-design-ai.json": { + "shape": "skills/gcp-to-aws/references/shared/schema-design-aws-ai.md", + "root_heading": "Shape" + }, "preferences.json": { "shape": "skills/gcp-to-aws/references/shared/schema-preferences.md", "root_heading": "Top-level shape", diff --git a/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json index 2f7a643dd..0a1e84ac6 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/aws-design-ai.schema.json @@ -82,7 +82,7 @@ } }, "oneOf": [ - {"properties": {"model_evidence": {"const": "observed"}, "models": {"minItems": 1}}}, + {"properties": {"model_evidence": {"const": "observed"}, "models": {"minItems": 1}, "model_observations": {"minItems": 1}}}, {"properties": {"model_evidence": {"const": "not_observed"}, "models": {"maxItems": 0}, "model_observations": {"maxItems": 0}}} ] }, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md new file mode 100644 index 000000000..0651025d0 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -0,0 +1,88 @@ +# Schema — aws-design-ai.json (GCP) + +> **Why this file exists.** `design-ai.md` produces `aws-design-ai.json`, and +> `estimate-ai.md` consumes it, so a prose-only contract drifts between the two by +> construction. This file is the single written contract for GCP's CURRENT +> `aws-design-ai.json` emission — the single-scalar `metadata.ai_source` shape +> `design-ai.md` writes today, not the shared `ai_sources[]`/`gateways[]` target shape in +> `skills/shared/ai/schema-design-aws-ai.md`. It is kept parallel to Azure's own shape doc +> at `skills/azure-to-aws/references/shared/schema-design-aws-ai.md` for the same reason: +> `scripts/artifact-contracts.json`'s `skills.gcp-to-aws.aws-design-ai.json` entry shadows +> the shared target-only schema with this file until `design-ai.md` is repointed. + +`aws-design-ai.json` is written by `phases/design/design-ai.md` **only when +`ai-workload-profile.json` exists**. + +## Shape + +```jsonc +{ + "phase": "design", + "focus": "ai", + "timestamp": "", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_source": "gemini", // "gemini" | "openai" | "anthropic" | "both" | "other" — summary.ai_source from the profile + "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) + "regional_validation": "checked" // checked | fallback_static + }, + "design_blocks": [], // one per workloads[] entry — see § design_blocks + "ai_architecture": {}, // see design-ai.md Part 6 field table + "regional_warnings": [], // ALWAYS present, [] when clean + "multi_model_warnings": [], // ALWAYS present, [] when single model + "agentic_design": null // present ONLY when agentic_profile.is_agentic +} +``` + +## design_blocks[] + +One row per confirmed workload — see `design-ai.md` Part 6 field table. + +```jsonc +{ + "workload_id": "wl_3a1f2c", // from the profile's workloads[]; preserve verbatim + "model_id": "gemini-2.5-flash", // the source model the workload used + "target_bedrock_model": "amazon.nova-lite-v1:0", // XOR target_aws_service + "target_aws_service": null, // XOR target_bedrock_model + "capability": "text_generation", // see § capability vocabulary + "capability_confidence": "medium", // high | medium | low + "rationale": "", + "confidence_warning": null // non-null string when capability_confidence == "low" +} +``` + +- **`target_bedrock_model` XOR `target_aws_service` — exactly one is non-null per row.** + Bedrock-model capabilities (`text_generation`, `structured_output`, `image_generation`, + `embedding`, `speech_to_text`, `text_to_speech`, `unknown`) set `target_bedrock_model` and + leave `target_aws_service` null. The three traditional-AI capabilities + (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` + (one of `"textract"`, `"rekognition"`, `"transcribe"`) and leave `target_bedrock_model` null. +- **`confidence_warning`** is a non-null string when `capability_confidence == "low"`; null + for `high`/`medium`. +- **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. + +## capability vocabulary + +Bedrock-model capabilities: `text_generation`, `structured_output`, `image_generation`, +`embedding`, `speech_to_text`, `text_to_speech`, `unknown`. +Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, +`image_analysis`, `speech_transcription`. + +## Validation Checklist + +- [ ] `metadata.ai_source` matches `summary.ai_source` from `ai-workload-profile.json`. +- [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / + `target_aws_service` non-null (XOR). +- [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) + has `target_bedrock_model: null` and a non-null `target_aws_service`. +- [ ] `confidence_warning` is non-null only when `capability_confidence == "low"`. + +## Status — GCP's current emission contract + +This file documents GCP's CURRENT scalar `aws-design-ai.json` emission as-is, the same role +`skills/azure-to-aws/references/shared/schema-design-aws-ai.md` plays for Azure. +`scripts/artifact-contracts.json`'s `skills.gcp-to-aws.aws-design-ai.json` entry shadows the +shared `json_schema` entry with this file's `Shape` block for exactly this reason: GCP's +actual runtime output has not changed. The shared, many-valued `ai_sources[]`/`gateways[]` +contract at `skills/shared/ai/schema-design-aws-ai.md` is the TARGET shape — repointing +`design-ai.md` to emit it is deferred runtime-adoption work, not done by this file. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index f3f4005ce..cdf8ced55 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -85,6 +85,10 @@ definition `aiSource`). Each entry: } ``` +The JSON Schema (`scripts/contracts/aws-design-ai.schema.json`) enforces `model_observations` +non-empty whenever `model_evidence == "observed"`, in addition to `models` non-empty — an +observed source can no longer carry an empty `model_observations[]`. + Semantic uniqueness is the `(provider, source_service)` pair; a design can carry multiple simultaneous sources (one Anthropic-sourced workload and one OpenAI-sourced workload in the same `aws-design-ai.json` is valid and expected — this is the shape change this canonicalization @@ -273,8 +277,12 @@ Azure's `aws-design-ai.json` is still `design-ai.md`, and `estimate-ai.md` read and assert today. `scripts/artifact-contracts.json`'s `azure-to-aws` entry for `aws-design-ai.json` shadows the shared `json_schema` entry with that file's `Shape` block for exactly this reason: Azure's actual runtime output has not changed. -GCP has no `aws-design-ai.json` contract today, so GCP's artifacts (if any) validate directly -against this canonical, target-only schema. +GCP now has its own shadow entry mirroring Azure's: `scripts/artifact-contracts.json`'s +`gcp-to-aws` entry for `aws-design-ai.json` points at +`skills/gcp-to-aws/references/shared/schema-design-aws-ai.md`, which names GCP's current +single-scalar `metadata.ai_source` emission (the shape `design-ai.md` writes today) as the +emission contract, with this file as the target. GCP's actual runtime output has not changed +either. This document (and its companion JSON Schema, `scripts/contracts/aws-design-ai.schema.json`) is the TARGET contract — the many-valued `ai_sources[]`/`gateways[]` shape neither skill's diff --git a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py index 1b86e9e0a..26ba69757 100644 --- a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py @@ -199,16 +199,17 @@ def test_fabricated_unknown_model_in_metadata_is_rejected() -> None: def test_observed_source_with_empty_model_observations_is_rejected_by_profile_semantics() -> None: - """The schema's observed branch (model_evidence: 'observed') only requires models[] - non-empty; it does not require model_observations[] non-empty (Draft-07 cannot express - the models[]/model_observations[] correspondence without $data). The projected-profile - semantic oracle (#415's _assert_profile_semantics, reused unmodified) already closes - this gap, so this is the lock: clearing model_observations on an observed source must - still pass the schema but fail the semantic oracle.""" + """The schema's observed branch (model_evidence: 'observed') now requires + model_observations[] non-empty, in addition to models[] non-empty, so clearing + model_observations on an observed source is rejected directly by the schema. The + projected-profile semantic oracle (#415's _assert_profile_semantics, reused unmodified) + remains as defense-in-depth for the exact-correspondence check (models[] keys == + model_observations[] keys) that minItems alone cannot express (Draft-07 cannot express + cross-array correspondence without $data).""" design = _base_design() anthropic = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") anthropic["model_observations"] = [] - assert _design_ai_findings(design) == [] # schema alone accepts this — expected, not the lock + assert _design_ai_findings(design) != [] # schema now rejects this directly profile = _project_to_profile(design) with pytest.raises(AssertionError): WORKLOAD_PROFILE._assert_profile_semantics(profile) @@ -251,6 +252,42 @@ def test_design_block_naming_an_untraceable_model_is_rejected() -> None: _assert_model_traceability(design) +def test_model_reachable_only_through_gateway_upstream_passes_traceability() -> None: + """claude-3-haiku-20240307 is removed from the Anthropic ai_sources[] entry's models + (and its matching model_observations entry, to keep the source internally consistent) + but remains on the gateway upstream's model_families[].models — proving the + gateway-upstream-walk branch of _assert_model_traceability, not just the direct + ai_sources[] branch, is what makes the design_blocks[] row naming it traceable.""" + design = copy.deepcopy(_load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "aws-design-ai.json")) + anthropic = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") + anthropic["models"].remove("claude-3-haiku-20240307") + anthropic["model_observations"] = [ + obs for obs in anthropic["model_observations"] if obs["normalized_model"] != "claude-3-haiku-20240307" + ] + _assert_model_traceability(design) # does NOT raise + + +def test_model_removed_from_both_source_and_gateway_upstream_fails_traceability() -> None: + """Starting from the same base as the gateway-upstream-only case, also drop + claude-3-haiku-20240307 from the gateway upstream's model_families[].models — now no + ai_sources[] entry or gateway upstream carries it, so the design_blocks[] row naming it + must be rejected.""" + design = copy.deepcopy(_load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "aws-design-ai.json")) + anthropic = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") + anthropic["models"].remove("claude-3-haiku-20240307") + anthropic["model_observations"] = [ + obs for obs in anthropic["model_observations"] if obs["normalized_model"] != "claude-3-haiku-20240307" + ] + for gateway in design["metadata"]["gateways"]: + for upstream in gateway["upstreams"]: + if upstream["provider"] == "anthropic": + for family in upstream["model_families"]: + if "claude-3-haiku-20240307" in family["models"]: + family["models"].remove("claude-3-haiku-20240307") + with pytest.raises(AssertionError): + _assert_model_traceability(design) + + # --------------------------------------------------------------------------- manifest shadowing (Azure) @@ -303,3 +340,58 @@ def test_azure_scalar_ai_source_fails_the_target_only_schema_with_no_skill() -> matched[0].json_schema, design, "", "design", matched[0].json_schema_path, design_findings ) assert design_findings != [] + + +# --------------------------------------------------------------------------- manifest shadowing (GCP) + + +def _gcp_scalar_design() -> dict: + """A document matching GCP's CURRENT single-scalar metadata.ai_source shape — the shape + GCP's design-ai.md actually emits today (skills/gcp-to-aws/references/shared/ + schema-design-aws-ai.md). Everything else matches the canonical top-level shape so only + `metadata` differs.""" + design = _base_design() + design["metadata"] = { + "ai_source": "gemini", + "bedrock_models_selected": design["metadata"]["bedrock_models_selected"], + "regional_validation": design["metadata"]["regional_validation"], + } + return design + + +def test_gcp_scalar_ai_source_validates_under_the_gcp_shape_contract() -> None: + """Validated AS skill=gcp-to-aws, the scalar document follows the GCP Shape contract at + skills/gcp-to-aws/references/shared/schema-design-aws-ai.md and is accepted — GCP's own + manifest entry shadows the shared target-only schema, mirroring Azure's entry.""" + manifest = VALIDATOR.load_manifest(MANIFEST_PATH) + findings: list = [] + contracts = VALIDATOR.build_contracts(manifest, "gcp-to-aws", findings) + matched = [c for c in contracts if c.artifact_glob == "aws-design-ai.json"] + assert len(matched) == 1 + assert matched[0].shape is not None, ( + "gcp-to-aws must shadow the shared aws-design-ai.json schema with its own Shape " + "contract, or GCP's actual scalar emission is validated against the target-only " + "schema and rejected" + ) + design = _gcp_scalar_design() + design_findings: list = [] + VALIDATOR.validate_shape(matched[0].shape.root, design, "", "design", matched[0].shape.label, design_findings) + assert design_findings == [] + + +def test_gcp_scalar_ai_source_fails_the_target_only_schema_with_no_skill() -> None: + """The SAME scalar document, validated with no skill (shared contracts only), fails the + target schema: additionalProperties:false on metadata plus missing + ai_sources/gateways/regional_validation-shaped required fields.""" + manifest = VALIDATOR.load_manifest(MANIFEST_PATH) + findings: list = [] + contracts = VALIDATOR.build_contracts(manifest, None, findings) + matched = [c for c in contracts if c.artifact_glob == "aws-design-ai.json"] + assert len(matched) == 1 + assert matched[0].json_schema is not None + design = _gcp_scalar_design() + design_findings: list = [] + VALIDATOR.validate_json_schema( + matched[0].json_schema, design, "", "design", matched[0].json_schema_path, design_findings + ) + assert design_findings != [] From 968c31aa0cff8f38b5ab1f039481b881aba5f793 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 6 Oct 2026 15:00:52 -0700 Subject: [PATCH 34/53] fix(aws-startup-advisor): require model_observations on observed ai-workload-profile sources --- plugins/aws-startup-advisor/fixtures/README.md | 2 +- .../scripts/contracts/ai-workload-profile.schema.json | 3 ++- .../tests/test_ai_workload_profile_contracts.py | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index a27b50e64..4739f6ee4 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -3,7 +3,7 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. -- `ai-workload-profiles/` — 4 representative target-only `ai-workload-profile.json` observation fixtures (not a Cartesian matrix): AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. +- `ai-workload-profiles/` — 4 representative target-only `ai-workload-profile.json` observation fixtures (not a Cartesian matrix): AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. `ai-workload-profile.schema.json`'s `aiSource` observed branch now also requires `model_observations` non-empty (`minItems: 1`) whenever `model_evidence == "observed"`, matching the identical rule already enforced for `aws-design-ai.schema.json`'s `metadata.ai_sources[]`. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json b/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json index 26ea0394b..df6cda897 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json @@ -84,7 +84,8 @@ { "properties": { "model_evidence": {"const": "observed"}, - "models": {"minItems": 1} + "models": {"minItems": 1}, + "model_observations": {"minItems": 1} } }, { diff --git a/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py index d147490c8..18792054c 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_workload_profile_contracts.py @@ -183,11 +183,13 @@ def _base_profile() -> dict: def test_empty_model_observations_on_an_observed_source_is_rejected() -> None: - """Lock in that _assert_profile_semantics rejects an observed source whose - model_observations has been emptied while models[] and model_evidence stay intact - (the JSON Schema alone cannot express this correspondence).""" + """The schema's observed branch now requires model_observations non-empty (minItems: 1), + so clearing model_observations on an observed source is rejected directly at the schema + level. _assert_profile_semantics remains as defense-in-depth for the exact-set-equality + check (model_observations keys == models[]) that minItems alone cannot express.""" profile = _base_profile() profile["ai_sources"][0]["model_observations"] = [] + assert _profile_findings(profile) != [] with pytest.raises(AssertionError): _assert_profile_semantics(profile) From 4e0c512406984963cd080a76c7e211e5df52255b Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 10:40:44 -0700 Subject: [PATCH 35/53] fix(aws-startup-advisor): sync vendored schema-design-aws-ai.md copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Azure and GCP vendored copies of schema-design-aws-ai.md had drifted from the canonical skills/shared/ai/schema-design-aws-ai.md — missing the PR's own round-2 fixes (the model_observations minItems note and the GCP manifest-shadow paragraph), which caused lint:vendored-parity to fail in CI. Re-ran tools/sync-vendored.py to bring both copies back to byte-identical with the canonical file. Verified: mise run build passes clean (lint:md, lint:manifests, lint:skill-parity, lint:spec, lint:skill-refs, lint:vendored-parity, lint:artifacts, gitleaks); pytest on test_aws_design_ai_contracts.py and test_ai_workload_profile_contracts.py — 50 passed. --- .../references/vendored/ai/schema-design-aws-ai.md | 12 ++++++++++-- .../references/vendored/ai/schema-design-aws-ai.md | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index f3f4005ce..cdf8ced55 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -85,6 +85,10 @@ definition `aiSource`). Each entry: } ``` +The JSON Schema (`scripts/contracts/aws-design-ai.schema.json`) enforces `model_observations` +non-empty whenever `model_evidence == "observed"`, in addition to `models` non-empty — an +observed source can no longer carry an empty `model_observations[]`. + Semantic uniqueness is the `(provider, source_service)` pair; a design can carry multiple simultaneous sources (one Anthropic-sourced workload and one OpenAI-sourced workload in the same `aws-design-ai.json` is valid and expected — this is the shape change this canonicalization @@ -273,8 +277,12 @@ Azure's `aws-design-ai.json` is still `design-ai.md`, and `estimate-ai.md` read and assert today. `scripts/artifact-contracts.json`'s `azure-to-aws` entry for `aws-design-ai.json` shadows the shared `json_schema` entry with that file's `Shape` block for exactly this reason: Azure's actual runtime output has not changed. -GCP has no `aws-design-ai.json` contract today, so GCP's artifacts (if any) validate directly -against this canonical, target-only schema. +GCP now has its own shadow entry mirroring Azure's: `scripts/artifact-contracts.json`'s +`gcp-to-aws` entry for `aws-design-ai.json` points at +`skills/gcp-to-aws/references/shared/schema-design-aws-ai.md`, which names GCP's current +single-scalar `metadata.ai_source` emission (the shape `design-ai.md` writes today) as the +emission contract, with this file as the target. GCP's actual runtime output has not changed +either. This document (and its companion JSON Schema, `scripts/contracts/aws-design-ai.schema.json`) is the TARGET contract — the many-valued `ai_sources[]`/`gateways[]` shape neither skill's diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index f3f4005ce..cdf8ced55 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -85,6 +85,10 @@ definition `aiSource`). Each entry: } ``` +The JSON Schema (`scripts/contracts/aws-design-ai.schema.json`) enforces `model_observations` +non-empty whenever `model_evidence == "observed"`, in addition to `models` non-empty — an +observed source can no longer carry an empty `model_observations[]`. + Semantic uniqueness is the `(provider, source_service)` pair; a design can carry multiple simultaneous sources (one Anthropic-sourced workload and one OpenAI-sourced workload in the same `aws-design-ai.json` is valid and expected — this is the shape change this canonicalization @@ -273,8 +277,12 @@ Azure's `aws-design-ai.json` is still `design-ai.md`, and `estimate-ai.md` read and assert today. `scripts/artifact-contracts.json`'s `azure-to-aws` entry for `aws-design-ai.json` shadows the shared `json_schema` entry with that file's `Shape` block for exactly this reason: Azure's actual runtime output has not changed. -GCP has no `aws-design-ai.json` contract today, so GCP's artifacts (if any) validate directly -against this canonical, target-only schema. +GCP now has its own shadow entry mirroring Azure's: `scripts/artifact-contracts.json`'s +`gcp-to-aws` entry for `aws-design-ai.json` points at +`skills/gcp-to-aws/references/shared/schema-design-aws-ai.md`, which names GCP's current +single-scalar `metadata.ai_source` emission (the shape `design-ai.md` writes today) as the +emission contract, with this file as the target. GCP's actual runtime output has not changed +either. This document (and its companion JSON Schema, `scripts/contracts/aws-design-ai.schema.json`) is the TARGET contract — the many-valued `ai_sources[]`/`gateways[]` shape neither skill's From f5a7c5c5e8818adb8eee9538555a6a0166a953d8 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 11:01:41 -0700 Subject: [PATCH 36/53] fix(aws-startup-advisor): add missing canonical source for telemetry vendoring PR #407 (merged to main) vendored identical telemetry/ files into azure-to-aws, gcp-to-aws, heroku-to-aws, and llm-to-bedrock but the lint:vendored-parity gate did not yet exist on main at that time, so no skills/shared/telemetry/ canonical source or README rows were ever added. Merging upstream/main into this branch (which carries the gate from PR #386) now surfaces that pre-existing gap. Add skills/shared/telemetry/ as the canonical source (identical across all four vendored copies, verified via md5sum), add README rows for the eleven telemetry files in the three skills with existing vendored READMEs, and create the vendored README for llm-to-bedrock (which vendors only the telemetry tree and had no README.md at all). Verified: python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check passes (90 files), and mise run build passes clean. --- .../references/vendored/README.md | 11 + .../gcp-to-aws/references/vendored/README.md | 11 + .../references/vendored/README.md | 11 + .../references/vendored/README.md | 35 ++ .../skills/shared/telemetry/PROTOCOL.md | 108 ++++ .../skills/shared/telemetry/cli.py | 45 ++ .../skills/shared/telemetry/consent/accept.py | 60 ++ .../skills/shared/telemetry/consent/cli.py | 87 +++ .../skills/shared/telemetry/consent/notice.py | 58 ++ .../shared/telemetry/consent/opt_out.py | 52 ++ .../skills/shared/telemetry/consent/record.py | 194 ++++++ .../telemetry/metric_emission/client.py | 289 +++++++++ .../telemetry/metric_emission/migration.py | 562 ++++++++++++++++++ .../metric_emission/migration_attributes.py | 525 ++++++++++++++++ .../skills/shared/telemetry/version.json | 3 + 15 files changed, 2051 insertions(+) create mode 100644 plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/PROTOCOL.md create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/cli.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/consent/accept.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/consent/cli.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/consent/notice.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/consent/opt_out.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/consent/record.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration_attributes.py create mode 100644 plugins/aws-startup-advisor/skills/shared/telemetry/version.json diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index 9407b1e9d..1d8fd7d26 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -44,3 +44,14 @@ if this skill needs one, add the copy **and** a row in the table below. | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | | `clarify/clarify-multicloud.md` | `skills/shared/clarify/clarify-multicloud.md` | | `clarify/clarify-region.md` | `skills/shared/clarify/clarify-region.md` | +| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | +| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | +| `telemetry/version.json` | `skills/shared/telemetry/version.json` | +| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | +| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | +| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | +| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | +| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | +| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | +| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | +| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index d10aa5cd5..751e63686 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -37,3 +37,14 @@ if this skill needs one, add the copy **and** a row in the table below. | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | | `clarify/clarify-multicloud.md` | `skills/shared/clarify/clarify-multicloud.md` | | `clarify/clarify-region.md` | `skills/shared/clarify/clarify-region.md` | +| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | +| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | +| `telemetry/version.json` | `skills/shared/telemetry/version.json` | +| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | +| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | +| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | +| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | +| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | +| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | +| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | +| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md index cb53a9dea..af5e05d1e 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md @@ -30,3 +30,14 @@ if this skill needs one, add the copy **and** a row in the table below. | `estimate/ri-sp-eligibility.md` | `skills/shared/estimate/ri-sp-eligibility.md` | | `pricing/aws-infra-pricing.json` | `skills/shared/pricing/aws-infra-pricing.json` | | `workshop/workshop-invariants.md` | `skills/shared/workshop/workshop-invariants.md` | +| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | +| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | +| `telemetry/version.json` | `skills/shared/telemetry/version.json` | +| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | +| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | +| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | +| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | +| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | +| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | +| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | +| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md new file mode 100644 index 000000000..32db7a29a --- /dev/null +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md @@ -0,0 +1,35 @@ +# Vendored shared files — DO NOT EDIT + +These files are **synced copies** of the plugin-level canonical source under +`plugins/aws-startup-advisor/skills/shared/`. They are vendored into this skill +so the skill folder is **self-contained** — it runs standalone (lifted out, zipped, +or used on its own) without reaching outside its own directory. + +**Do not hand-edit anything in this directory.** Edit the canonical source instead, +then bring every vendored copy back in sync: + +```sh +# from the repository root +python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy +python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) +``` + +CI fails when a vendored copy differs from its canonical source, has no canonical source, +is listed in the table below but missing on disk, or is on disk but not listed below. So an +edit to a `skills/shared/` file this skill vendors, or a deleted copy, cannot merge +unnoticed. What CI cannot see is a **new** `skills/shared/` file that no skill vendors yet — +if this skill needs one, add the copy **and** a row in the table below. + +| Vendored path | Canonical source | +| ---------------------------------------------------- | ------------------------------------------------------------------- | +| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | +| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | +| `telemetry/version.json` | `skills/shared/telemetry/version.json` | +| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | +| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | +| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | +| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | +| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | +| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | +| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | +| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/PROTOCOL.md b/plugins/aws-startup-advisor/skills/shared/telemetry/PROTOCOL.md new file mode 100644 index 000000000..bbdd77e85 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/PROTOCOL.md @@ -0,0 +1,108 @@ +# Migration telemetry on CLI hosts + +This protocol applies to Azure, GCP, Heroku, and LLM-to-Bedrock runs under +`.migration/`. It does not extend telemetry to `.agent-advisor/` runs. + +## Resolve the reporting mode + +Before starting or resuming this skill, resolve the absolute directory containing +its `SKILL.md`. The runtime is bundled in that skill's +`references/vendored/telemetry/` directory. Use Python 3.8 or newer (`python3`, +`python`, or `py -3` on Windows). If Python, the bundle, or permission to run the +command is unavailable, skip telemetry and continue the requested work. + +Run the read-only check, substituting the actual absolute skill directory: + +```sh +python3 "/references/vendored/telemetry/cli.py" status +``` + +Use `reportingMode` from its JSON output, not the model's name or the agent's +self-description. The check uses the shared client's host environment markers +and the runtime's install layout. Claude Code and Cursor select `hook` only when +the enclosing plugin has reachable migration-hook configuration and runtime. +A standalone skill without those plugin files selects `cli`, including on +Claude Code and Cursor. Other sources also select `cli`; unknown sources remain +`OTHER`. Explicit Codex/Kiro markers take precedence over an inherited +`CURSOR_AGENT` marker. Do not set or clear host markers to change this result. + +- `hook`: do not invoke CLI reconciliation or duplicate the host's consent + exchange. Existing hooks own reporting. Missing or delayed hook activity does + not change this policy: finding the configured files does not prove that the + host enabled or successfully ran its hooks. +- `cli`: follow the consent exchange below, then reconcile at the state + boundaries. A missing `pluginVersion`, invalid output, or command failure means + telemetry is unavailable; continue the skill without reporting. + +Recheck on resume, since the user may continue in a different host. Do not inspect +snapshot timestamps to select the reporting mode. Use absolute paths in every +tool call, or assign and use shell variables within the same invocation. + +## Reuse the existing consent exchange + +The only consent record is `~/.aws-startup-advisor/plugin-telemetry.json`. +Do not copy old project consent, assume an answer, or edit the record yourself. + +- `consentStatus: "ACCEPTED"`: reporting is allowed. +- `consentStatus: "OPT_OUT"`: do not ask again or report. Continue the skill. +- `consentStatus: null`: if the notice has not already been raised in this session, + run the bundled `consent/cli.py show`. Relay its output VERBATIM, including its + question, and end that turn. Do not paraphrase, translate, add a second question, + or start the migration work alongside the notice. + +Use the actual absolute skill directory: + +```sh +python3 "/references/vendored/telemetry/consent/cli.py" show +``` + +After the user replies, run the bundled `consent/accept.py` only for an explicit +acknowledgement, or `consent/opt_out.py` for an explicit opt-out. Then continue the +original request without asking the user to repeat it. If they ignore the notice +or ask to proceed, record nothing, continue without telemetry, and do not repeat +the notice in this session. A failure to save the choice is not consent and must +not block the migration. A later opt-out uses the same `consent/opt_out.py`. + +For an explicit acknowledgement: + +```sh +python3 "/references/vendored/telemetry/consent/accept.py" +``` + +For an explicit opt-out: + +```sh +python3 "/references/vendored/telemetry/consent/opt_out.py" +``` + +These are the same notice and writers the plugin's hooks use. Telemetry consent +does not authorize cloud discovery, deployment, or other migration actions. +The first unanswered notice deliberately uses one turn. Failures and declines +do not block migration, and an ignored notice is not asked again in that session. + +## Reconcile persisted state + +In `cli` mode, from the project root containing `.migration/`, run: + +```sh +python3 "/references/vendored/telemetry/cli.py" reconcile +``` + +Run this in the main workflow after initialization writes `.phase-status.json`, +after each phase-status update, after a gate failure is recorded and before +returning, and after selecting and validating a run on resume. Include sidebar +updates, decision-only completion, and completion after opted-in Generate. For +LLM-to-Bedrock, also reconcile after creating or resuming its own `.bedrock-*` run +and after writing its completion. + +The emitter derives events and attribution from disk and rechecks consent. +Do not construct HTTP payloads, create new run IDs for reporting, edit snapshots, +or manually retry a failed request. Later boundaries reconcile unreported events. +Preserve the existing one-completion-per-runMode and gate-failure semantics. +The CLI fallback reports migration activity; it does not synthesize skill +invocation events. Reporting failure never changes a phase result or blocks work. + +CLI reporting has no automatic session-end callback. If the session ends between +a state write and reconciliation, the next resume or explicit `reconcile` can +report that saved transition. No final sweep is guaranteed if the run is never +resumed. A separate session-end command would not create a host callback. diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/cli.py b/plugins/aws-startup-advisor/skills/shared/telemetry/cli.py new file mode 100644 index 000000000..dc3cb33e7 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/cli.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Standalone skill entry: read-only status, or consent-gated reconciliation. + + python3 cli.py status + python3 cli.py reconcile + +Status chooses the migration reporting policy from the shared client's host +markers. It does not send events, create consent, or inspect migration artifacts. +The reporting action rechecks the same policy inside the existing emitter. +""" + +import json +import sys +from pathlib import Path + +TELEMETRY = Path(__file__).resolve().parent +sys.path[:0] = [str(TELEMETRY / "consent"), str(TELEMETRY / "metric_emission")] + + +def main(argv): + # Imports can fail when a standalone bundle is incomplete. Keep them inside + # the guarded invocation so optional telemetry still exits without blocking. + import client + import migration + import record + + if argv == ["status"]: + print(json.dumps({ + "source": client.detect_source(), + "reportingMode": client.migration_reporting_mode(), + "consentStatus": record.consent_status(), + "pluginVersion": client.plugin_version(), + })) + return 0 + if argv == ["reconcile"]: + return migration.main(["--reconcile", "--via", "cli"]) + print("usage: cli.py {status|reconcile}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except Exception: + sys.exit(0) # telemetry must not block the skill diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/consent/accept.py b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/accept.py new file mode 100644 index 000000000..26bf180ab --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/accept.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Record that the user ACKNOWLEDGED the usage-data notice. + + python3 scripts/telemetry/consent/accept.py + +Run only after `cli.py show` has printed the notice and the user has acknowledged +it in their own words. Nothing here can verify that, which is why accept and +opt-out are two scripts rather than one script with a flag: the agent's action is +unambiguous in the transcript and a host can permission-gate them separately. + +Writes {"installId": "", "consentStatus": "ACCEPTED"}, minting the install ID +on first write and preserving any existing one. Exits non-zero and explains itself +if the write fails, so a failure is never silently taken as consent. + +Then sends a `consentRecorded` event. The record is written first and is what gates +every future emission, so a failed POST costs one event and nothing else. It cannot +fail this script and is not mentioned in the output — a user who just said yes +learns nothing from being told whether one HTTPS request succeeded. +""" + +import sys +from pathlib import Path + +_TELEMETRY = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(_TELEMETRY / "consent"), str(_TELEMETRY / "metric_emission")] + +import client # noqa: E402 (paths set above so these resolve in-plugin) +import record # noqa: E402 + + +def main(): + try: + written = record.write_state(record.ACCEPTED) + except OSError as exc: + print( + "Could not record your choice at %s: %s\n" + "Nothing was saved, so no usage data will be collected. The notice " + "will be shown again next session." % (record.state_path(), exc), + file=sys.stderr, + ) + return 1 + + client.send_consent_recorded() # fire and forget + + print( + "Acknowledged — usage data collection is ON.\n" + "Recorded at %s (install ID %s).\n" + "\n" + "%s" + % ( + record.state_path(), + written["installId"], + record.opt_out_help(), + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/consent/cli.py b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/cli.py new file mode 100644 index 000000000..36808b564 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/cli.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Print the usage-data notice, or report its status. + + python3 scripts/telemetry/consent/cli.py show # print the notice text + python3 scripts/telemetry/consent/cli.py status # the setting, for a human + +`show` exists so the user reads the notice as *tool output* rather than as model +prose: tool output reaches the terminal byte for byte, and a paraphrased legal +notice is not the notice. It is read-only and mints no install ID — printing the +text is not acknowledgement. Only `accept.py` and `opt_out.py` write. + +`status` always exits 0: "not recorded" is a state, not an error. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import record # noqa: E402 (path set above so this resolves in-plugin) + +USAGE = "usage: cli.py {show|status}" + +CONSENT_DIR = Path(__file__).resolve().parent + + +def status_report(): + """One human-readable paragraph describing the current setting. + + Reads the record and nothing else — there is no environment variable to rank + against it, so what this prints is what the plugin acts on. + """ + state = record.read_state() + status = state["consentStatus"] if state else None + + if status == record.ACCEPTED: + return ( + "Usage data collection: ON. You acknowledged the notice.\n" + "Install ID: %s\n" + "Recorded at: %s\n" + "\n" + "%s" + % ( + state["installId"], + record.state_path(), + record.opt_out_help(), + ) + ) + + if status == record.OPT_OUT: + return ( + "Usage data collection: OFF. You opted out, and nothing is sent.\n" + "Recorded at: %s\n" + 'To turn it back on, run: python3 "%s/accept.py"' + % (record.state_path(), CONSENT_DIR) + ) + + # Also what a corrupt record reports: the gate treats the two identically. + return ( + "Usage data collection: OFF. Nothing is recorded yet, so nothing is " + "sent.\n" + "The notice will be shown at the start of your next session.\n" + "Expected record location: %s" % record.state_path() + ) + + +def main(argv): + if len(argv) != 1 or argv[0] in ("-h", "--help"): + print(USAGE, file=sys.stderr) + return 0 if argv and argv[0] in ("-h", "--help") else 2 + + command = argv[0] + + if command == "show": + print(record.disclaimer(), end="") # already ends with a newline + return 0 + + if command == "status": + print(status_report()) + return 0 + + print("%s\n%s" % ("unknown command: %s" % command, USAGE), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/consent/notice.py b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/notice.py new file mode 100644 index 000000000..4f40de54b --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/notice.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""The approved usage-data notice. Text only — no state, no filesystem, no imports +but textwrap, so legal can review this file without reading any logic. + +Do not rephrase, reorder or trim NOTICE_BODY. Line breaks are the only formatting +liberty, and they are applied at print time rather than written here. +""" + +import textwrap + +NOTICE_TITLE = "AWS Startup Advisor plugin — usage data notice" + +# One unwrapped paragraph; render() wraps it. Do not wrap it here: {opt_out} is +# substituted before wrapping, so a line that fits in this source need not fit in +# the output. +NOTICE_BODY = ( + "You've installed the AWS Startup Advisor plugin, which provides skills that " + "help you build on AWS through your AI coding agent. To help us improve it, " + "we'd like to collect optional usage data about how the plugin is used. What " + "we collect: which skill you invoked, the plugin version, the host tool " + "you're using, a randomly generated install identifier, and, for migration " + "skills, the source cloud provider and which migration phase you start and " + "complete. The usage data above is not linked to your identity. You can " + "opt-out by {opt_out}. Opting out does not change any plugin functionality." +) + +# The approved source ends in "[Acknowledge]", a GUI button label. In a terminal +# that renders as a dead string asking the user nothing, so it becomes a question +# they can answer in words. +ACKNOWLEDGE_PROMPT = ( + 'Do you acknowledge this notice? Reply "yes" to acknowledge, or "opt out" if ' + "you would rather no usage data was collected. Either answer is fine, and " + "nothing is collected until you answer." +) + +# TODO(StartupEngBlend-3621): confirm both substitutions with legal. + +# 79, not 80, so a terminal reserving a cell for the cursor does not re-wrap. +WRAP_WIDTH = 79 + + +def _wrap(text): + # break_* off so a path or variable name is never split across lines. + return textwrap.fill( + text, + width=WRAP_WIDTH, + break_long_words=False, + break_on_hyphens=False, + ) + + +def render(opt_out): + """The notice as the user must see it. + + `opt_out` completes "You can opt-out by ...". Substitute first, then wrap. + """ + body = _wrap(NOTICE_BODY.format(opt_out=opt_out)) + return "%s\n\n%s\n\n%s\n" % (NOTICE_TITLE, body, _wrap(ACKNOWLEDGE_PROMPT)) diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/consent/opt_out.py b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/opt_out.py new file mode 100644 index 000000000..49e29b1d3 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/opt_out.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Opt the user out of usage-data collection. + + python3 scripts/telemetry/consent/opt_out.py + +The command the notice names, and the only way to opt out — there is no +environment variable, because the record file is the single source of truth. +Writes {"installId": "", "consentStatus": "OPT_OUT"}. + +Recording the opt-out is also what stops the session-start hook raising the notice +again, so it has to be written down as deliberately as an acknowledgement. An +install ID is written too, so a later change of heart is not a new install; nothing +emits it, since every telemetry path is gated on `is_accepted()`. + +Sends nothing. `ConsentRecordedDetails` is empty and cannot say which way the user +answered, so a request from here would be indistinguishable from a yes. + +The plugin is fully functional after opting out. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import record # noqa: E402 (path set above so this resolves in-plugin) + + +def main(): + try: + record.write_state(record.OPT_OUT) + except OSError as exc: + print( + "Could not record your choice at %s: %s\n" + "No usage data will be collected either way — the plugin only emits " + "when an acknowledged record exists. But because nothing was saved, " + "the notice will be shown again next session." % (record.state_path(), exc), + file=sys.stderr, + ) + return 1 + + print( + "You are opted out — usage data collection is OFF and no data will be " + "sent.\n" + "Recorded at %s so you are not asked again.\n" + "Every feature of the plugin still works." % record.state_path() + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/consent/record.py b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/record.py new file mode 100644 index 000000000..f4a7ce8b0 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/consent/record.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""The consent record: one reader, one writer, one source of truth. + + ~/.aws-startup-advisor/plugin-telemetry.json + {"installId": "", "consentStatus": "ACCEPTED" | "OPT_OUT"} + +That file is the only thing that decides whether telemetry may be emitted. There +is no environment variable, so a user cannot end up opted out according to their +shell and opted in according to their disk. + +Opting out is a change to that file, and the notice says so in those words, so a +hand-edited record is a supported input and not a corrupt one. `opt_out.py` does +the same edit in one step and keeps the install ID, which a hand-edit cannot. + +Fail-closed on read, fail-loud on write. A missing, unreadable or schema-invalid +record reads as *no consent*, never as acceptance. Writes raise, so `accept.py` +cannot report success it did not achieve. + +`notice.py` owns the wording and knows nothing about any of this; `disclaimer()` +is the seam between the two. +""" + +import json +import os +import re +import sys +import tempfile +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import notice # noqa: E402 (path set above so this resolves in-plugin) + +STATE_DIR_NAME = ".aws-startup-advisor" +STATE_FILE_NAME = "plugin-telemetry.json" + +# Two answers, and only ACCEPTED permits emitting anything. +ACCEPTED = "ACCEPTED" +OPT_OUT = "OPT_OUT" + +VALID_STATUSES = (ACCEPTED, OPT_OUT) + +# The service's UUID shape, from model/types/scalars.smithy. installId is sent +# verbatim as a @required UUID, so a record holding anything else is unusable. +_UUID_PATTERN = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + +# The record as a human should see it: `~`, not an expanded home directory. +DISPLAY_STATE_PATH = "~/%s/%s" % (STATE_DIR_NAME, STATE_FILE_NAME) + +# The opt-out the notice names: a config setting, not a command. It is prose, so +# it wraps; it names no absolute install path; and it needs no Python. +OPT_OUT_INSTRUCTION = 'setting "consentStatus" to "%s" in %s' % ( + OPT_OUT, + DISPLAY_STATE_PATH, +) + +# The one-step equivalent, for a human reading script output rather than the +# notice. It preserves the install ID, which a hand-edit cannot. `py -3` on +# Windows, where python3 is usually not on PATH at all. +OPT_OUT_SCRIPT = Path(__file__).resolve().parent / "opt_out.py" +INTERPRETER = "py -3" if os.name == "nt" else "python3" +OPT_OUT_COMMAND = '%s "%s"' % (INTERPRETER, OPT_OUT_SCRIPT) + + +def disclaimer(): + """The notice, with the opt-out mechanism filled in.""" + return notice.render(opt_out=OPT_OUT_INSTRUCTION) + + +def opt_out_help(): + """The copy-pasteable opt-out, for a human reading script output.""" + return "\n".join( + [ + "To opt out later, run:", + " %s" % OPT_OUT_COMMAND, + 'Or edit %s by hand and set "consentStatus" to "%s".' + % (state_path(), OPT_OUT), + "That file is the only thing that decides whether anything is sent.", + "Opting out disables no plugin features.", + ] + ) + + +def state_dir(): + return Path.home() / STATE_DIR_NAME + + +def state_path(): + return state_dir() / STATE_FILE_NAME + + +def _valid_install_id(value): + return isinstance(value, str) and bool(_UUID_PATTERN.match(value)) + + +def read_state(): + """The record, or None if absent, unparseable, or invalid. + + In order: the file exists, parses as JSON, is an object, and carries an allowed + `consentStatus`. Every caller has to treat "cannot establish consent" and "no + consent" identically, so failure is None rather than an exception. + + `installId` is required only when the status is ACCEPTED, because that is the + only status that sends anything and it is sent verbatim as a @required UUID. + The notice tells the user to opt out by setting `consentStatus` to OPT_OUT in + this file; demanding a UUID they have no way to invent would make that + instruction produce a record we then read as corrupt, and the notice would be + raised at them again next session. + """ + # utf-8-sig, not utf-8: the notice tells the user to edit this file, and a + # Windows editor adds a BOM that json.load rejects. Reading that as "nothing + # recorded" would raise the notice again at someone who just did what it asked. + # It reads a BOM-less file identically, so this costs nothing. + try: + with open(state_path(), "r", encoding="utf-8-sig") as handle: + data = json.load(handle) + except (OSError, ValueError): + return None + + if not isinstance(data, dict): + return None + if data.get("consentStatus") not in VALID_STATUSES: + return None + if data["consentStatus"] == ACCEPTED and not _valid_install_id( + data.get("installId") + ): + return None + return data + + +def consent_status(): + """'ACCEPTED', 'OPT_OUT', or None when nothing is decided.""" + state = read_state() + return state["consentStatus"] if state else None + + +def is_accepted(): + """The single gate every telemetry path must call. + + Deliberately not `!= OPT_OUT`: a missing or corrupt record must block too. + """ + return consent_status() == ACCEPTED + + +def write_state(status): + """Record `status`, return the record written, raise on failure. + + Atomic, so an interrupted write cannot leave a half-written record that + `read_state` rejects and the hook then re-prompts on forever. An existing + `installId` is preserved, so changing the answer is not a new install — but + only if it is a UUID, since a hand-edited record may carry no ID at all or + something the service would reject. + """ + if status not in VALID_STATUSES: + raise ValueError( + "consentStatus must be one of %s, got %r" % (VALID_STATUSES, status) + ) + + existing = read_state() or {} + install_id = existing.get("installId") + if not _valid_install_id(install_id): + install_id = str(uuid.uuid4()) + record = {"installId": install_id, "consentStatus": status} + + directory = state_dir() + directory.mkdir(parents=True, exist_ok=True) + + # Renamed into place from the same directory, so the replace is atomic. + handle = tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=str(directory), + prefix=STATE_FILE_NAME + ".", + suffix=".tmp", + delete=False, + ) + try: + with handle: + json.dump(record, handle, indent=2) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(handle.name, str(state_path())) + except BaseException: + try: + os.unlink(handle.name) + except OSError: + pass + raise + + return record diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py new file mode 100644 index 000000000..b26247789 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""POST a PluginTelemetryEvent to the Startups Advisor IDE Extension API. + + POST /v1/plugin-telemetry-event + { + "installId": "", # @required, UUID shape + "source": "CLAUDE_CODE", # @required, PluginSource enum + "pluginVersion": "2.0.2", # @required, ExtensionVersion shape + "occurredAt": 1759276800000, # @required, epoch milliseconds + "pluginTelemetryEvent": {...} # the PluginTelemetryEvent union + } + +The endpoint has no authorizer — the gateway stamps `auth: {type: NONE}` and an +`AnyPrincipal` allow on `execute-api:Invoke` — so this needs no credentials, no +SigV4 and no boto3, and has no third-party imports. + +Fire and forget. `post_event` swallows everything and reports failure to nobody: a +telemetry POST is not something the user asked for and must not be able to fail an +operation they did ask for. Every status is treated the same way — a 403 or 400 is +dropped exactly like a timeout, so nothing here depends on which the service sends. + +The consent gate is re-checked in `post_event`, so a new call site cannot emit by +forgetting to ask. +""" + +import json +import os +import socket +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +_TELEMETRY = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(_TELEMETRY / "consent"), str(_TELEMETRY / "metric_emission")] + +import record # noqa: E402 (paths set above so this resolves in-plugin) + +def _plugin_root(): + """Recognize canonical/plugin-bundled layouts; never treat references/ as a plugin.""" + if _TELEMETRY.parent.name == "scripts": + return _TELEMETRY.parent.parent + if (len(_TELEMETRY.parents) > 4 + and _TELEMETRY.parent.name == "vendored" + and _TELEMETRY.parents[1].name == "references" + and _TELEMETRY.parents[3].name == "skills"): + return _TELEMETRY.parents[4] + return None + + +PLUGIN_ROOT = _plugin_root() + +# Prod is the only stage a customer should reach; beta and gamma are the service +# team's, which is what the override is for. It chooses where an event goes, never +# whether one is sent. +DEFAULT_ENDPOINT = ( + "https://us-east-1.prod.startup-advisor-extension.saws.activate.aws.dev" + "/v1/plugin-telemetry-event" +) +ENDPOINT_ENV = "AWS_STARTUP_ADVISOR_PLUGIN_TELEMETRY_ENDPOINT" + +# Short, because these run inline in an agent turn with a human waiting. +TIMEOUT_SECONDS = 3.0 + +# post_event_status's answer when no connection could be made at all (offline, +# DNS, refused): unlike a timeout, nothing can have reached the service. +UNREACHABLE = 0 + +SOURCE_CLAUDE_CODE = "CLAUDE_CODE" +SOURCE_CODEX = "CODEX" +SOURCE_CURSOR = "CURSOR" +SOURCE_KIRO = "KIRO" +SOURCE_OTHER = "OTHER" + +# First marker present wins. Claude Code and Cursor Agent CLI markers have been +# verified against running hosts; an unmatched host reports OTHER rather than a plausible guess, +# because a wrong attribution silently moves one host's numbers into another's. +# TODO(StartupEngBlend-3621): confirm the other three. +# Cursor's hook-specific markers precede Claude compatibility variables. +# CURSOR_AGENT can be inherited by a nested CLI, so explicit Codex/Kiro markers +# precede that weaker marker. +_HOST_MARKERS = ( + ("CURSOR_VERSION", SOURCE_CURSOR), # verified in Cursor's hook environment + ("CURSOR_PROJECT_DIR", SOURCE_CURSOR), # verified in Cursor's hook environment + ("CURSOR_TRACE_ID", SOURCE_CURSOR), # unverified + ("CODEX_SANDBOX", SOURCE_CODEX), # unverified + ("KIRO_IDE", SOURCE_KIRO), # unverified + ("CURSOR_AGENT", SOURCE_CURSOR), # verified in Cursor Agent CLI + ("CLAUDECODE", SOURCE_CLAUDE_CODE), # verified + ("CLAUDE_CODE_ENTRYPOINT", SOURCE_CLAUDE_CODE), # verified +) + +# The PluginSkillId enum, mirrored from model/types/plugin-telemetry.smithy. An +# unmodelled member is a 400 for the whole request, so it is checked here, last, +# before anything goes on the wire. +PLUGIN_SKILL_IDS = frozenset( + { + "ARCHITECT_FOR_STARTUPS", + "KNOWLEDGE_BASE_FOR_STARTUPS", + "PROMPT_LIBRARY_FOR_STARTUPS", + "START_BUILDING_FOR_STARTUPS", + "AZURE_TO_AWS", + "GCP_TO_AWS", + "HEROKU_TO_AWS", + "LLM_TO_BEDROCK", + "AGENT_ADVISOR", + "TF_BEST_PRACTICES", + "CONTEXTUAL_OFFERS_FOR_STARTUPS", + "OPERATE_ON_AWS", + } +) + +EVENT_SKILL_INVOKED = "SKILL_INVOKED" + +# A bundle reports its own generated version. Only recognized plugin layouts +# may fall back to plugin manifests; unrelated references/plugin.json is ignored. +MANIFEST_PATHS = (_TELEMETRY / "version.json",) +if PLUGIN_ROOT is not None: + MANIFEST_PATHS += ( + PLUGIN_ROOT / ".claude-plugin" / "plugin.json", + PLUGIN_ROOT / ".cursor-plugin" / "plugin.json", + PLUGIN_ROOT / "plugin.json", + ) + + +def endpoint(): + """The URL to POST to, honoring the override.""" + return os.environ.get(ENDPOINT_ENV, "").strip() or DEFAULT_ENDPOINT + + +def detect_source(): + """Best-effort PluginSource for the host, or OTHER.""" + for name, source in _HOST_MARKERS: + if os.environ.get(name, "").strip(): + return source + return SOURCE_OTHER + + +def migration_reporting_mode(): + """Use configured plugin hooks when reachable, otherwise the bundled CLI. + + This checks the install layout, not whether a host enabled or ran its hooks. + """ + source = detect_source() + manifest_name = { + SOURCE_CLAUDE_CODE: ".claude-plugin/plugin.json", + SOURCE_CURSOR: ".cursor-plugin/plugin.json", + }.get(source) + if not manifest_name or PLUGIN_ROOT is None: + return "cli" + try: + manifest = json.loads((PLUGIN_ROOT / manifest_name).read_text(encoding="utf-8")) + if manifest.get("name") != "aws-startup-advisor": + return "cli" + hook_file = (PLUGIN_ROOT / manifest["hooks"]).resolve() + hook_file.relative_to(PLUGIN_ROOT.resolve()) + hooks = json.loads(hook_file.read_text(encoding="utf-8"))["hooks"] + script = "scripts/telemetry/metric_emission/migration.py" + if not (PLUGIN_ROOT / script).is_file(): + return "cli" + events = ("PostToolUse", "Stop") if source == SOURCE_CLAUDE_CODE else ( + "afterFileEdit", "afterShellExecution", "stop", + ) + for event in events: + for entry in hooks.get(event, []): + commands = entry.get("hooks", []) if source == SOURCE_CLAUDE_CODE else [entry] + if any(script in command.get("command", "") for command in commands): + return "hook" + except (OSError, ValueError, KeyError, TypeError, AttributeError): + pass + return "cli" + + +def plugin_version(): + """The plugin's version, or None. + + None is a refusal to send, not a default: a made-up "0.0.0" would pass the + semver pattern and misattribute the event to a version that does not exist. + """ + for manifest in MANIFEST_PATHS: + try: + with open(manifest, "r", encoding="utf-8") as handle: + version = json.load(handle).get("version") + except (OSError, ValueError): + continue + if isinstance(version, str) and version.strip(): + return version.strip() + return None + + +def build_payload(event, install_id, now_ms=None): + """The request body, or None if a required field is missing.""" + version = plugin_version() + if not install_id or not version: + return None + + return { + "installId": install_id, + "source": detect_source(), + "pluginVersion": version, + # The service drops timestamps more than five minutes ahead, silently and + # with a 200, so a fast clock loses its telemetry with no signal here. + "occurredAt": int((now_ms if now_ms is not None else time.time() * 1000)), + "pluginTelemetryEvent": event, + } + + +def post_event_status(event, install_id, url, timeout=TIMEOUT_SECONDS): + """POST one event and report the HTTP status; UNREACHABLE when no connection + could be made; None when nothing was sent or the outcome is unknown (a + timeout, a connection dropped mid-request); never raises. + + `install_id` and `url` are passed in so a caller that has already read and + validated the record is not making this re-derive it. The status lets a + caller tell a refusal it may retry (403, 429, 5xx) from one it may not (400). + + Everything is inside the try, not just the request: a malformed URL, an + unserializable event or an unreadable manifest must fail the same silent way + an unreachable network does. + """ + try: + if not record.is_accepted(): + return None + + payload = build_payload(event, install_id) + if payload is None: + return None + + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + except urllib.error.URLError as error: + # Raised before any response: the connection was never made, unless + # the reason is a timeout, which may have struck after the send. + if isinstance(error.reason, (socket.timeout, TimeoutError)): + return None + return UNREACHABLE + except BaseException: + return None + + +def post_event(event, install_id, url): + """POST one event. True only if the service accepted it; never raises.""" + status = post_event_status(event, install_id, url) + return status is not None and 200 <= status < 300 + + +def send_event(event): + """POST one event, reading the install ID and endpoint for the caller.""" + # .get: an opted-out record is valid without an installId, and post_event + # re-checks consent anyway, so None here simply means nothing is sent. + state = record.read_state() or {} + return post_event(event, state.get("installId"), endpoint()) + + +def emit_skill_invocation_metric(install_id, skill_id, url): + """Report that `skill_id` was invoked. + + A skill id that is not a PluginSkillId member is dropped here rather than + sent. The caller maps a host's skill name onto the enum and may legitimately + have no match, so that is an ordinary outcome, not an error. + """ + if skill_id not in PLUGIN_SKILL_IDS: + return False + + return post_event( + {"skillInvoked": {"skill": skill_id, "eventName": EVENT_SKILL_INVOKED}}, + install_id, + url, + ) + + +def send_consent_recorded(): + """Report that consent was recorded. Only ever sent on acknowledgement. + + `ConsentRecordedDetails` is empty and cannot say which way the user answered, + so a request from `opt_out.py` would be indistinguishable from a yes — besides + being a request made on behalf of someone who just said no. + """ + return send_event({"consentRecorded": {}}) diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration.py b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration.py new file mode 100644 index 000000000..95c06426a --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration.py @@ -0,0 +1,562 @@ +#!/usr/bin/env python3 +"""Hook: report a migration run's progress as MigrationActivity telemetry. + + python3 scripts/telemetry/metric_emission/migration.py # PostToolUse (Write|Edit|Bash) + python3 scripts/telemetry/metric_emission/migration.py --reconcile # Stop + python3 scripts/telemetry/metric_emission/migration.py --session-end # SessionEnd + python3 scripts/telemetry/metric_emission/migration.py --reconcile --via cli # other hosts + +Nothing is inferred from the conversation and no skill instruction has to be +followed. The migration skills maintain `.migration//.phase-status.json` (and +`.gate-failures.json`) for their own gate enforcement; this script reads those +files, diffs them against a snapshot it keeps beside them, and POSTs one event per +transition found: RUN_STARTED, PHASE_COMPLETED, GATE_FAILED, RUN_COMPLETED. The +diff is idempotent and the run is locked while it runs, so overlapping hooks +cannot report the same transition twice. RUN_COMPLETED is sent once per runMode: +a run that ends decision-only and is later executed ends again, under +DECIDE_AND_EXECUTE, so consumers read a run's runMode from its latest +RUN_COMPLETED and count completed runs by distinct runId. The same ending is +never repeated, even after a confirmed re-entry; a held ending the run has +since outgrown is superseded by the new one rather than re-sent. + +Consent is `record.is_accepted()`, the plugin's single gate; the install id comes +from the same record. Attribution is read from disk: a run names its owner in +`owning_skill`, and a run without one, or with one outside the migration set, +emits nothing. + +Fail-open: every path exits 0 and swallows every error. A refusal the service may +later withdraw (403 while the launch gate is closed, 429, 5xx), or no connection +at all, leaves that event unrecorded so only it is re-sent; anything the service +took, or refused for good (400), is never repeated. +""" + +import json +import os +import re +import shutil +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +_TELEMETRY = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(_TELEMETRY / "consent"), str(_TELEMETRY / "metric_emission")] + +import client # noqa: E402 (paths set above so these resolve in-plugin) +import migration_attributes as attrs # noqa: E402 +import record # noqa: E402 + +MIGRATION_SKILLS = frozenset({"AZURE_TO_AWS", "GCP_TO_AWS", "HEROKU_TO_AWS", "LLM_TO_BEDROCK"}) + +# A hook killed by the host mid-run leaves its lock behind; anything older than +# this is treated as abandoned. +LOCK_STALE_SECONDS = 60.0 +POST_TIMEOUT_SECONDS = 3.0 +# Claude Code gives all SessionEnd hooks one shared budget, 1.5 s by default. A +# longer per-hook timeout can raise it, but a hook killed at the budget would leave +# accepted events unrecorded, so the sweep budgets itself to the default instead; +# the reserve leaves room for the snapshot write. +SESSION_END_BUDGET_SECONDS = 1.5 +SESSION_END_RESERVE_SECONDS = 0.4 +STDIN_TIMEOUT_SECONDS = 2.0 + +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) + +STATUS_FILE = ".phase-status.json" +GATE_FILE = ".gate-failures.json" +SNAPSHOT_FILE = ".telemetry-snapshot.json" +LOCK_DIR = ".telemetry-lock" + +SNAPSHOT_UNREADABLE = object() + + +# ---------------------------------------------------------------- utilities + + +def as_uuid(value): + """A lower-case UUID string, or None: the data lake accepts only lower case, + while macOS uuidgen (what the skills run) mints upper case.""" + if isinstance(value, str) and UUID_RE.match(value): + return value.lower() + return None + + +def read_snapshot(path): + """None when absent, SNAPSHOT_UNREADABLE when present but not JSON. A missing + snapshot means a new run; a corrupt one must not, or the run would be + re-reported from RUN_STARTED. The caller rebuilds it from the current state + without sending, so the run reports again from its next transition.""" + if not path.exists(): + return None + try: + with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle) + except (OSError, ValueError): + return SNAPSHOT_UNREADABLE + return data if isinstance(data, dict) else SNAPSHOT_UNREADABLE + + +def write_json(path, value): + """Temp file plus rename, so a hook killed at the host's timeout cannot leave + a half-written file where read_snapshot would find it.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name("%s.%d.tmp" % (path.name, os.getpid())) + with open(tmp, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2) + handle.write("\n") + os.replace(tmp, path) + + +def read_hook_payload(stream=None, timeout=STDIN_TIMEOUT_SECONDS): + """The hook's stdin parsed as a dict, or {}. Read on a daemon thread so a host + that holds the pipe open cannot stall the turn.""" + stream = sys.stdin if stream is None else stream + box = {} + + def read(): + try: + box["raw"] = stream.read() + except Exception: + pass + + reader = threading.Thread(target=read, daemon=True) + reader.start() + reader.join(timeout) + try: + payload = json.loads(box.get("raw") or "") + except ValueError: + return {} + return payload if isinstance(payload, dict) else {} + + +def via_mode(argv): + """`--via cli` marks an invocation made from a skill instruction instead of a hook.""" + try: + return "cli" if argv[argv.index("--via") + 1] == "cli" else "hook" + except (ValueError, IndexError): + return "hook" + + +# ------------------------------------------------------------ run discovery + + +def find_run_dirs(start_dir): + """Run directories (.migration// with a state file) for the nearest + .migration tree, walking up a bounded number of levels so runs from parent or + sibling projects are not mixed in.""" + runs = [] + directory = Path(start_dir or os.getcwd()).resolve() + for _ in range(4): + root = directory / ".migration" + try: + entries = list(os.scandir(root)) + except OSError: + parent = directory.parent + if parent == directory: + break + directory = parent + continue + for entry in entries: + if entry.is_dir() and (Path(entry.path) / STATUS_FILE).exists(): + runs.append(Path(entry.path)) + break + return runs + + +# ------------------------------------------------------------------ the lock + + +def acquire_lock(run_dir): + """An exclusive per-run lock (directory creation as atomic test-and-set). + Overlapping hooks would otherwise read the same "before" state and report the + same transitions twice; this is the one duplication defence no downstream + layer can substitute for.""" + lock = Path(run_dir) / LOCK_DIR + try: + lock.mkdir() + return lock + except FileExistsError: + try: + if time.time() - lock.stat().st_mtime > LOCK_STALE_SECONDS: + shutil.rmtree(lock, ignore_errors=True) + lock.mkdir() + return lock + except OSError: + pass + return None # the holder is about to report the same transitions + except OSError: + return None # unwritable run directory: emit nothing + + +def release_lock(lock): + shutil.rmtree(lock, ignore_errors=True) + + +# ------------------------------------------------------------------- the diff + + +def gate_failure_entries(gate_failures): + """The skill records each failed gate in .gate-failures.json, keyed by phase + (see INTERPRETER.md, "Recording a failed gate"). Only phases the model knows + are reportable; a null, scalar or array entry is not a failure.""" + if not isinstance(gate_failures, dict): + return [] + entries = [] + for name, entry in gate_failures.items(): + if not isinstance(entry, dict): + continue + phase = str(name).upper() + if phase in attrs.PHASES: + entries.append((phase, entry)) + return entries + + +BACKBONE = ("discover", "clarify", "design", "estimate", "generate") + + +def run_ended(status): + """(ended, runMode). The skills flip `run_mode` to decide_and_execute before + Generate runs and may leave `current_phase: complete` from the decision-only + finish in place, so an executed ending counts only once Generate is done. + + `current_phase` is optional in the shared state schema and authoritative when + present. Without it the owning skill evaluates the backbone in order and is + complete only when every backbone phase, Generate included, is completed; + Generate stays opt-in, so a decision-only run that omits the field has not + ended. Sidebars (workshop, feedback) never decide completion.""" + run_mode = attrs.map_enum(attrs.RUN_MODE, status.get("run_mode")) + phases = status.get("phases") or {} + generate = str(phases.get("generate", "")).lower() + executing = run_mode == "DECIDE_AND_EXECUTE" and generate != "completed" + current = status.get("current_phase") + if current is None: + ended = all(str(phases.get(name, "")).lower() == "completed" for name in BACKBONE) + else: + ended = current == "complete" + return ended and not executing, run_mode + + +def reported_run_modes(snapshot): + """Every runMode this run has already been reported ending under. Older + snapshots kept only the last one under completedRunMode.""" + snapshot = snapshot or {} + modes = snapshot.get("completedRunModes") + if isinstance(modes, list): + return [m for m in modes if isinstance(m, str)] + last = snapshot.get("completedRunMode") + return [last] if isinstance(last, str) else [] + + +def diff_events(status, snapshot, gate_failures): + """One event per transition between the snapshot and the state file, plus one + GATE_FAILED per phase newly present in the gate record (a repeat failure of + the same phase is not a new event; its later success is). A run ends once + per runMode: a decision-only finish the user later turns into an executed + one is reported again, under DECIDE_AND_EXECUTE, and each mode at most once.""" + events = [] + if snapshot is None or snapshot.get("started") is False: + events.append({"eventName": "RUN_STARTED"}) + reported = set((snapshot or {}).get("gateFailures") or []) + for phase, entry in gate_failure_entries(gate_failures): + if phase in reported: + continue + event = {"eventName": "GATE_FAILED", "phase": phase, "status": "FAILED"} + reason = attrs.map_enum(attrs.FAILURE_REASON, entry.get("reason")) + if reason: + event["failureReason"] = reason + events.append(event) + before = (snapshot or {}).get("phases") or {} + for name, state in (status.get("phases") or {}).items(): + if before.get(name) == state: + continue + mapped = attrs.RESOLVED_STATUS.get(str(state).lower()) + phase = str(name).upper() + if not mapped or phase not in attrs.PHASES: + continue + events.append({"eventName": "PHASE_COMPLETED", "phase": phase, "status": mapped, "key": name}) + ended, run_mode = run_ended(status) + reported_modes = reported_run_modes(snapshot) + # A workshop re-entry clears run_mode and reopens the decision gate while the + # run stays completed; a mode already reported is never reported again. + ended_again = bool(run_mode and reported_modes and run_mode not in reported_modes) + if ended and (not (snapshot or {}).get("completed") or ended_again): + event = {"eventName": "RUN_COMPLETED", "status": "SUCCESS"} + if run_mode: + event["runMode"] = run_mode + events.append(event) + return events + + +def build_activity(event, ctx): + """The `migrationActivity` union member for `event`.""" + activity = {"eventName": event["eventName"], "skill": ctx["skill"]} + if ctx.get("initiatingSkill"): + activity["initiatingSkill"] = ctx["initiatingSkill"] + activity["runId"] = ctx["runId"] + if ctx.get("sessionId"): + activity["sessionId"] = ctx["sessionId"] + if event.get("phase"): + activity["phase"] = event["phase"] + if event.get("status"): + activity["status"] = event["status"] + try: + attributes = attrs.derive_attributes(ctx["runDir"], ctx["skill"], event, ctx["status"]) + except Exception: + attributes = None # an artifact of an unexpected shape costs its attributes, never the event + if attributes: + activity["attributes"] = attributes + return {"migrationActivity": activity} + + +def is_held(status): + """Outcomes that mean the service did not take the event but may later: no + connection at all (offline, DNS), 403 (a closed launch gate, or a WAF rate + limit), 429 and 5xx. Anything else is the loss the design tolerates rather + than risk a duplicate: a 400 the event would earn again, and a timeout or a + dropped connection (None), after which the service may have taken it.""" + return status == client.UNREACHABLE or status in (403, 429) or (status is not None and status >= 500) + + +# ------------------------------------------------------------------ per run + + +def process_run(run_dir, session_id, session_end, url, deadline, edited_path, via): + if time.time() >= deadline: + return # out of budget: untouched, so the next trigger reports it + run_dir = Path(run_dir) + + def read_state(): + status = attrs.read_json(run_dir / STATUS_FILE) + if not isinstance(status, dict) or not status.get("migration_id"): + return None, None + return status, attrs.read_json(run_dir / GATE_FILE) + + status, gate_failures = read_state() + if status is None: + return + + # Attribution is read from disk, never from an argument: a run that declares + # no owner, or an owner outside the migration set, emits nothing (fail + # closed) rather than emitting under the wrong skill. + skill = status.get("owning_skill") + if skill not in MIGRATION_SKILLS: + return + + # Consent is the plugin's one gate; without it nothing is read into a + # snapshot either, so a user who never agreed, or opted out, leaves no trace. + state = record.read_state() + if not state or state.get("consentStatus") != record.ACCEPTED: + return + install_id = as_uuid(state.get("installId")) + if not install_id: + return + + snapshot_file = run_dir / SNAPSHOT_FILE + lock = acquire_lock(run_dir) + if lock is None: + return + try: + # Read again under the lock. Hooks run concurrently: one that read the + # state before another reported a newer transition, and took the lock + # only after that report, would otherwise diff against stale state and + # move the snapshot backwards, and the next reconcile would re-send the + # newer transition. + status, gate_failures = read_state() + if status is None or status.get("owning_skill") != skill: + return + snapshot = read_snapshot(snapshot_file) + rebuild = snapshot is SNAPSHOT_UNREADABLE + if rebuild: + snapshot = None # re-observed below, without sending + valid_session_id = as_uuid(session_id) + + # Teardown sweeps only the session that wrote the snapshot; a run last + # touched by another session is that session's to report. + owner_before = (snapshot or {}).get("sessionId") + if session_end and owner_before and valid_session_id and owner_before != valid_session_id: + return + + # Identifiers read back from customer-editable files are validated, not + # trusted: the service rejects the whole event on one malformed UUID. A + # run directory is one lifecycle, so the id the snapshot already reports + # under stays authoritative: a state file rebuilt after corruption + # carries a fresh run_id, which must not split the run. + run_id = as_uuid((snapshot or {}).get("runId")) or as_uuid(status.get("run_id")) or str(uuid.uuid4()) + + def in_run(path): + if not path: + return False + try: + Path(path).resolve().relative_to(run_dir.resolve()) # is_relative_to needs 3.9 + return True + except (OSError, ValueError): + return False + + modes_before = reported_run_modes(snapshot) + + def write_snapshot(phases, completed, gates, started, owner, completed_modes): + value = {"runId": run_id} + if owner: + value["sessionId"] = owner + if started is False: + value["started"] = False + value.update({"phases": phases, "gateFailures": gates, "completed": completed, "via": via, + "updatedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}) + if completed_modes: + value["completedRunModes"] = completed_modes # every runMode the run has been reported ending under + write_json(snapshot_file, value) + + def observe(phases, completed, gates, completed_modes): + """Record the state last observed, whether or not anything was sent. + Ownership moves to this session only when it changed the run or + edited a file inside it; a hook that merely scanned past leaves the + owner alone. Written only when something changed.""" + changed = ( + snapshot is None + or (snapshot.get("phases") or {}) != phases + or bool(snapshot.get("completed")) != completed + or modes_before != completed_modes + or (snapshot.get("gateFailures") or []) != gates + ) + touched = changed or in_run(edited_path) + owner = (valid_session_id or owner_before) if touched else (owner_before or valid_session_id) + if not changed and snapshot.get("sessionId") == owner: + return + write_snapshot(phases, completed, gates, (snapshot or {}).get("started"), owner, completed_modes) + + if rebuild: + # A corrupt snapshot (not this script's: its write is atomic) cannot + # say what was reported. Those transitions are lost; recording the + # current state lets the run report again from the next one. + ended, run_mode = run_ended(status) + observe(status.get("phases") or {}, ended, [phase for phase, _ in gate_failure_entries(gate_failures)], + [run_mode] if ended and run_mode else []) + return + + events = diff_events(status, snapshot, gate_failures) + known_gates = list(dict.fromkeys((snapshot or {}).get("gateFailures") or [])) + if not events: + # Nothing to send, but the observation may have changed: a phase reset + # (a confirmed re-entry sets downstream phases back to pending) or a + # run taken over by this session. Record it, or the phase's next + # completion would read as already reported and this session's + # teardown would skip the run. + observe(status.get("phases") or {}, bool((snapshot or {}).get("completed")), known_gates, modes_before) + return + + ctx = { + "runDir": run_dir, + "status": status, + "skill": skill, + # Only a known migration skill may be named as the invoker; anything + # else is dropped rather than risk rejecting the whole event. + "initiatingSkill": status.get("initiated_by") if status.get("initiated_by") in MIGRATION_SKILLS else None, + "runId": run_id, + "sessionId": valid_session_id, + } + + # Concurrently, under one budget: sent serially, a reconcile catching a + # whole run could outlive the host's hook timeout and never reach the + # snapshot write, so the next trigger would repeat the batch. + activities = [build_activity(event, ctx) for event in events] + timeout = max(0.05, min(POST_TIMEOUT_SECONDS, deadline - time.time())) + with ThreadPoolExecutor(max_workers=max(1, len(events))) as pool: + results = list(pool.map( + lambda activity: client.post_event_status(activity, install_id, url, timeout=timeout), + activities, + )) + held = [event for event, result in zip(events, results) if is_held(result)] + if len(held) == len(events): + return # nothing taken: nothing recorded + + # The snapshot records exactly what the service took. A held phase keeps + # its previous state so only that transition is re-sent; a held + # RUN_STARTED, GATE_FAILED or RUN_COMPLETED is re-sent alone. Accepted + # events are never repeated, even when a later trigger replays the held ones. + phases = dict(status.get("phases") or {}) + before = (snapshot or {}).get("phases") or {} + for event in held: + if event["eventName"] != "PHASE_COMPLETED": + continue + if event["key"] in before: + phases[event["key"]] = before[event["key"]] + else: + phases.pop(event["key"], None) + run_started = next((e for e in events if e["eventName"] == "RUN_STARTED"), None) + run_completed = next((e for e in events if e["eventName"] == "RUN_COMPLETED"), None) + sent_gates = [e["phase"] for e in events if e["eventName"] == "GATE_FAILED" and e not in held] + ended_now = bool(run_completed and run_completed not in held) + modes = list(modes_before) + if ended_now and run_completed.get("runMode") and run_completed["runMode"] not in modes: + modes.append(run_completed["runMode"]) + write_snapshot( + phases, + bool((snapshot or {}).get("completed")) or ended_now, + list(dict.fromkeys(known_gates + sent_gates)), + not (run_started and run_started in held), + valid_session_id or owner_before, + modes, + ) + finally: + release_lock(lock) + + +# ---------------------------------------------------------------------- main + + +def reconcile(start_dir, session_id=None, session_end=False, edited_path=None, url=None, via="hook"): + """Report every run reachable from `start_dir`. Returns nothing; never raises.""" + url = url or client.endpoint() + deadline = time.time() + SESSION_END_BUDGET_SECONDS - SESSION_END_RESERVE_SECONDS if session_end else float("inf") + for run_dir in find_run_dirs(start_dir): + try: + process_run(run_dir, session_id, session_end, url, deadline, edited_path, via) + except BaseException: + continue # one run's trouble must not silence the others + + +def main(argv, stdin=None): + cli = via_mode(argv) == "cli" + if cli and client.migration_reporting_mode() == "hook": + return 0 # Claude Code and Cursor retain their existing hook path. + if cli and client.plugin_version() is None: + return 0 # do not advance a snapshot when a standalone bundle is incomplete + session_end = "--session-end" in argv + # Explicit CLI calls use cwd; they have no hook payload and must not wait + # for stdin or accept a caller-supplied hook session/project identity. + payload = {} if cli else read_hook_payload(stdin) + # Claude Code sends session_id on every hook; Cursor sends conversation_id. + session_id = payload.get("session_id") or payload.get("conversation_id") + tool_input = payload.get("tool_input") if isinstance(payload.get("tool_input"), dict) else {} + edited_path = tool_input.get("file_path") or payload.get("file_path") + if not isinstance(edited_path, str): + edited_path = None + + # Post-write fast path: an edit outside any .migration tree is no work at all. + # A Bash payload carries a command, not a file path, and the agent does write + # state files through the shell (a heredoc instead of the Write tool), so it + # falls through to a reconcile of every run under the hook's cwd: a few file + # reads, and nothing is sent unless a transition is new. + if not session_end and "--reconcile" not in argv and edited_path and ".migration" not in edited_path: + return 0 + + # Start discovery at the directory that owns the .migration tree, so an edit + # deep inside a run's artifacts still resolves to its run. + marker = edited_path.find("%s.migration%s" % (os.sep, os.sep)) if edited_path else -1 + roots = payload.get("workspace_roots") + start_dir = ( + edited_path[:marker] if marker != -1 else None + ) or payload.get("cwd") or (roots[0] if isinstance(roots, list) and roots else None) or os.getcwd() + + reconcile(start_dir, session_id=session_id, session_end=session_end, edited_path=edited_path, via=via_mode(argv)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except BaseException: + sys.exit(0) # telemetry must never make the user's work look like it failed diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration_attributes.py b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration_attributes.py new file mode 100644 index 000000000..ff7ef7a08 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/migration_attributes.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +"""Attributes for a migration telemetry event, derived from the run's own artifacts. + +Every value is a lookup over files the migration skills already write into +`.migration//`: no inference, no free text. Unknown or unmodelled values drop +the attribute, never the event, because the service rejects a whole event on one +bad enum member. Phase facts attach only to the PHASE_COMPLETED event of the +phase that produced them, so no aggregation double counts; `complexityTier` is +the exception, riding every Design-onward and terminal event because it segments +the funnel. + +Vocabularies mirror `MigrationEventAttributes` in migration-telemetry.smithy. +""" + +import json +import re +from pathlib import Path + +PHASES = frozenset( + {"DISCOVER", "CLARIFY", "DESIGN", "ESTIMATE", "WORKSHOP", "GENERATE", "FEEDBACK"} +) +RESOLVED_STATUS = { + "completed": "SUCCESS", + "skipped": "SKIPPED", + "not_applicable": "NOT_APPLICABLE", + "failed": "FAILED", +} +RUN_MODE = {"decide": "DECIDE", "decide_and_execute": "DECIDE_AND_EXECUTE"} +# Mirrors the reason= constants of the interpreter's GATE_FAIL line. +FAILURE_REASON = { + "missing": "MISSING", + "invalid": "INVALID", + "stale_downstream": "STALE_DOWNSTREAM", +} + +PRICING_SOURCE = { + "live": "LIVE", + "cached": "CACHED", + "cached_fallback": "CACHED_FALLBACK", + "cached_stale": "CACHED_STALE", + "unavailable": "UNAVAILABLE", +} +RECOMMENDATION_OUTCOME = { + "go": "GO", + "conditional_go": "CONDITIONAL_GO", + "defer": "DEFER", + "defer_for_evidence": "DEFER", + "stay": "STAY", +} +CLARIFY_MODE = {"wizard": "WIZARD", "full": "FULL", "fast_path": "FAST"} +# estimated_from_token_volume is the AI route (it prices tokens, not +# infrastructure). `preferences` is handled by preference_spend_basis, since a +# clarify answer the skill filled in itself is not the customer's figure. +SPEND_BASIS = { + "billing_data": "BILLING_DATA", + "inventory_estimate": "INVENTORY_ESTIMATE", + "live_prices_plus_cache": "LIVE_PRICES_PLUS_CACHE", + "pricing_cache": "PRICING_CACHE", + "user_provided": "USER_PROVIDED", + "unavailable": "UNAVAILABLE", + "estimated_from_token_volume": "TOKEN_VOLUME_ESTIMATE", + # azure-to-aws's baseline rungs (estimate-infra.md): a stated figure, a Cost + # Management export or RDfA consumption data, or a figure derived from SKUs. + "user_stated": "USER_PROVIDED", + "cost_management_export": "BILLING_DATA", + "consumption_data": "BILLING_DATA", + "derived_from_skus": "INVENTORY_ESTIMATE", +} +AI_SOURCE = {"openai": "OPENAI", "anthropic": "ANTHROPIC", "gemini": "GCP", "azure_openai": "AZURE", "other": "OTHER"} +COMPLEXITY_TIER = frozenset({"SMALL", "MEDIUM", "LARGE"}) +# The discover preview's coarser signal, used before a tiered artifact exists. +COMPLEXITY_SIGNAL = {"likely_simple": "SMALL", "standard": "MEDIUM", "complex": "LARGE"} +RECOMMENDATION_CONFIDENCE = frozenset({"LOW", "MEDIUM", "HIGH"}) +VALIDATION_STATUS = frozenset({"PASSED", "PASSED_DEGRADED_OFFLINE", "FAILED"}) +COMPLIANCE = frozenset({"NONE", "UNKNOWN", "HIPAA", "PCI", "SOC2", "GDPR", "CCPA", "FEDRAMP"}) +COMPLIANCE_ALIAS = {"PCI_DSS": "PCI", "SOC_2": "SOC2", "FED_RAMP": "FEDRAMP"} +COMPLIANCE_ACCEPTED = COMPLIANCE | frozenset(COMPLIANCE_ALIAS) +AVAILABILITY = frozenset({"SINGLE_AZ", "MULTI_AZ", "MULTI_AZ_HA", "MULTI_REGION"}) +CUTOVER_STRATEGY = frozenset( + {"MAINTENANCE_WINDOW_WEEKLY", "MAINTENANCE_WINDOW_MONTHLY", "FLEXIBLE", "ZERO_DOWNTIME"} +) +DATABASE_TRAFFIC = frozenset({"STEADY", "READ_HEAVY", "WRITE_HEAVY"}) +# gcp's clarify writes "write-heavy-global" for the write-heavy answer. +DATABASE_TRAFFIC_ALIAS = {"WRITE_HEAVY_GLOBAL": "WRITE_HEAVY"} +DATABASE_TRAFFIC_ACCEPTED = DATABASE_TRAFFIC | frozenset(DATABASE_TRAFFIC_ALIAS) +COMPUTE_POSTURE = frozenset( + {"EKS_MANAGED", "EKS_OR_ECS", "ECS_FARGATE", "EKS", "ECS", "ELASTIC_BEANSTALK"} +) +TARGET_REGION = frozenset( + { + "US_EAST_1", "US_EAST_2", "US_WEST_1", "US_WEST_2", "CA_CENTRAL_1", + "EU_WEST_1", "EU_WEST_2", "EU_WEST_3", "EU_CENTRAL_1", "EU_NORTH_1", + "AP_SOUTH_1", "AP_SOUTHEAST_1", "AP_SOUTHEAST_2", "AP_NORTHEAST_1", + "AP_NORTHEAST_2", "AP_NORTHEAST_3", "SA_EAST_1", + } +) +# The skills write db_size as a human range ("<10GB", "10-100GB"), not a token. +DB_SIZE = { + "<10gb": "DB_UNDER_10GB", + "10-100gb": "DB_10_100GB", + "100-500gb": "DB_100_500GB", + ">500gb": "DB_OVER_500GB", + "unknown": "UNKNOWN", +} +PROJECTED_COST_MAX = 10_000_000 # model @range +RESOURCE_COUNT_MAX = 10_000 # model @range + +AI_TYPE = re.compile( + r"vertex|aiplatform|notebooks|discovery_engine|automl|ml_engine|dialogflow|document_ai|cognitive|machine_?learning|bedrock|sagemaker|comprehend" +) +DB_TYPE = re.compile( + r"sql|postgres|mysql|mongo|redis|firestore|spanner|bigtable|datastore|memorystore|alloydb|cosmos|rds|aurora|dynamo|elasticache|documentdb" +) + +# Where a route records the source-platform spend, in preference order. +SOURCE_SPEND_KEYS = ( + "gcp_monthly_spend", + "gcp_monthly", + "gcp_monthly_usd", + "gcp_monthly_baseline", + "current_gcp_monthly", + "total_monthly_spend", + "gcp_total_monthly", + "total_monthly", + "gcp_monthly_ai_spend", + "total_current_ai_monthly", + "heroku_monthly", + "heroku_monthly_estimated", + "heroku_monthly_baseline", + "current_heroku_monthly", + "azure_monthly", + "azure_monthly_spend", + "azure_monthly_baseline", + "current_azure_monthly", +) + +SKILL_INVENTORY = { + "AZURE_TO_AWS": {"inventory": "azure-resource-inventory.json", "provider": "AZURE"}, + "GCP_TO_AWS": {"inventory": "gcp-resource-inventory.json", "provider": "GCP"}, + "HEROKU_TO_AWS": {"inventory": "heroku-resource-inventory.json", "provider": "HEROKU"}, +} +ESTIMATE_FILES = ("estimation-infra.json", "estimation-ai.json", "estimation-billing.json") +DESIGN_ONWARD = frozenset({"DESIGN", "ESTIMATE", "WORKSHOP", "GENERATE", "FEEDBACK"}) + + +def read_json(path): + """The parsed file, or None when absent or unparseable.""" + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def map_enum(table, value): + """Look `value` up case-insensitively; only the table's own keys count.""" + if value is None: + return None + return table.get(str(value).lower()) + + +def to_enum(members, value): + """Normalise a skill-written value ("multi-az-ha", "us-east-1") to UPPER_SNAKE + and admit it only if the model declares it.""" + if value is None: + return None + key = re.sub(r"[^A-Z0-9]+", "_", str(value).strip().upper()).strip("_") + return key if key in members else None + + +def _is_number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def to_pricing_source(raw): + if raw is None: + return None + is_object = isinstance(raw, dict) + mapped = map_enum(PRICING_SOURCE, raw.get("status") if is_object else raw) + if not mapped: + return None + stale = is_object and (raw.get("fallback_staleness") or {}).get("is_stale") is True + return "CACHED_STALE" if mapped == "CACHED" and stale else mapped + + +def to_spend_band(amount): + if isinstance(amount, str): + try: + amount = float(amount) + except ValueError: + return None + if not _is_number(amount) or amount != amount or amount < 0: + return None + if amount < 100: + return "UNDER_100" + if amount < 1000: + return "FROM_100_TO_1K" + if amount < 10000: + return "FROM_1K_TO_10K" + return "OVER_10K" + + +def to_projected_usd(value): + """Integer USD within the model's range; anything else is omitted rather than + clamped, since a clamped cost would read as a real figure.""" + if isinstance(value, str): + try: + value = float(value) + except ValueError: + return None + if not _is_number(value) or value != value or value < 0 or value > PROJECTED_COST_MAX: + return None + return int(round(value)) + + +def to_accuracy_band(text): + """"±5-10%", "±20-30%+", "±5% (billing)": the upper bound decides the band.""" + if not isinstance(text, str): + return None + match = re.search(r"±?\s*(\d+)(?:\s*-\s*(\d+))?\s*%", text) + if not match: + return None + upper = int(match.group(2) or match.group(1)) + if upper <= 10: + return "HIGH" + if upper <= 20: + return "MEDIUM" + return "LOW" + + +def to_coverage(value): + """"100%", "97%", 100: FULL at 100, NEAR_COMPLETE from 90, PARTIAL below.""" + if value is None or value == "": + return None + try: + number = float(value) if _is_number(value) else float(str(value).replace("%", "").strip()) + except ValueError: + return None + if number >= 100: + return "FULL" + if number >= 90: + return "NEAR_COMPLETE" + return "PARTIAL" + + +def constraint_value(preferences, key): + """A design constraint is an object with the interpreted value under `value` + (gcp) or `default` (heroku's compute_target); older artifacts hold the bare + value. Heroku records region, compliance and availability as plain values + under `global` instead.""" + if not isinstance(preferences, dict): + return None + raw = (preferences.get("design_constraints") or {}).get(key) + if raw is None: + raw = (preferences.get("global") or {}).get(key) + if raw is None: + return None + if isinstance(raw, dict): + return raw.get("value", raw.get("default")) + return raw + + +def to_compliance_list(raw): + """An explicit empty list is the customer's confirmed "no requirements", which + the model spells NONE; only an absent answer is omitted.""" + if raw is None: + return None + if isinstance(raw, list) and len(raw) == 0: + return ["NONE"] + values = raw if isinstance(raw, list) else [raw] + out = [] + for value in values: + key = to_enum(COMPLIANCE_ACCEPTED, value) + if key: + member = COMPLIANCE_ALIAS.get(key, key) + if member not in out: + out.append(member) + return out or None + + +def preference_spend_basis(run_dir): + """An estimate that cites "preferences" for the source spend is only as good as + the clarify answer behind it: the customer's own answer is USER_PROVIDED, one + the skill filled in itself is DEFAULTED, one extracted from the customer's + billing export is BILLING_DATA, and anything without provenance is left out.""" + preferences = read_json(Path(run_dir) / "preferences.json") or {} + spend = (preferences.get("design_constraints") or {}).get("gcp_monthly_spend") or {} + if not isinstance(spend, dict): + return None + source = str(spend.get("source") or "") + chosen_by = spend.get("chosen_by") + if chosen_by == "default" or source.startswith("default"): + return "DEFAULTED" + if chosen_by == "user": + return "USER_PROVIDED" + if chosen_by == "extracted" and source.startswith("billing"): + return "BILLING_DATA" + return None + + +def resource_types(resources): + """The resource types as one lower-cased string. gcp writes `type`, heroku + `resource_type` (with the add-on service under config), azure the canonical + ARM type under `azure_type` (Microsoft.DocumentDB/databaseAccounts).""" + parts = [] + for resource in resources: + if not isinstance(resource, dict): + continue + kind = resource.get("type") or resource.get("resource_type") or resource.get("azure_type") or "" + addon = (resource.get("config") or {}).get("addon_service") if isinstance(resource.get("config"), dict) else "" + parts.append("%s %s" % (kind, addon or "")) + return " ".join(parts).lower() + + +def source_provider(run_dir, spec): + """The migration's source: the skill's inventory file names it; an AI-only run + has none and takes the provider from its workload profile.""" + if spec and (Path(run_dir) / spec["inventory"]).exists(): + return spec["provider"] + profile = read_json(Path(run_dir) / "ai-workload-profile.json") or {} + return map_enum(AI_SOURCE, (profile.get("summary") or {}).get("ai_source")) + + +def cost_containers(estimate): + """Where a route records the source-platform cost, in preference order: the + infra and AI routes use current_costs, the billing-only route gcp_baseline, + and heroku's assembler also rolls it into financial_summary.""" + return [ + estimate.get(key) + for key in ("current_costs", "gcp_baseline", "cost_comparison", "financial_summary") + if isinstance(estimate.get(key), dict) + ] + + +def projected_cost(estimate, tier, option): + """The three cost tiers are written flat (aws_monthly_*), with older shapes + nesting them under option_*/tiers.""" + costs = estimate.get("projected_costs") or {} + value = costs.get("aws_monthly_%s" % tier) + if value is None: + value = (costs.get(option) or {}).get("aws_monthly") if isinstance(costs.get(option), dict) else None + if value is None: + tiers = costs.get("tiers") or {} + value = (tiers.get(tier) or {}).get("monthly") if isinstance(tiers.get(tier), dict) else None + return to_projected_usd(value) + + +def complexity_tier(run_dir, status): + """Written by generate (gcp) or estimate (heroku); before either is complete, + the discover preview's coarser signal. A confirmed re-entry resets downstream + phases to pending but leaves their artifacts on disk, so an artifact counts + only while its phase is completed.""" + phases = (status or {}).get("phases") or {} + tiered = [] + if phases.get("generate") == "completed": + tiered += ["generation-infra.json", "generation-billing.json", "generation-ai.json"] + if phases.get("estimate") == "completed": + tiered.append("estimation-infra.json") + for name in tiered: + artifact = read_json(Path(run_dir) / name) or {} + raw = artifact.get("complexity_tier") + if raw is None: + raw = (artifact.get("estimation_summary") or {}).get("complexity_tier") + tier = to_enum(COMPLEXITY_TIER, raw) + if tier: + return tier + preview = read_json(Path(run_dir) / "migration-preview.json") or {} + return map_enum(COMPLEXITY_SIGNAL, preview.get("complexity_signal")) + + +def derive_attributes(run_dir, skill, event, status): + """The attributes object for `event`, or None when there is nothing to say.""" + run_dir = Path(run_dir) + attributes = {} + spec = SKILL_INVENTORY.get(skill) + provider = source_provider(run_dir, spec) + if provider: + attributes["sourceProvider"] = provider + + phase_event = event.get("eventName") == "PHASE_COMPLETED" + phase = event.get("phase") + + if phase_event and phase == "DISCOVER": + inventory = read_json(run_dir / spec["inventory"]) if spec else None + resources = inventory if isinstance(inventory, list) else (inventory or {}).get("resources") if isinstance(inventory, dict) else None + # App-code AI detection lives only here; it is what lets an AI workload + # with no AI resource (an SDK called from application code) report hasAi. + profile = read_json(run_dir / "ai-workload-profile.json") + models = (profile or {}).get("models") + if isinstance(models, list): + ai_from_profile = len(models) > 0 + else: + ai_from_profile = ((profile or {}).get("summary") or {}).get("total_models_detected", 0) > 0 + if isinstance(resources, list): + attributes["resourceCount"] = min(len(resources), RESOURCE_COUNT_MAX) + types = resource_types(resources) + attributes["hasDatabase"] = bool(DB_TYPE.search(types)) + attributes["hasAi"] = bool(AI_TYPE.search(types)) or ai_from_profile + elif profile is not None: + attributes["hasAi"] = ai_from_profile + coverage = to_coverage(((inventory or {}).get("summary") or {}).get("classification_coverage") if isinstance(inventory, dict) else None) + if coverage: + attributes["classificationCoverage"] = coverage + + if phase_event and phase == "CLARIFY": + preferences = read_json(run_dir / "preferences.json") or {} + metadata = preferences.get("metadata") or {} + clarify_mode = "AI_ONLY" if metadata.get("migration_type") == "ai-only" else map_enum(CLARIFY_MODE, metadata.get("clarify_mode")) + if clarify_mode: + attributes["clarifyMode"] = clarify_mode + compliance = to_compliance_list(constraint_value(preferences, "compliance")) + if compliance: + attributes["compliance"] = compliance + availability = to_enum(AVAILABILITY, constraint_value(preferences, "availability")) + if availability: + attributes["availability"] = availability + cutover = to_enum(CUTOVER_STRATEGY, constraint_value(preferences, "cutover_strategy")) + if cutover: + attributes["cutoverStrategy"] = cutover + db_size = map_enum(DB_SIZE, re.sub(r"\s+", "", str(constraint_value(preferences, "db_size") or ""))) + if db_size: + attributes["dbSize"] = db_size + traffic_key = to_enum(DATABASE_TRAFFIC_ACCEPTED, constraint_value(preferences, "database_traffic")) + if traffic_key: + attributes["databaseTraffic"] = DATABASE_TRAFFIC_ALIAS.get(traffic_key, traffic_key) + # gcp asks about kubernetes, heroku about a compute target; same decision. + posture_raw = constraint_value(preferences, "kubernetes") + if posture_raw is None: + posture_raw = constraint_value(preferences, "compute_target") + posture = to_enum(COMPUTE_POSTURE, posture_raw) + if posture: + attributes["computePosture"] = posture + region = to_enum(TARGET_REGION, constraint_value(preferences, "target_region")) + if region: + attributes["targetRegion"] = region + + if phase_event and phase == "ESTIMATE": + estimates = [e for e in (read_json(run_dir / name) for name in ESTIMATE_FILES) if isinstance(e, dict)] + for estimate in estimates: + recommendation = estimate.get("recommendation") or {} + if "recommendationOutcome" not in attributes: + outcome = map_enum(RECOMMENDATION_OUTCOME, recommendation.get("outcome")) + if outcome: + attributes["recommendationOutcome"] = outcome + if "recommendationConfidence" not in attributes: + confidence = to_enum(RECOMMENDATION_CONFIDENCE, recommendation.get("confidence")) + if confidence: + attributes["recommendationConfidence"] = confidence + if "pricingSource" not in attributes: + # The billing-only route records provenance under metadata. + raw = estimate.get("pricing_source") + if raw is None: + raw = (estimate.get("metadata") or {}).get("pricing_source") + if raw is None: + raw = (estimate.get("projected_costs") or {}).get("pricing_source") + pricing = to_pricing_source(raw) + if pricing: + attributes["pricingSource"] = pricing + if "estimateAccuracyBand" not in attributes: + accuracy = (estimate.get("current_costs") or {}).get("accuracy") + if accuracy is None: + accuracy = estimate.get("accuracy_confidence") + band = to_accuracy_band(accuracy) + if band: + attributes["estimateAccuracyBand"] = band + if "billingDataAvailable" not in attributes: + available = (estimate.get("migration_cost_considerations") or {}).get("billing_data_available") + if isinstance(available, bool): + attributes["billingDataAvailable"] = available + if "awsProjectedCostBalanced" not in attributes: + for tier, option, key in ( + ("optimized", "option_c_optimized", "awsProjectedCostOptimized"), + ("balanced", "option_b_balanced", "awsProjectedCostBalanced"), + ("premium", "option_a_premium", "awsProjectedCostPremium"), + ): + usd = projected_cost(estimate, tier, option) + if usd is not None: + attributes[key] = usd + # spendBand and spendBasis travel as a pair: a band without its basis is + # indistinguishable from a measured figure. The basis comes from the cost + # container's own `source`; when a route omits it, confirmed billing data + # is the one case that can still be named. A basis alone is harmless and + # is still reported. + for estimate in estimates: + containers = cost_containers(estimate) + basis = None + for container in containers: + source = container.get("source") + basis = preference_spend_basis(run_dir) if source == "preferences" else map_enum(SPEND_BASIS, source) + if basis: + break + if not basis and (estimate.get("migration_cost_considerations") or {}).get("billing_data_available") is True: + basis = "BILLING_DATA" + band = None + for container in containers: + for key in SOURCE_SPEND_KEYS: + band = to_spend_band(container.get(key)) + if band: + break + if band: + break + if basis and band: + attributes["spendBand"] = band + attributes["spendBasis"] = basis + break + if basis and "spendBasis" not in attributes: + attributes["spendBasis"] = basis + + if phase_event and phase == "GENERATE": + report = read_json(run_dir / "validation-report.json") or {} + validation = to_enum(VALIDATION_STATUS, report.get("status")) + if validation: + attributes["validationStatus"] = validation + + from_design_onward = (phase_event or event.get("eventName") == "GATE_FAILED") and phase in DESIGN_ONWARD + if from_design_onward or event.get("eventName") == "RUN_COMPLETED": + tier = complexity_tier(run_dir, status) + if tier: + attributes["complexityTier"] = tier + + if event.get("runMode"): + attributes["runMode"] = event["runMode"] + if event.get("failureReason"): + attributes["failureReason"] = event["failureReason"] + return attributes or None diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/version.json b/plugins/aws-startup-advisor/skills/shared/telemetry/version.json new file mode 100644 index 000000000..22ba7b20d --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/version.json @@ -0,0 +1,3 @@ +{ + "version": "2.0.3" +} From 383c755f44b593213552693c62f4d77bd1b8cfbe Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 18:03:52 -0700 Subject: [PATCH 37/53] fix(aws-startup-advisor): keep telemetry under its bundle gate Co-authored-by: Cursor --- .../tools/sync-vendored.py | 29 +++++++++++++++---- .../tools/test_sync_vendored.py | 19 ++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py index 26d038ce6..2692bfc66 100644 --- a/plugins/aws-startup-advisor/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -14,8 +14,8 @@ Rules ----- 1. Every file under `plugins//skills//references/vendored/` - (README.md excluded) has a canonical `plugins//skills/shared/` and is - byte-identical to it. + (README.md and the independently generated `telemetry/` bundle excluded) has a + canonical `plugins//skills/shared/` and is byte-identical to it. 2. Each vendored directory's README.md carries a `| Vendored path | Canonical source |` table. Every row must exist on disk (a deleted copy is caught), every file on disk must have a row (the README stays honest), and the row's canonical source must be @@ -52,6 +52,7 @@ VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} +SKIP_TOP_LEVEL_DIRS = {"telemetry"} def _rel(p: Path) -> str: @@ -62,6 +63,20 @@ def _rel(p: Path) -> str: return str(p) +def _is_managed_file(path: Path, vend_dir: Path) -> bool: + """Whether this gate owns a file in references/vendored/. + + Telemetry bundles are generated from scripts/telemetry/ by sync_bundles.py, + not from skills/shared/, so that subtree has its own parity gate. + """ + rel = path.relative_to(vend_dir) + return ( + path.is_file() + and path.name not in SKIP_NAMES + and rel.parts[0] not in SKIP_TOP_LEVEL_DIRS + ) + + _ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|\s*$", re.M) @@ -93,11 +108,15 @@ def check_manifest(plugin_dir: Path) -> List[str]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue + on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if _is_managed_file(p, vend)} + if not (vend / "README.md").is_file() and not on_disk: + # A vendored tree owned entirely by another generator (currently + # telemetry) has no skills/shared manifest for this gate to check. + continue rows, errs = readme_manifest(vend) errors.extend(errs) if not rows: continue - on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if p.is_file() and p.name not in SKIP_NAMES} for rel in sorted(set(rows) - on_disk): errors.append( f"{_rel(vend / rel)}: listed in {_rel(vend / 'README.md')} but missing on disk — " @@ -124,9 +143,7 @@ def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue - for f in sorted(p for p in vend.rglob("*") if p.is_file()): - if f.name in SKIP_NAMES: - continue + for f in sorted(p for p in vend.rglob("*") if _is_managed_file(p, vend)): rel = f.relative_to(vend) out.append((f, shared / rel, rel)) return out diff --git a/plugins/aws-startup-advisor/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py index f7188b952..8c582c60b 100644 --- a/plugins/aws-startup-advisor/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -109,6 +109,25 @@ def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): assert sv.check_manifest(plugin) == [] +def test_telemetry_bundle_is_owned_by_its_separate_sync_gate(tmp_path: Path): + """Telemetry is generated from scripts/telemetry, not skills/shared.""" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" + (vend / "telemetry").mkdir() + (vend / "telemetry" / "PROTOCOL.md").write_text("generated bundle\n") + (vend / "telemetry" / "version.json").write_text('{"version": "1.0.0"}\n') + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + (vend / "pricing.json").write_text('{"a": 1}\n') + telemetry_only = plugin / "skills" / "telemetry-only" / "references" / "vendored" / "telemetry" + telemetry_only.mkdir(parents=True) + (telemetry_only / "cli.py").write_text("# generated bundle\n") + + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] + + def test_cli_check_exit_codes(tmp_path: Path): plugin = _tree(tmp_path) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], From 86f39d927badbbc2e036a095822d62d3a197c74e Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 18:03:52 -0700 Subject: [PATCH 38/53] fix(aws-startup-advisor): keep telemetry under its bundle gate Co-authored-by: Cursor --- .../tools/sync-vendored.py | 29 +++++++++++++++---- .../tools/test_sync_vendored.py | 19 ++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py index 26d038ce6..2692bfc66 100644 --- a/plugins/aws-startup-advisor/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -14,8 +14,8 @@ Rules ----- 1. Every file under `plugins//skills//references/vendored/` - (README.md excluded) has a canonical `plugins//skills/shared/` and is - byte-identical to it. + (README.md and the independently generated `telemetry/` bundle excluded) has a + canonical `plugins//skills/shared/` and is byte-identical to it. 2. Each vendored directory's README.md carries a `| Vendored path | Canonical source |` table. Every row must exist on disk (a deleted copy is caught), every file on disk must have a row (the README stays honest), and the row's canonical source must be @@ -52,6 +52,7 @@ VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} +SKIP_TOP_LEVEL_DIRS = {"telemetry"} def _rel(p: Path) -> str: @@ -62,6 +63,20 @@ def _rel(p: Path) -> str: return str(p) +def _is_managed_file(path: Path, vend_dir: Path) -> bool: + """Whether this gate owns a file in references/vendored/. + + Telemetry bundles are generated from scripts/telemetry/ by sync_bundles.py, + not from skills/shared/, so that subtree has its own parity gate. + """ + rel = path.relative_to(vend_dir) + return ( + path.is_file() + and path.name not in SKIP_NAMES + and rel.parts[0] not in SKIP_TOP_LEVEL_DIRS + ) + + _ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|\s*$", re.M) @@ -93,11 +108,15 @@ def check_manifest(plugin_dir: Path) -> List[str]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue + on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if _is_managed_file(p, vend)} + if not (vend / "README.md").is_file() and not on_disk: + # A vendored tree owned entirely by another generator (currently + # telemetry) has no skills/shared manifest for this gate to check. + continue rows, errs = readme_manifest(vend) errors.extend(errs) if not rows: continue - on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if p.is_file() and p.name not in SKIP_NAMES} for rel in sorted(set(rows) - on_disk): errors.append( f"{_rel(vend / rel)}: listed in {_rel(vend / 'README.md')} but missing on disk — " @@ -124,9 +143,7 @@ def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue - for f in sorted(p for p in vend.rglob("*") if p.is_file()): - if f.name in SKIP_NAMES: - continue + for f in sorted(p for p in vend.rglob("*") if _is_managed_file(p, vend)): rel = f.relative_to(vend) out.append((f, shared / rel, rel)) return out diff --git a/plugins/aws-startup-advisor/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py index f7188b952..8c582c60b 100644 --- a/plugins/aws-startup-advisor/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -109,6 +109,25 @@ def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): assert sv.check_manifest(plugin) == [] +def test_telemetry_bundle_is_owned_by_its_separate_sync_gate(tmp_path: Path): + """Telemetry is generated from scripts/telemetry, not skills/shared.""" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" + (vend / "telemetry").mkdir() + (vend / "telemetry" / "PROTOCOL.md").write_text("generated bundle\n") + (vend / "telemetry" / "version.json").write_text('{"version": "1.0.0"}\n') + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + (vend / "pricing.json").write_text('{"a": 1}\n') + telemetry_only = plugin / "skills" / "telemetry-only" / "references" / "vendored" / "telemetry" + telemetry_only.mkdir(parents=True) + (telemetry_only / "cli.py").write_text("# generated bundle\n") + + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] + + def test_cli_check_exit_codes(tmp_path: Path): plugin = _tree(tmp_path) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], From 5dc6df826b121203576ce73b35ed8941029e56b3 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 18:03:52 -0700 Subject: [PATCH 39/53] fix(aws-startup-advisor): keep telemetry under its bundle gate Co-authored-by: Cursor --- .../tools/sync-vendored.py | 29 +++++++++++++++---- .../tools/test_sync_vendored.py | 19 ++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py index 26d038ce6..2692bfc66 100644 --- a/plugins/aws-startup-advisor/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -14,8 +14,8 @@ Rules ----- 1. Every file under `plugins//skills//references/vendored/` - (README.md excluded) has a canonical `plugins//skills/shared/` and is - byte-identical to it. + (README.md and the independently generated `telemetry/` bundle excluded) has a + canonical `plugins//skills/shared/` and is byte-identical to it. 2. Each vendored directory's README.md carries a `| Vendored path | Canonical source |` table. Every row must exist on disk (a deleted copy is caught), every file on disk must have a row (the README stays honest), and the row's canonical source must be @@ -52,6 +52,7 @@ VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} +SKIP_TOP_LEVEL_DIRS = {"telemetry"} def _rel(p: Path) -> str: @@ -62,6 +63,20 @@ def _rel(p: Path) -> str: return str(p) +def _is_managed_file(path: Path, vend_dir: Path) -> bool: + """Whether this gate owns a file in references/vendored/. + + Telemetry bundles are generated from scripts/telemetry/ by sync_bundles.py, + not from skills/shared/, so that subtree has its own parity gate. + """ + rel = path.relative_to(vend_dir) + return ( + path.is_file() + and path.name not in SKIP_NAMES + and rel.parts[0] not in SKIP_TOP_LEVEL_DIRS + ) + + _ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|\s*$", re.M) @@ -93,11 +108,15 @@ def check_manifest(plugin_dir: Path) -> List[str]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue + on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if _is_managed_file(p, vend)} + if not (vend / "README.md").is_file() and not on_disk: + # A vendored tree owned entirely by another generator (currently + # telemetry) has no skills/shared manifest for this gate to check. + continue rows, errs = readme_manifest(vend) errors.extend(errs) if not rows: continue - on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if p.is_file() and p.name not in SKIP_NAMES} for rel in sorted(set(rows) - on_disk): errors.append( f"{_rel(vend / rel)}: listed in {_rel(vend / 'README.md')} but missing on disk — " @@ -124,9 +143,7 @@ def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue - for f in sorted(p for p in vend.rglob("*") if p.is_file()): - if f.name in SKIP_NAMES: - continue + for f in sorted(p for p in vend.rglob("*") if _is_managed_file(p, vend)): rel = f.relative_to(vend) out.append((f, shared / rel, rel)) return out diff --git a/plugins/aws-startup-advisor/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py index f7188b952..8c582c60b 100644 --- a/plugins/aws-startup-advisor/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -109,6 +109,25 @@ def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): assert sv.check_manifest(plugin) == [] +def test_telemetry_bundle_is_owned_by_its_separate_sync_gate(tmp_path: Path): + """Telemetry is generated from scripts/telemetry, not skills/shared.""" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" + (vend / "telemetry").mkdir() + (vend / "telemetry" / "PROTOCOL.md").write_text("generated bundle\n") + (vend / "telemetry" / "version.json").write_text('{"version": "1.0.0"}\n') + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + (vend / "pricing.json").write_text('{"a": 1}\n') + telemetry_only = plugin / "skills" / "telemetry-only" / "references" / "vendored" / "telemetry" + telemetry_only.mkdir(parents=True) + (telemetry_only / "cli.py").write_text("# generated bundle\n") + + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] + + def test_cli_check_exit_codes(tmp_path: Path): plugin = _tree(tmp_path) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], From eb997ff1b1b4d23df70abdf0f8cfd41b39748e82 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 18:03:52 -0700 Subject: [PATCH 40/53] fix(aws-startup-advisor): keep telemetry under its bundle gate Co-authored-by: Cursor --- .../tools/sync-vendored.py | 29 +++++++++++++++---- .../tools/test_sync_vendored.py | 19 ++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py index 26d038ce6..2692bfc66 100644 --- a/plugins/aws-startup-advisor/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -14,8 +14,8 @@ Rules ----- 1. Every file under `plugins//skills//references/vendored/` - (README.md excluded) has a canonical `plugins//skills/shared/` and is - byte-identical to it. + (README.md and the independently generated `telemetry/` bundle excluded) has a + canonical `plugins//skills/shared/` and is byte-identical to it. 2. Each vendored directory's README.md carries a `| Vendored path | Canonical source |` table. Every row must exist on disk (a deleted copy is caught), every file on disk must have a row (the README stays honest), and the row's canonical source must be @@ -52,6 +52,7 @@ VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} +SKIP_TOP_LEVEL_DIRS = {"telemetry"} def _rel(p: Path) -> str: @@ -62,6 +63,20 @@ def _rel(p: Path) -> str: return str(p) +def _is_managed_file(path: Path, vend_dir: Path) -> bool: + """Whether this gate owns a file in references/vendored/. + + Telemetry bundles are generated from scripts/telemetry/ by sync_bundles.py, + not from skills/shared/, so that subtree has its own parity gate. + """ + rel = path.relative_to(vend_dir) + return ( + path.is_file() + and path.name not in SKIP_NAMES + and rel.parts[0] not in SKIP_TOP_LEVEL_DIRS + ) + + _ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|\s*$", re.M) @@ -93,11 +108,15 @@ def check_manifest(plugin_dir: Path) -> List[str]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue + on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if _is_managed_file(p, vend)} + if not (vend / "README.md").is_file() and not on_disk: + # A vendored tree owned entirely by another generator (currently + # telemetry) has no skills/shared manifest for this gate to check. + continue rows, errs = readme_manifest(vend) errors.extend(errs) if not rows: continue - on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if p.is_file() and p.name not in SKIP_NAMES} for rel in sorted(set(rows) - on_disk): errors.append( f"{_rel(vend / rel)}: listed in {_rel(vend / 'README.md')} but missing on disk — " @@ -124,9 +143,7 @@ def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue - for f in sorted(p for p in vend.rglob("*") if p.is_file()): - if f.name in SKIP_NAMES: - continue + for f in sorted(p for p in vend.rglob("*") if _is_managed_file(p, vend)): rel = f.relative_to(vend) out.append((f, shared / rel, rel)) return out diff --git a/plugins/aws-startup-advisor/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py index f7188b952..8c582c60b 100644 --- a/plugins/aws-startup-advisor/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -109,6 +109,25 @@ def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): assert sv.check_manifest(plugin) == [] +def test_telemetry_bundle_is_owned_by_its_separate_sync_gate(tmp_path: Path): + """Telemetry is generated from scripts/telemetry, not skills/shared.""" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" + (vend / "telemetry").mkdir() + (vend / "telemetry" / "PROTOCOL.md").write_text("generated bundle\n") + (vend / "telemetry" / "version.json").write_text('{"version": "1.0.0"}\n') + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + (vend / "pricing.json").write_text('{"a": 1}\n') + telemetry_only = plugin / "skills" / "telemetry-only" / "references" / "vendored" / "telemetry" + telemetry_only.mkdir(parents=True) + (telemetry_only / "cli.py").write_text("# generated bundle\n") + + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] + + def test_cli_check_exit_codes(tmp_path: Path): plugin = _tree(tmp_path) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], From 3e44b06d81a496b5995e5acc81e0d0d70869bfff Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Wed, 7 Oct 2026 18:03:52 -0700 Subject: [PATCH 41/53] fix(aws-startup-advisor): keep telemetry under its bundle gate Co-authored-by: Cursor --- .../tools/sync-vendored.py | 29 +++++++++++++++---- .../tools/test_sync_vendored.py | 19 ++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/plugins/aws-startup-advisor/tools/sync-vendored.py b/plugins/aws-startup-advisor/tools/sync-vendored.py index 26d038ce6..2692bfc66 100644 --- a/plugins/aws-startup-advisor/tools/sync-vendored.py +++ b/plugins/aws-startup-advisor/tools/sync-vendored.py @@ -14,8 +14,8 @@ Rules ----- 1. Every file under `plugins//skills//references/vendored/` - (README.md excluded) has a canonical `plugins//skills/shared/` and is - byte-identical to it. + (README.md and the independently generated `telemetry/` bundle excluded) has a + canonical `plugins//skills/shared/` and is byte-identical to it. 2. Each vendored directory's README.md carries a `| Vendored path | Canonical source |` table. Every row must exist on disk (a deleted copy is caught), every file on disk must have a row (the README stays honest), and the row's canonical source must be @@ -52,6 +52,7 @@ VENDORED_SUBDIR = Path("references") / "vendored" SHARED_DIRNAME = "shared" SKIP_NAMES = {"README.md"} +SKIP_TOP_LEVEL_DIRS = {"telemetry"} def _rel(p: Path) -> str: @@ -62,6 +63,20 @@ def _rel(p: Path) -> str: return str(p) +def _is_managed_file(path: Path, vend_dir: Path) -> bool: + """Whether this gate owns a file in references/vendored/. + + Telemetry bundles are generated from scripts/telemetry/ by sync_bundles.py, + not from skills/shared/, so that subtree has its own parity gate. + """ + rel = path.relative_to(vend_dir) + return ( + path.is_file() + and path.name not in SKIP_NAMES + and rel.parts[0] not in SKIP_TOP_LEVEL_DIRS + ) + + _ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|\s*$", re.M) @@ -93,11 +108,15 @@ def check_manifest(plugin_dir: Path) -> List[str]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue + on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if _is_managed_file(p, vend)} + if not (vend / "README.md").is_file() and not on_disk: + # A vendored tree owned entirely by another generator (currently + # telemetry) has no skills/shared manifest for this gate to check. + continue rows, errs = readme_manifest(vend) errors.extend(errs) if not rows: continue - on_disk = {str(p.relative_to(vend)) for p in vend.rglob("*") if p.is_file() and p.name not in SKIP_NAMES} for rel in sorted(set(rows) - on_disk): errors.append( f"{_rel(vend / rel)}: listed in {_rel(vend / 'README.md')} but missing on disk — " @@ -124,9 +143,7 @@ def vendored_files(plugin_dir: Path) -> List[Tuple[Path, Path, Path]]: vend = skill / VENDORED_SUBDIR if skill.name == SHARED_DIRNAME or not vend.is_dir(): continue - for f in sorted(p for p in vend.rglob("*") if p.is_file()): - if f.name in SKIP_NAMES: - continue + for f in sorted(p for p in vend.rglob("*") if _is_managed_file(p, vend)): rel = f.relative_to(vend) out.append((f, shared / rel, rel)) return out diff --git a/plugins/aws-startup-advisor/tools/test_sync_vendored.py b/plugins/aws-startup-advisor/tools/test_sync_vendored.py index f7188b952..8c582c60b 100644 --- a/plugins/aws-startup-advisor/tools/test_sync_vendored.py +++ b/plugins/aws-startup-advisor/tools/test_sync_vendored.py @@ -109,6 +109,25 @@ def test_new_shared_file_nobody_vendors_is_not_an_error(tmp_path: Path): assert sv.check_manifest(plugin) == [] +def test_telemetry_bundle_is_owned_by_its_separate_sync_gate(tmp_path: Path): + """Telemetry is generated from scripts/telemetry, not skills/shared.""" + plugin = _tree(tmp_path) + vend = plugin / "skills" / "alpha" / "references" / "vendored" + (vend / "telemetry").mkdir() + (vend / "telemetry" / "PROTOCOL.md").write_text("generated bundle\n") + (vend / "telemetry" / "version.json").write_text('{"version": "1.0.0"}\n') + (vend / "orphan.md").unlink() + readme = vend / "README.md" + readme.write_text("\n".join(l for l in readme.read_text().splitlines() if "orphan" not in l) + "\n") + (vend / "pricing.json").write_text('{"a": 1}\n') + telemetry_only = plugin / "skills" / "telemetry-only" / "references" / "vendored" / "telemetry" + telemetry_only.mkdir(parents=True) + (telemetry_only / "cli.py").write_text("# generated bundle\n") + + assert sv.check_plugin(plugin) == [] + assert sv.check_manifest(plugin) == [] + + def test_cli_check_exit_codes(tmp_path: Path): plugin = _tree(tmp_path) r = subprocess.run([sys.executable, str(SCRIPT), "--check", "--plugin-root", str(plugin)], From d94ac696f61fc810d85658e89896c1245ca2bf27 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Thu, 8 Oct 2026 21:57:31 -0700 Subject: [PATCH 42/53] fix(aws-startup-advisor): match the shared telemetry client to the bundle The parity check treats skills/shared as canonical. That copy was missing the plugin User-Agent the vendored bundles already send, so the check failed. --- .../skills/shared/telemetry/metric_emission/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py index b26247789..94cbff5d8 100644 --- a/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py +++ b/plugins/aws-startup-advisor/skills/shared/telemetry/metric_emission/client.py @@ -67,6 +67,8 @@ def _plugin_root(): # DNS, refused): unlike a timeout, nothing can have reached the service. UNREACHABLE = 0 +USER_AGENT = "aws-startup-advisor-plugin" + SOURCE_CLAUDE_CODE = "CLAUDE_CODE" SOURCE_CODEX = "CODEX" SOURCE_CURSOR = "CURSOR" @@ -230,7 +232,7 @@ def post_event_status(event, install_id, url, timeout=TIMEOUT_SECONDS): request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, + headers={"Content-Type": "application/json", "User-Agent": USER_AGENT}, method="POST", ) try: From 590640ffe929892969b19cbae1bcc4d2bda98b3d Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 9 Oct 2026 08:55:08 -0700 Subject: [PATCH 43/53] fix(aws-startup-advisor): address PR #410 Cursor review findings Three independently reproduced findings: 1. clarify-ai-only.md's "preferences.json (AI-only)" example contradicted its own prose: ai_framework.value was a bare string while Q1 is documented as multi-select with a ["direct"] default, and the example omitted the mandated metadata.report_caveats[] and never used the source/question_id provenance alternative the prose referenced. Fixed the example to match an array value and added report_caveats; clarified the prose that question_id is this flow's sole provenance field. 2. _assert_scenario_semantics deduplicated gateway upstreams on their raw (provider, source_service-or-None) key BEFORE resolving a service-less upstream to its actual source, so an explicit-service upstream and an overlapping service-less upstream for the same single retained source had different raw keys and both passed. Now dedupes on the resolved source's own (provider, source_service) key. Added a parametrized regression test covering both insertion orderings. 3. strip_jsonc attributed a trailing // comment to the textually-last "key": match on a line, which broke for an inline array/object value that closes before the comment ("call_sites": [{ "file": ..., "line": 1 }], // REQUIRED resolved to "line", not "call_sites"), so omitting the entire required field passed silently. Now tracks bracket depth and binds the comment to the key whose own value's depth matches (or, for a value still open at end of line, is one level shallower than) the depth at the comment. Added unit tests on strip_jsonc directly plus an end-to-end MISSING_REQUIRED regression against the real contract. All three reproduced against the live PR head before fixing. 104 tests pass in the two affected files; full mise run build, lint:artifacts (74 checked, 0 errors), lint:vendored-parity, diff --check, and merge-tree against upstream/main are all clean. --- .../scripts/validate-artifacts.py | 73 +++++++++++++++- .../phases/clarify/clarify-ai-only.md | 12 ++- .../test_ai_migration_matrix_contracts.py | 59 +++++++++++-- .../tests/test_validate_artifacts.py | 84 +++++++++++++++++++ 4 files changed, 211 insertions(+), 17 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index de3817855..cd7d685aa 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -152,10 +152,75 @@ def strip_jsonc(text: str) -> Tuple[str, Dict[int, str], Dict[int, str]]: if idx is not None: comments[i] = line[idx + 2:].strip() line = line[:idx].rstrip() - # the key a trailing comment annotates is the LAST `"key":` opened on the line - key_matches = re.findall(r'"((?:[^"\\]|\\.)*)"\s*:', line) - if key_matches: - keys[i] = key_matches[-1] + # The key a trailing comment annotates is the LAST `"key":` opened on the line + # WHOSE OWN NESTING DEPTH MATCHES THE DEPTH AT THE END OF THE LINE — i.e. a key + # that is still "current" by the time the comment is reached, not merely the + # last key-like string that happens to appear earliest in the text. Two cases + # must resolve differently and both are exercised by tests: + # - `"routing_provenance": "table", // REQUIRED` (depth stays open on this + # key's own value, never closed back out before the comment) → the comment + # belongs to `routing_provenance` itself, the last key opened, matching the + # original design for a key immediately followed by its own `// REQUIRED`. + # - `"call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED` (the inline + # array/object value is fully opened AND closed before the comment, so depth + # returns to the level `call_sites` itself was opened at) → the comment + # belongs to `call_sites`, the enclosing field, not to `line` (a key nested + # inside that value, whose own depth no longer matches the end-of-line depth). + # Tracking bracket depth while re-scanning the comment-stripped line, and keeping + # the LAST key seen at each depth, implements this without changing the plain + # same-depth case (several flat `"a": 1, "b": 2 // REQUIRED` siblings still + # resolve to the last one, `b`, because nothing nests and depth never changes). + depth = 0 + last_key_at_depth: Dict[int, str] = {} + k_in_str = False + k_esc = False + pos = 0 + key_start: Optional[int] = None + string_open_pos = 0 + while pos < len(line): + ch = line[pos] + if k_in_str: + if k_esc: + k_esc = False + elif ch == "\\": + k_esc = True + elif ch == '"': + k_in_str = False + key_start = pos # candidate string just closed at `pos` + pos += 1 + continue + if ch == '"': + k_in_str = True + string_open_pos = pos + pos += 1 + continue + if ch in "{[": + depth += 1 + pos += 1 + continue + if ch in "}]": + depth -= 1 + pos += 1 + continue + if ch == ":" and key_start is not None: + # the string that just closed, immediately followed by `:` (ignoring + # whitespace), is a key opened at the CURRENT depth (before this colon's + # value opens any further nesting of its own) + between = line[key_start + 1:pos] + if between.strip() == "": + last_key_at_depth[depth] = line[string_open_pos + 1:key_start] + if ch not in '"': + key_start = None + pos += 1 + # A key's value can legitimately still be OPEN at end-of-line (e.g. + # `"workloads": [ // REQUIRED` whose `]` closes on a later line) — that key was + # recorded one level shallower than `depth` (the depth its own `[`/`{` pushed to). + # Prefer an exact depth match (the fully-closed-on-this-line case); fall back to + # `depth - 1` (the still-open-at-end-of-line case) only when no exact match exists. + if depth in last_key_at_depth: + keys[i] = last_key_at_depth[depth] + elif (depth - 1) in last_key_at_depth: + keys[i] = last_key_at_depth[depth - 1] out_lines.append(line) joined = "\n".join(out_lines) # bare `...` ellipses in arrays/objects (`["Q1", "Q2", ...]`) → a placeholder string diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index 263f0b618..bb2950e24 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -146,8 +146,11 @@ Write `$MIGRATION_DIR/preferences.json`: the downstream source of truth (design-ai reads it, not the profile). Each constraint carries the full clarify field shape (`value`, `chosen_by` ∈ -`user|extracted|default|derived`, `prompt`, `design_consequence`, `source`/`question_id`). No -nulls. Delete `preferences-draft.json` after writing. +`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`). Every +constraint in this flow is tied to a specific numbered question, so `question_id` is always +present and `source` (the inventory/billing-derived provenance tag used in +`clarify-assemble.md`'s infra flow) is not part of this AI-only shape. No nulls. Delete +`preferences-draft.json` after writing. ## preferences.json (AI-only) @@ -159,7 +162,8 @@ nulls. Delete `preferences-draft.json` after writing. "clarify_mode": "full", // fast_path | full "questions_asked": [], "questions_defaulted": [], - "questions_extracted": [] + "questions_extracted": [], + "report_caveats": [] // REQUIRED — [] when none; Q1.5 skip/default appends a caveat here }, "design_constraints": { // REQUIRED — region and compliance only; no infra constraints "target_region": { // REQUIRED @@ -188,7 +192,7 @@ nulls. Delete `preferences-draft.json` after writing. }, "ai_constraints": { // REQUIRED "ai_framework": { // REQUIRED - "value": "direct", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" + "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" }, "ai_priority": { // REQUIRED "value": "balanced", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q17" diff --git a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py index dde38f84c..7843c6818 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_ai_migration_matrix_contracts.py @@ -117,17 +117,18 @@ def _assert_scenario_semantics(snapshot: dict) -> None: sources[key] = source for gateway in dimensions["gateways"]: - upstream_keys = set() + resolved_keys = set() for upstream in gateway["upstreams"]: - key = (upstream["provider"], upstream.get("source_service")) - assert key not in upstream_keys, ( - f"duplicate or overlapping gateway upstream: {key}" - ) - upstream_keys.add(key) - + # Resolve the upstream's optional provenance to its effective source FIRST. + # Deduplicating on the raw (provider, source_service-or-None) tuple before + # resolution lets an explicit-service upstream and a service-less upstream + # that both resolve to the same single retained source carry different keys + # and silently pass as "distinct" — exactly the overlapping association the + # uniqueness contract forbids. if "source_service" in upstream: - source = sources.get((upstream["provider"], upstream["source_service"])) - assert source is not None, f"unmatched gateway upstream: {key}" + raw_key = (upstream["provider"], upstream["source_service"]) + source = sources.get(raw_key) + assert source is not None, f"unmatched gateway upstream: {raw_key}" else: candidates = [ source @@ -139,6 +140,15 @@ def _assert_scenario_semantics(snapshot: dict) -> None: ) source = candidates[0] + # Check the EFFECTIVE provider/service pair — the resolved source's own key, + # not the upstream's possibly-service-less raw key — so the explicit-service + # and service-less spellings of the same source collide correctly. + resolved_key = (source["provider"], source["source_service"]) + assert resolved_key not in resolved_keys, ( + f"duplicate or overlapping gateway upstream: {resolved_key}" + ) + resolved_keys.add(resolved_key) + families = set(source["model_families"]) models = set(source["models"]) seen_families = set() @@ -571,6 +581,37 @@ def test_duplicate_or_overlapping_gateway_upstreams_are_rejected(overlap: str) - assert _scenario_validation_findings(scenario) +@pytest.mark.parametrize("explicit_service_first", [True, False]) +def test_overlapping_explicit_and_service_less_upstream_for_one_retained_source_is_rejected( + explicit_service_first: bool, +) -> None: + """A gateway with exactly one retained ``openai``/``openai_api`` source must not accept + both an explicit ``source_service: "openai_api"`` upstream and a service-less copy with + the same family/models. The two specs have different raw (provider, source_service) + keys but resolve to the identical source — the uniqueness contract is about the + *resolved* association, not the upstream's raw provenance spelling. Both argument + orderings are checked: the dedup bug this guards against was order-independent (each + upstream's raw key differed from the other's regardless of which was added first), so + this proves the fix does not depend on insertion order. + """ + scenario = copy.deepcopy(_cases()["ai-only-openrouter"][1]) + gateway = scenario["dimensions"]["gateways"][0] + explicit_service_upstream = copy.deepcopy(gateway["upstreams"][0]) + service_less_upstream = { + "provider": "openai", + "model_families": copy.deepcopy(explicit_service_upstream["model_families"]), + } + if explicit_service_first: + gateway["upstreams"] = [explicit_service_upstream, service_less_upstream] + else: + gateway["upstreams"] = [service_less_upstream, explicit_service_upstream] + if not _scenario_validation_findings(scenario): + with pytest.raises(AssertionError): + _assert_scenario_semantics(scenario) + else: + assert _scenario_validation_findings(scenario) + + @pytest.mark.parametrize( "mutation", [ diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index b3b7a57da..4cd39d1ae 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -46,6 +46,31 @@ def test_strip_jsonc_handles_ellipsis_and_bool_alternation(): assert json.loads(out) == {"a": ["Q1", "..."], "b": True} +def test_strip_jsonc_binds_a_trailing_comment_to_the_enclosing_field_not_a_nested_key(): + """`"call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED` must attribute the + comment to `call_sites` (the field this line actually declares), not to `line` (the + last key opened on the line, which is nested two levels deeper inside the inline + array/object value). Picking the textually-last key regardless of depth let a + `// REQUIRED` on an inline array/object value silently annotate one of its own nested + keys instead of the enclosing field, so the enclosing field was never marked required. + """ + text = '{\n "call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED\n}' + _, comments, keys = va.strip_jsonc(text) + assert comments[1] == "REQUIRED" + assert keys[1] == "call_sites" + + +def test_strip_jsonc_still_binds_to_the_last_of_several_flat_sibling_keys(): + """Several sibling keys packed on one flat line (no nesting) must still resolve to the + LAST one, preserving the original intent for the common `"a": 1, "b": 2 // REQUIRED` + case this attribution logic was designed for. + """ + text = '{\n "a": 1, "b": 2 // REQUIRED\n}' + _, comments, keys = va.strip_jsonc(text) + assert comments[1] == "REQUIRED" + assert keys[1] == "b" + + # --------------------------------------------------------------------------- template inference @@ -393,6 +418,65 @@ def test_injected_off_contract_key_fails(tmp_path: Path): for e in report["errors"]), report["errors"] +def _ai_only_preferences_example() -> dict: + """Load the real `## preferences.json (AI-only)` example from + `clarify-ai-only.md`, filling in its `chosen_by` placeholder so it is a concrete, + schema-valid document rather than a documentation template.""" + doc = ( + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "phases" / "clarify" + / "clarify-ai-only.md" + ).read_text() + import re + m = re.search(r"## preferences\.json \(AI-only\)\s*```jsonc\n(.*?)\n```", doc, re.S) + lines = [] + for line in m.group(1).split("\n"): + idx = line.find(" //") + lines.append(line[:idx] if idx != -1 else line) + data = json.loads("\n".join(lines)) + + def fix_chosen_by(node): + if isinstance(node, dict): + if node.get("chosen_by") == "user|extracted|default|derived": + node["chosen_by"] = "extracted" + for v in node.values(): + fix_chosen_by(v) + elif isinstance(node, list): + for v in node: + fix_chosen_by(v) + + fix_chosen_by(data) + return data + + +def test_required_inline_array_field_cannot_be_omitted_on_either_route(tmp_path: Path): + """Finding 3 regression. `clarify-ai-only.md`'s `workloads[].call_sites` is declared + `// REQUIRED` as `"call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED` — an + inline array of objects on one line. Before the `strip_jsonc` key-attribution fix, + the trailing comment bound to `line` (the last key opened on the line, nested inside + the array) instead of `call_sites` (the enclosing field), so omitting the entire + `call_sites` array passed validation with no MISSING_REQUIRED finding. `priority` is + the control: a plain top-level sibling on the same object, correctly flagged either + way, proving the fix did not just start over-flagging unrelated keys.""" + manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") + + base = _ai_only_preferences_example() + for mutate, missing_path in ( + (lambda d: d["workloads"][0].pop("call_sites"), "workloads[0].call_sites"), + (lambda d: d["workloads"][0].pop("priority"), "workloads[0].priority"), + ): + import copy + data = copy.deepcopy(base) + mutate(data) + run = tmp_path / missing_path.replace("[", "_").replace("]", "").replace(".", "_") + run.mkdir() + (run / "preferences.json").write_text(json.dumps(data)) + findings = [] + va.validate_run_dir(run, "azure-to-aws", manifest, findings) + assert any( + f.code == "MISSING_REQUIRED" and f.path == missing_path for f in findings + ), (missing_path, [f.as_dict() for f in findings]) + + def test_empty_phases_fails_min_properties(tmp_path: Path): """Review finding on #385: phase-status.schema.json has `minProperties: 1` on `phases` and the subset silently skipped it, so `phases: {}` passed.""" From 1182828b8c75b3175f63ba868d081ca067a90177 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 9 Oct 2026 09:14:27 -0700 Subject: [PATCH 44/53] fix(aws-startup-advisor): address PR #416 Cursor review findings Five independently reproduced findings: 1. The GCP shadow contract (schema-design-aws-ai.md) put phase/focus/timestamp at the top level, but design-ai.md's own "top-level fields" table nests all three (plus ai_source, bedrock_models_selected) inside metadata -- GCP does not split them the way Azure's parallel shape does. The example also omitted design_blocks[].honest_assessment, required ("not_applicable") on the three traditional-AI capability rows per design-ai.md's Validation Checklist. Fixed the example's nesting and added the conditional honest_assessment field; corrected test_aws_design_ai_contracts.py's _gcp_scalar_design() fixture, which built from the canonical target shape with only metadata swapped and so never exercised the actual producer envelope. 2. test_neutral_matrix_shared_artifacts_extend_the_mapped_fixture_count assumed only ai-migration-matrix contributes unmapped "shared" fixtures, but ai-workload-profiles/ (#415) and aws-design-ai/ (#416) are unmapped too. Generalized the assertion to count every unmapped fixture family. 3. clarify-ai-only.md's AI-only example contradicted its own prose (ai_framework as a bare string vs. the documented ["direct"] multi-select default, missing metadata.report_caveats); clarify-ai.md's mixed-flow example omitted its own documented Q21 ai_latency row. Fixed both examples; clarified clarify-ai-only .md's prose that question_id is this flow's sole provenance field. 4. test_positive_fixtures_project_exactly_into_pr415_semantics only ran the weaker profile-level oracle (_assert_profile_semantics), not the stronger matrix-level gateway/source correspondence oracle (_assert_scenario_semantics) that the dedicated negative mutation test already proves is strict. A model dropped from only a gateway upstream in a copied positive fixture passed all existing positive-fixture checks. Now runs both oracles on every positive fixture. 5. artifact-contracts.json's extra_shapes entries point preferences.json's AI fields at clarify-ai.md's "Step 3: Rows returned" example, which showed bare per-fragment values ("ai_priority": "balanced") instead of the final assembled {disposition, value, default} envelope clarify-assemble.md's own rules require and every sibling fragment (e.g. clarify-global.md) already documents in that wrapped form. A properly assembled mixed-Azure artifact failed TYPE_MISMATCH/UNKNOWN_KEY on every AI row. Rewrote the example to show the wrapped envelope, matching the established sibling-fragment convention. All five reproduced against the live PR head before fixing. 181 tests pass in the plugin's full suite; mise run build, lint:artifacts (80 checked, 0 errors), lint:vendored-parity (90 files), diff --check, and merge-tree against upstream/main are all clean. --- .../phases/clarify/clarify-ai-only.md | 12 ++- .../references/phases/clarify/clarify-ai.md | 73 ++++++++++++++----- .../references/shared/schema-design-aws-ai.md | 22 ++++-- .../tests/test_aws_design_ai_contracts.py | 55 +++++++++++--- .../tests/test_validate_artifacts.py | 15 +++- 5 files changed, 138 insertions(+), 39 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index 263f0b618..bb2950e24 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -146,8 +146,11 @@ Write `$MIGRATION_DIR/preferences.json`: the downstream source of truth (design-ai reads it, not the profile). Each constraint carries the full clarify field shape (`value`, `chosen_by` ∈ -`user|extracted|default|derived`, `prompt`, `design_consequence`, `source`/`question_id`). No -nulls. Delete `preferences-draft.json` after writing. +`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`). Every +constraint in this flow is tied to a specific numbered question, so `question_id` is always +present and `source` (the inventory/billing-derived provenance tag used in +`clarify-assemble.md`'s infra flow) is not part of this AI-only shape. No nulls. Delete +`preferences-draft.json` after writing. ## preferences.json (AI-only) @@ -159,7 +162,8 @@ nulls. Delete `preferences-draft.json` after writing. "clarify_mode": "full", // fast_path | full "questions_asked": [], "questions_defaulted": [], - "questions_extracted": [] + "questions_extracted": [], + "report_caveats": [] // REQUIRED — [] when none; Q1.5 skip/default appends a caveat here }, "design_constraints": { // REQUIRED — region and compliance only; no infra constraints "target_region": { // REQUIRED @@ -188,7 +192,7 @@ nulls. Delete `preferences-draft.json` after writing. }, "ai_constraints": { // REQUIRED "ai_framework": { // REQUIRED - "value": "direct", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" + "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" }, "ai_priority": { // REQUIRED "value": "balanced", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q17" diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md index a4bfe41bf..7df830cd7 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md @@ -101,32 +101,69 @@ Q16–Q22, and write `"workloads": []` when none. ## Step 3: Rows returned +Every row is merged into the final `preferences.json` by `clarify-assemble.md`'s Step +"Assemble the artifact" rules 1–2: it carries `disposition`, `value`, and `default`, exactly +like every other fragment's rows (see `clarify-global.md`'s `## Rows returned` for the same +convention). A row whose value came from its default stays **PROPOSED** — never promoted to +DETECTED (DETECTED means read from the estate; Design's rationale and the report distinguish +"you chose" from "we assumed"). A `DETECTED` row additionally carries `source` naming where +the value was read from (per `schema-preferences.md`'s `allowed_anywhere` provenance keys). + ```jsonc { - "ai_framework": ["direct"], // DETECTED | PROPOSED - "ai_monthly_spend": "$500-$2K", // DETECTED | PROPOSED - "ai_priority": "balanced", // PROPOSED - "ai_critical_feature": null, // PROPOSED - "ai_token_volume": "low", // PROPOSED - "ai_model_baseline": "gpt-4o", // DETECTED | PROPOSED - "ai_vision": false, // DETECTED | PROPOSED - "ai_complexity": "moderate", // PROPOSED - "startup_program_status": null, // ESSENTIAL — value null until answered (the Gate-2 completion gate) - "ai_constraints": { // agentic block present ONLY when agentic_profile.is_agentic + "ai_framework": { "disposition": "PROPOSED", "value": ["direct"], "default": ["direct"] }, + "ai_monthly_spend": { + "disposition": "DETECTED", + "value": "$500-$2K", + "default": "$500-$2K", + "source": "current_costs.monthly_ai_spend" + }, + "ai_priority": { "disposition": "PROPOSED", "value": "balanced", "default": "balanced" }, + "ai_critical_feature": { "disposition": "PROPOSED", "value": null, "default": null }, + "ai_token_volume": { "disposition": "PROPOSED", "value": "low", "default": "low" }, + "ai_model_baseline": { + "disposition": "DETECTED", + "value": "gpt-4o", + "default": "gpt-4o", + "source": "models[] confidence >= 0.8" + }, + "ai_vision": { + "disposition": "DETECTED", + "value": false, + "default": false, + "source": "capabilities_summary.vision" + }, + "ai_latency": { "disposition": "PROPOSED", "value": "important", "default": "important" }, + "ai_complexity": { "disposition": "PROPOSED", "value": "moderate", "default": "moderate" }, + "startup_program_status": { + "disposition": "ESSENTIAL", + "value": null, // null until answered (the Gate-2 completion gate) + "default": null + }, + "ai_constraints": { // agentic block present ONLY when agentic_profile.is_agentic — rows, not raw values, same disposition/value/default wrapping "agentic": { - "migration_approach": "undecided", - "memory_requirement": "session", - "task_duration": "medium", - "incremental_migration": false + "migration_approach": { + "disposition": "PROPOSED", + "value": "undecided", + "default": "undecided" + }, + "memory_requirement": { + "disposition": "PROPOSED", + "value": "session", + "default": "session" + }, + "task_duration": { "disposition": "PROPOSED", "value": "medium", "default": "medium" }, + "incremental_migration": { + "disposition": "PROPOSED", + "value": false, + "default": false + } } }, - "workloads": [] // the confirmed array (persist rule above) + "workloads": [] // NOT a row — the confirmed array itself (persist rule above), same as clarify-ai-only.md's workloads[] } ``` -A value taken from its default stays **PROPOSED** — never promoted to DETECTED (DETECTED means -read from the estate; Design's rationale and the report distinguish "you chose" from "we assumed"). - ### `workloads[]` ```jsonc diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md index 0651025d0..a546f0323 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -17,15 +17,18 @@ ```jsonc { - "phase": "design", - "focus": "ai", - "timestamp": "", - "source_profile": "ai-workload-profile.json", "metadata": { + // design-ai.md's own "top-level fields" table: `metadata` is `phase`, `focus`, + // `ai_source`, `bedrock_models_selected`, `timestamp` — all five nested here, not + // split across the top level and `metadata` the way Azure's parallel shape does. + "phase": "design", + "focus": "ai", "ai_source": "gemini", // "gemini" | "openai" | "anthropic" | "both" | "other" — summary.ai_source from the profile "bedrock_models_selected": [], // aws_model_id strings chosen (Bedrock targets only) - "regional_validation": "checked" // checked | fallback_static + "timestamp": "", + "regional_validation": "checked" // checked | fallback_static — ONLY present when the fallback-static path fires (design-ai.md Step 0.5); absent on the normal checked path }, + "source_profile": "ai-workload-profile.json", "design_blocks": [], // one per workloads[] entry — see § design_blocks "ai_architecture": {}, // see design-ai.md Part 6 field table "regional_warnings": [], // ALWAYS present, [] when clean @@ -47,7 +50,8 @@ One row per confirmed workload — see `design-ai.md` Part 6 field table. "capability": "text_generation", // see § capability vocabulary "capability_confidence": "medium", // high | medium | low "rationale": "", - "confidence_warning": null // non-null string when capability_confidence == "low" + "confidence_warning": null, // non-null string when capability_confidence == "low" + "honest_assessment": "not_applicable" // ONLY present on traditional-AI rows (document_extraction/image_analysis/speech_transcription), always "not_applicable" there; absent on Bedrock-model rows, whose overall assessment lives at ai_architecture.honest_assessment instead } ``` @@ -57,6 +61,12 @@ One row per confirmed workload — see `design-ai.md` Part 6 field table. leave `target_aws_service` null. The three traditional-AI capabilities (`document_extraction`, `image_analysis`, `speech_transcription`) set `target_aws_service` (one of `"textract"`, `"rekognition"`, `"transcribe"`) and leave `target_bedrock_model` null. +- **`honest_assessment`** is present on `design_blocks[]` rows ONLY for the three + traditional-AI capabilities, and is always `"not_applicable"` there — these are + feature/service swaps, not a cost-driven model migration decision. Bedrock-model rows do + not carry a per-row `honest_assessment`; the overall assessment is + `ai_architecture.honest_assessment` instead (one value for the whole design, driven by the + weakest model). - **`confidence_warning`** is a non-null string when `capability_confidence == "low"`; null for `high`/`medium`. - **Input order preserved:** `design_blocks[]` order matches the profile's `workloads[]` order. diff --git a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py index 26ba69757..2a81a4b3e 100644 --- a/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_aws_design_ai_contracts.py @@ -101,6 +101,16 @@ def test_positive_fixtures_project_exactly_into_pr415_semantics(fixture_path: Pa profile = _project_to_profile(design) assert WORKLOAD_PROFILE._profile_findings(profile) == [] WORKLOAD_PROFILE._assert_profile_semantics(profile) + # `_assert_profile_semantics` checks source observations and duplicate upstream + # identities, but not whether a gateway upstream's retained models/families actually + # match its resolved source's full set — that stronger correspondence oracle lives one + # layer up, in #410's `_assert_scenario_semantics`. Run it here too so every POSITIVE + # fixture is protected by the same oracle the dedicated negative mutation test already + # proves is strict (`test_dropped_model_under_a_resolved_gateway_upstream_is_rejected`), + # not only the weaker profile-level check. + WORKLOAD_PROFILE.MATRIX._assert_scenario_semantics( + WORKLOAD_PROFILE._project_to_scenario(profile) + ) def test_multi_provider_fixture_carries_two_simultaneous_direct_sources() -> None: @@ -346,17 +356,42 @@ def test_azure_scalar_ai_source_fails_the_target_only_schema_with_no_skill() -> def _gcp_scalar_design() -> dict: - """A document matching GCP's CURRENT single-scalar metadata.ai_source shape — the shape - GCP's design-ai.md actually emits today (skills/gcp-to-aws/references/shared/ - schema-design-aws-ai.md). Everything else matches the canonical top-level shape so only - `metadata` differs.""" - design = _base_design() - design["metadata"] = { - "ai_source": "gemini", - "bedrock_models_selected": design["metadata"]["bedrock_models_selected"], - "regional_validation": design["metadata"]["regional_validation"], + """A document matching GCP's CURRENT producer envelope — the shape `design-ai.md`'s own + "top-level fields" table documents and `design-ai.md`'s Validation Checklist requires, + not the canonical target shape with only `metadata` swapped. Per that table, `metadata` + nests `phase`, `focus`, `ai_source`, `bedrock_models_selected`, and `timestamp` (GCP does + NOT split those across the top level and `metadata` the way the canonical/Azure shape + does), and `source_profile` stays top-level alongside `design_blocks`/`ai_architecture`/ + `regional_warnings`/`multi_model_warnings`/`agentic_design`. One `design_blocks[]` row + uses a traditional-AI capability so `honest_assessment: "not_applicable"` is exercised + (design-ai.md's Validation Checklist requires it on exactly those rows).""" + return { + "metadata": { + "phase": "design", + "focus": "ai", + "ai_source": "gemini", + "bedrock_models_selected": ["anthropic.claude-sonnet-5"], + "timestamp": "2026-10-06T00:00:00Z", + }, + "source_profile": "ai-workload-profile.json", + "design_blocks": [ + { + "workload_id": "wl_doc1", + "model_id": "gemini-1.5-pro-vision", + "target_bedrock_model": None, + "target_aws_service": "textract", + "capability": "document_extraction", + "capability_confidence": "high", + "rationale": "Traditional-AI capability swap, not a model migration.", + "confidence_warning": None, + "honest_assessment": "not_applicable", + } + ], + "ai_architecture": {}, + "regional_warnings": [], + "multi_model_warnings": [], + "agentic_design": None, } - return design def test_gcp_scalar_ai_source_validates_under_the_gcp_shape_contract() -> None: diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index b3b7a57da..9434aaebd 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -334,17 +334,30 @@ def test_neutral_matrix_shared_artifacts_extend_the_mapped_fixture_count(): assert matrix_checked == len(shared_artifacts) assert matrix_findings == [] + # Several fixture families besides ai-migration-matrix are unmapped (no + # fixture_prefix entry) and so validate only against artifact-contracts.json's + # "shared" section too -- e.g. ai-workload-profiles/ (#415) and aws-design-ai/ + # (#416). Count every unmapped family's shared artifacts, not only the matrix's, + # so this assertion does not silently drift whenever a future PR adds another one. mapped_checked = 0 mapped_findings = [] + unmapped_shared_count = 0 for fixture_dir in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): skill = va.skill_for_fixture(fixture_dir.name, manifest) if skill is not None: mapped_checked += va.validate_run_dir(fixture_dir, skill, manifest, mapped_findings) + else: + unmapped_shared_count += len( + [p for p in va.find_artifacts(fixture_dir) if p.name in manifest["shared"]] + ) + assert unmapped_shared_count >= len(shared_artifacts), ( + "the matrix's own shared artifacts must be a subset of every unmapped family's total" + ) r = _run("--fixtures", "--json") assert r.returncode == 0, r.stdout + r.stderr report = json.loads(r.stdout) - assert report["checked_artifacts"] - mapped_checked == len(shared_artifacts) + assert report["checked_artifacts"] - mapped_checked == unmapped_shared_count def test_invalid_neutral_shared_artifacts_fail_their_schemas(tmp_path: Path): From 4f69ea2269bf3eaedf68cf2a553b47d1f8fab09a Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 9 Oct 2026 10:28:24 -0700 Subject: [PATCH 45/53] fix(aws-startup-advisor): address PR #411 Cursor review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four independently reproduced findings: 1. The pre-split Azure cache had its own "### Embeddings — Bedrock" target table (Titan Text Embeddings v1/v2, Cohere Embed v4/English v3/Multilingual v3) nested under GCP's "## Bedrock Models (On-Demand)" heading, but it is a different subsection than the one the parity script's byte-equality check covers, so the split dropped it entirely. estimate-ai.md's design flow still references this table and still recommends Titan v2 as the default embedding target. Moved the table into the canonical cache (scoped out of the main section's byte-equality check, matching the existing Claude Fable exception pattern) and the model_rows() quick-reference scanner (its Dimensions column coincidentally parses as a numeric output price for three rows), synced all three vendors, and added a dedicated row-level parity check. 2. clarify-ai-only.md's AI-only example contradicted its own prose (bare-string ai_framework vs. the documented ["direct"] array default, missing metadata.report_caveats) and route_shapes does not inherit the parent preferences.json entry's allowed_anywhere -- so (REQUIRED on every DETECTED-equivalent row per schema-preferences.md's own fundamental rule) was rejected on this route even though the general contract requires it. Fixed the example and added allowed_anywhere to the AI-only route_shapes entry; added source to this route's extracted rows and corrected the prose accordingly. Added three documented-shape acceptance cases to the existing producer-route test. 3. artifact-contracts.json's extra_shapes pointed preferences.json's mixed-flow AI fields at clarify-ai.md's raw per-fragment "Step 3: Rows returned" values instead of the final assembled {disposition, value, default} envelope clarify-assemble.md's rules require and every sibling fragment (e.g. clarify-global.md) already documents in that wrapped form; the example also omitted the documented Q21 ai_latency row. Rewrote the example to match the established convention, with source on DETECTED rows per finding 2's rule. 4. strip_jsonc attributed a trailing comment to the textually-last "key": match on a line, which broke for the committed workloads shape "call_sites": [{ "file": ..., "line": 1 }], // REQUIRED (resolved to "line", not "call_sites"), so omitting the entire required field passed silently in both Azure preferences routes. Now tracks bracket depth and binds the comment to the key whose own depth matches the end-of-line depth. Added strip_jsonc unit tests plus end-to-end MISSING_REQUIRED regressions on the real contract. All four reproduced against the live PR head before fixing. 73 tests pass in the plugin's full suite plus 23 in llm-to-bedrock's pricing suite; mise run build, lint:artifacts (53 checked, 0 errors), lint:vendored-parity (47 files), diff --check, and merge-tree against upstream/main are all clean. --- .../scripts/artifact-contracts.json | 16 +++- .../scripts/check-bedrock-pricing-parity.py | 48 +++++++++++- .../scripts/validate-artifacts.py | 67 ++++++++++++++++- .../vendored/ai/bedrock-pricing-cache.md | 22 ++++++ .../phases/clarify/clarify-ai-only.md | 18 +++-- .../references/phases/clarify/clarify-ai.md | 74 ++++++++++++++----- .../vendored/ai/bedrock-pricing-cache.md | 22 ++++++ .../vendored/ai/bedrock-pricing-cache.md | 22 ++++++ .../skills/shared/ai/bedrock-pricing-cache.md | 22 ++++++ .../tests/test_validate_artifacts.py | 57 ++++++++++++++ 10 files changed, 336 insertions(+), 32 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index ad2101864..e089a8141 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -235,7 +235,21 @@ "when_value": "ai-only", "shape": "skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md", "root_heading": "preferences.json (AI-only)", - "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required." + "allowed_anywhere": [ + "source", + "reason", + "context", + "note", + "unanswered", + "blocks_phase", + "conflict", + "forced_by", + "residency_warning", + "detail", + "default_basis", + "deferred_to_generate" + ], + "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required. route_shapes does not inherit the parent entry's allowed_anywhere (build_shape_contract is called with only this route object), so it is repeated here -- schema-preferences.md's row-annotation vocabulary, especially `source` REQUIRED on a chosen_by: extracted row, applies to this route's rows the same as the infra route's." } ] }, diff --git a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py index 23387af85..539e337f0 100755 --- a/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py +++ b/plugins/aws-startup-advisor/scripts/check-bedrock-pricing-parity.py @@ -74,7 +74,18 @@ def metadata(text: str) -> dict[str, str]: def model_rows(text: str) -> dict[str, tuple[str, str]]: rows = {} - for line in section(text, "Bedrock Models (On-Demand)").splitlines(): + bedrock_section = section(text, "Bedrock Models (On-Demand)") + # Exclude the "### Embeddings — Bedrock" subsection: it is an input-only table (price, + # Dimensions) rather than this scanner's assumed input/output price-pair shape, and + # three of its five Dimensions values (single numbers like "1536") coincidentally pass + # the same numeric fullmatch as a real output price, which would otherwise mix + # embedding rows into the generative-model lookup-key comparison below. + embed_match = re.search(r"(?m)^### Embeddings — Bedrock.*$", bedrock_section) + if embed_match: + embed_end = re.search(r"(?m)^### ", bedrock_section[embed_match.end() :]) + stop = embed_match.end() + embed_end.start() if embed_end else len(bedrock_section) + bedrock_section = bedrock_section[: embed_match.start()] + bedrock_section[stop:] + for line in bedrock_section.splitlines(): if not line.startswith("|"): continue cells = [cell.strip() for cell in line.strip().strip("|").split("|")] @@ -160,7 +171,18 @@ def main() -> int: old_bedrock = section(old, "Bedrock Models (On-Demand)") new_bedrock = section(canonical, "Bedrock Models (On-Demand)") - assert new_bedrock.rstrip() == old_bedrock.rstrip(), "Bedrock section changed during extraction" + # The GCP-origin "## Bedrock Models (On-Demand)" section itself nests the + # "### Embeddings — Bedrock" subsection reinstated below (it was previously only in + # Azure's cache, under the same heading level, and was lost entirely during the + # original split). Strip exactly that known, reviewed addition before the + # byte-equality check, the same way the Claude Fable exception below is scoped to one + # named, reviewed change rather than widened wholesale. + new_bedrock_without_embeddings = re.sub( + r"(?s)### Embeddings — Bedrock \(per 1M input tokens, US East\).*?(?=### Stability AI)", + "", + new_bedrock, + ) + assert new_bedrock_without_embeddings.rstrip() == old_bedrock.rstrip(), "Bedrock section changed during extraction" old_metadata = metadata(old) canonical_metadata = metadata(canonical) for field in ("Last updated", "Region", "Currency"): @@ -201,6 +223,28 @@ def main() -> int: assert markdown_data_rows(section(azure_cache, "Source Provider Pricing (for Migration Comparison)")) == markdown_data_rows(section(azure_old, "Source Provider Pricing (for Migration Comparison)")), "Azure source-provider rate rows changed" assert "any Bedrock rate change into\n `references/vendored/ai/bedrock-pricing-cache.md`" in azure_openai + # The pre-split Azure cache's own "Embeddings — Bedrock" target table (Titan/Cohere) is + # NOT part of GCP's "Bedrock Models (On-Demand)" section the checks above already cover, + # so a prior split silently dropped it entirely instead of moving it to the canonical + # cache. Prove every target row survived the move byte-for-byte and that each embedding + # model ID is still reachable from the canonical cache's own quick-reference rows. + def subsection(text: str, heading: str) -> str: + """Like `section()` but for a `### ` subsection nested inside a `## ` section.""" + match = re.search(rf"(?m)^### {re.escape(heading)}.*$", text) + assert match, f"missing subsection: {heading}" + end = re.search(r"(?m)^#{2,3} ", text[match.end() :]) + stop = match.end() + end.start() if end else len(text) + return text[match.start() : stop] + + azure_old_embed_rows = markdown_data_rows(subsection(azure_old, "Embeddings — Bedrock")) + assert azure_old_embed_rows, "pre-split Azure cache's Embeddings — Bedrock table is missing; nothing to compare against" + canonical_embed_rows = markdown_data_rows(subsection(canonical, "Embeddings — Bedrock")) + assert canonical_embed_rows == azure_old_embed_rows, "Bedrock embedding target rows were not preserved during the split" + for vendored_copy in VENDORED: + assert markdown_data_rows(subsection(vendored_copy.read_text(), "Embeddings — Bedrock")) == azure_old_embed_rows, ( + f"Bedrock embedding target rows drifted in vendored copy: {vendored_copy}" + ) + canonical_bytes = CANONICAL.read_bytes() for copy in VENDORED: assert copy.read_bytes() == canonical_bytes, f"vendored drift: {copy}" diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index 03ef685bb..0cc4840c6 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -152,10 +152,69 @@ def strip_jsonc(text: str) -> Tuple[str, Dict[int, str], Dict[int, str]]: if idx is not None: comments[i] = line[idx + 2:].strip() line = line[:idx].rstrip() - # the key a trailing comment annotates is the LAST `"key":` opened on the line - key_matches = re.findall(r'"((?:[^"\\]|\\.)*)"\s*:', line) - if key_matches: - keys[i] = key_matches[-1] + # The key a trailing comment annotates is the LAST `"key":` opened on the line + # WHOSE OWN NESTING DEPTH MATCHES THE DEPTH AT THE END OF THE LINE -- i.e. a key + # that is still "current" by the time the comment is reached, not merely the + # last key-like string that happens to appear earliest in the text. Two cases + # must resolve differently: + # - `"routing_provenance": "table", // REQUIRED` (depth stays open on this + # key's own value, never closed back out before the comment) -> the comment + # belongs to `routing_provenance` itself, the last key opened. + # - `"call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED` (the inline + # array/object value is fully opened AND closed before the comment, so depth + # returns to the level `call_sites` itself was opened at) -> the comment + # belongs to `call_sites`, the enclosing field, not to `line` (a key nested + # inside that value, whose own depth no longer matches the end-of-line depth). + # A key's value can also legitimately still be OPEN at end-of-line (e.g. + # `"workloads": [ // REQUIRED` whose `]` closes on a later line) -- that key was + # recorded one level shallower than `depth` (the depth its own `[`/`{` pushed + # to). Prefer an exact depth match; fall back to `depth - 1` for that case. + depth = 0 + last_key_at_depth: Dict[int, str] = {} + k_in_str = False + k_esc = False + pos = 0 + key_start: Optional[int] = None + string_open_pos = 0 + while pos < len(line): + ch = line[pos] + if k_in_str: + if k_esc: + k_esc = False + elif ch == "\\": + k_esc = True + elif ch == '"': + k_in_str = False + key_start = pos # candidate string just closed at `pos` + pos += 1 + continue + if ch == '"': + k_in_str = True + string_open_pos = pos + pos += 1 + continue + if ch in "{[": + depth += 1 + pos += 1 + continue + if ch in "}]": + depth -= 1 + pos += 1 + continue + if ch == ":" and key_start is not None: + # the string that just closed, immediately followed by `:` (ignoring + # whitespace), is a key opened at the CURRENT depth (before this colon's + # value opens any further nesting of its own) + between = line[key_start + 1:pos] + if between.strip() == "": + last_key_at_depth[depth] = line[string_open_pos + 1:key_start] + if ch not in '"': + key_start = None + pos += 1 + if depth in last_key_at_depth: + keys[i] = last_key_at_depth[depth] + elif (depth - 1) in last_key_at_depth: + keys[i] = last_key_at_depth[depth - 1] out_lines.append(line) joined = "\n".join(out_lines) # bare `...` ellipses in arrays/objects (`["Q1", "Q2", ...]`) → a placeholder string diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md index f2e1b199d..eb7b73b46 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md @@ -94,6 +94,28 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | | Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | +### Embeddings — Bedrock (per 1M input tokens, US East) + +Embedding models are **input-only** — priced per 1M input tokens, no output charge. Use for +`embedding` capability workloads (RAG corpora, semantic search, FAQ retrieval). A migration from +an OpenAI/Azure embedding deployment lands on one of these; note the **dimension** must match (or +the corpus must be re-embedded and any similarity threshold recalibrated). + +| Model | Model ID | Provider | Input $/1M | Dimensions | Tier | Status | +| ---------------------------- | ----------------------------- | -------- | ---------- | ----------------- | -------- | ------ | +| Titan Text Embeddings v2 | amazon.titan-embed-text-v2:0 | Amazon | 0.02 | 1024/512/256 | default | active | +| Titan Text Embeddings v1 | amazon.titan-embed-text-v1 | Amazon | 0.10 | 1536 | legacy | active | +| Cohere Embed v4 | cohere.embed-v4:0 | Cohere | 0.12 | 1536/1024/512/256 | flagship | active | +| Cohere Embed English v3 | cohere.embed-english-v3 | Cohere | 0.10 | 1024 | mid | active | +| Cohere Embed Multilingual v3 | cohere.embed-multilingual-v3 | Cohere | 0.10 | 1024 | mid | active | + +**Default target for a migrating OpenAI/Azure embedding workload:** Titan Text Embeddings v2 +(`amazon.titan-embed-text-v2:0`) — cheapest, configurable dimensions (1024 default; 512/256 for +cost/latency). Choose Cohere Embed v4 when the source used a large-dimension model and matrix +compatibility or multilingual quality matters. Titan v2 output dimension is configurable, so map +`text-embedding-3-large` (3072-dim) or `-small` (1536-dim) to Titan 1024 and **re-embed** — there +is no dimension-preserving swap. See `references/vendored/ai/ai-openai-to-bedrock.md`. + ### Stability AI — Image Generation (per image, US East) Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index 263f0b618..f289a3ff9 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -146,8 +146,11 @@ Write `$MIGRATION_DIR/preferences.json`: the downstream source of truth (design-ai reads it, not the profile). Each constraint carries the full clarify field shape (`value`, `chosen_by` ∈ -`user|extracted|default|derived`, `prompt`, `design_consequence`, `source`/`question_id`). No -nulls. Delete `preferences-draft.json` after writing. +`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`), plus +`source` on every `chosen_by: "extracted"` row — per `schema-preferences.md`'s rule that the +`DETECTED`-equivalent disposition must say what in the estate was read, so a value taken from +discovery is never indistinguishable from a promoted default. No nulls. Delete +`preferences-draft.json` after writing. ## preferences.json (AI-only) @@ -159,7 +162,8 @@ nulls. Delete `preferences-draft.json` after writing. "clarify_mode": "full", // fast_path | full "questions_asked": [], "questions_defaulted": [], - "questions_extracted": [] + "questions_extracted": [], + "report_caveats": [] // REQUIRED — [] when none; Q1.5 skip/default appends a caveat here }, "design_constraints": { // REQUIRED — region and compliance only; no infra constraints "target_region": { // REQUIRED @@ -188,22 +192,22 @@ nulls. Delete `preferences-draft.json` after writing. }, "ai_constraints": { // REQUIRED "ai_framework": { // REQUIRED - "value": "direct", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" + "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16", "source": "integration.pattern + integration.gateway_type" }, "ai_priority": { // REQUIRED "value": "balanced", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q17" }, "ai_monthly_spend": { // REQUIRED - "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18" + "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18", "source": "current_costs.monthly_ai_spend" }, "cross_cloud": { // REQUIRED "value": "latency-acceptable", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q4" }, "ai_model_baseline": { // REQUIRED - "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19" + "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19", "source": "models[] confidence >= 0.8" }, "ai_vision": { // REQUIRED - "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20" + "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20", "source": "integration.capabilities_summary.vision" }, "ai_token_volume": { // REQUIRED "value": "low", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q21" diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md index a4bfe41bf..42206c2bb 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md @@ -101,32 +101,70 @@ Q16–Q22, and write `"workloads": []` when none. ## Step 3: Rows returned +Every row is merged into the final `preferences.json` by `clarify-assemble.md`'s Step +"Assemble the artifact" rules 1–2: it carries `disposition`, `value`, and `default`, exactly +like every other fragment's rows (see `clarify-global.md`'s `## Rows returned` for the same +convention). A value taken from its default stays **PROPOSED** — never promoted to DETECTED +(DETECTED means read from the estate; Design's rationale and the report distinguish "you +chose" from "we assumed"). A `DETECTED` row additionally carries `source` naming where the +value was read from — `schema-preferences.md`'s rule that the justification key is `source` +and is REQUIRED on `DETECTED` applies here the same as the infra flow. + ```jsonc { - "ai_framework": ["direct"], // DETECTED | PROPOSED - "ai_monthly_spend": "$500-$2K", // DETECTED | PROPOSED - "ai_priority": "balanced", // PROPOSED - "ai_critical_feature": null, // PROPOSED - "ai_token_volume": "low", // PROPOSED - "ai_model_baseline": "gpt-4o", // DETECTED | PROPOSED - "ai_vision": false, // DETECTED | PROPOSED - "ai_complexity": "moderate", // PROPOSED - "startup_program_status": null, // ESSENTIAL — value null until answered (the Gate-2 completion gate) - "ai_constraints": { // agentic block present ONLY when agentic_profile.is_agentic + "ai_framework": { "disposition": "DETECTED", "value": ["direct"], "default": ["direct"], "source": "integration.pattern + integration.gateway_type" }, + "ai_monthly_spend": { + "disposition": "DETECTED", + "value": "$500-$2K", + "default": "$500-$2K", + "source": "current_costs.monthly_ai_spend" + }, + "ai_priority": { "disposition": "PROPOSED", "value": "balanced", "default": "balanced" }, + "ai_critical_feature": { "disposition": "PROPOSED", "value": null, "default": null }, + "ai_token_volume": { "disposition": "PROPOSED", "value": "low", "default": "low" }, + "ai_model_baseline": { + "disposition": "DETECTED", + "value": "gpt-4o", + "default": "gpt-4o", + "source": "models[] confidence >= 0.8" + }, + "ai_vision": { + "disposition": "DETECTED", + "value": false, + "default": false, + "source": "capabilities_summary.vision" + }, + "ai_latency": { "disposition": "PROPOSED", "value": "important", "default": "important" }, + "ai_complexity": { "disposition": "PROPOSED", "value": "moderate", "default": "moderate" }, + "startup_program_status": { + "disposition": "ESSENTIAL", + "value": null, // null until answered (the Gate-2 completion gate) + "default": null + }, + "ai_constraints": { // agentic block present ONLY when agentic_profile.is_agentic — rows, not raw values, same disposition/value/default wrapping "agentic": { - "migration_approach": "undecided", - "memory_requirement": "session", - "task_duration": "medium", - "incremental_migration": false + "migration_approach": { + "disposition": "PROPOSED", + "value": "undecided", + "default": "undecided" + }, + "memory_requirement": { + "disposition": "PROPOSED", + "value": "session", + "default": "session" + }, + "task_duration": { "disposition": "PROPOSED", "value": "medium", "default": "medium" }, + "incremental_migration": { + "disposition": "PROPOSED", + "value": false, + "default": false + } } }, - "workloads": [] // the confirmed array (persist rule above) + "workloads": [] // NOT a row — the confirmed array itself (persist rule above), same as clarify-ai-only.md's workloads[] } ``` -A value taken from its default stays **PROPOSED** — never promoted to DETECTED (DETECTED means -read from the estate; Design's rationale and the report distinguish "you chose" from "we assumed"). - ### `workloads[]` ```jsonc diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md index f2e1b199d..eb7b73b46 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -94,6 +94,28 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | | Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | +### Embeddings — Bedrock (per 1M input tokens, US East) + +Embedding models are **input-only** — priced per 1M input tokens, no output charge. Use for +`embedding` capability workloads (RAG corpora, semantic search, FAQ retrieval). A migration from +an OpenAI/Azure embedding deployment lands on one of these; note the **dimension** must match (or +the corpus must be re-embedded and any similarity threshold recalibrated). + +| Model | Model ID | Provider | Input $/1M | Dimensions | Tier | Status | +| ---------------------------- | ----------------------------- | -------- | ---------- | ----------------- | -------- | ------ | +| Titan Text Embeddings v2 | amazon.titan-embed-text-v2:0 | Amazon | 0.02 | 1024/512/256 | default | active | +| Titan Text Embeddings v1 | amazon.titan-embed-text-v1 | Amazon | 0.10 | 1536 | legacy | active | +| Cohere Embed v4 | cohere.embed-v4:0 | Cohere | 0.12 | 1536/1024/512/256 | flagship | active | +| Cohere Embed English v3 | cohere.embed-english-v3 | Cohere | 0.10 | 1024 | mid | active | +| Cohere Embed Multilingual v3 | cohere.embed-multilingual-v3 | Cohere | 0.10 | 1024 | mid | active | + +**Default target for a migrating OpenAI/Azure embedding workload:** Titan Text Embeddings v2 +(`amazon.titan-embed-text-v2:0`) — cheapest, configurable dimensions (1024 default; 512/256 for +cost/latency). Choose Cohere Embed v4 when the source used a large-dimension model and matrix +compatibility or multilingual quality matters. Titan v2 output dimension is configurable, so map +`text-embedding-3-large` (3072-dim) or `-small` (1536-dim) to Titan 1024 and **re-embed** — there +is no dimension-preserving swap. See `references/vendored/ai/ai-openai-to-bedrock.md`. + ### Stability AI — Image Generation (per image, US East) Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md index f2e1b199d..eb7b73b46 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -94,6 +94,28 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | | Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | +### Embeddings — Bedrock (per 1M input tokens, US East) + +Embedding models are **input-only** — priced per 1M input tokens, no output charge. Use for +`embedding` capability workloads (RAG corpora, semantic search, FAQ retrieval). A migration from +an OpenAI/Azure embedding deployment lands on one of these; note the **dimension** must match (or +the corpus must be re-embedded and any similarity threshold recalibrated). + +| Model | Model ID | Provider | Input $/1M | Dimensions | Tier | Status | +| ---------------------------- | ----------------------------- | -------- | ---------- | ----------------- | -------- | ------ | +| Titan Text Embeddings v2 | amazon.titan-embed-text-v2:0 | Amazon | 0.02 | 1024/512/256 | default | active | +| Titan Text Embeddings v1 | amazon.titan-embed-text-v1 | Amazon | 0.10 | 1536 | legacy | active | +| Cohere Embed v4 | cohere.embed-v4:0 | Cohere | 0.12 | 1536/1024/512/256 | flagship | active | +| Cohere Embed English v3 | cohere.embed-english-v3 | Cohere | 0.10 | 1024 | mid | active | +| Cohere Embed Multilingual v3 | cohere.embed-multilingual-v3 | Cohere | 0.10 | 1024 | mid | active | + +**Default target for a migrating OpenAI/Azure embedding workload:** Titan Text Embeddings v2 +(`amazon.titan-embed-text-v2:0`) — cheapest, configurable dimensions (1024 default; 512/256 for +cost/latency). Choose Cohere Embed v4 when the source used a large-dimension model and matrix +compatibility or multilingual quality matters. Titan v2 output dimension is configurable, so map +`text-embedding-3-large` (3072-dim) or `-small` (1536-dim) to Titan 1024 and **re-embed** — there +is no dimension-preserving swap. See `references/vendored/ai/ai-openai-to-bedrock.md`. + ### Stability AI — Image Generation (per image, US East) Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md index f2e1b199d..eb7b73b46 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md @@ -94,6 +94,28 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | Jurassic-2 Ultra | — | AI21 Labs | 18.80 | 18.80 | — | legacy | legacy | | Jamba-Instruct | — | AI21 Labs | 0.50 | 0.70 | — | mid | active | +### Embeddings — Bedrock (per 1M input tokens, US East) + +Embedding models are **input-only** — priced per 1M input tokens, no output charge. Use for +`embedding` capability workloads (RAG corpora, semantic search, FAQ retrieval). A migration from +an OpenAI/Azure embedding deployment lands on one of these; note the **dimension** must match (or +the corpus must be re-embedded and any similarity threshold recalibrated). + +| Model | Model ID | Provider | Input $/1M | Dimensions | Tier | Status | +| ---------------------------- | ----------------------------- | -------- | ---------- | ----------------- | -------- | ------ | +| Titan Text Embeddings v2 | amazon.titan-embed-text-v2:0 | Amazon | 0.02 | 1024/512/256 | default | active | +| Titan Text Embeddings v1 | amazon.titan-embed-text-v1 | Amazon | 0.10 | 1536 | legacy | active | +| Cohere Embed v4 | cohere.embed-v4:0 | Cohere | 0.12 | 1536/1024/512/256 | flagship | active | +| Cohere Embed English v3 | cohere.embed-english-v3 | Cohere | 0.10 | 1024 | mid | active | +| Cohere Embed Multilingual v3 | cohere.embed-multilingual-v3 | Cohere | 0.10 | 1024 | mid | active | + +**Default target for a migrating OpenAI/Azure embedding workload:** Titan Text Embeddings v2 +(`amazon.titan-embed-text-v2:0`) — cheapest, configurable dimensions (1024 default; 512/256 for +cost/latency). Choose Cohere Embed v4 when the source used a large-dimension model and matrix +compatibility or multilingual quality matters. Titan v2 output dimension is configurable, so map +`text-embedding-3-large` (3072-dim) or `-small` (1536-dim) to Titan 1024 and **re-embed** — there +is no dimension-preserving swap. See `references/vendored/ai/ai-openai-to-bedrock.md`. + ### Stability AI — Image Generation (per image, US East) Active image generation models on Bedrock. Priced **per image** (not per token). Use for `image_generation` capability workloads. diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index fb1a0bf17..9cc70804b 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -46,6 +46,31 @@ def test_strip_jsonc_handles_ellipsis_and_bool_alternation(): assert json.loads(out) == {"a": ["Q1", "..."], "b": True} +def test_strip_jsonc_binds_a_trailing_comment_to_the_enclosing_field_not_a_nested_key(): + """`"call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED` must attribute the + comment to `call_sites` (the field this line actually declares), not to `line` (the + last key opened on the line, which is nested two levels deeper inside the inline + array/object value). Picking the textually-last key regardless of depth let a + `// REQUIRED` on an inline array/object value silently annotate one of its own nested + keys instead of the enclosing field, so the enclosing field was never marked required. + """ + text = '{\n "call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED\n}' + _, comments, keys = va.strip_jsonc(text) + assert comments[1] == "REQUIRED" + assert keys[1] == "call_sites" + + +def test_strip_jsonc_still_binds_to_the_last_of_several_flat_sibling_keys(): + """Several sibling keys packed on one flat line (no nesting) must still resolve to the + LAST one, preserving the original intent for the common `"a": 1, "b": 2 // REQUIRED` + case this attribution logic was designed for. + """ + text = '{\n "a": 1, "b": 2 // REQUIRED\n}' + _, comments, keys = va.strip_jsonc(text) + assert comments[1] == "REQUIRED" + assert keys[1] == "b" + + # --------------------------------------------------------------------------- template inference @@ -668,6 +693,38 @@ def run(name, data): invented = json.loads(json.dumps(ai)) invented["metadata"]["invented"] = 1 assert ("UNKNOWN_KEY", "metadata.invented") in run("ai-invented", invented) + + # Review finding: Q1 is documented as select-all with a ["direct"] array default, + # skipped/defaulted Q1.5 compliance is documented as appending a + # metadata.report_caveats entry, and Step 3's prose permits per-constraint `source` + # provenance. Each of these three documented variations independently used to fail + # against the committed example (ai_framework typed as a bare string, report_caveats + # absent from metadata, source never accepted) even though the literal, unmodified + # example passed. Prove the producer contract now actually accepts the documented + # shapes it describes, not only its own single literal example. + multi_framework = json.loads(json.dumps(ai)) + multi_framework["ai_constraints"]["ai_framework"]["value"] = ["direct", "langchain"] + assert run("ai-multi-framework", multi_framework) == set(), run("ai-multi-framework", multi_framework) + with_caveat = json.loads(json.dumps(ai)) + with_caveat["metadata"]["report_caveats"] = ["Compliance requirements were not confirmed by the user"] + assert run("ai-caveat", with_caveat) == set(), run("ai-caveat", with_caveat) + with_source = json.loads(json.dumps(ai)) + with_source["design_constraints"]["target_region"]["source"] = "discovery.ai-workload-profile.json" + assert run("ai-source", with_source) == set(), run("ai-source", with_source) + + # Review finding: workloads[0].call_sites is declared `"call_sites": [{ "file": ..., + # "line": 1 }], // REQUIRED` -- an inline array of objects on one line. Before the + # strip_jsonc key-attribution fix, the trailing comment bound to `line` (the last key + # opened on the line, nested inside the array) instead of `call_sites` (the enclosing + # field), so omitting the entire call_sites array passed with no finding in both Azure + # preferences routes. model_id is the control: a plain sibling, correctly flagged + # either way, proving the fix did not just start over-flagging unrelated keys. + ai_no_call_sites = json.loads(json.dumps(ai)) + del ai_no_call_sites["workloads"][0]["call_sites"] + assert ("MISSING_REQUIRED", "workloads[0].call_sites") in run("ai-no-call-sites", ai_no_call_sites) + ai_no_model_id = json.loads(json.dumps(ai)) + del ai_no_model_id["workloads"][0]["model_id"] + assert ("MISSING_REQUIRED", "workloads[0].model_id") in run("ai-no-model-id", ai_no_model_id) # the infra verdict stays required on the infra route no_status = json.loads(json.dumps(infra)) no_status.pop("clarify_status", None) From 8603d087bcca8e0bcb4aab861770124a6fcdb4a4 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 9 Oct 2026 10:33:01 -0700 Subject: [PATCH 46/53] fix(aws-startup-advisor): require source on AI-only DETECTED-equivalent rows Correction to an earlier fix on this PR. While fixing the identical finding on PR #411, confirmed via schema-preferences.md's own fundamental rule ("the justification key is source, and it is REQUIRED on DETECTED") that source should be ACCEPTED (and is in fact required) on clarify-ai-only.md's chosen_by: "extracted" rows -- the AI-only route's naming for the same DETECTED concept. The earlier fix on this PR incorrectly concluded source was "not part of this AI-only shape" and only clarified the prose to say so, rather than fixing the real root cause: route_shapes builds a shape contract from only the route object, not merged with the parent preferences.json entry, so it never inherited that entry's allowed_anywhere vocabulary (source, reason, context, etc.). Added allowed_anywhere to the AI-only route_shapes manifest entry (mirroring the parent preferences.json entry's), added source to the example's chosen_by: "extracted" rows (ai_framework, ai_monthly_spend, ai_model_baseline, ai_vision), and corrected the prose to say source is required on those rows. Added three acceptance cases (multi-value framework, report_caveats present, source present) to the existing producer-route test, replacing the test case that had asserted source must still fail. 136 tests pass in the plugin's full suite; mise run build, lint:artifacts (74 checked, 0 errors), diff --check, and merge-tree against upstream/main are all clean. --- .../scripts/artifact-contracts.json | 16 +++++++++++++++- .../phases/clarify/clarify-ai-only.md | 16 ++++++++-------- .../tests/test_validate_artifacts.py | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 1bbe3608e..b4bb61c80 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -238,7 +238,21 @@ "when_value": "ai-only", "shape": "skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md", "root_heading": "preferences.json (AI-only)", - "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required." + "allowed_anywhere": [ + "source", + "reason", + "context", + "note", + "unanswered", + "blocks_phase", + "conflict", + "forced_by", + "residency_warning", + "detail", + "default_basis", + "deferred_to_generate" + ], + "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required. route_shapes does not inherit the parent entry's allowed_anywhere (build_shape_contract is called with only this route object), so it is repeated here -- schema-preferences.md's row-annotation vocabulary, especially `source` REQUIRED on a chosen_by: extracted row, applies to this route's rows the same as the infra route's." } ] }, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index bb2950e24..f289a3ff9 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -146,10 +146,10 @@ Write `$MIGRATION_DIR/preferences.json`: the downstream source of truth (design-ai reads it, not the profile). Each constraint carries the full clarify field shape (`value`, `chosen_by` ∈ -`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`). Every -constraint in this flow is tied to a specific numbered question, so `question_id` is always -present and `source` (the inventory/billing-derived provenance tag used in -`clarify-assemble.md`'s infra flow) is not part of this AI-only shape. No nulls. Delete +`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`), plus +`source` on every `chosen_by: "extracted"` row — per `schema-preferences.md`'s rule that the +`DETECTED`-equivalent disposition must say what in the estate was read, so a value taken from +discovery is never indistinguishable from a promoted default. No nulls. Delete `preferences-draft.json` after writing. ## preferences.json (AI-only) @@ -192,22 +192,22 @@ present and `source` (the inventory/billing-derived provenance tag used in }, "ai_constraints": { // REQUIRED "ai_framework": { // REQUIRED - "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" + "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16", "source": "integration.pattern + integration.gateway_type" }, "ai_priority": { // REQUIRED "value": "balanced", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q17" }, "ai_monthly_spend": { // REQUIRED - "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18" + "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18", "source": "current_costs.monthly_ai_spend" }, "cross_cloud": { // REQUIRED "value": "latency-acceptable", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q4" }, "ai_model_baseline": { // REQUIRED - "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19" + "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19", "source": "models[] confidence >= 0.8" }, "ai_vision": { // REQUIRED - "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20" + "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20", "source": "integration.capabilities_summary.vision" }, "ai_token_volume": { // REQUIRED "value": "low", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q21" diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 4cd39d1ae..272511814 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -808,6 +808,24 @@ def run(name, data): invented = json.loads(json.dumps(ai)) invented["metadata"]["invented"] = 1 assert ("UNKNOWN_KEY", "metadata.invented") in run("ai-invented", invented) + + # Q1 is documented as select-all with a ["direct"] array default, skipped/defaulted + # Q1.5 compliance is documented as appending a metadata.report_caveats entry, and + # source is REQUIRED on every chosen_by: "extracted" row per schema-preferences.md's + # own fundamental rule -- but route_shapes does not inherit the parent preferences + # .json entry's allowed_anywhere, so source was previously rejected on this route even + # though the general contract requires it. Prove the producer contract now accepts + # the documented shapes it describes, not only its own single literal example. + multi_framework = json.loads(json.dumps(ai)) + multi_framework["ai_constraints"]["ai_framework"]["value"] = ["direct", "langchain"] + assert run("ai-multi-framework", multi_framework) == set(), run("ai-multi-framework", multi_framework) + with_caveat = json.loads(json.dumps(ai)) + with_caveat["metadata"]["report_caveats"] = ["Compliance requirements were not confirmed by the user"] + assert run("ai-caveat", with_caveat) == set(), run("ai-caveat", with_caveat) + with_source = json.loads(json.dumps(ai)) + with_source["design_constraints"]["target_region"]["source"] = "discovery.ai-workload-profile.json" + assert run("ai-source", with_source) == set(), run("ai-source", with_source) + # the infra verdict stays required on the infra route no_status = json.loads(json.dumps(infra)) no_status.pop("clarify_status", None) From 8675295ee8e0a9eac1056348275ec6dd8133196f Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 9 Oct 2026 10:35:13 -0700 Subject: [PATCH 47/53] fix(aws-startup-advisor): require source on AI-only DETECTED-equivalent rows Correction to an earlier fix on this PR. While fixing the identical finding on PR #411, confirmed via schema-preferences.md's own fundamental rule ("the justification key is source, and it is REQUIRED on DETECTED") that source should be ACCEPTED (and is in fact required) on clarify-ai-only.md's chosen_by: "extracted" rows -- the AI-only route's naming for the same DETECTED concept. The earlier fix on this PR incorrectly concluded source was "not part of this AI-only shape" and only clarified the prose to say so, rather than fixing the real root cause: route_shapes builds a shape contract from only the route object, not merged with the parent preferences.json entry, so it never inherited that entry's allowed_anywhere vocabulary (source, reason, context, etc.). Added allowed_anywhere to the AI-only route_shapes manifest entry (mirroring the parent preferences.json entry's), added source to the example's chosen_by: "extracted" rows, and corrected the prose. Added three acceptance cases (multi-value framework, report_caveats present, source present) to the existing producer-route test. 181 tests pass in the plugin's full suite; mise run build, lint:artifacts (80 checked, 0 errors), diff --check, and merge-tree against upstream/main are all clean. --- .../scripts/artifact-contracts.json | 16 +++++++++++++++- .../phases/clarify/clarify-ai-only.md | 16 ++++++++-------- .../tests/test_validate_artifacts.py | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 62239448a..99d966a5e 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -248,7 +248,21 @@ "when_value": "ai-only", "shape": "skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md", "root_heading": "preferences.json (AI-only)", - "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required." + "allowed_anywhere": [ + "source", + "reason", + "context", + "note", + "unanswered", + "blocks_phase", + "conflict", + "forced_by", + "residency_warning", + "detail", + "default_basis", + "deferred_to_generate" + ], + "_why": "clarify.md Step 0 sends an AI-only run through clarify-ai-only.md and skips the infra assembler. That file is a different preferences layout; infra requiredness (clarify_status) does not apply, and the AI fields stay required. route_shapes does not inherit the parent entry's allowed_anywhere (build_shape_contract is called with only this route object), so it is repeated here -- schema-preferences.md's row-annotation vocabulary, especially `source` REQUIRED on a chosen_by: extracted row, applies to this route's rows the same as the infra route's." } ] }, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index bb2950e24..f289a3ff9 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -146,10 +146,10 @@ Write `$MIGRATION_DIR/preferences.json`: the downstream source of truth (design-ai reads it, not the profile). Each constraint carries the full clarify field shape (`value`, `chosen_by` ∈ -`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`). Every -constraint in this flow is tied to a specific numbered question, so `question_id` is always -present and `source` (the inventory/billing-derived provenance tag used in -`clarify-assemble.md`'s infra flow) is not part of this AI-only shape. No nulls. Delete +`user|extracted|default|derived`, `prompt`, `design_consequence`, `question_id`), plus +`source` on every `chosen_by: "extracted"` row — per `schema-preferences.md`'s rule that the +`DETECTED`-equivalent disposition must say what in the estate was read, so a value taken from +discovery is never indistinguishable from a promoted default. No nulls. Delete `preferences-draft.json` after writing. ## preferences.json (AI-only) @@ -192,22 +192,22 @@ present and `source` (the inventory/billing-derived provenance tag used in }, "ai_constraints": { // REQUIRED "ai_framework": { // REQUIRED - "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16" + "value": ["direct"], "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q16", "source": "integration.pattern + integration.gateway_type" }, "ai_priority": { // REQUIRED "value": "balanced", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q17" }, "ai_monthly_spend": { // REQUIRED - "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18" + "value": "$500-$2K", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q18", "source": "current_costs.monthly_ai_spend" }, "cross_cloud": { // REQUIRED "value": "latency-acceptable", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q4" }, "ai_model_baseline": { // REQUIRED - "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19" + "value": "gpt-4o", "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q19", "source": "models[] confidence >= 0.8" }, "ai_vision": { // REQUIRED - "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20" + "value": false, "chosen_by": "extracted", "prompt": "", "design_consequence": "", "question_id": "Q20", "source": "integration.capabilities_summary.vision" }, "ai_token_volume": { // REQUIRED "value": "low", "chosen_by": "default", "prompt": "", "design_consequence": "", "question_id": "Q21" diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 9434aaebd..fe06a2069 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -737,6 +737,24 @@ def run(name, data): invented = json.loads(json.dumps(ai)) invented["metadata"]["invented"] = 1 assert ("UNKNOWN_KEY", "metadata.invented") in run("ai-invented", invented) + + # Q1 is documented as select-all with a ["direct"] array default, skipped/defaulted + # Q1.5 compliance is documented as appending a metadata.report_caveats entry, and + # source is REQUIRED on every chosen_by: "extracted" row per schema-preferences.md's + # own fundamental rule -- but route_shapes does not inherit the parent preferences + # .json entry's allowed_anywhere, so source was previously rejected on this route even + # though the general contract requires it. Prove the producer contract now accepts + # the documented shapes it describes, not only its own single literal example. + multi_framework = json.loads(json.dumps(ai)) + multi_framework["ai_constraints"]["ai_framework"]["value"] = ["direct", "langchain"] + assert run("ai-multi-framework", multi_framework) == set(), run("ai-multi-framework", multi_framework) + with_caveat = json.loads(json.dumps(ai)) + with_caveat["metadata"]["report_caveats"] = ["Compliance requirements were not confirmed by the user"] + assert run("ai-caveat", with_caveat) == set(), run("ai-caveat", with_caveat) + with_source = json.loads(json.dumps(ai)) + with_source["design_constraints"]["target_region"]["source"] = "discovery.ai-workload-profile.json" + assert run("ai-source", with_source) == set(), run("ai-source", with_source) + # the infra verdict stays required on the infra route no_status = json.loads(json.dumps(infra)) no_status.pop("clarify_status", None) From 133163d400a539847d57276a45b31f6b076b590c Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 9 Oct 2026 11:07:12 -0700 Subject: [PATCH 48/53] fix(aws-startup-advisor): address PR #410 follow-up Cursor review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independently reproduced findings: 1. Regression in the strip_jsonc depth-tracking fix: the unconditional 'if ch not in "\"\": key_start = None' reset the pending key on ANY non-quote character, including whitespace between a closed key string and its ':'. 'priority' : "balanced", // REQUIRED (space or tab before the colon) lost the key entirely, so omitting workloads[0].priority stopped producing MISSING_REQUIRED. Fixed by exempting whitespace from the reset; only a genuinely different character invalidates the pending key. Added parametrized unit tests for space/tab/double-space before the colon plus a control proving a real intervening token still invalidates. 2. artifact-contracts.json's extra_shapes still pointed preferences.json's mixed-flow AI fields at clarify-ai.md's raw per-fragment 'Step 3: Rows returned' values instead of the final assembled {disposition, value, default} envelope clarify-assemble.md's rules require (the same finding already fixed on #411/#416, not yet addressed here); the example also omitted the documented Q21 ai_latency row. Rewrote the example to match the established sibling-fragment convention, with source on DETECTED rows per the #411 correction. 3. Both clarify-ai.md and clarify-ai-only.md persisted workloads[].capability: "chat", but schema-discover-ai.md's capability enum is text_generation/structured_output/image_generation/embedding/… and design-ai.md has no routing case for "chat". Changed both examples to text_generation and added a test asserting both producer examples use a value from the discovery/design vocabulary. All three reproduced against the live PR head before fixing. 141 tests pass in the plugin's full suite; mise run build, lint:artifacts (74 checked, 0 errors), diff --check, and merge-tree against upstream/main are all clean. --- .../scripts/validate-artifacts.py | 6 +- .../phases/clarify/clarify-ai-only.md | 2 +- .../references/phases/clarify/clarify-ai.md | 76 +++++++++++++----- .../tests/test_validate_artifacts.py | 77 ++++++++++++++++++- 4 files changed, 139 insertions(+), 22 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index cd7d685aa..4efccbbbf 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -209,7 +209,11 @@ def strip_jsonc(text: str) -> Tuple[str, Dict[int, str], Dict[int, str]]: between = line[key_start + 1:pos] if between.strip() == "": last_key_at_depth[depth] = line[string_open_pos + 1:key_start] - if ch not in '"': + key_start = None + elif ch not in '"' and not ch.isspace(): + # Whitespace/tabs between a closed key string and its `:` must NOT drop + # the pending key (`"priority" : ...` is still a key, just spaced out) — + # only a genuinely different, non-whitespace character invalidates it. key_start = None pos += 1 # A key's value can legitimately still be OPEN at end-of-line (e.g. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index f289a3ff9..0e4f237a7 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -230,7 +230,7 @@ discovery is never indistinguishable from a promoted default. No nulls. Delete "workload_id": "wl-1", // REQUIRED "model_id": "gpt-4o", // REQUIRED "sdk_method": "chat.completions", // REQUIRED - "capability": "chat", // REQUIRED + "capability": "text_generation", // REQUIRED — the schema-discover-ai.md / design-ai.md capability enum (text_generation, structured_output, image_generation, embedding, …), not an ad hoc "chat" value "capability_confidence": "high", // REQUIRED "structured_output": false, // REQUIRED "call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md index a4bfe41bf..52382626c 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md @@ -101,32 +101,70 @@ Q16–Q22, and write `"workloads": []` when none. ## Step 3: Rows returned +Every row is merged into the final `preferences.json` by `clarify-assemble.md`'s Step +"Assemble the artifact" rules 1–2: it carries `disposition`, `value`, and `default`, exactly +like every other fragment's rows (see `clarify-global.md`'s `## Rows returned` for the same +convention). A value taken from its default stays **PROPOSED** — never promoted to DETECTED +(DETECTED means read from the estate; Design's rationale and the report distinguish "you +chose" from "we assumed"). A `DETECTED` row additionally carries `source` naming where the +value was read from — `schema-preferences.md`'s rule that the justification key is `source` +and is REQUIRED on `DETECTED` applies here the same as the infra flow. + ```jsonc { - "ai_framework": ["direct"], // DETECTED | PROPOSED - "ai_monthly_spend": "$500-$2K", // DETECTED | PROPOSED - "ai_priority": "balanced", // PROPOSED - "ai_critical_feature": null, // PROPOSED - "ai_token_volume": "low", // PROPOSED - "ai_model_baseline": "gpt-4o", // DETECTED | PROPOSED - "ai_vision": false, // DETECTED | PROPOSED - "ai_complexity": "moderate", // PROPOSED - "startup_program_status": null, // ESSENTIAL — value null until answered (the Gate-2 completion gate) - "ai_constraints": { // agentic block present ONLY when agentic_profile.is_agentic + "ai_framework": { "disposition": "DETECTED", "value": ["direct"], "default": ["direct"], "source": "integration.pattern + integration.gateway_type" }, + "ai_monthly_spend": { + "disposition": "DETECTED", + "value": "$500-$2K", + "default": "$500-$2K", + "source": "current_costs.monthly_ai_spend" + }, + "ai_priority": { "disposition": "PROPOSED", "value": "balanced", "default": "balanced" }, + "ai_critical_feature": { "disposition": "PROPOSED", "value": null, "default": null }, + "ai_token_volume": { "disposition": "PROPOSED", "value": "low", "default": "low" }, + "ai_model_baseline": { + "disposition": "DETECTED", + "value": "gpt-4o", + "default": "gpt-4o", + "source": "models[] confidence >= 0.8" + }, + "ai_vision": { + "disposition": "DETECTED", + "value": false, + "default": false, + "source": "capabilities_summary.vision" + }, + "ai_latency": { "disposition": "PROPOSED", "value": "important", "default": "important" }, + "ai_complexity": { "disposition": "PROPOSED", "value": "moderate", "default": "moderate" }, + "startup_program_status": { + "disposition": "ESSENTIAL", + "value": null, // null until answered (the Gate-2 completion gate) + "default": null + }, + "ai_constraints": { // agentic block present ONLY when agentic_profile.is_agentic — rows, not raw values, same disposition/value/default wrapping "agentic": { - "migration_approach": "undecided", - "memory_requirement": "session", - "task_duration": "medium", - "incremental_migration": false + "migration_approach": { + "disposition": "PROPOSED", + "value": "undecided", + "default": "undecided" + }, + "memory_requirement": { + "disposition": "PROPOSED", + "value": "session", + "default": "session" + }, + "task_duration": { "disposition": "PROPOSED", "value": "medium", "default": "medium" }, + "incremental_migration": { + "disposition": "PROPOSED", + "value": false, + "default": false + } } }, - "workloads": [] // the confirmed array (persist rule above) + "workloads": [] // NOT a row — the confirmed array itself (persist rule above), same as clarify-ai-only.md's workloads[] } ``` -A value taken from its default stays **PROPOSED** — never promoted to DETECTED (DETECTED means -read from the estate; Design's rationale and the report distinguish "you chose" from "we assumed"). - ### `workloads[]` ```jsonc @@ -135,7 +173,7 @@ read from the estate; Design's rationale and the report distinguish "you chose" "workload_id": "wl-1", // REQUIRED "model_id": "gpt-4o", // REQUIRED "sdk_method": "chat.completions", // REQUIRED - "capability": "chat", // REQUIRED + "capability": "text_generation", // REQUIRED — the schema-discover-ai.md / design-ai.md capability enum (text_generation, structured_output, image_generation, embedding, …), not an ad hoc "chat" value "capability_confidence": "high", // REQUIRED "structured_output": false, // REQUIRED "call_sites": [{ "file": "app.py", "line": 1 }], // REQUIRED diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 272511814..fcc7b58e2 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -14,6 +14,8 @@ import sys from pathlib import Path +import pytest + PLUGIN_ROOT = Path(__file__).resolve().parent.parent SCRIPT = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" @@ -71,6 +73,32 @@ def test_strip_jsonc_still_binds_to_the_last_of_several_flat_sibling_keys(): assert keys[1] == "b" +@pytest.mark.parametrize("separator", [" ", "\t", " "]) +def test_strip_jsonc_keeps_the_pending_key_across_whitespace_before_the_colon(separator: str): + """`"priority" : "balanced", // REQUIRED` (a space or tab between the closed key + string and its `:`) must still bind the comment to `priority`. The depth-tracking fix + unconditionally reset the pending key on any character that was not a `"`, including + whitespace encountered before the colon was reached, so a spaced key lost its + REQUIRED annotation entirely. Only a genuinely different, non-whitespace character + (not whitespace waiting for `:`) may invalidate the pending key. + """ + text = f'{{\n "priority"{separator}: "balanced", // REQUIRED\n}}' + _, comments, keys = va.strip_jsonc(text) + assert comments[1] == "REQUIRED" + assert keys[1] == "priority" + + +def test_strip_jsonc_still_invalidates_a_key_followed_by_a_real_token_before_the_colon(): + """A non-whitespace, non-colon character between a closed key string and a `:` + elsewhere on the line (not actually this key's own colon) must still drop the + pending key -- the whitespace exemption must not silently accept arbitrary content. + """ + text = '{\n "a" junk "b": 1 // REQUIRED\n}' + _, comments, keys = va.strip_jsonc(text) + assert comments[1] == "REQUIRED" + assert keys[1] == "b" + + # --------------------------------------------------------------------------- template inference @@ -788,7 +816,17 @@ def run(name, data): "--skill", "azure-to-aws", "--no-baseline", "--json")) assert ("MISSING_REQUIRED", "clarify_status") not in run("infra", infra) - assert run("mixed", mixed) - run("infra", infra) == set() + mixed_errors = run("mixed", mixed) + assert mixed_errors - run("infra", infra) == set() + # The mixed-route AI rows are imported directly from clarify-ai.md's own "Step 3: + # Rows returned" example (via _doc_example above), so they must now be the final + # assembled {disposition, value, default} envelope clarify-assemble.md's merge + # rules require, not the bare per-fragment values the example previously showed + # ("ai_priority": "balanced" with no wrapper). Assert zero AI-specific errors + # explicitly, not merely "no new errors beyond infra's own pre-existing ones" -- + # confirms the AI rows genuinely validate, not that two broken sets happen to match. + assert not any(path.startswith(("ai_", "startup_program_status", "ai_constraints")) + for _, path in mixed_errors), mixed_errors bad = json.loads(json.dumps(mixed)) bad["not_a_preference"] = 1 assert ("UNKNOWN_KEY", "not_a_preference") in run("mixed-bad", bad) @@ -826,12 +864,49 @@ def run(name, data): with_source["design_constraints"]["target_region"]["source"] = "discovery.ai-workload-profile.json" assert run("ai-source", with_source) == set(), run("ai-source", with_source) + # Q21 (ai_latency) is documented in clarify-ai.md's own prose and "Who consumes + # these" table but, like every other top-level AI row here, was never marked + # `// REQUIRED` in the contract itself (only `workloads[]`'s own fields are). + # The review finding is that ADDING the documented row used to fail UNKNOWN_KEY + # because the example omitted it; prove the now-fixed example accepts it (part of + # the `mixed` assertions above) and that it is a genuinely typed row, not that this + # top-level field is contractually required (it is not). + assert "ai_latency" in mixed and mixed["ai_latency"]["value"] == "important" + # the infra verdict stays required on the infra route no_status = json.loads(json.dumps(infra)) no_status.pop("clarify_status", None) assert ("MISSING_REQUIRED", "clarify_status") in run("infra-status", no_status) +def test_ai_workload_capability_uses_the_discovery_design_vocabulary(tmp_path: Path): + """Review finding: clarify-ai-only.md's and clarify-ai.md's workloads[] examples both + persisted `capability: "chat"`, but schema-discover-ai.md's capability enum is + `text_generation`, `structured_output`, `image_generation`, `embedding`, … -- "chat" + is not a member, and design-ai.md has no routing case for it. Clarify's persisted + workloads[] is Design's source of truth, so a published example using an off-vocabulary + value is a real source-contract mismatch even though the shape validator (which only + checks type, not enum membership against a different file) accepts it either way. + """ + discover_vocab_text = ( + PLUGIN_ROOT / "skills/azure-to-aws/references/shared/schema-discover-ai.md" + ).read_text() + assert "`capability` enum: `text_generation`" in discover_vocab_text + + def run(name, data): + return _errors(_run("--run-dir", str(_write_run(tmp_path, name, "preferences.json", data)), + "--skill", "azure-to-aws", "--no-baseline", "--json")) + + ai = _doc_example(PLUGIN_ROOT / "skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md", + "preferences.json (AI-only)") + assert ai["workloads"][0]["capability"] == "text_generation", ai["workloads"][0]["capability"] + assert run("ai-only-capability", ai) == set(), run("ai-only-capability", ai) + + mixed_workloads = _doc_example(PLUGIN_ROOT / "skills/azure-to-aws/references/phases/clarify/clarify-ai.md", + "workloads[]") + assert mixed_workloads[0]["capability"] == "text_generation", mixed_workloads[0]["capability"] + + def test_gcp_drift_values_keep_their_json_types(tmp_path: Path): """Review finding on #387: the live-discovery example typed drift values as strings, so a numeric or boolean config change failed.""" From 2055f4e40007437576a0e64c7a5384f3fd5a3b6c Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 11 Oct 2026 21:21:56 -0700 Subject: [PATCH 49/53] refactor(aws-startup-advisor): move vendored-parity check into plugin dispatcher --- .github/workflows/artifact-contracts.yml | 27 ------------------------ .github/workflows/vendored-parity.yml | 27 ------------------------ plugins/aws-startup-advisor/mise.toml | 2 +- 3 files changed, 1 insertion(+), 55 deletions(-) delete mode 100644 .github/workflows/artifact-contracts.yml delete mode 100644 .github/workflows/vendored-parity.yml diff --git a/.github/workflows/artifact-contracts.yml b/.github/workflows/artifact-contracts.yml deleted file mode 100644 index 0c2a52f5f..000000000 --- a/.github/workflows/artifact-contracts.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Freshness/consistency gate for aws-startup-advisor's fixture artifact contracts. -# -# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every -# PR. A failing check is the notification: it shows on the pull request. -# Checks only the shipped fixture corpus against its published schema -# contracts (see plugins/aws-startup-advisor/scripts/validate-artifacts.py); -# does not touch anything outside the plugin. -name: Artifact contracts -on: - pull_request: - paths: - - "plugins/aws-startup-advisor/**" - - ".github/workflows/artifact-contracts.yml" - workflow_dispatch: {} -permissions: - actions: none - contents: none -jobs: - validate: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Validate fixture artifacts against their schema contracts - run: python3 plugins/aws-startup-advisor/scripts/validate-artifacts.py --fixtures diff --git a/.github/workflows/vendored-parity.yml b/.github/workflows/vendored-parity.yml deleted file mode 100644 index cf8b62bff..000000000 --- a/.github/workflows/vendored-parity.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Parity gate for aws-startup-advisor's vendored/shared skill files. -# -# Runs on pull requests that touch plugins/aws-startup-advisor/, not on every -# PR. A failing check is the notification: it shows on the pull request. -# Checks only that each skill's references/vendored/ copies are byte-identical -# to skills/shared/ (see plugins/aws-startup-advisor/tools/sync-vendored.py); -# does not touch anything outside the plugin. -name: Vendored parity -on: - pull_request: - paths: - - "plugins/aws-startup-advisor/**" - - ".github/workflows/vendored-parity.yml" - workflow_dispatch: {} -permissions: - actions: none - contents: none -jobs: - check: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Check vendored copies are in sync with skills/shared/ - run: python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check diff --git a/plugins/aws-startup-advisor/mise.toml b/plugins/aws-startup-advisor/mise.toml index 074fc3664..551e984b4 100644 --- a/plugins/aws-startup-advisor/mise.toml +++ b/plugins/aws-startup-advisor/mise.toml @@ -8,7 +8,7 @@ run = "python3 tools/sync-vendored.py --check" [tasks.ci] description = "Run startup advisor CI checks" -depends = ["lint:model-id", "test"] +depends = ["lint:model-id", "lint:artifacts", "lint:vendored-parity", "test"] [tasks."lint:model-id"] description = "Check model references against the lifecycle registry" From 863bb46e7b695c0fb56b21ef7bddb3abc5984992 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 11 Oct 2026 22:12:31 -0700 Subject: [PATCH 50/53] fix(aws-startup-advisor): drop stale telemetry rows from vendored READMEs The merged-in sync-vendored.py now excludes telemetry/ from its managed file set (that subtree has its own parity gate, generated from scripts/telemetry/ via sync_bundles.py, not copied from skills/shared/). This branch's vendored READMEs still carried telemetry rows from before that exclusion landed, so every telemetry row failed as 'listed but missing on disk' once rglob stopped counting those files as managed. Removed the telemetry rows from azure-to-aws, gcp-to-aws, and heroku-to-aws's vendored READMEs (each still vendors other, real skills/shared/ files). Deleted llm-to-bedrock's vendored README entirely, since telemetry was the only thing it ever listed and upstream/main carries no README for that skill's vendored dir either. Verified: mise -C plugins/aws-startup-advisor run ci passes (vendored- parity: 46 files OK; artifacts, model-id, skill-refs, and 272 tests all green). mise run build from repo root clean. --- .../references/vendored/README.md | 11 ------ .../gcp-to-aws/references/vendored/README.md | 11 ------ .../references/vendored/README.md | 11 ------ .../references/vendored/README.md | 35 ------------------- 4 files changed, 68 deletions(-) delete mode 100644 plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index 1d8fd7d26..9407b1e9d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -44,14 +44,3 @@ if this skill needs one, add the copy **and** a row in the table below. | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | | `clarify/clarify-multicloud.md` | `skills/shared/clarify/clarify-multicloud.md` | | `clarify/clarify-region.md` | `skills/shared/clarify/clarify-region.md` | -| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | -| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | -| `telemetry/version.json` | `skills/shared/telemetry/version.json` | -| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | -| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | -| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | -| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | -| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | -| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | -| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | -| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 751e63686..d10aa5cd5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -37,14 +37,3 @@ if this skill needs one, add the copy **and** a row in the table below. | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | | `clarify/clarify-multicloud.md` | `skills/shared/clarify/clarify-multicloud.md` | | `clarify/clarify-region.md` | `skills/shared/clarify/clarify-region.md` | -| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | -| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | -| `telemetry/version.json` | `skills/shared/telemetry/version.json` | -| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | -| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | -| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | -| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | -| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | -| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | -| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | -| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md index af5e05d1e..cb53a9dea 100644 --- a/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/heroku-to-aws/references/vendored/README.md @@ -30,14 +30,3 @@ if this skill needs one, add the copy **and** a row in the table below. | `estimate/ri-sp-eligibility.md` | `skills/shared/estimate/ri-sp-eligibility.md` | | `pricing/aws-infra-pricing.json` | `skills/shared/pricing/aws-infra-pricing.json` | | `workshop/workshop-invariants.md` | `skills/shared/workshop/workshop-invariants.md` | -| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | -| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | -| `telemetry/version.json` | `skills/shared/telemetry/version.json` | -| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | -| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | -| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | -| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | -| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | -| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | -| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | -| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md deleted file mode 100644 index 32db7a29a..000000000 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/vendored/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Vendored shared files — DO NOT EDIT - -These files are **synced copies** of the plugin-level canonical source under -`plugins/aws-startup-advisor/skills/shared/`. They are vendored into this skill -so the skill folder is **self-contained** — it runs standalone (lifted out, zipped, -or used on its own) without reaching outside its own directory. - -**Do not hand-edit anything in this directory.** Edit the canonical source instead, -then bring every vendored copy back in sync: - -```sh -# from the repository root -python3 plugins/aws-startup-advisor/tools/sync-vendored.py # copies skills/shared/ over every vendored copy -python3 plugins/aws-startup-advisor/tools/sync-vendored.py --check # what CI runs (mise run lint:vendored-parity) -``` - -CI fails when a vendored copy differs from its canonical source, has no canonical source, -is listed in the table below but missing on disk, or is on disk but not listed below. So an -edit to a `skills/shared/` file this skill vendors, or a deleted copy, cannot merge -unnoticed. What CI cannot see is a **new** `skills/shared/` file that no skill vendors yet — -if this skill needs one, add the copy **and** a row in the table below. - -| Vendored path | Canonical source | -| ---------------------------------------------------- | ------------------------------------------------------------------- | -| `telemetry/PROTOCOL.md` | `skills/shared/telemetry/PROTOCOL.md` | -| `telemetry/cli.py` | `skills/shared/telemetry/cli.py` | -| `telemetry/version.json` | `skills/shared/telemetry/version.json` | -| `telemetry/consent/accept.py` | `skills/shared/telemetry/consent/accept.py` | -| `telemetry/consent/cli.py` | `skills/shared/telemetry/consent/cli.py` | -| `telemetry/consent/notice.py` | `skills/shared/telemetry/consent/notice.py` | -| `telemetry/consent/opt_out.py` | `skills/shared/telemetry/consent/opt_out.py` | -| `telemetry/consent/record.py` | `skills/shared/telemetry/consent/record.py` | -| `telemetry/metric_emission/client.py` | `skills/shared/telemetry/metric_emission/client.py` | -| `telemetry/metric_emission/migration.py` | `skills/shared/telemetry/metric_emission/migration.py` | -| `telemetry/metric_emission/migration_attributes.py` | `skills/shared/telemetry/metric_emission/migration_attributes.py` | From 73c135389a43031d5d56605161fe0218a4af7331 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 11 Oct 2026 22:48:08 -0700 Subject: [PATCH 51/53] fix(aws-startup-advisor): account for every neutral fixture directory, not just ai-migration-matrix test_neutral_matrix_shared_artifacts_extend_the_mapped_fixture_count hardcoded the assumption that ai-migration-matrix was the only fixture directory with no skill prefix (gcp-/heroku-/azure-), so --fixtures' checked_artifacts total only needed one 'shared artifacts' bucket subtracted out. This PR's own fixtures/ai-workload-profiles/ directory is a second such neutral directory (its 4 ai-workload-profile.json files are themselves registered as a 'shared' contract in artifact-contracts.json), so --fixtures counts them in checked_artifacts, but the test's formula never added them to the subtracted total -- failing with 'assert (78 - 53) == 21' instead of the correct 25. Generalized the test to sum shared-artifact counts and validate_run_dir checks across every fixture directory that skill_for_fixture maps to None, not just the matrix by name. This also guards against the same accounting gap recurring if a third neutral fixture directory is added later. Verification: the fixed test passes; mise -C plugins/aws-startup-advisor run ci passes in full (vendored-parity 44 files, artifacts 78 checked/0 errors, model-id, skill-refs, 246 tests); mise run build from repo root clean; git diff --check clean. --- .../tests/test_validate_artifacts.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index b3b7a57da..fbc974ac7 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -321,6 +321,12 @@ def test_fixtures_pass_with_baseline_and_no_stale_entries(): def test_neutral_matrix_shared_artifacts_extend_the_mapped_fixture_count(): + """Every fixture directory with no skill prefix (`ai-migration-matrix`, + `ai-workload-profiles`, ...) is "neutral": its artifacts are checked against + the manifest's `shared` contracts only, with no `--skill` passed. `--fixtures` + walks and counts all of them, so the accounting below must sum across every + neutral directory, not just the matrix — a second neutral directory (as + `ai-workload-profiles` is) is not a special case, it is the general one.""" manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") fixtures_root = PLUGIN_ROOT / "fixtures" matrix = fixtures_root / "ai-migration-matrix" @@ -335,16 +341,25 @@ def test_neutral_matrix_shared_artifacts_extend_the_mapped_fixture_count(): assert matrix_findings == [] mapped_checked = 0 + neutral_checked = 0 + neutral_shared_count = 0 mapped_findings = [] + neutral_findings = [] for fixture_dir in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): skill = va.skill_for_fixture(fixture_dir.name, manifest) if skill is not None: mapped_checked += va.validate_run_dir(fixture_dir, skill, manifest, mapped_findings) + else: + neutral_checked += va.validate_run_dir(fixture_dir, None, manifest, neutral_findings) + neutral_shared_count += sum( + 1 for p in va.find_artifacts(fixture_dir) if p.name in manifest["shared"]) + assert neutral_findings == [] + assert neutral_checked == neutral_shared_count r = _run("--fixtures", "--json") assert r.returncode == 0, r.stdout + r.stderr report = json.loads(r.stdout) - assert report["checked_artifacts"] - mapped_checked == len(shared_artifacts) + assert report["checked_artifacts"] - mapped_checked == neutral_shared_count def test_invalid_neutral_shared_artifacts_fail_their_schemas(tmp_path: Path): From 608b612c01dacf65932a755e50b3b2592d383262 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Tue, 13 Oct 2026 18:17:06 -0700 Subject: [PATCH 52/53] test(aws-startup-advisor): pin Estimate and Generate AI contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Target-only, additive JSON Schemas for estimation-ai.json and generation-ai.json, stacked on merged #415 (ai-workload-profile) and #416 (aws-design-ai). cost_by_source[] / artifacts_by_source[] carry the ai_sources[]/gateways[] traceability chain forward per-source, per-model; model_observations[] intentionally not carried forward (identity already resolved by Design). No runtime wiring, no producer change — estimate-ai.md / generate-ai.md / generate-artifacts-ai.md are untouched in both skills. Added: - scripts/contracts/estimation-ai.schema.json, generation-ai.schema.json - skills/shared/ai/schema-estimate-ai.md, schema-generate-ai.md (vendored byte-identically into gcp-to-aws and azure-to-aws only) - fixtures/estimation-ai/{multi-provider-direct,openrouter-gateway-multi-upstream}/estimation-ai.json - fixtures/generation-ai/{multi-provider-direct,openrouter-gateway-multi-upstream}/generation-ai.json - tests/test_estimation_ai_contracts.py, test_generation_ai_contracts.py (positive validation, malformed/duplicate/overlapping rejection, untraceable/dropped-model negatives, cross-phase projection into aws-design-ai.json and estimation-ai.json, vendored byte-identity) Registered two 'shared' entries in artifact-contracts.json, two fixture bullets in fixtures/README.md, two rows in skills/shared/ai/README.md, and a scripts/ clause in README.md. Verification run from plugins/aws-startup-advisor (all commands executed for real, output below): uv run --with pytest python -m pytest tests/ -q -> 429 passed (from the worktree root) mise run build -> lint:manifests, lint:spec, lint:skill-parity, lint:md, security all PASS mise run ci -> lint:artifacts: PASS — 84 artifact(s) checked, 0 error(s) lint:vendored-parity: PASS — 53 vendored file(s) byte-identical lint:skill-refs: PASS — 1339 reference(s) checked, 0 missing lint:md: 0 error(s) lint:model-id: OK (18 retired IDs enforced) test: 429 passed test:heroku-source-review: 44 passed --- plugins/aws-startup-advisor/README.md | 2 +- .../aws-startup-advisor/fixtures/README.md | 2 + .../multi-provider-direct/estimation-ai.json | 126 +++++++++ .../estimation-ai.json | 139 ++++++++++ .../multi-provider-direct/generation-ai.json | 74 +++++ .../generation-ai.json | 78 ++++++ .../scripts/artifact-contracts.json | 6 + .../contracts/estimation-ai.schema.json | 183 +++++++++++++ .../contracts/generation-ai.schema.json | 145 ++++++++++ .../references/vendored/README.md | 2 + .../vendored/ai/schema-estimate-ai.md | 161 +++++++++++ .../vendored/ai/schema-generate-ai.md | 131 +++++++++ .../gcp-to-aws/references/vendored/README.md | 2 + .../vendored/ai/schema-estimate-ai.md | 161 +++++++++++ .../vendored/ai/schema-generate-ai.md | 131 +++++++++ .../skills/shared/ai/README.md | 2 + .../skills/shared/ai/schema-estimate-ai.md | 161 +++++++++++ .../skills/shared/ai/schema-generate-ai.md | 131 +++++++++ .../tests/test_estimation_ai_contracts.py | 253 ++++++++++++++++++ .../tests/test_generation_ai_contracts.py | 237 ++++++++++++++++ 20 files changed, 2126 insertions(+), 1 deletion(-) create mode 100644 plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json create mode 100644 plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json create mode 100644 plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json create mode 100644 plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json create mode 100644 plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json create mode 100644 plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json create mode 100644 plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md create mode 100644 plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md create mode 100644 plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md create mode 100644 plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md create mode 100644 plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md create mode 100644 plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md create mode 100644 plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py create mode 100644 plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py diff --git a/plugins/aws-startup-advisor/README.md b/plugins/aws-startup-advisor/README.md index ff25b7da9..c68f0f016 100644 --- a/plugins/aws-startup-advisor/README.md +++ b/plugins/aws-startup-advisor/README.md @@ -39,7 +39,7 @@ Alongside the skills, the plugin ships supporting material the skills load on de - [`skills/shared/`](skills/shared/) — Canonical reference material shared across skills rather than an invocable skill of its own: the phase-workflow interpreter contract, estimation schemas and complexity tiers, a snapshot of AWS infrastructure rates, phase-status schema, what-if workshop invariants, shared Clarify question fragments, and the AI-migration guardrails and Bedrock model references. Each consuming skill carries a byte-identical copy under its own `references/vendored/` so the skill folder stays self-contained. - [`agents/`](agents/) — Seven Claude Code subagent definitions: two generic, phase-agnostic migration phase workers (read/write and read/write/shell tiers) plus five specialists for the Bedrock rewrite flow (code analyzer, code rewriter, log ingestor, prompt evaluator, report generator). - [`tools/`](tools/) — Contains `model-staleness.py`, which checks the Bedrock model-lifecycle registry's freshness contract (`**Last updated**` / `**Staleness window**`) and that every vendored copy under `skills/*/references/vendored/ai/` is byte-identical to `skills/shared/ai/ai-model-lifecycle.md` (`--strict` also fails on a stale snapshot). -- [`scripts/`](scripts/) — Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter (`emit-plan-json.py`), `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/telemetry/` collects optional usage data: nothing is sent until you accept, no skill depends on it, and `/usage-data` shows the setting or opts out. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. `scripts/contracts/aws-design-ai.schema.json`, canonicalized from Azure's prior single-scalar `metadata.ai_source` contract into `skills/shared/ai/schema-design-aws-ai.md`, likewise defines a target-only `aws-design-ai.json` design contract whose `metadata.ai_sources[]`/`metadata.gateways[]` mirror the workload-profile contract exactly; `design-ai.md` in both skills still writes against the prior per-skill shape, so this is a target, not a claim about current output. +- [`scripts/`](scripts/) — Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter (`emit-plan-json.py`), `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/telemetry/` collects optional usage data: nothing is sent until you accept, no skill depends on it, and `/usage-data` shows the setting or opts out. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. `scripts/contracts/aws-design-ai.schema.json`, canonicalized from Azure's prior single-scalar `metadata.ai_source` contract into `skills/shared/ai/schema-design-aws-ai.md`, likewise defines a target-only `aws-design-ai.json` design contract whose `metadata.ai_sources[]`/`metadata.gateways[]` mirror the workload-profile contract exactly; `design-ai.md` in both skills still writes against the prior per-skill shape, so this is a target, not a claim about current output. `scripts/contracts/estimation-ai.schema.json` and `scripts/contracts/generation-ai.schema.json`, canonicalized into `skills/shared/ai/schema-estimate-ai.md`/`schema-generate-ai.md`, likewise define target-only `estimation-ai.json`/`generation-ai.json` contracts whose `cost_by_source[]`/`artifacts_by_source[]` carry a per-source/per-model traceability chain back through `aws-design-ai.json`'s `metadata.ai_sources[]`/`gateways[]`; `estimate-ai.md` and `generate-ai.md`/`generate-artifacts-ai.md` in both skills still write against their prior per-skill shapes, so these are targets, not claims about current output. - [`fixtures/`](fixtures/) — Reference and regression fixtures for the migration report and estimation artifacts, used to keep generated output within the documented contract. `fixtures/azure-iac-terraform/` additionally carries deterministic asserters that pin the Azure Discover, Clarify, Design, and Estimate contracts against a committed synthetic estate. ## MCP Servers diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 106bb2c54..9e9fa62f1 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -5,6 +5,8 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `ai-workload-profiles/` — 4 representative target-only `ai-workload-profile.json` observation fixtures (not a Cartesian matrix): AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. `ai-workload-profile.schema.json`'s `aiSource` observed branch now also requires `model_observations` non-empty (`minItems: 1`) whenever `model_evidence == "observed"`, matching the identical rule already enforced for `aws-design-ai.schema.json`'s `metadata.ai_sources[]`. - `aws-design-ai/` — 2 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway) and `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. +- `estimation-ai/` — 2 representative target-only `estimation-ai.json` cost-estimate fixtures, nested as `/estimation-ai.json` so `lint:artifacts` discovers them: `multi-provider-direct` and `openrouter-gateway-multi-upstream`, reusing #416's `aws-design-ai/` scenario names and the same providers/models. `cost_by_source[]` carries forward `aws-design-ai.json`'s `metadata.ai_sources[]`/`gateways[]` identity pattern so a per-source/per-model estimated cost traces back to the exact Design entry; `model_observations[]` is intentionally not carried forward (the identity is already resolved by Design). `test_estimation_ai_contracts.py` proves both fixtures project into (are a subset of) the matching `aws-design-ai//aws-design-ai.json`, and exercises negative mutations — duplicate/overlapping source identity, OpenRouter named as a direct provider, a fabricated untraceable model, and a model dropped under a resolved gateway upstream. Schema validity here does not imply an implemented pricing lookup, model mapping, or runtime support, and does not claim either skill's current `estimate-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. +- `generation-ai/` — 2 representative target-only `generation-ai.json` migration-plan fixtures, nested as `/generation-ai.json`, mirroring `estimation-ai/`'s two case names. `artifacts_by_source[]` carries the traceability chain one phase forward: a per-source/per-model generated-artifact reference traces back to the exact `estimation-ai.json → cost_by_source[]` entry it was produced from. `test_generation_ai_contracts.py` proves both fixtures project into the matching `estimation-ai//estimation-ai.json`, and exercises the same negative-mutation categories one phase forward. Schema validity here does not imply an implemented adapter, rollout mechanism, or runtime support, and does not claim either skill's current `generate-ai.md`/`generate-artifacts-ai.md` emits this shape. - `heroku-workshop/` — Heroku what-if workshop seed + arm64 reprice snapshot + `check_expected_workshop.py` - `heroku-nonweb-scaling/` — Heroku Design seed + golden output for the Horizontal Non-Web Capacity Guard (`eco`/`basic` worker at `quantity: 2` → Fargate) + `check_expected_nonweb_scaling.py` - `gcp-workshop/` — GCP what-if workshop seed + graviton reprice + `check_expected_workshop.py` diff --git a/plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json b/plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json new file mode 100644 index 000000000..20a1ad653 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json @@ -0,0 +1,126 @@ +{ + "phase": "estimate", + "timestamp": "2026-09-08T03:10:00Z", + "pricing_source": "cached", + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", + "monthly_ai_spend": 1200.0, + "services": ["anthropic_api", "openai_api"] + }, + "token_volume": { + "source": "tier_table", + "monthly_input_tokens": 60000000, + "monthly_output_tokens": 40000000, + "ratio": "60/40" + }, + "model_comparison": [ + { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 1320.0, + "vs_current": "+10%", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + }, + { + "model": "openai.gpt-5.6-terra", + "monthly_cost": 660.0, + "vs_current": "-12%", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + } + ], + "recommended_model": { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 1320.0, + "breakdown": {"input": 900.0, "output": 420.0, "embeddings": 0.0}, + "rationale": "Same-model migration for the Anthropic-sourced workload; current recommended model and provider priority favor Claude." + }, + "backup_model": { + "model": "anthropic.claude-haiku-4-5-v1:0", + "monthly_cost": 480.0, + "rationale": "Lower-cost fallback within the same model family." + }, + "cost_comparison": { + "current_source_monthly": 1200.0, + "projected_bedrock_monthly": 1980.0, + "monthly_difference": 780.0, + "annual_difference": 9360.0, + "percent_change": 65.0 + }, + "migration_cost_considerations": { + "categories": [], + "complexity_factors": ["integration.pattern = \"direct_sdk\" -> moderate SDK and API pattern changes"], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": { + "monthly_cost_delta": 780.0, + "annual_cost_delta": 9360.0, + "justification": "Combined Anthropic and OpenAI Bedrock costs are higher than current direct-API spend; non-cost benefits justify the move.", + "non_cost_benefits": ["IAM/VPC/PrivateLink/KMS/CloudTrail governance", "usage counting toward existing AWS commitments"] + }, + "optimization_opportunities": [ + { + "opportunity": "Prompt caching", + "type": "prompt_caching", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "up to 90% on cached input tokens", + "commitment": "none", + "implementation_effort": "low", + "available": "day_1", + "description": "Repeated system prompts qualify for cache-read pricing on both Claude and GPT-5.6." + }, + { + "opportunity": "Model downsizing / tiering", + "type": "model_tiering", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "60-87%", + "commitment": "none", + "implementation_effort": "low", + "available": "day_1", + "description": "High-volume, premium model selected; a lower tier may suffice for simple queries." + } + ], + "optimized_projection": { + "monthly_with_optimizations": 1400.0, + "vs_current": "+17%", + "note": "Best-case projection assuming prompt caching and selective model tiering." + }, + "recommendation": { + "path": "migrate_optimized", + "path_label": "Migrate to Bedrock", + "migrate_if": "Governance, residency, and commitment-counting benefits outweigh the modest cost premium.", + "stay_if": "Cost is the only priority and no non-cost driver applies.", + "confidence": "high", + "rationale": "Both Bedrock targets carry a modest premium over direct-API pricing, offset by governance and ecosystem benefits; both source models are strong_migrate per the design assessment." + }, + "cost_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "monthly_cost": 1320.0, + "vs_source_percent": 10.0 + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "monthly_cost": 660.0, + "vs_source_percent": -12.0 + } + ] + } + ], + "services_not_estimated": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json b/plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json new file mode 100644 index 000000000..5cbcb4225 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json @@ -0,0 +1,139 @@ +{ + "phase": "estimate", + "timestamp": "2026-09-08T03:10:00Z", + "pricing_source": "cached", + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", + "monthly_ai_spend": 2400.0, + "services": ["openrouter"] + }, + "token_volume": { + "source": "tier_table", + "monthly_input_tokens": 600000000, + "monthly_output_tokens": 400000000, + "ratio": "60/40" + }, + "model_comparison": [ + { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 13200.0, + "vs_current": "favorable", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + }, + { + "model": "anthropic.claude-haiku-4-5-v1:0", + "monthly_cost": 1100.0, + "vs_current": "favorable", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + }, + { + "model": "openai.gpt-5.6-terra", + "monthly_cost": 8800.0, + "vs_current": "mixed", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + } + ], + "recommended_model": { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 13200.0, + "breakdown": {"input": 9000.0, "output": 4200.0, "embeddings": 0.0}, + "rationale": "Primary chat tier routed through OpenRouter to Claude Sonnet; same-family Bedrock target is favorable vs. router pricing." + }, + "backup_model": { + "model": "anthropic.claude-haiku-4-5-v1:0", + "monthly_cost": 1100.0, + "rationale": "Low-cost tier backup within the same model family." + }, + "cost_comparison": { + "current_source_monthly": 2400.0, + "projected_bedrock_monthly": 23100.0, + "monthly_difference": 20700.0, + "annual_difference": 248400.0, + "percent_change": 862.5 + }, + "migration_cost_considerations": { + "categories": [], + "complexity_factors": ["integration.pattern = \"framework\" -> lower integration touch surface", "summary.total_models_detected > 3 -> multi-model coordination"], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": { + "monthly_cost_delta": 20700.0, + "annual_cost_delta": 248400.0, + "justification": "Direct per-model Bedrock pricing is favorable vs. the router's blended markup once volume is normalized across all three upstream models.", + "non_cost_benefits": ["model flexibility (100+ models)", "AWS ecosystem (Guardrails, Knowledge Bases, AgentCore)"] + }, + "optimization_opportunities": [ + { + "opportunity": "Multi-model tiered routing", + "type": "multi_model_routing", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "60-87%", + "commitment": "none", + "implementation_effort": "low", + "available": "day_1", + "description": "High-volume workload with a 3-tier routing strategy already recorded in the design artifact." + }, + { + "opportunity": "Intelligent Prompt Routing", + "type": "intelligent_prompt_routing", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "up to 30%", + "commitment": "none", + "implementation_effort": "low", + "available": "after_2_weeks_production_traffic", + "description": "Anthropic family router is available for the Claude Sonnet/Haiku tiers once production traffic has run for 2+ weeks." + } + ], + "optimized_projection": { + "monthly_with_optimizations": 15000.0, + "vs_current": "favorable", + "note": "Best-case projection assuming multi-model tiered routing is fully adopted." + }, + "recommendation": { + "path": "migrate_optimized", + "path_label": "Migrate to Bedrock", + "migrate_if": "Direct per-model Bedrock pricing beats the router's blended markup at this volume.", + "stay_if": "Router flexibility across many providers is a higher priority than per-model cost.", + "confidence": "high", + "rationale": "All three router upstreams land on favorable or mixed Bedrock pricing, and the design assessment is strong_migrate across the board." + }, + "cost_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "monthly_cost": 13200.0, + "vs_source_percent": null + }, + { + "model": "claude-3-haiku-20240307", + "monthly_cost": 1100.0, + "vs_source_percent": null + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "monthly_cost": 8800.0, + "vs_source_percent": null + } + ] + } + ], + "services_not_estimated": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json b/plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json new file mode 100644 index 000000000..347fe40a6 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json @@ -0,0 +1,74 @@ +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "2026-09-08T04:00:00Z", + "migration_plan": { + "duration_drivers": ["multiple models (adapter + evaluation per model)", "framework integration"], + "approach": "direct_sdk_adapter", + "phases": [ + {"name": "Phase 1", "sequence": 1, "activities": ["Enable Bedrock access", "create IAM role", "develop provider adapter with feature flag", "unit test"]}, + {"name": "Phase 2", "sequence": 2, "activities": ["Deploy to staging", "run A/B comparison", "measure latency/quality/cost", "tune prompts"]}, + {"name": "Phase 3", "sequence": 3, "activities": ["Gradual rollout (10% -> 50% -> 100%)", "monitor", "disable source provider after 48h stable"]} + ], + "models_to_migrate": ["claude-3-5-sonnet-20240620", "gpt-4o"] + }, + "step_by_step_guide": { + "languages": ["python"], + "primary_pattern": "direct_sdk", + "files_to_modify": ["src/ai/anthropic_client.py", "src/ai/openai_client.py"], + "dependency_changes": "boto3 for the Anthropic client; openai>=2.45.0 with aws_bedrock_token_generator for the OpenAI client." + }, + "rollback_plan": { + "mechanism": "feature_flag", + "flag_name": "AI_PROVIDER", + "default_value": "anthropic_api", + "rollback_time": "instant", + "triggers": ["quality below threshold", "P95 latency > 2x baseline", "error rate > 1% for 5 min", "cost per request > 3x source"] + }, + "monitoring": { + "dashboards": ["Request volume by provider", "latency comparison (P50/P95/P99)", "error rates", "token usage", "cost tracking", "quality scores"], + "alerting_rules": [ + {"severity": "Critical", "condition": "Error rate > 5% for 2 min", "action": "auto-rollback"}, + {"severity": "High", "condition": "Latency P95 > 3x baseline for 5 min", "action": "page on-call"} + ] + }, + "production_readiness_checklist": [ + "Bedrock model access enabled", + "IAM role with bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream", + "Provider adapter deployed and tested in staging", + "A/B test with >= 100 representative prompts", + "Rollback procedure documented and tested" + ], + "success_criteria": { + "quality": {"category": "Quality", "criteria": "Response quality", "target": ">= 90% of source baseline"}, + "latency": {"category": "Latency", "criteria": "P95", "target": "Within 2x of source"}, + "cost": {"category": "Cost", "criteria": "Monthly", "target": "Within 20% of estimation-ai.json"} + }, + "recommendation": { + "approach": "Direct SDK adapter per source, rolled out gradually with a shared feature flag.", + "confidence": "high", + "key_risks": ["multi-model coordination across two simultaneous sources"] + }, + "artifacts_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "generated_files": ["src/ai/anthropic_client.py"] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "generated_files": ["src/ai/openai_client.py"] + } + ] + } + ] +} diff --git a/plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json b/plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json new file mode 100644 index 000000000..c9246f5cc --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json @@ -0,0 +1,78 @@ +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "2026-09-08T04:00:00Z", + "migration_plan": { + "duration_drivers": ["multiple models (adapter + evaluation per model)", "custom inference pipeline"], + "approach": "gateway_config_swap", + "phases": [ + {"name": "Phase 1", "sequence": 1, "activities": ["Replace OpenRouter SDK call sites with per-provider Bedrock clients"]}, + {"name": "Phase 2", "sequence": 2, "activities": ["Deploy to staging", "run A/B comparison per tier"]}, + {"name": "Phase 3", "sequence": 3, "activities": ["Gradual rollout per tier", "monitor", "disable router after 48h stable"]} + ], + "models_to_migrate": ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307", "gpt-4o"] + }, + "step_by_step_guide": { + "languages": ["python"], + "primary_pattern": "framework", + "files_to_modify": ["src/ai/router_client.py"], + "dependency_changes": "Replace the OpenRouter SDK call sites with per-provider Bedrock clients (boto3 Converse for Anthropic, Mantle Responses for OpenAI)." + }, + "rollback_plan": { + "mechanism": "feature_flag", + "flag_name": "AI_PROVIDER", + "default_value": "openrouter", + "rollback_time": "instant", + "triggers": ["quality below threshold", "P95 latency > 2x baseline", "error rate > 1% for 5 min", "cost per request > 3x source"] + }, + "monitoring": { + "dashboards": ["Request volume by provider", "latency comparison (P50/P95/P99)", "error rates", "token usage", "cost tracking", "quality scores"], + "alerting_rules": [ + {"severity": "Critical", "condition": "Error rate > 5% for 2 min", "action": "auto-rollback"}, + {"severity": "Medium", "condition": "Daily cost > 2x projected", "action": "notify finance channel"} + ] + }, + "production_readiness_checklist": [ + "Bedrock model access enabled for all three tiers", + "IAM role with bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream", + "Provider adapters deployed and tested in staging for each tier", + "A/B test with >= 100 representative prompts per tier", + "Rollback procedure documented and tested" + ], + "success_criteria": { + "quality": {"category": "Quality", "criteria": "Response quality", "target": ">= 90% of source baseline"}, + "latency": {"category": "Latency", "criteria": "P95", "target": "Within 2x of source"}, + "cost": {"category": "Cost", "criteria": "Monthly", "target": "Within 20% of estimation-ai.json"} + }, + "recommendation": { + "approach": "Replace the OpenRouter gateway with direct per-provider Bedrock clients, keeping the existing 3-tier routing strategy.", + "confidence": "high", + "key_risks": ["multi-upstream gateway replacement touches three models across two providers simultaneously"] + }, + "artifacts_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "generated_files": ["src/ai/router_client.py"] + }, + { + "model": "claude-3-haiku-20240307", + "generated_files": ["src/ai/router_client.py"] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "generated_files": ["src/ai/router_client.py"] + } + ] + } + ] +} diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 9f8cbb8bf..eb84f05a9 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -17,6 +17,12 @@ }, "aws-design-ai.json": { "json_schema": "scripts/contracts/aws-design-ai.schema.json" + }, + "estimation-ai.json": { + "json_schema": "scripts/contracts/estimation-ai.schema.json" + }, + "generation-ai.json": { + "json_schema": "scripts/contracts/generation-ai.schema.json" } }, "skills": { diff --git a/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json new file mode 100644 index 000000000..6b70832c4 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json @@ -0,0 +1,183 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:estimation-ai", + "title": "estimation-ai.json (canonical, target-only estimate contract)", + "description": "Provider-neutral, target-only contract for the estimation-ai.json AI cost estimate artifact. Structural validity does NOT imply an implemented pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either skill's current estimate-ai.md actually emits this shape — this is an estimate-time target. cost_by_source[] carries forward the aws-design-ai.json metadata.ai_sources[]/gateways[] identity pattern so a per-source/per-model estimated cost traces back to the exact Design entry it was computed from.", + "type": "object", + "required": [ + "phase", + "timestamp", + "pricing_source", + "accuracy_confidence", + "current_costs", + "token_volume", + "model_comparison", + "recommended_model", + "backup_model", + "cost_comparison", + "migration_cost_considerations", + "roi_analysis", + "optimization_opportunities", + "optimized_projection", + "recommendation", + "cost_by_source", + "services_not_estimated" + ], + "additionalProperties": false, + "properties": { + "phase": {"const": "estimate"}, + "timestamp": {"type": "string", "minLength": 1}, + "pricing_source": {"enum": ["cached", "unavailable"]}, + "accuracy_confidence": {"type": "string", "minLength": 1}, + "current_costs": { + "type": "object", + "required": ["source", "monthly_ai_spend", "services"], + "additionalProperties": false, + "properties": { + "source": {"type": "string", "minLength": 1}, + "monthly_ai_spend": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "services": {"type": "array"} + } + }, + "token_volume": {"type": "object"}, + "model_comparison": {"type": "array"}, + "recommended_model": {"type": "object"}, + "backup_model": {"type": "object"}, + "embeddings": {"type": "object"}, + "cost_comparison": { + "type": "object", + "required": [ + "current_source_monthly", + "projected_bedrock_monthly", + "monthly_difference", + "annual_difference", + "percent_change" + ], + "additionalProperties": false, + "properties": { + "current_source_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "projected_bedrock_monthly": {"type": "number"}, + "monthly_difference": {"type": "number"}, + "annual_difference": {"type": "number"}, + "percent_change": {"oneOf": [{"type": "null"}, {"type": "number"}]} + } + }, + "migration_cost_considerations": { + "type": "object", + "required": ["categories", "complexity_factors", "note"], + "additionalProperties": false, + "properties": { + "categories": {"type": "array", "maxItems": 0}, + "complexity_factors": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "note": {"type": "string", "minLength": 1} + } + }, + "roi_analysis": {"type": "object"}, + "optimization_opportunities": { + "type": "array", + "items": {"$ref": "#/definitions/optimizationOpportunity"} + }, + "optimized_projection": {"type": "object"}, + "recommendation": {"$ref": "#/definitions/recommendation"}, + "cost_by_source": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceCostEntry"} + }, + "services_not_estimated": { + "type": "array", + "items": {"$ref": "#/definitions/serviceNotEstimated"} + } + }, + "definitions": { + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "sourceCostModel": { + "type": "object", + "required": ["model", "monthly_cost", "vs_source_percent"], + "additionalProperties": false, + "properties": { + "model": {"$ref": "#/definitions/normalizedModel"}, + "monthly_cost": {"type": "number"}, + "vs_source_percent": {"oneOf": [{"type": "null"}, {"type": "number"}]} + } + }, + "sourceCostEntry": { + "description": "A per-source, per-model estimated cost entry. (provider, source_service) MUST exist in the aws-design-ai.json metadata.ai_sources[]/gateways[].upstreams[] this estimate was built from, and every models[].model MUST be one of that source's models[] — see schema-estimate-ai.md's cost_by_source[] traceability rule.", + "type": "object", + "required": ["provider", "source_service", "models"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "models": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceCostModel"} + } + } + }, + "optimizationOpportunity": { + "type": "object", + "required": [ + "opportunity", + "type", + "target_services", + "commitment", + "potential_savings_monthly", + "potential_savings_percent", + "available", + "implementation_effort", + "description" + ], + "additionalProperties": false, + "properties": { + "opportunity": {"type": "string", "minLength": 1}, + "type": { + "enum": [ + "model_tiering", + "prompt_caching", + "batch_api", + "intelligent_prompt_routing", + "provisioned_throughput", + "input_token_reduction", + "multi_model_routing" + ] + }, + "target_services": {"type": "array", "items": {"const": "Bedrock"}, "minItems": 1}, + "commitment": {"type": "string", "minLength": 1}, + "potential_savings_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "potential_savings_percent": {"type": "string", "minLength": 1}, + "available": {"type": "string", "minLength": 1}, + "implementation_effort": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1} + } + }, + "recommendation": { + "type": "object", + "required": ["path", "path_label", "migrate_if", "stay_if", "confidence", "rationale"], + "additionalProperties": false, + "properties": { + "path": {"enum": ["migrate_optimized", "migrate_phased", "stay"]}, + "path_label": {"type": "string", "minLength": 1}, + "migrate_if": {"type": "string", "minLength": 1}, + "stay_if": {"type": "string", "minLength": 1}, + "confidence": {"enum": ["high", "medium", "low"]}, + "rationale": {"type": "string", "minLength": 1} + } + }, + "serviceNotEstimated": { + "type": "object", + "required": ["workload_id", "target_aws_service", "reason"], + "additionalProperties": false, + "properties": { + "workload_id": {"type": "string", "minLength": 1}, + "target_aws_service": { + "enum": ["textract", "rekognition", "transcribe", "comprehend", "translate", "polly", "sagemaker"] + }, + "reason": {"const": "not_token_priced"} + } + } + } +} diff --git a/plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json new file mode 100644 index 000000000..dd9fdabc1 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:generation-ai", + "title": "generation-ai.json (canonical, target-only generate contract)", + "description": "Provider-neutral, target-only contract for the generation-ai.json AI migration-plan artifact. Structural validity does NOT imply an implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that either skill's current generate-ai.md/generate-artifacts-ai.md actually emits this shape — this is a generate-time target. artifacts_by_source[] carries the traceability chain one phase forward from estimation-ai.json's cost_by_source[] so a per-source/per-model generated-artifact reference traces back to the exact Estimate entry it was produced from.", + "type": "object", + "required": [ + "phase", + "generation_source", + "timestamp", + "migration_plan", + "step_by_step_guide", + "rollback_plan", + "monitoring", + "production_readiness_checklist", + "success_criteria", + "recommendation", + "artifacts_by_source" + ], + "additionalProperties": false, + "properties": { + "phase": {"const": "generate"}, + "generation_source": {"const": "ai"}, + "timestamp": {"type": "string", "minLength": 1}, + "migration_plan": { + "type": "object", + "required": ["duration_drivers", "approach", "phases", "models_to_migrate"], + "additionalProperties": false, + "properties": { + "duration_drivers": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "approach": {"type": "string", "minLength": 1}, + "phases": {"type": "array"}, + "models_to_migrate": {"type": "array", "items": {"$ref": "#/definitions/normalizedModel"}} + } + }, + "step_by_step_guide": { + "type": "object", + "required": ["languages", "primary_pattern", "files_to_modify", "dependency_changes"], + "additionalProperties": false, + "properties": { + "languages": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "primary_pattern": {"type": "string", "minLength": 1}, + "files_to_modify": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "dependency_changes": {"type": "string", "minLength": 1} + } + }, + "rollback_plan": { + "type": "object", + "required": ["mechanism", "flag_name", "default_value", "rollback_time", "triggers"], + "additionalProperties": false, + "properties": { + "mechanism": {"const": "feature_flag"}, + "flag_name": {"type": "string", "minLength": 1}, + "default_value": {"type": "string", "minLength": 1}, + "rollback_time": {"type": "string", "minLength": 1}, + "triggers": {"type": "array", "items": {"type": "string", "minLength": 1}, "minItems": 1} + } + }, + "monitoring": { + "type": "object", + "required": ["dashboards", "alerting_rules"], + "additionalProperties": false, + "properties": { + "dashboards": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "alerting_rules": { + "type": "array", + "items": { + "type": "object", + "required": ["severity", "condition", "action"], + "additionalProperties": false, + "properties": { + "severity": {"enum": ["Critical", "High", "Medium", "Low"]}, + "condition": {"type": "string", "minLength": 1}, + "action": {"type": "string", "minLength": 1} + } + } + } + } + }, + "production_readiness_checklist": { + "type": "array", + "minItems": 5, + "items": {"type": "string", "minLength": 1} + }, + "success_criteria": { + "type": "object", + "required": ["quality", "latency", "cost"], + "additionalProperties": false, + "properties": { + "quality": {"type": "object"}, + "latency": {"type": "object"}, + "cost": {"type": "object"} + } + }, + "recommendation": { + "type": "object", + "required": ["approach", "confidence", "key_risks"], + "additionalProperties": false, + "properties": { + "approach": {"type": "string", "minLength": 1}, + "confidence": {"enum": ["high", "medium", "low"]}, + "key_risks": {"type": "array", "items": {"type": "string", "minLength": 1}} + } + }, + "artifacts_by_source": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceArtifactEntry"} + } + }, + "definitions": { + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "sourceArtifactModel": { + "type": "object", + "required": ["model", "generated_files"], + "additionalProperties": false, + "properties": { + "model": {"$ref": "#/definitions/normalizedModel"}, + "generated_files": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + }, + "sourceArtifactEntry": { + "description": "A per-source, per-model generated-artifact reference entry. (provider, source_service) MUST exist in the estimation-ai.json cost_by_source[] this plan was built from, and every models[].model MUST be one of that source's models[] — see schema-generate-ai.md's artifacts_by_source[] traceability rule.", + "type": "object", + "required": ["provider", "source_service", "models"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "models": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceArtifactModel"} + } + } + } + } +} diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index 1751964d3..4e8df69c5 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -40,6 +40,8 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | | `ai/sdk-capability-map.json` | `skills/shared/ai/sdk-capability-map.json` | | `ai/schema-design-aws-ai.md` | `skills/shared/ai/schema-design-aws-ai.md` | +| `ai/schema-estimate-ai.md` | `skills/shared/ai/schema-estimate-ai.md` | +| `ai/schema-generate-ai.md` | `skills/shared/ai/schema-generate-ai.md` | | `clarify/clarify-availability.md` | `skills/shared/clarify/clarify-availability.md` | | `clarify/clarify-compliance.md` | `skills/shared/clarify/clarify-compliance.md` | | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md new file mode 100644 index 000000000..098b0607b --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md @@ -0,0 +1,161 @@ +# Schema — estimation-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design-ai.json` had a prose-only contract that drifted between +> GCP and Azure until #416 canonicalized it (`schema-design-aws-ai.md`). `estimation-ai.json` is +> the same risk one phase over: `estimate-ai.md` produces it (both GCP's +> `skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +> `skills/azure-to-aws/references/phases/estimate/estimate-ai.md`), and `generate-ai.md` / +> `generate-artifacts-ai.md` consume it, with each skill's own "Schema — top-level fields" +> table carrying the same field set under GCP-/Azure-named keys (`gcp_monthly_ai_spend` vs +> `azure_monthly_ai_spend`, `current_gcp_monthly` vs `current_azure_monthly`). This file is the +> single, provider-neutral written contract, using neutral field names +> (`current_costs.monthly_ai_spend`, `cost_comparison.current_source_monthly`) so one JSON +> Schema (`scripts/contracts/estimation-ai.schema.json`) serves both skills without a GCP/Azure +> branch — the same provider-neutral framing `ai-workload-profile.schema.json` (#415) and +> `aws-design-ai.schema.json` (#416) already use. Each consuming skill carries a byte-identical +> vendored copy at `references/vendored/ai/schema-estimate-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410), +> `ai-workload-profile.schema.json` (#415), and `aws-design-ai.schema.json` (#416).** Structural +> validity against this document or its companion JSON Schema does NOT imply an implemented +> pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either +> skill's current `estimate-ai.md` actually emits this shape yet — NEITHER skill's `estimate-ai.md` +> currently emits the neutral field names (`current_costs.monthly_ai_spend`, +> `cost_comparison.current_source_monthly`, `cost_by_source[]`); current GCP/Azure output still +> uses the GCP-/Azure-named fields documented in each skill's own `estimate-ai.md`, unchanged by +> this document. No producer has been repointed to this canonical contract in this change — that +> is deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "estimate", + "timestamp": "", + "pricing_source": "cached", // cached | unavailable + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", // provenance label — see each skill's estimate-ai.md Part 1 + "monthly_ai_spend": 1200.0, // provider-neutral name for gcp_monthly_ai_spend / azure_monthly_ai_spend + "services": [] + }, + "token_volume": {}, + "model_comparison": [], + "recommended_model": {}, + "backup_model": {}, + "embeddings": {}, // present only when the workload uses embeddings + "cost_comparison": { + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly + "projected_bedrock_monthly": 1320.0, + "monthly_difference": 120.0, + "annual_difference": 1440.0, + "percent_change": 10.0 + }, + "migration_cost_considerations": { + "categories": [], // ALWAYS [] — human/pro-services costs are out of scope + "complexity_factors": [], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": {}, + "optimization_opportunities": [], // see § optimization_opportunities[] + "optimized_projection": {}, + "recommendation": {}, // see § recommendation + "cost_by_source": [], // many-valued — see § cost_by_source[] + "services_not_estimated": [] // ALWAYS present, [] when every workload is token-priced +} +``` + +## cost_by_source[] + +Carries the design-time identity chain forward into the estimate: each entry's +`(provider, source_service)` pair MUST exist in the `aws-design-ai.json` the estimate was built +from (either a direct `metadata.ai_sources[]` entry, or a `metadata.gateways[].upstreams[]` +entry), and every `models[].model` MUST be one of that source's `models[]` (direct) or +`model_families[].models` (gateway upstream). This is additive — it does not replace +`model_comparison[]`, which remains the per-model Bedrock-vs-source comparison for ALL viable +models regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source + "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `aws-design-ai.json`'s +`metadata.ai_sources[]` — a duplicate or overlapping pair is a contract violation, not a +presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted cost still +attributes to the fronted provider's `(provider, source_service)` pair, mirroring how +`aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. + +## Design-decision: no `model_observations[]` here + +`model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in +`ai-workload-profile.schema.json` and `aws-design-ai.schema.json` to carry an identity that is +still being established at Discover/Design time. By Estimate, the identity is already resolved — +`aws-design-ai.json`'s `metadata.ai_sources[]`/`models[]` is the resolved, trusted identity list, +and this contract's `cost_by_source[]` entries key off `(provider, source_service, model)` triples +copied FROM that resolved list, not re-observed. Carrying `model_observations[]` forward into +`estimation-ai.json` would re-introduce an uncertainty concept that Design has already resolved. +This is intentionally dropped, not forgotten. + +## optimization_opportunities[] + +Each entry's `type` is REQUIRED and is one of `model_tiering`, `prompt_caching`, `batch_api`, +`intelligent_prompt_routing`, `provisioned_throughput`, `input_token_reduction`, +`multi_model_routing` (see each skill's `estimate-ai.md` Part 6 for the full vocabulary and +sequencing notes). `target_services` is always `["Bedrock"]`. `potential_savings_percent` is +REQUIRED on every entry — populated even when `potential_savings_monthly` is `null`, so a merged +report table is never blank. + +## recommendation + +`path` is REQUIRED and is one of `migrate_optimized`, `migrate_phased`, `stay` (see each skill's +`estimate-ai.md` Part 7 for the full decision logic). When the Design artifact's +`ai_architecture.honest_assessment` is `recommend_stay`, `recommendation.path` MUST be `stay` +regardless of cost numbers. + +## services_not_estimated[] + +ALWAYS present (`[]` when every workload is token-priced). Per entry: +`{ workload_id, target_aws_service, reason: "not_token_priced" }` — `design_blocks[]` rows with +`target_aws_service` set (Textract, Rekognition, Transcribe) are per-page/per-image/per-minute +priced, not token priced, and are listed here instead of silently omitted. + +## Validation Checklist + +- [ ] `cost_by_source[]` is non-empty whenever `aws-design-ai.json`'s `metadata.ai_sources[]` is + non-empty, and each entry's `(provider, source_service)` pair is unique. +- [ ] Every `cost_by_source[]` entry's `(provider, source_service)` pair exists in the matching + `aws-design-ai.json`'s `metadata.ai_sources[]` or a `metadata.gateways[].upstreams[]` entry. +- [ ] Every `cost_by_source[].models[].model` is one of that source's `models[]` (direct) or + `model_families[].models` (gateway upstream) in the matching `aws-design-ai.json`. +- [ ] No entry in `cost_by_source[]` has `provider: "openrouter"`. +- [ ] `recommendation` is present with non-empty `path`, `path_label`, `migrate_if`, `stay_if`, + and `rationale`; `path` is one of `migrate_optimized`, `migrate_phased`, `stay`. +- [ ] If Design `honest_assessment` is `recommend_stay`, `recommendation.path` is `stay`. +- [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, + `potential_savings_percent`, and `available`. +- [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] `services_not_estimated[]` is present (`[]` when clean). + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. Both GCP's +`skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +`skills/azure-to-aws/references/phases/estimate/estimate-ai.md` still write against their own +"Schema — top-level fields" table with GCP-/Azure-named fields (`current_costs.gcp_monthly_ai_spend` +/ `current_costs.azure_monthly_ai_spend`, `cost_comparison.current_gcp_monthly` / +`cost_comparison.current_azure_monthly`), and neither emits `cost_by_source[]`. This document +(and its companion JSON Schema, `scripts/contracts/estimation-ai.schema.json`) is the TARGET +contract. Repointing either skill's `estimate-ai.md` to emit this shape is deferred +runtime-adoption work, named explicitly so a future PR does not silently assume today's runtime +output already satisfies this contract. `generate-ai.md` and `generate-artifacts-ai.md` are +unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md new file mode 100644 index 000000000..d8925c48c --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md @@ -0,0 +1,131 @@ +# Schema — generation-ai.json (canonical, target-only) + +> **Why this file exists.** `estimation-ai.json` just gained a single, provider-neutral written +> contract (`schema-estimate-ai.md`, #389 follow-on). `generation-ai.json` is the same risk one +> phase further: GCP's `skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the +> `generation-ai.json` section of Azure's +> `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` each carry their own +> "Schema — top-level fields" table for the same artifact. This file is the single, +> provider-neutral written contract for `generation-ai.json`, matching +> `schema-estimate-ai.md`'s own canonicalization pattern so GCP and Azure share one definition +> instead of drifting independently. Each consuming skill carries a byte-identical vendored copy +> at `references/vendored/ai/schema-generate-ai.md`. +> +> **This is a target-only contract, like `schema-estimate-ai.md` and the rest of this +> contract family (#410, #415, #416).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/generation-ai.schema.json`) does NOT imply an +> implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that +> either skill's current `generate-ai.md` / `generate-artifacts-ai.md` actually emits this shape +> yet. No producer has been repointed to this canonical contract in this change — that is +> deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "", + "migration_plan": { + "duration_drivers": [], + "approach": "", + "phases": [], + "models_to_migrate": [] // normalizedModel[] — covers every model from aws-design-ai.json + }, + "step_by_step_guide": { + "languages": [], + "primary_pattern": "", + "files_to_modify": [], + "dependency_changes": "" + }, + "rollback_plan": { + "mechanism": "feature_flag", // MUST be this exact value + "flag_name": "", + "default_value": "", // free-form pre-cutover provider identifier, e.g. "vertex_ai" | "azure_openai" — not an enum + "rollback_time": "", + "triggers": [] + }, + "monitoring": { + "dashboards": [], + "alerting_rules": [] // each: { severity, condition, action } + }, + "production_readiness_checklist": [], // >= 5 string items + "success_criteria": { + "quality": {}, + "latency": {}, + "cost": {} + }, + "recommendation": { + "approach": "", + "confidence": "high | medium | low", + "key_risks": [] + }, + "artifacts_by_source": [] // many-valued — see § artifacts_by_source[] +} +``` + +## artifacts_by_source[] + +Carries the traceability chain one phase forward from `estimation-ai.json`'s `cost_by_source[]`: +each entry's `(provider, source_service)` pair MUST exist in the `estimation-ai.json` +`cost_by_source[]` the migration plan was built from, and every `models[].model` MUST be one of +that source's `models[]`. This is additive — it does not replace `migration_plan.models_to_migrate[]`, +which remains the flat list of every model being migrated regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching estimation-ai.json cost_by_source[] entry + "generated_files": ["src/ai/anthropic_client.py"] // files this migration plan touches for this specific model + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `estimation-ai.json`'s +`cost_by_source[]`. `provider` is never `"openrouter"` — an OpenRouter-fronted artifact still +attributes to the fronted provider's `(provider, source_service)` pair. + +## Design-decision: no `model_observations[]` here + +Same rationale as `schema-estimate-ai.md`: by Generate, the model identity has been resolved +twice over (Design, then Estimate). `artifacts_by_source[]` entries key off +`(provider, source_service, model)` triples copied FROM `estimation-ai.json`'s `cost_by_source[]`, +not re-observed. Carrying `model_observations[]` forward here would re-introduce an uncertainty +concept that was resolved two phases earlier. This is intentionally dropped, not forgotten. + +## rollback_plan + +`mechanism` MUST be the literal string `"feature_flag"` — this contract does not model any other +rollback mechanism. `default_value` is a free-form string naming the pre-cutover provider +identifier (e.g. GCP's `"vertex_ai"`, Azure's `"azure_openai"`); both skills already treat it as +a free-form string, not an enum, and this contract does not special-case the value. + +## Validation Checklist + +- [ ] `migration_plan.models_to_migrate` covers all models from the matching `aws-design-ai.json`. +- [ ] `artifacts_by_source[]` is non-empty whenever the matching `estimation-ai.json`'s + `cost_by_source[]` is non-empty, and each entry's `(provider, source_service)` pair is + unique. +- [ ] Every `artifacts_by_source[]` entry's `(provider, source_service)` pair exists in the + matching `estimation-ai.json`'s `cost_by_source[]`. +- [ ] Every `artifacts_by_source[].models[].model` is one of that source's `cost_by_source[]` + `models[].model` entries in the matching `estimation-ai.json`. +- [ ] No entry in `artifacts_by_source[]` has `provider: "openrouter"`. +- [ ] `rollback_plan.mechanism` is `"feature_flag"`. +- [ ] `production_readiness_checklist` has at least 5 items. +- [ ] `success_criteria` covers `quality`, `latency`, and `cost`. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. GCP's +`skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the `generation-ai.json` +section of Azure's `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` +still write against their own "Schema — top-level fields" tables, and neither emits +`artifacts_by_source[]`. This document (and its companion JSON Schema, +`scripts/contracts/generation-ai.schema.json`) is the TARGET contract. Repointing either skill's +generate phase to emit this shape is deferred runtime-adoption work, named explicitly so a +future PR does not silently assume today's runtime output already satisfies this contract. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index e904edc3d..5444c69ee 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -33,6 +33,8 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | | `ai/sdk-capability-map.json` | `skills/shared/ai/sdk-capability-map.json` | | `ai/schema-design-aws-ai.md` | `skills/shared/ai/schema-design-aws-ai.md` | +| `ai/schema-estimate-ai.md` | `skills/shared/ai/schema-estimate-ai.md` | +| `ai/schema-generate-ai.md` | `skills/shared/ai/schema-generate-ai.md` | | `clarify/clarify-availability.md` | `skills/shared/clarify/clarify-availability.md` | | `clarify/clarify-compliance.md` | `skills/shared/clarify/clarify-compliance.md` | | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md new file mode 100644 index 000000000..098b0607b --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md @@ -0,0 +1,161 @@ +# Schema — estimation-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design-ai.json` had a prose-only contract that drifted between +> GCP and Azure until #416 canonicalized it (`schema-design-aws-ai.md`). `estimation-ai.json` is +> the same risk one phase over: `estimate-ai.md` produces it (both GCP's +> `skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +> `skills/azure-to-aws/references/phases/estimate/estimate-ai.md`), and `generate-ai.md` / +> `generate-artifacts-ai.md` consume it, with each skill's own "Schema — top-level fields" +> table carrying the same field set under GCP-/Azure-named keys (`gcp_monthly_ai_spend` vs +> `azure_monthly_ai_spend`, `current_gcp_monthly` vs `current_azure_monthly`). This file is the +> single, provider-neutral written contract, using neutral field names +> (`current_costs.monthly_ai_spend`, `cost_comparison.current_source_monthly`) so one JSON +> Schema (`scripts/contracts/estimation-ai.schema.json`) serves both skills without a GCP/Azure +> branch — the same provider-neutral framing `ai-workload-profile.schema.json` (#415) and +> `aws-design-ai.schema.json` (#416) already use. Each consuming skill carries a byte-identical +> vendored copy at `references/vendored/ai/schema-estimate-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410), +> `ai-workload-profile.schema.json` (#415), and `aws-design-ai.schema.json` (#416).** Structural +> validity against this document or its companion JSON Schema does NOT imply an implemented +> pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either +> skill's current `estimate-ai.md` actually emits this shape yet — NEITHER skill's `estimate-ai.md` +> currently emits the neutral field names (`current_costs.monthly_ai_spend`, +> `cost_comparison.current_source_monthly`, `cost_by_source[]`); current GCP/Azure output still +> uses the GCP-/Azure-named fields documented in each skill's own `estimate-ai.md`, unchanged by +> this document. No producer has been repointed to this canonical contract in this change — that +> is deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "estimate", + "timestamp": "", + "pricing_source": "cached", // cached | unavailable + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", // provenance label — see each skill's estimate-ai.md Part 1 + "monthly_ai_spend": 1200.0, // provider-neutral name for gcp_monthly_ai_spend / azure_monthly_ai_spend + "services": [] + }, + "token_volume": {}, + "model_comparison": [], + "recommended_model": {}, + "backup_model": {}, + "embeddings": {}, // present only when the workload uses embeddings + "cost_comparison": { + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly + "projected_bedrock_monthly": 1320.0, + "monthly_difference": 120.0, + "annual_difference": 1440.0, + "percent_change": 10.0 + }, + "migration_cost_considerations": { + "categories": [], // ALWAYS [] — human/pro-services costs are out of scope + "complexity_factors": [], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": {}, + "optimization_opportunities": [], // see § optimization_opportunities[] + "optimized_projection": {}, + "recommendation": {}, // see § recommendation + "cost_by_source": [], // many-valued — see § cost_by_source[] + "services_not_estimated": [] // ALWAYS present, [] when every workload is token-priced +} +``` + +## cost_by_source[] + +Carries the design-time identity chain forward into the estimate: each entry's +`(provider, source_service)` pair MUST exist in the `aws-design-ai.json` the estimate was built +from (either a direct `metadata.ai_sources[]` entry, or a `metadata.gateways[].upstreams[]` +entry), and every `models[].model` MUST be one of that source's `models[]` (direct) or +`model_families[].models` (gateway upstream). This is additive — it does not replace +`model_comparison[]`, which remains the per-model Bedrock-vs-source comparison for ALL viable +models regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source + "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `aws-design-ai.json`'s +`metadata.ai_sources[]` — a duplicate or overlapping pair is a contract violation, not a +presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted cost still +attributes to the fronted provider's `(provider, source_service)` pair, mirroring how +`aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. + +## Design-decision: no `model_observations[]` here + +`model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in +`ai-workload-profile.schema.json` and `aws-design-ai.schema.json` to carry an identity that is +still being established at Discover/Design time. By Estimate, the identity is already resolved — +`aws-design-ai.json`'s `metadata.ai_sources[]`/`models[]` is the resolved, trusted identity list, +and this contract's `cost_by_source[]` entries key off `(provider, source_service, model)` triples +copied FROM that resolved list, not re-observed. Carrying `model_observations[]` forward into +`estimation-ai.json` would re-introduce an uncertainty concept that Design has already resolved. +This is intentionally dropped, not forgotten. + +## optimization_opportunities[] + +Each entry's `type` is REQUIRED and is one of `model_tiering`, `prompt_caching`, `batch_api`, +`intelligent_prompt_routing`, `provisioned_throughput`, `input_token_reduction`, +`multi_model_routing` (see each skill's `estimate-ai.md` Part 6 for the full vocabulary and +sequencing notes). `target_services` is always `["Bedrock"]`. `potential_savings_percent` is +REQUIRED on every entry — populated even when `potential_savings_monthly` is `null`, so a merged +report table is never blank. + +## recommendation + +`path` is REQUIRED and is one of `migrate_optimized`, `migrate_phased`, `stay` (see each skill's +`estimate-ai.md` Part 7 for the full decision logic). When the Design artifact's +`ai_architecture.honest_assessment` is `recommend_stay`, `recommendation.path` MUST be `stay` +regardless of cost numbers. + +## services_not_estimated[] + +ALWAYS present (`[]` when every workload is token-priced). Per entry: +`{ workload_id, target_aws_service, reason: "not_token_priced" }` — `design_blocks[]` rows with +`target_aws_service` set (Textract, Rekognition, Transcribe) are per-page/per-image/per-minute +priced, not token priced, and are listed here instead of silently omitted. + +## Validation Checklist + +- [ ] `cost_by_source[]` is non-empty whenever `aws-design-ai.json`'s `metadata.ai_sources[]` is + non-empty, and each entry's `(provider, source_service)` pair is unique. +- [ ] Every `cost_by_source[]` entry's `(provider, source_service)` pair exists in the matching + `aws-design-ai.json`'s `metadata.ai_sources[]` or a `metadata.gateways[].upstreams[]` entry. +- [ ] Every `cost_by_source[].models[].model` is one of that source's `models[]` (direct) or + `model_families[].models` (gateway upstream) in the matching `aws-design-ai.json`. +- [ ] No entry in `cost_by_source[]` has `provider: "openrouter"`. +- [ ] `recommendation` is present with non-empty `path`, `path_label`, `migrate_if`, `stay_if`, + and `rationale`; `path` is one of `migrate_optimized`, `migrate_phased`, `stay`. +- [ ] If Design `honest_assessment` is `recommend_stay`, `recommendation.path` is `stay`. +- [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, + `potential_savings_percent`, and `available`. +- [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] `services_not_estimated[]` is present (`[]` when clean). + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. Both GCP's +`skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +`skills/azure-to-aws/references/phases/estimate/estimate-ai.md` still write against their own +"Schema — top-level fields" table with GCP-/Azure-named fields (`current_costs.gcp_monthly_ai_spend` +/ `current_costs.azure_monthly_ai_spend`, `cost_comparison.current_gcp_monthly` / +`cost_comparison.current_azure_monthly`), and neither emits `cost_by_source[]`. This document +(and its companion JSON Schema, `scripts/contracts/estimation-ai.schema.json`) is the TARGET +contract. Repointing either skill's `estimate-ai.md` to emit this shape is deferred +runtime-adoption work, named explicitly so a future PR does not silently assume today's runtime +output already satisfies this contract. `generate-ai.md` and `generate-artifacts-ai.md` are +unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md new file mode 100644 index 000000000..d8925c48c --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md @@ -0,0 +1,131 @@ +# Schema — generation-ai.json (canonical, target-only) + +> **Why this file exists.** `estimation-ai.json` just gained a single, provider-neutral written +> contract (`schema-estimate-ai.md`, #389 follow-on). `generation-ai.json` is the same risk one +> phase further: GCP's `skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the +> `generation-ai.json` section of Azure's +> `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` each carry their own +> "Schema — top-level fields" table for the same artifact. This file is the single, +> provider-neutral written contract for `generation-ai.json`, matching +> `schema-estimate-ai.md`'s own canonicalization pattern so GCP and Azure share one definition +> instead of drifting independently. Each consuming skill carries a byte-identical vendored copy +> at `references/vendored/ai/schema-generate-ai.md`. +> +> **This is a target-only contract, like `schema-estimate-ai.md` and the rest of this +> contract family (#410, #415, #416).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/generation-ai.schema.json`) does NOT imply an +> implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that +> either skill's current `generate-ai.md` / `generate-artifacts-ai.md` actually emits this shape +> yet. No producer has been repointed to this canonical contract in this change — that is +> deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "", + "migration_plan": { + "duration_drivers": [], + "approach": "", + "phases": [], + "models_to_migrate": [] // normalizedModel[] — covers every model from aws-design-ai.json + }, + "step_by_step_guide": { + "languages": [], + "primary_pattern": "", + "files_to_modify": [], + "dependency_changes": "" + }, + "rollback_plan": { + "mechanism": "feature_flag", // MUST be this exact value + "flag_name": "", + "default_value": "", // free-form pre-cutover provider identifier, e.g. "vertex_ai" | "azure_openai" — not an enum + "rollback_time": "", + "triggers": [] + }, + "monitoring": { + "dashboards": [], + "alerting_rules": [] // each: { severity, condition, action } + }, + "production_readiness_checklist": [], // >= 5 string items + "success_criteria": { + "quality": {}, + "latency": {}, + "cost": {} + }, + "recommendation": { + "approach": "", + "confidence": "high | medium | low", + "key_risks": [] + }, + "artifacts_by_source": [] // many-valued — see § artifacts_by_source[] +} +``` + +## artifacts_by_source[] + +Carries the traceability chain one phase forward from `estimation-ai.json`'s `cost_by_source[]`: +each entry's `(provider, source_service)` pair MUST exist in the `estimation-ai.json` +`cost_by_source[]` the migration plan was built from, and every `models[].model` MUST be one of +that source's `models[]`. This is additive — it does not replace `migration_plan.models_to_migrate[]`, +which remains the flat list of every model being migrated regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching estimation-ai.json cost_by_source[] entry + "generated_files": ["src/ai/anthropic_client.py"] // files this migration plan touches for this specific model + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `estimation-ai.json`'s +`cost_by_source[]`. `provider` is never `"openrouter"` — an OpenRouter-fronted artifact still +attributes to the fronted provider's `(provider, source_service)` pair. + +## Design-decision: no `model_observations[]` here + +Same rationale as `schema-estimate-ai.md`: by Generate, the model identity has been resolved +twice over (Design, then Estimate). `artifacts_by_source[]` entries key off +`(provider, source_service, model)` triples copied FROM `estimation-ai.json`'s `cost_by_source[]`, +not re-observed. Carrying `model_observations[]` forward here would re-introduce an uncertainty +concept that was resolved two phases earlier. This is intentionally dropped, not forgotten. + +## rollback_plan + +`mechanism` MUST be the literal string `"feature_flag"` — this contract does not model any other +rollback mechanism. `default_value` is a free-form string naming the pre-cutover provider +identifier (e.g. GCP's `"vertex_ai"`, Azure's `"azure_openai"`); both skills already treat it as +a free-form string, not an enum, and this contract does not special-case the value. + +## Validation Checklist + +- [ ] `migration_plan.models_to_migrate` covers all models from the matching `aws-design-ai.json`. +- [ ] `artifacts_by_source[]` is non-empty whenever the matching `estimation-ai.json`'s + `cost_by_source[]` is non-empty, and each entry's `(provider, source_service)` pair is + unique. +- [ ] Every `artifacts_by_source[]` entry's `(provider, source_service)` pair exists in the + matching `estimation-ai.json`'s `cost_by_source[]`. +- [ ] Every `artifacts_by_source[].models[].model` is one of that source's `cost_by_source[]` + `models[].model` entries in the matching `estimation-ai.json`. +- [ ] No entry in `artifacts_by_source[]` has `provider: "openrouter"`. +- [ ] `rollback_plan.mechanism` is `"feature_flag"`. +- [ ] `production_readiness_checklist` has at least 5 items. +- [ ] `success_criteria` covers `quality`, `latency`, and `cost`. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. GCP's +`skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the `generation-ai.json` +section of Azure's `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` +still write against their own "Schema — top-level fields" tables, and neither emits +`artifacts_by_source[]`. This document (and its companion JSON Schema, +`scripts/contracts/generation-ai.schema.json`) is the TARGET contract. Repointing either skill's +generate phase to emit this shape is deferred runtime-adoption work, named explicitly so a +future PR does not silently assume today's runtime output already satisfies this contract. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/README.md b/plugins/aws-startup-advisor/skills/shared/ai/README.md index 9948541bf..daa862dc7 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/README.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/README.md @@ -20,6 +20,8 @@ before opening a pull request. | `sdk-capability-map.json` | SDK method → capability lookup used by app-code discovery | | `bedrock-pricing-cache.md` | Provider-neutral Bedrock model and service pricing | | `schema-design-aws-ai.md` | Target-only `aws-design-ai.json` contract; `metadata.ai_sources[]`/`gateways[]` mirror `ai-workload-profile.schema.json` exactly | +| `schema-estimate-ai.md` | Target-only `estimation-ai.json` contract; `cost_by_source[]` traces per-source/per-model cost back to `aws-design-ai.json` | +| `schema-generate-ai.md` | Target-only `generation-ai.json` contract; `artifacts_by_source[]` traces per-source/per-model generated artifacts back to `estimation-ai.json` | Deliberately NOT here: a source-cloud's own mapping guide (e.g. gcp-to-aws's `ai-gemini-to-bedrock.md`) and its `schema-discover-ai.md`, both of which are diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md new file mode 100644 index 000000000..098b0607b --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md @@ -0,0 +1,161 @@ +# Schema — estimation-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design-ai.json` had a prose-only contract that drifted between +> GCP and Azure until #416 canonicalized it (`schema-design-aws-ai.md`). `estimation-ai.json` is +> the same risk one phase over: `estimate-ai.md` produces it (both GCP's +> `skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +> `skills/azure-to-aws/references/phases/estimate/estimate-ai.md`), and `generate-ai.md` / +> `generate-artifacts-ai.md` consume it, with each skill's own "Schema — top-level fields" +> table carrying the same field set under GCP-/Azure-named keys (`gcp_monthly_ai_spend` vs +> `azure_monthly_ai_spend`, `current_gcp_monthly` vs `current_azure_monthly`). This file is the +> single, provider-neutral written contract, using neutral field names +> (`current_costs.monthly_ai_spend`, `cost_comparison.current_source_monthly`) so one JSON +> Schema (`scripts/contracts/estimation-ai.schema.json`) serves both skills without a GCP/Azure +> branch — the same provider-neutral framing `ai-workload-profile.schema.json` (#415) and +> `aws-design-ai.schema.json` (#416) already use. Each consuming skill carries a byte-identical +> vendored copy at `references/vendored/ai/schema-estimate-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410), +> `ai-workload-profile.schema.json` (#415), and `aws-design-ai.schema.json` (#416).** Structural +> validity against this document or its companion JSON Schema does NOT imply an implemented +> pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either +> skill's current `estimate-ai.md` actually emits this shape yet — NEITHER skill's `estimate-ai.md` +> currently emits the neutral field names (`current_costs.monthly_ai_spend`, +> `cost_comparison.current_source_monthly`, `cost_by_source[]`); current GCP/Azure output still +> uses the GCP-/Azure-named fields documented in each skill's own `estimate-ai.md`, unchanged by +> this document. No producer has been repointed to this canonical contract in this change — that +> is deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "estimate", + "timestamp": "", + "pricing_source": "cached", // cached | unavailable + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", // provenance label — see each skill's estimate-ai.md Part 1 + "monthly_ai_spend": 1200.0, // provider-neutral name for gcp_monthly_ai_spend / azure_monthly_ai_spend + "services": [] + }, + "token_volume": {}, + "model_comparison": [], + "recommended_model": {}, + "backup_model": {}, + "embeddings": {}, // present only when the workload uses embeddings + "cost_comparison": { + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly + "projected_bedrock_monthly": 1320.0, + "monthly_difference": 120.0, + "annual_difference": 1440.0, + "percent_change": 10.0 + }, + "migration_cost_considerations": { + "categories": [], // ALWAYS [] — human/pro-services costs are out of scope + "complexity_factors": [], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": {}, + "optimization_opportunities": [], // see § optimization_opportunities[] + "optimized_projection": {}, + "recommendation": {}, // see § recommendation + "cost_by_source": [], // many-valued — see § cost_by_source[] + "services_not_estimated": [] // ALWAYS present, [] when every workload is token-priced +} +``` + +## cost_by_source[] + +Carries the design-time identity chain forward into the estimate: each entry's +`(provider, source_service)` pair MUST exist in the `aws-design-ai.json` the estimate was built +from (either a direct `metadata.ai_sources[]` entry, or a `metadata.gateways[].upstreams[]` +entry), and every `models[].model` MUST be one of that source's `models[]` (direct) or +`model_families[].models` (gateway upstream). This is additive — it does not replace +`model_comparison[]`, which remains the per-model Bedrock-vs-source comparison for ALL viable +models regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source + "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `aws-design-ai.json`'s +`metadata.ai_sources[]` — a duplicate or overlapping pair is a contract violation, not a +presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted cost still +attributes to the fronted provider's `(provider, source_service)` pair, mirroring how +`aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. + +## Design-decision: no `model_observations[]` here + +`model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in +`ai-workload-profile.schema.json` and `aws-design-ai.schema.json` to carry an identity that is +still being established at Discover/Design time. By Estimate, the identity is already resolved — +`aws-design-ai.json`'s `metadata.ai_sources[]`/`models[]` is the resolved, trusted identity list, +and this contract's `cost_by_source[]` entries key off `(provider, source_service, model)` triples +copied FROM that resolved list, not re-observed. Carrying `model_observations[]` forward into +`estimation-ai.json` would re-introduce an uncertainty concept that Design has already resolved. +This is intentionally dropped, not forgotten. + +## optimization_opportunities[] + +Each entry's `type` is REQUIRED and is one of `model_tiering`, `prompt_caching`, `batch_api`, +`intelligent_prompt_routing`, `provisioned_throughput`, `input_token_reduction`, +`multi_model_routing` (see each skill's `estimate-ai.md` Part 6 for the full vocabulary and +sequencing notes). `target_services` is always `["Bedrock"]`. `potential_savings_percent` is +REQUIRED on every entry — populated even when `potential_savings_monthly` is `null`, so a merged +report table is never blank. + +## recommendation + +`path` is REQUIRED and is one of `migrate_optimized`, `migrate_phased`, `stay` (see each skill's +`estimate-ai.md` Part 7 for the full decision logic). When the Design artifact's +`ai_architecture.honest_assessment` is `recommend_stay`, `recommendation.path` MUST be `stay` +regardless of cost numbers. + +## services_not_estimated[] + +ALWAYS present (`[]` when every workload is token-priced). Per entry: +`{ workload_id, target_aws_service, reason: "not_token_priced" }` — `design_blocks[]` rows with +`target_aws_service` set (Textract, Rekognition, Transcribe) are per-page/per-image/per-minute +priced, not token priced, and are listed here instead of silently omitted. + +## Validation Checklist + +- [ ] `cost_by_source[]` is non-empty whenever `aws-design-ai.json`'s `metadata.ai_sources[]` is + non-empty, and each entry's `(provider, source_service)` pair is unique. +- [ ] Every `cost_by_source[]` entry's `(provider, source_service)` pair exists in the matching + `aws-design-ai.json`'s `metadata.ai_sources[]` or a `metadata.gateways[].upstreams[]` entry. +- [ ] Every `cost_by_source[].models[].model` is one of that source's `models[]` (direct) or + `model_families[].models` (gateway upstream) in the matching `aws-design-ai.json`. +- [ ] No entry in `cost_by_source[]` has `provider: "openrouter"`. +- [ ] `recommendation` is present with non-empty `path`, `path_label`, `migrate_if`, `stay_if`, + and `rationale`; `path` is one of `migrate_optimized`, `migrate_phased`, `stay`. +- [ ] If Design `honest_assessment` is `recommend_stay`, `recommendation.path` is `stay`. +- [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, + `potential_savings_percent`, and `available`. +- [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] `services_not_estimated[]` is present (`[]` when clean). + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. Both GCP's +`skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +`skills/azure-to-aws/references/phases/estimate/estimate-ai.md` still write against their own +"Schema — top-level fields" table with GCP-/Azure-named fields (`current_costs.gcp_monthly_ai_spend` +/ `current_costs.azure_monthly_ai_spend`, `cost_comparison.current_gcp_monthly` / +`cost_comparison.current_azure_monthly`), and neither emits `cost_by_source[]`. This document +(and its companion JSON Schema, `scripts/contracts/estimation-ai.schema.json`) is the TARGET +contract. Repointing either skill's `estimate-ai.md` to emit this shape is deferred +runtime-adoption work, named explicitly so a future PR does not silently assume today's runtime +output already satisfies this contract. `generate-ai.md` and `generate-artifacts-ai.md` are +unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md new file mode 100644 index 000000000..d8925c48c --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md @@ -0,0 +1,131 @@ +# Schema — generation-ai.json (canonical, target-only) + +> **Why this file exists.** `estimation-ai.json` just gained a single, provider-neutral written +> contract (`schema-estimate-ai.md`, #389 follow-on). `generation-ai.json` is the same risk one +> phase further: GCP's `skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the +> `generation-ai.json` section of Azure's +> `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` each carry their own +> "Schema — top-level fields" table for the same artifact. This file is the single, +> provider-neutral written contract for `generation-ai.json`, matching +> `schema-estimate-ai.md`'s own canonicalization pattern so GCP and Azure share one definition +> instead of drifting independently. Each consuming skill carries a byte-identical vendored copy +> at `references/vendored/ai/schema-generate-ai.md`. +> +> **This is a target-only contract, like `schema-estimate-ai.md` and the rest of this +> contract family (#410, #415, #416).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/generation-ai.schema.json`) does NOT imply an +> implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that +> either skill's current `generate-ai.md` / `generate-artifacts-ai.md` actually emits this shape +> yet. No producer has been repointed to this canonical contract in this change — that is +> deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "", + "migration_plan": { + "duration_drivers": [], + "approach": "", + "phases": [], + "models_to_migrate": [] // normalizedModel[] — covers every model from aws-design-ai.json + }, + "step_by_step_guide": { + "languages": [], + "primary_pattern": "", + "files_to_modify": [], + "dependency_changes": "" + }, + "rollback_plan": { + "mechanism": "feature_flag", // MUST be this exact value + "flag_name": "", + "default_value": "", // free-form pre-cutover provider identifier, e.g. "vertex_ai" | "azure_openai" — not an enum + "rollback_time": "", + "triggers": [] + }, + "monitoring": { + "dashboards": [], + "alerting_rules": [] // each: { severity, condition, action } + }, + "production_readiness_checklist": [], // >= 5 string items + "success_criteria": { + "quality": {}, + "latency": {}, + "cost": {} + }, + "recommendation": { + "approach": "", + "confidence": "high | medium | low", + "key_risks": [] + }, + "artifacts_by_source": [] // many-valued — see § artifacts_by_source[] +} +``` + +## artifacts_by_source[] + +Carries the traceability chain one phase forward from `estimation-ai.json`'s `cost_by_source[]`: +each entry's `(provider, source_service)` pair MUST exist in the `estimation-ai.json` +`cost_by_source[]` the migration plan was built from, and every `models[].model` MUST be one of +that source's `models[]`. This is additive — it does not replace `migration_plan.models_to_migrate[]`, +which remains the flat list of every model being migrated regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching estimation-ai.json cost_by_source[] entry + "generated_files": ["src/ai/anthropic_client.py"] // files this migration plan touches for this specific model + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `estimation-ai.json`'s +`cost_by_source[]`. `provider` is never `"openrouter"` — an OpenRouter-fronted artifact still +attributes to the fronted provider's `(provider, source_service)` pair. + +## Design-decision: no `model_observations[]` here + +Same rationale as `schema-estimate-ai.md`: by Generate, the model identity has been resolved +twice over (Design, then Estimate). `artifacts_by_source[]` entries key off +`(provider, source_service, model)` triples copied FROM `estimation-ai.json`'s `cost_by_source[]`, +not re-observed. Carrying `model_observations[]` forward here would re-introduce an uncertainty +concept that was resolved two phases earlier. This is intentionally dropped, not forgotten. + +## rollback_plan + +`mechanism` MUST be the literal string `"feature_flag"` — this contract does not model any other +rollback mechanism. `default_value` is a free-form string naming the pre-cutover provider +identifier (e.g. GCP's `"vertex_ai"`, Azure's `"azure_openai"`); both skills already treat it as +a free-form string, not an enum, and this contract does not special-case the value. + +## Validation Checklist + +- [ ] `migration_plan.models_to_migrate` covers all models from the matching `aws-design-ai.json`. +- [ ] `artifacts_by_source[]` is non-empty whenever the matching `estimation-ai.json`'s + `cost_by_source[]` is non-empty, and each entry's `(provider, source_service)` pair is + unique. +- [ ] Every `artifacts_by_source[]` entry's `(provider, source_service)` pair exists in the + matching `estimation-ai.json`'s `cost_by_source[]`. +- [ ] Every `artifacts_by_source[].models[].model` is one of that source's `cost_by_source[]` + `models[].model` entries in the matching `estimation-ai.json`. +- [ ] No entry in `artifacts_by_source[]` has `provider: "openrouter"`. +- [ ] `rollback_plan.mechanism` is `"feature_flag"`. +- [ ] `production_readiness_checklist` has at least 5 items. +- [ ] `success_criteria` covers `quality`, `latency`, and `cost`. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. GCP's +`skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the `generation-ai.json` +section of Azure's `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` +still write against their own "Schema — top-level fields" tables, and neither emits +`artifacts_by_source[]`. This document (and its companion JSON Schema, +`scripts/contracts/generation-ai.schema.json`) is the TARGET contract. Repointing either skill's +generate phase to emit this shape is deferred runtime-adoption work, named explicitly so a +future PR does not silently assume today's runtime output already satisfies this contract. diff --git a/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py new file mode 100644 index 000000000..9da4441a0 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py @@ -0,0 +1,253 @@ +"""Contract tests for the target-only, canonical estimation-ai.json contract. + +Pins the schema's self-validity and proves that `cost_by_source[]` projects into +(is a subset of) the matching `aws-design-ai.json`'s `metadata.ai_sources[]` / +`metadata.gateways[].upstreams[]` it was built from, reusing #416's own validator helpers +as the oracle — the same way #416 reused #415's helpers. Target-only: no producer emits +this shape yet (`estimate-ai.md` is unchanged in both skills); no runtime behavior is +asserted. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "estimation-ai" +DESIGN_FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "aws-design-ai" +ESTIMATION_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "estimation-ai.schema.json" +VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +SYNC_VENDORED_PATH = PLUGIN_ROOT / "tools" / "sync-vendored.py" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +VALIDATOR = _load_module("estimation_ai__validator", VALIDATOR_PATH) + +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*/estimation-ai.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _estimation_ai_schema() -> dict: + return _load(ESTIMATION_AI_SCHEMA) + + +def _estimation_ai_findings(estimate: dict) -> list: + findings = [] + VALIDATOR.validate_json_schema(_estimation_ai_schema(), estimate, "", "estimate", str(ESTIMATION_AI_SCHEMA), findings) + return findings + + +def _base_estimate() -> dict: + return copy.deepcopy(_load(FIXTURE_ROOT / "multi-provider-direct" / "estimation-ai.json")) + + +def _matching_design(case: str) -> dict: + return _load(DESIGN_FIXTURE_ROOT / case / "aws-design-ai.json") + + +# --------------------------------------------------------------------------- schema self-validity + +def test_schema_has_no_unsupported_keywords() -> None: + assert VALIDATOR.unsupported_keywords(_estimation_ai_schema()) == [] + + +def test_schema_is_valid_json_and_draft07_shaped() -> None: + schema = _estimation_ai_schema() + assert schema["$schema"] == "http://json-schema.org/draft-07/schema#" + assert schema["type"] == "object" + + +# --------------------------------------------------------------------------- positive fixtures + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_validate_against_estimation_ai_schema(fixture_path: Path) -> None: + estimate = _load(fixture_path) + assert _estimation_ai_findings(estimate) == [] + + +def test_multi_provider_fixture_carries_two_simultaneous_direct_sources() -> None: + estimate = _load(FIXTURE_ROOT / "multi-provider-direct" / "estimation-ai.json") + providers = {s["provider"] for s in estimate["cost_by_source"]} + assert providers == {"anthropic", "openai"} + + +def test_gateway_fixture_carries_multi_upstream_openrouter_fronting_two_providers() -> None: + estimate = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "estimation-ai.json") + claude = next(s for s in estimate["cost_by_source"] if s["provider"] == "anthropic") + assert len(claude["models"]) == 2 + openai_source = next(s for s in estimate["cost_by_source"] if s["provider"] == "openai") + assert len(openai_source["models"]) == 1 + + +# --------------------------------------------------------------------------- cross-phase projection + + +def _assert_cost_by_source_projects_into_design(estimate: dict, design: dict) -> None: + """Every `cost_by_source[]` entry's `(provider, source_service, models[])` MUST project + into (be a subset of) the matching `aws-design-ai.json`'s `metadata.ai_sources[]` / + `metadata.gateways[].upstreams[]` — same 'projects exactly/subset' style as + #416's `test_positive_fixtures_project_exactly_into_pr415_semantics`.""" + direct_models: dict[tuple, set] = {} + for source in design["metadata"]["ai_sources"]: + direct_models[(source["provider"], source["source_service"])] = set(source["models"]) + upstream_models: dict[tuple, set] = {} + for gateway in design["metadata"]["gateways"]: + for upstream in gateway["upstreams"]: + key = (upstream["provider"], upstream["source_service"]) + models = upstream_models.setdefault(key, set()) + for family in upstream["model_families"]: + models.update(family["models"]) + + for entry in estimate["cost_by_source"]: + key = (entry["provider"], entry["source_service"]) + allowed = direct_models.get(key) or upstream_models.get(key) + assert allowed is not None, ( + f"cost_by_source[] entry {key!r} traces to no aws-design-ai.json " + f"metadata.ai_sources[] entry or metadata.gateways[] upstream" + ) + entry_models = {m["model"] for m in entry["models"]} + assert entry_models <= allowed, ( + f"cost_by_source[] entry {key!r} names models {entry_models - allowed!r} " + f"not present in the matching aws-design-ai.json source" + ) + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_project_into_matching_design_source(fixture_path: Path) -> None: + estimate = _load(fixture_path) + design = _matching_design(fixture_path.parent.name) + _assert_cost_by_source_projects_into_design(estimate, design) + + +def test_untraceable_model_under_resolved_design_source_is_rejected() -> None: + """Mirrors #416's `test_design_block_naming_an_untraceable_model_is_rejected` one phase + forward: a cost_by_source[] entry naming a model not present in the matching + aws-design-ai.json's resolved ai_sources[]/gateways[] is a fabricated/untraceable model.""" + estimate = _base_estimate() + design = _matching_design("multi-provider-direct") + estimate["cost_by_source"][0]["models"][0]["model"] = "made-up-model" + with pytest.raises(AssertionError): + _assert_cost_by_source_projects_into_design(estimate, design) + + +def test_dropped_model_under_a_resolved_gateway_upstream_is_rejected() -> None: + """Mirrors #416's own test of the same name one phase forward: a model that is removed + from BOTH the gateway upstream and the direct source in aws-design-ai.json is no longer + traceable, so a cost_by_source[] entry still naming it must be rejected.""" + estimate = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "estimation-ai.json") + design = copy.deepcopy(_matching_design("openrouter-gateway-multi-upstream")) + anthropic_source = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") + anthropic_source["models"].remove("claude-3-haiku-20240307") + anthropic_source["model_observations"] = [ + obs for obs in anthropic_source["model_observations"] if obs["normalized_model"] != "claude-3-haiku-20240307" + ] + for gateway in design["metadata"]["gateways"]: + for upstream in gateway["upstreams"]: + if upstream["provider"] == "anthropic": + for family in upstream["model_families"]: + if "claude-3-haiku-20240307" in family["models"]: + family["models"].remove("claude-3-haiku-20240307") + with pytest.raises(AssertionError): + _assert_cost_by_source_projects_into_design(estimate, design) + + +# --------------------------------------------------------------------------- negative mutations + + +def test_duplicate_cost_by_source_identity_is_rejected() -> None: + estimate = _base_estimate() + estimate["cost_by_source"].append(copy.deepcopy(estimate["cost_by_source"][0])) + assert _estimation_ai_findings(estimate) != [] + + +def _assert_cost_by_source_identity_is_unique(estimate: dict) -> None: + """uniqueItems on cost_by_source[] only rejects byte-identical duplicates; two entries + with the SAME (provider, source_service) pair but different payloads (an overlapping + identity, not an exact duplicate) need this semantic check instead, mirroring + #415's/#416's own key-based duplicate/overlap check for ai_sources[].""" + seen = set() + for entry in estimate["cost_by_source"]: + key = (entry["provider"], entry["source_service"]) + assert key not in seen, f"duplicate or overlapping cost_by_source identity: {key}" + seen.add(key) + + +def test_positive_fixtures_have_unique_cost_by_source_identity() -> None: + for fixture_path in POSITIVE_FIXTURES: + _assert_cost_by_source_identity_is_unique(_load(fixture_path)) + + +def test_overlapping_cost_by_source_identity_is_rejected() -> None: + """A second entry with the SAME (provider, source_service) pair but different model + content is still a duplicate/overlapping identity, not a legitimate second entry — + schema-level uniqueItems alone cannot catch this (see helper docstring), so the semantic + helper above is the lock.""" + estimate = _base_estimate() + overlapping = copy.deepcopy(estimate["cost_by_source"][0]) + overlapping["models"][0]["monthly_cost"] = 1.0 + estimate["cost_by_source"].append(overlapping) + with pytest.raises(AssertionError): + _assert_cost_by_source_identity_is_unique(estimate) + + +def test_openrouter_as_a_direct_cost_by_source_provider_is_rejected() -> None: + estimate = _base_estimate() + estimate["cost_by_source"][0]["provider"] = "openrouter" + assert _estimation_ai_findings(estimate) != [] + + +def test_malformed_cost_by_source_entry_missing_models_is_rejected() -> None: + estimate = _base_estimate() + del estimate["cost_by_source"][0]["models"] + assert _estimation_ai_findings(estimate) != [] + + +def test_recommendation_requires_a_valid_path_enum() -> None: + estimate = _base_estimate() + estimate["recommendation"]["path"] = "not_a_real_path" + assert _estimation_ai_findings(estimate) != [] + + +def test_optimization_opportunity_missing_type_is_rejected() -> None: + estimate = _base_estimate() + del estimate["optimization_opportunities"][0]["type"] + assert _estimation_ai_findings(estimate) != [] + + +def test_services_not_estimated_entry_requires_not_token_priced_reason() -> None: + estimate = _base_estimate() + estimate["services_not_estimated"] = [ + {"workload_id": "wl_doc1", "target_aws_service": "textract", "reason": "something_else"} + ] + assert _estimation_ai_findings(estimate) != [] + + +# --------------------------------------------------------------------------- vendored byte-identity + + +def test_vendored_copies_are_byte_identical() -> None: + """Reuses tools/sync-vendored.py --check (the dedicated CI gate) rather than + re-implementing filecmp logic inline, matching the project's existing pattern.""" + result = subprocess.run( + [sys.executable, str(SYNC_VENDORED_PATH), "--check"], + cwd=PLUGIN_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py new file mode 100644 index 000000000..175f53168 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py @@ -0,0 +1,237 @@ +"""Contract tests for the target-only, canonical generation-ai.json contract. + +Pins the schema's self-validity and proves that `artifacts_by_source[]` projects into +(is a subset of) the matching `estimation-ai.json`'s `cost_by_source[]` it was built from — +one phase forward from `test_estimation_ai_contracts.py`'s own projection into +`aws-design-ai.json`. Target-only: no producer emits this shape yet (`generate-ai.md` / +`generate-artifacts-ai.md` are unchanged in both skills); no runtime behavior is asserted. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "generation-ai" +ESTIMATION_FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "estimation-ai" +GENERATION_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "generation-ai.schema.json" +VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +SYNC_VENDORED_PATH = PLUGIN_ROOT / "tools" / "sync-vendored.py" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +VALIDATOR = _load_module("generation_ai__validator", VALIDATOR_PATH) + +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*/generation-ai.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _generation_ai_schema() -> dict: + return _load(GENERATION_AI_SCHEMA) + + +def _generation_ai_findings(generation: dict) -> list: + findings = [] + VALIDATOR.validate_json_schema(_generation_ai_schema(), generation, "", "generation", str(GENERATION_AI_SCHEMA), findings) + return findings + + +def _base_generation() -> dict: + return copy.deepcopy(_load(FIXTURE_ROOT / "multi-provider-direct" / "generation-ai.json")) + + +def _matching_estimation(case: str) -> dict: + return _load(ESTIMATION_FIXTURE_ROOT / case / "estimation-ai.json") + + +# --------------------------------------------------------------------------- schema self-validity + +def test_schema_has_no_unsupported_keywords() -> None: + assert VALIDATOR.unsupported_keywords(_generation_ai_schema()) == [] + + +def test_schema_is_valid_json_and_draft07_shaped() -> None: + schema = _generation_ai_schema() + assert schema["$schema"] == "http://json-schema.org/draft-07/schema#" + assert schema["type"] == "object" + + +# --------------------------------------------------------------------------- positive fixtures + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_validate_against_generation_ai_schema(fixture_path: Path) -> None: + generation = _load(fixture_path) + assert _generation_ai_findings(generation) == [] + + +def test_multi_provider_fixture_carries_two_simultaneous_direct_sources() -> None: + generation = _load(FIXTURE_ROOT / "multi-provider-direct" / "generation-ai.json") + providers = {s["provider"] for s in generation["artifacts_by_source"]} + assert providers == {"anthropic", "openai"} + + +def test_gateway_fixture_carries_multi_upstream_openrouter_fronting_two_providers() -> None: + generation = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "generation-ai.json") + claude = next(s for s in generation["artifacts_by_source"] if s["provider"] == "anthropic") + assert len(claude["models"]) == 2 + openai_source = next(s for s in generation["artifacts_by_source"] if s["provider"] == "openai") + assert len(openai_source["models"]) == 1 + + +# --------------------------------------------------------------------------- cross-phase projection + + +def _assert_artifacts_by_source_projects_into_estimation(generation: dict, estimation: dict) -> None: + """Every `artifacts_by_source[]` entry's `(provider, source_service, models[])` MUST + project into (be a subset of) the matching `estimation-ai.json`'s `cost_by_source[]` — + one phase forward from the estimate-level projection into `aws-design-ai.json`.""" + allowed_models: dict[tuple, set] = {} + for entry in estimation["cost_by_source"]: + key = (entry["provider"], entry["source_service"]) + allowed_models[key] = {m["model"] for m in entry["models"]} + + for entry in generation["artifacts_by_source"]: + key = (entry["provider"], entry["source_service"]) + allowed = allowed_models.get(key) + assert allowed is not None, ( + f"artifacts_by_source[] entry {key!r} traces to no estimation-ai.json " + f"cost_by_source[] entry" + ) + entry_models = {m["model"] for m in entry["models"]} + assert entry_models <= allowed, ( + f"artifacts_by_source[] entry {key!r} names models {entry_models - allowed!r} " + f"not present in the matching estimation-ai.json cost_by_source[] entry" + ) + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_project_into_matching_estimation_source(fixture_path: Path) -> None: + generation = _load(fixture_path) + estimation = _matching_estimation(fixture_path.parent.name) + _assert_artifacts_by_source_projects_into_estimation(generation, estimation) + + +def test_untraceable_model_under_resolved_estimation_source_is_rejected() -> None: + """Mirrors the estimate-level untraceable-model negative one phase forward: an + artifacts_by_source[] entry naming a model not present in the matching + estimation-ai.json's cost_by_source[] is a fabricated/untraceable model.""" + generation = _base_generation() + estimation = _matching_estimation("multi-provider-direct") + generation["artifacts_by_source"][0]["models"][0]["model"] = "made-up-model" + with pytest.raises(AssertionError): + _assert_artifacts_by_source_projects_into_estimation(generation, estimation) + + +def test_dropped_model_under_a_resolved_gateway_upstream_is_rejected() -> None: + """Mirrors the estimate-level dropped-model negative one phase forward: a model removed + from the matching estimation-ai.json's cost_by_source[] entry is no longer traceable, so + an artifacts_by_source[] entry still naming it must be rejected.""" + generation = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "generation-ai.json") + estimation = copy.deepcopy(_matching_estimation("openrouter-gateway-multi-upstream")) + anthropic_entry = next(s for s in estimation["cost_by_source"] if s["provider"] == "anthropic") + anthropic_entry["models"] = [ + m for m in anthropic_entry["models"] if m["model"] != "claude-3-haiku-20240307" + ] + with pytest.raises(AssertionError): + _assert_artifacts_by_source_projects_into_estimation(generation, estimation) + + +# --------------------------------------------------------------------------- negative mutations + + +def test_duplicate_artifacts_by_source_identity_is_rejected() -> None: + generation = _base_generation() + generation["artifacts_by_source"].append(copy.deepcopy(generation["artifacts_by_source"][0])) + assert _generation_ai_findings(generation) != [] + + +def _assert_artifacts_by_source_identity_is_unique(generation: dict) -> None: + """uniqueItems on artifacts_by_source[] only rejects byte-identical duplicates; two + entries with the SAME (provider, source_service) pair but different payloads (an + overlapping identity, not an exact duplicate) need this semantic check instead, + mirroring the equivalent helper in test_estimation_ai_contracts.py.""" + seen = set() + for entry in generation["artifacts_by_source"]: + key = (entry["provider"], entry["source_service"]) + assert key not in seen, f"duplicate or overlapping artifacts_by_source identity: {key}" + seen.add(key) + + +def test_positive_fixtures_have_unique_artifacts_by_source_identity() -> None: + for fixture_path in POSITIVE_FIXTURES: + _assert_artifacts_by_source_identity_is_unique(_load(fixture_path)) + + +def test_overlapping_artifacts_by_source_identity_is_rejected() -> None: + """A second entry with the SAME (provider, source_service) pair but different generated + file content is still a duplicate/overlapping identity, not a legitimate second entry — + schema-level uniqueItems alone cannot catch this (see helper docstring), so the semantic + helper above is the lock.""" + generation = _base_generation() + overlapping = copy.deepcopy(generation["artifacts_by_source"][0]) + overlapping["models"][0]["generated_files"] = ["src/ai/other_client.py"] + generation["artifacts_by_source"].append(overlapping) + with pytest.raises(AssertionError): + _assert_artifacts_by_source_identity_is_unique(generation) + + +def test_openrouter_as_a_direct_artifacts_by_source_provider_is_rejected() -> None: + generation = _base_generation() + generation["artifacts_by_source"][0]["provider"] = "openrouter" + assert _generation_ai_findings(generation) != [] + + +def test_malformed_artifacts_by_source_entry_missing_models_is_rejected() -> None: + generation = _base_generation() + del generation["artifacts_by_source"][0]["models"] + assert _generation_ai_findings(generation) != [] + + +def test_rollback_plan_mechanism_must_be_feature_flag() -> None: + generation = _base_generation() + generation["rollback_plan"]["mechanism"] = "manual_cutover" + assert _generation_ai_findings(generation) != [] + + +def test_production_readiness_checklist_requires_at_least_five_items() -> None: + generation = _base_generation() + generation["production_readiness_checklist"] = generation["production_readiness_checklist"][:2] + assert _generation_ai_findings(generation) != [] + + +def test_generated_files_entry_requires_at_least_one_file() -> None: + generation = _base_generation() + generation["artifacts_by_source"][0]["models"][0]["generated_files"] = [] + assert _generation_ai_findings(generation) != [] + + +# --------------------------------------------------------------------------- vendored byte-identity + + +def test_vendored_copies_are_byte_identical() -> None: + """Reuses tools/sync-vendored.py --check (the dedicated CI gate) rather than + re-implementing filecmp logic inline, matching the project's existing pattern (and + test_estimation_ai_contracts.py's own choice to leave this check to the dedicated tool).""" + result = subprocess.run( + [sys.executable, str(SYNC_VENDORED_PATH), "--check"], + cwd=PLUGIN_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr From 237810cd9228a3bbe7f3e9a84c9cc18a847c88c9 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 19 Oct 2026 15:07:22 -0700 Subject: [PATCH 53/53] fix: address AI artifact contract review Keep target-only estimate and generate schemas out of named skill run validation while retaining neutral fixture and self-check coverage. Preserve unavailable estimate costs and dependent deltas as null, with synced documentation and regression coverage. Verification: - affected pytest modules: 87 passed - startup-advisor artifact validation: 84 checked, 0 errors - vendored parity: 53 files byte-identical - startup-advisor CI: 435 Python tests passed; 57 Node tests passed; all lint and validation tasks passed --- .../scripts/artifact-contracts.json | 8 ++-- .../contracts/estimation-ai.schema.json | 8 ++-- .../scripts/validate-artifacts.py | 14 ++++-- .../vendored/ai/schema-estimate-ai.md | 24 +++++++--- .../vendored/ai/schema-estimate-ai.md | 24 +++++++--- .../skills/shared/ai/schema-estimate-ai.md | 24 +++++++--- .../tests/test_estimation_ai_contracts.py | 40 ++++++++++++++++ .../tests/test_validate_artifacts.py | 47 +++++++++++++++++++ 8 files changed, 161 insertions(+), 28 deletions(-) diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index eb84f05a9..cac36b77b 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -1,5 +1,5 @@ { - "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (exact or word-boundary prefix match on the heading text, backticks ignored — 'graviton' matches '`graviton` block …' but not '`graviton_profile` …'); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", + "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'fixture_only' keeps a target contract available to neutral fixtures and self-checking without applying it to current named-skill runs. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (exact or word-boundary prefix match on the heading text, backticks ignored — 'graviton' matches '`graviton` block …' but not '`graviton_profile` …'); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", "fixture_prefixes": { "gcp-": "gcp-to-aws", "heroku-": "heroku-to-aws", @@ -19,10 +19,12 @@ "json_schema": "scripts/contracts/aws-design-ai.schema.json" }, "estimation-ai.json": { - "json_schema": "scripts/contracts/estimation-ai.schema.json" + "json_schema": "scripts/contracts/estimation-ai.schema.json", + "fixture_only": true }, "generation-ai.json": { - "json_schema": "scripts/contracts/generation-ai.schema.json" + "json_schema": "scripts/contracts/generation-ai.schema.json", + "fixture_only": true } }, "skills": { diff --git a/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json index 6b70832c4..5f77f8045 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json @@ -56,9 +56,9 @@ "additionalProperties": false, "properties": { "current_source_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, - "projected_bedrock_monthly": {"type": "number"}, - "monthly_difference": {"type": "number"}, - "annual_difference": {"type": "number"}, + "projected_bedrock_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "monthly_difference": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "annual_difference": {"oneOf": [{"type": "null"}, {"type": "number"}]}, "percent_change": {"oneOf": [{"type": "null"}, {"type": "number"}]} } }, @@ -98,7 +98,7 @@ "additionalProperties": false, "properties": { "model": {"$ref": "#/definitions/normalizedModel"}, - "monthly_cost": {"type": "number"}, + "monthly_cost": {"oneOf": [{"type": "null"}, {"type": "number"}]}, "vs_source_percent": {"oneOf": [{"type": "null"}, {"type": "number"}]} } }, diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index c23719aa3..42ecd6383 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -1019,9 +1019,17 @@ def _admit_schema_properties(node: Node, schema: Any, source: str) -> None: _admit_schema_properties(node.keys[k], sub, source) -def build_contracts(manifest: Dict[str, Any], skill: Optional[str], findings: List[Finding]) -> List[Contract]: +def build_contracts( + manifest: Dict[str, Any], + skill: Optional[str], + findings: List[Finding], + include_fixture_only: bool = False, +) -> List[Contract]: specs: Dict[str, Any] = {} - specs.update(manifest.get("shared", {})) + for glob_, spec in manifest.get("shared", {}).items(): + if skill is not None and spec.get("fixture_only") and not include_fixture_only: + continue + specs[glob_] = spec if skill is not None: specs.update(manifest.get("skills", {}).get(skill, {})) contracts: List[Contract] = [] @@ -1178,7 +1186,7 @@ def main(argv: Optional[List[str]] = None) -> int: if args.self_check: for skill in manifest.get("skills", {}): - build_contracts(manifest, skill, findings) + build_contracts(manifest, skill, findings, include_fixture_only=True) elif args.fixtures: fixtures_root = PLUGIN_ROOT / "fixtures" for d in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md index 098b0607b..1feda673d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md @@ -45,11 +45,11 @@ "backup_model": {}, "embeddings": {}, // present only when the workload uses embeddings "cost_comparison": { - "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly - "projected_bedrock_monthly": 1320.0, - "monthly_difference": 120.0, - "annual_difference": 1440.0, - "percent_change": 10.0 + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly; number or null + "projected_bedrock_monthly": 1320.0, // number or null + "monthly_difference": 120.0, // number or null + "annual_difference": 1440.0, // number or null + "percent_change": 10.0 // number or null }, "migration_cost_considerations": { "categories": [], // ALWAYS [] — human/pro-services costs are out of scope @@ -82,7 +82,7 @@ models regardless of source attribution. "models": [ { "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source - "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source; null when its target rate is unavailable "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) } ] @@ -95,6 +95,16 @@ presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted c attributes to the fronted provider's `(provider, source_service)` pair, mirroring how `aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. +## Unknown costs and null propagation + +Unavailable monetary values are JSON `null`, never numeric zero. If the source baseline is +unknown, `current_costs.monthly_ai_spend` and `cost_comparison.current_source_monthly` are `null`, +and all differences derived from that baseline (`monthly_difference`, `annual_difference`, and +`percent_change`) are also `null`. If a target rate is unavailable, set +`cost_comparison.projected_bedrock_monthly` and the affected +`cost_by_source[].models[].monthly_cost` values to `null`; every difference that depends on that +unknown target is `null` as well. Numeric `0` is reserved for a cost that is known to be zero. + ## Design-decision: no `model_observations[]` here `model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in @@ -144,6 +154,8 @@ priced, not token priced, and are listed here instead of silently omitted. - [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, `potential_savings_percent`, and `available`. - [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] Unknown source or target costs and their dependent deltas are `null`; known zero-dollar + costs remain numeric `0`. - [ ] `services_not_estimated[]` is present (`[]` when clean). ## Status — target-only, no producer yet diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md index 098b0607b..1feda673d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md @@ -45,11 +45,11 @@ "backup_model": {}, "embeddings": {}, // present only when the workload uses embeddings "cost_comparison": { - "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly - "projected_bedrock_monthly": 1320.0, - "monthly_difference": 120.0, - "annual_difference": 1440.0, - "percent_change": 10.0 + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly; number or null + "projected_bedrock_monthly": 1320.0, // number or null + "monthly_difference": 120.0, // number or null + "annual_difference": 1440.0, // number or null + "percent_change": 10.0 // number or null }, "migration_cost_considerations": { "categories": [], // ALWAYS [] — human/pro-services costs are out of scope @@ -82,7 +82,7 @@ models regardless of source attribution. "models": [ { "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source - "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source; null when its target rate is unavailable "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) } ] @@ -95,6 +95,16 @@ presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted c attributes to the fronted provider's `(provider, source_service)` pair, mirroring how `aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. +## Unknown costs and null propagation + +Unavailable monetary values are JSON `null`, never numeric zero. If the source baseline is +unknown, `current_costs.monthly_ai_spend` and `cost_comparison.current_source_monthly` are `null`, +and all differences derived from that baseline (`monthly_difference`, `annual_difference`, and +`percent_change`) are also `null`. If a target rate is unavailable, set +`cost_comparison.projected_bedrock_monthly` and the affected +`cost_by_source[].models[].monthly_cost` values to `null`; every difference that depends on that +unknown target is `null` as well. Numeric `0` is reserved for a cost that is known to be zero. + ## Design-decision: no `model_observations[]` here `model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in @@ -144,6 +154,8 @@ priced, not token priced, and are listed here instead of silently omitted. - [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, `potential_savings_percent`, and `available`. - [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] Unknown source or target costs and their dependent deltas are `null`; known zero-dollar + costs remain numeric `0`. - [ ] `services_not_estimated[]` is present (`[]` when clean). ## Status — target-only, no producer yet diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md index 098b0607b..1feda673d 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md @@ -45,11 +45,11 @@ "backup_model": {}, "embeddings": {}, // present only when the workload uses embeddings "cost_comparison": { - "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly - "projected_bedrock_monthly": 1320.0, - "monthly_difference": 120.0, - "annual_difference": 1440.0, - "percent_change": 10.0 + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly; number or null + "projected_bedrock_monthly": 1320.0, // number or null + "monthly_difference": 120.0, // number or null + "annual_difference": 1440.0, // number or null + "percent_change": 10.0 // number or null }, "migration_cost_considerations": { "categories": [], // ALWAYS [] — human/pro-services costs are out of scope @@ -82,7 +82,7 @@ models regardless of source attribution. "models": [ { "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source - "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source; null when its target rate is unavailable "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) } ] @@ -95,6 +95,16 @@ presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted c attributes to the fronted provider's `(provider, source_service)` pair, mirroring how `aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. +## Unknown costs and null propagation + +Unavailable monetary values are JSON `null`, never numeric zero. If the source baseline is +unknown, `current_costs.monthly_ai_spend` and `cost_comparison.current_source_monthly` are `null`, +and all differences derived from that baseline (`monthly_difference`, `annual_difference`, and +`percent_change`) are also `null`. If a target rate is unavailable, set +`cost_comparison.projected_bedrock_monthly` and the affected +`cost_by_source[].models[].monthly_cost` values to `null`; every difference that depends on that +unknown target is `null` as well. Numeric `0` is reserved for a cost that is known to be zero. + ## Design-decision: no `model_observations[]` here `model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in @@ -144,6 +154,8 @@ priced, not token priced, and are listed here instead of silently omitted. - [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, `potential_savings_percent`, and `available`. - [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] Unknown source or target costs and their dependent deltas are `null`; known zero-dollar + costs remain numeric `0`. - [ ] `services_not_estimated[]` is present (`[]` when clean). ## Status — target-only, no producer yet diff --git a/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py index 9da4441a0..1585f992d 100644 --- a/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py +++ b/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py @@ -95,6 +95,46 @@ def test_gateway_fixture_carries_multi_upstream_openrouter_fronting_two_provider assert len(openai_source["models"]) == 1 +def test_unknown_source_baseline_propagates_null_to_dependent_deltas() -> None: + estimate = _base_estimate() + estimate["current_costs"]["monthly_ai_spend"] = None + comparison = estimate["cost_comparison"] + comparison["current_source_monthly"] = None + comparison["monthly_difference"] = None + comparison["annual_difference"] = None + comparison["percent_change"] = None + for source in estimate["cost_by_source"]: + for model in source["models"]: + model["vs_source_percent"] = None + + assert _estimation_ai_findings(estimate) == [] + assert comparison["projected_bedrock_monthly"] is not None + + comparison["projected_bedrock_monthly"] = 0 + assert comparison["projected_bedrock_monthly"] == 0 + assert comparison["projected_bedrock_monthly"] is not None + assert _estimation_ai_findings(estimate) == [] + + comparison["projected_bedrock_monthly"] = "0" + assert _estimation_ai_findings(estimate) != [] + + +def test_unavailable_target_pricing_preserves_unknown_costs_as_null() -> None: + estimate = _base_estimate() + estimate["pricing_source"] = "unavailable" + comparison = estimate["cost_comparison"] + comparison["projected_bedrock_monthly"] = None + comparison["monthly_difference"] = None + comparison["annual_difference"] = None + comparison["percent_change"] = None + for source in estimate["cost_by_source"]: + for model in source["models"]: + model["monthly_cost"] = None + model["vs_source_percent"] = None + + assert _estimation_ai_findings(estimate) == [] + + # --------------------------------------------------------------------------- cross-phase projection diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 20ce3c526..9ebb18a62 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -468,6 +468,53 @@ def test_neutral_fixture_skips_provider_artifacts_and_expected_snapshots(tmp_pat assert [p.name for p in va.find_artifacts(tmp_path)] == ["gcp-resource-inventory.json"] +@pytest.mark.parametrize("skill", ["gcp-to-aws", "azure-to-aws"]) +@pytest.mark.parametrize("artifact", ["estimation-ai.json", "generation-ai.json"]) +def test_target_only_ai_contracts_do_not_validate_current_skill_outputs( + tmp_path: Path, skill: str, artifact: str +): + """The canonical AI estimate/generate schemas are fixture-only until the producer + documents adopt their neutral fields and per-source traceability arrays.""" + source_prefix = "gcp" if skill == "gcp-to-aws" else "azure" + if artifact == "estimation-ai.json": + current = { + "phase": "estimate", + "timestamp": "2026-09-08T03:10:00Z", + "pricing_source": "cached", + "current_costs": { + "source": "billing", + f"{source_prefix}_monthly_ai_spend": 100.0, + "services": [], + }, + "cost_comparison": { + f"current_{source_prefix}_monthly": 100.0, + "projected_bedrock_monthly": 110.0, + "monthly_difference": 10.0, + "annual_difference": 120.0, + "percent_change": 10.0, + }, + } + else: + current = { + "phase": "generate", + "generation_source": "ai", + "timestamp": "2026-09-08T04:00:00Z", + "migration_plan": {"approach": "provider_adapter", "phases": [], "models_to_migrate": []}, + "step_by_step_guide": {}, + "rollback_plan": {"mechanism": "feature_flag", "flag_name": "AI_PROVIDER"}, + "monitoring": {}, + "production_readiness_checklist": [], + "success_criteria": {}, + "recommendation": {}, + } + + run = _write_run(tmp_path, f"{skill}-{artifact}", artifact, current) + findings = [] + manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") + assert va.validate_run_dir(run, skill, manifest, findings) == 0 + assert [(finding.code, finding.path) for finding in findings] == [("NO_CONTRACT", "")] + + def test_injected_off_contract_key_fails(tmp_path: Path): """The regression this tool exists for: a producer emits a key its contract never documented. Copy a golden run, plant the key, expect exit 1 naming the path."""