diff --git a/plugins/aws-startup-advisor/README.md b/plugins/aws-startup-advisor/README.md index b944c203b..39fb65c21 100644 --- a/plugins/aws-startup-advisor/README.md +++ b/plugins/aws-startup-advisor/README.md @@ -39,7 +39,7 @@ Alongside the skills, the plugin ships supporting material the skills load on de - [`skills/shared/`](skills/shared/) — Canonical reference material shared across skills rather than an invocable skill of its own: the phase-workflow interpreter contract, estimation schemas and complexity tiers, a snapshot of AWS infrastructure rates, phase-status schema, what-if workshop invariants, shared Clarify question fragments, and the AI-migration guardrails and Bedrock model references. Each consuming skill carries a byte-identical copy under its own `references/vendored/` so the skill folder stays self-contained. - [`agents/`](agents/) — Seven Claude Code subagent definitions: two generic, phase-agnostic migration phase workers (read/write and read/write/shell tiers) plus five specialists for the Bedrock rewrite flow (code analyzer, code rewriter, log ingestor, prompt evaluator, report generator). - [`tools/`](tools/) — Contains `model-staleness.py`, which checks the Bedrock model-lifecycle registry's freshness contract (`**Last updated**` / `**Staleness window**`) and that every vendored copy under `skills/*/references/vendored/ai/` is byte-identical to `skills/shared/ai/ai-model-lifecycle.md` (`--strict` also fails on a stale snapshot). -- [`scripts/`](scripts/) — Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter (`emit-plan-json.py`), `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/telemetry/` collects optional usage data: nothing is sent until you accept, no skill depends on it, and `/usage-data` shows the setting or opts out. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. `scripts/contracts/aws-design-ai.schema.json`, canonicalized from Azure's prior single-scalar `metadata.ai_source` contract into `skills/shared/ai/schema-design-aws-ai.md`, likewise defines a target-only `aws-design-ai.json` design contract whose `metadata.ai_sources[]`/`metadata.gateways[]` mirror the workload-profile contract exactly; `design-ai.md` in both skills still writes against the prior per-skill shape, so this is a target, not a claim about current output. `scripts/contracts/ai-capability-catalog.schema.json` defines the target-only contract for `skills/shared/ai/ai-capability-catalog.json`, a static source-provider/gateway capability catalog; its `implemented_adapter`/`route` fields are null everywhere and it asserts no runtime support. +- [`scripts/`](scripts/) — Python helpers that validate generated artifacts, emit run summaries, and lint Bedrock model references: a migration report validator, a startup-program artifact check, a plan summary emitter (`emit-plan-json.py`), `validate-artifacts.py`, which checks every phase artifact in a run directory (or the fixture goldens, in CI) against the contract its `schema-*.md` / JSON Schema publishes — unknown keys, missing required keys, type and enum drift — and `model-id-lint.py`, which fails when a file outside the model catalog names a Bedrock model the lifecycle registry marks excluded or removed, or a fabricated dated Sonnet 4.6 / Opus 4.8 ID. `scripts/artifact-contracts.json` maps artifacts to contracts; `scripts/artifact-contracts-baseline.json` is the explained burn-down list of known contract gaps. The Heroku flow's report validator ships inside `skills/heroku-to-aws/scripts/`. `scripts/telemetry/` collects optional usage data: nothing is sent until you accept, no skill depends on it, and `/usage-data` shows the setting or opts out. `scripts/contracts/ai-workload-profile.schema.json` additionally defines a target-only, provider-neutral observation contract for a future `ai-workload-profile.json` normalization output; it has no producer yet and asserts no runtime support. `scripts/contracts/aws-design-ai.schema.json`, canonicalized from Azure's prior single-scalar `metadata.ai_source` contract into `skills/shared/ai/schema-design-aws-ai.md`, likewise defines a target-only `aws-design-ai.json` design contract whose `metadata.ai_sources[]`/`metadata.gateways[]` mirror the workload-profile contract exactly; `design-ai.md` in both skills still writes against the prior per-skill shape, so this is a target, not a claim about current output. `scripts/contracts/ai-capability-catalog.schema.json` defines the target-only contract for `skills/shared/ai/ai-capability-catalog.json`, a static source-provider/gateway capability catalog; its `implemented_adapter`/`route` fields are null everywhere and it asserts no runtime support. `scripts/contracts/estimation-ai.schema.json` and `scripts/contracts/generation-ai.schema.json`, canonicalized into `skills/shared/ai/schema-estimate-ai.md`/`schema-generate-ai.md`, likewise define target-only `estimation-ai.json`/`generation-ai.json` contracts whose `cost_by_source[]`/`artifacts_by_source[]` carry a per-source/per-model traceability chain back through `aws-design-ai.json`'s `metadata.ai_sources[]`/`gateways[]`; `estimate-ai.md` and `generate-ai.md`/`generate-artifacts-ai.md` in both skills still write against their prior per-skill shapes, so these are targets, not claims about current output. - [`fixtures/`](fixtures/) — Reference and regression fixtures for the migration report and estimation artifacts, used to keep generated output within the documented contract. `fixtures/azure-iac-terraform/` additionally carries deterministic asserters that pin the Azure Discover, Clarify, Design, and Estimate contracts against a committed synthetic estate. ## MCP Servers diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index 3aeac28c3..ab217a87d 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -5,6 +5,8 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `ai-workload-profiles/` — 4 representative target-only `ai-workload-profile.json` observation fixtures (not a Cartesian matrix): AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. `ai-workload-profile.schema.json`'s `aiSource` observed branch now also requires `model_observations` non-empty (`minItems: 1`) whenever `model_evidence == "observed"`, matching the identical rule already enforced for `aws-design-ai.schema.json`'s `metadata.ai_sources[]`. - `aws-design-ai/` — 2 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway) and `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. +- `estimation-ai/` — 2 representative target-only `estimation-ai.json` cost-estimate fixtures, nested as `/estimation-ai.json` so `lint:artifacts` discovers them: `multi-provider-direct` and `openrouter-gateway-multi-upstream`, reusing #416's `aws-design-ai/` scenario names and the same providers/models. `cost_by_source[]` carries forward `aws-design-ai.json`'s `metadata.ai_sources[]`/`gateways[]` identity pattern so a per-source/per-model estimated cost traces back to the exact Design entry; `model_observations[]` is intentionally not carried forward (the identity is already resolved by Design). `test_estimation_ai_contracts.py` proves both fixtures project into (are a subset of) the matching `aws-design-ai//aws-design-ai.json`, and exercises negative mutations — duplicate/overlapping source identity, OpenRouter named as a direct provider, a fabricated untraceable model, and a model dropped under a resolved gateway upstream. Schema validity here does not imply an implemented pricing lookup, model mapping, or runtime support, and does not claim either skill's current `estimate-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. +- `generation-ai/` — 2 representative target-only `generation-ai.json` migration-plan fixtures, nested as `/generation-ai.json`, mirroring `estimation-ai/`'s two case names. `artifacts_by_source[]` carries the traceability chain one phase forward: a per-source/per-model generated-artifact reference traces back to the exact `estimation-ai.json → cost_by_source[]` entry it was produced from. `test_generation_ai_contracts.py` proves both fixtures project into the matching `estimation-ai//estimation-ai.json`, and exercises the same negative-mutation categories one phase forward. Schema validity here does not imply an implemented adapter, rollout mechanism, or runtime support, and does not claim either skill's current `generate-ai.md`/`generate-artifacts-ai.md` emits this shape. - `azure-rdfa-report*/` — producer-shaped RDfA source fixtures: inventory rows, metrics batches wrapped in the upstream `{ "Metrics": [...] }` envelope, corrupt/multi-ZIP cases, fixed/autoscaled AKS pools, canonical VM OS-disk fields, and an ML workspace. diff --git a/plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json b/plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json new file mode 100644 index 000000000..20a1ad653 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/estimation-ai/multi-provider-direct/estimation-ai.json @@ -0,0 +1,126 @@ +{ + "phase": "estimate", + "timestamp": "2026-09-08T03:10:00Z", + "pricing_source": "cached", + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", + "monthly_ai_spend": 1200.0, + "services": ["anthropic_api", "openai_api"] + }, + "token_volume": { + "source": "tier_table", + "monthly_input_tokens": 60000000, + "monthly_output_tokens": 40000000, + "ratio": "60/40" + }, + "model_comparison": [ + { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 1320.0, + "vs_current": "+10%", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + }, + { + "model": "openai.gpt-5.6-terra", + "monthly_cost": 660.0, + "vs_current": "-12%", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + } + ], + "recommended_model": { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 1320.0, + "breakdown": {"input": 900.0, "output": 420.0, "embeddings": 0.0}, + "rationale": "Same-model migration for the Anthropic-sourced workload; current recommended model and provider priority favor Claude." + }, + "backup_model": { + "model": "anthropic.claude-haiku-4-5-v1:0", + "monthly_cost": 480.0, + "rationale": "Lower-cost fallback within the same model family." + }, + "cost_comparison": { + "current_source_monthly": 1200.0, + "projected_bedrock_monthly": 1980.0, + "monthly_difference": 780.0, + "annual_difference": 9360.0, + "percent_change": 65.0 + }, + "migration_cost_considerations": { + "categories": [], + "complexity_factors": ["integration.pattern = \"direct_sdk\" -> moderate SDK and API pattern changes"], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": { + "monthly_cost_delta": 780.0, + "annual_cost_delta": 9360.0, + "justification": "Combined Anthropic and OpenAI Bedrock costs are higher than current direct-API spend; non-cost benefits justify the move.", + "non_cost_benefits": ["IAM/VPC/PrivateLink/KMS/CloudTrail governance", "usage counting toward existing AWS commitments"] + }, + "optimization_opportunities": [ + { + "opportunity": "Prompt caching", + "type": "prompt_caching", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "up to 90% on cached input tokens", + "commitment": "none", + "implementation_effort": "low", + "available": "day_1", + "description": "Repeated system prompts qualify for cache-read pricing on both Claude and GPT-5.6." + }, + { + "opportunity": "Model downsizing / tiering", + "type": "model_tiering", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "60-87%", + "commitment": "none", + "implementation_effort": "low", + "available": "day_1", + "description": "High-volume, premium model selected; a lower tier may suffice for simple queries." + } + ], + "optimized_projection": { + "monthly_with_optimizations": 1400.0, + "vs_current": "+17%", + "note": "Best-case projection assuming prompt caching and selective model tiering." + }, + "recommendation": { + "path": "migrate_optimized", + "path_label": "Migrate to Bedrock", + "migrate_if": "Governance, residency, and commitment-counting benefits outweigh the modest cost premium.", + "stay_if": "Cost is the only priority and no non-cost driver applies.", + "confidence": "high", + "rationale": "Both Bedrock targets carry a modest premium over direct-API pricing, offset by governance and ecosystem benefits; both source models are strong_migrate per the design assessment." + }, + "cost_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "monthly_cost": 1320.0, + "vs_source_percent": 10.0 + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "monthly_cost": 660.0, + "vs_source_percent": -12.0 + } + ] + } + ], + "services_not_estimated": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json b/plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json new file mode 100644 index 000000000..5cbcb4225 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/estimation-ai/openrouter-gateway-multi-upstream/estimation-ai.json @@ -0,0 +1,139 @@ +{ + "phase": "estimate", + "timestamp": "2026-09-08T03:10:00Z", + "pricing_source": "cached", + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", + "monthly_ai_spend": 2400.0, + "services": ["openrouter"] + }, + "token_volume": { + "source": "tier_table", + "monthly_input_tokens": 600000000, + "monthly_output_tokens": 400000000, + "ratio": "60/40" + }, + "model_comparison": [ + { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 13200.0, + "vs_current": "favorable", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + }, + { + "model": "anthropic.claude-haiku-4-5-v1:0", + "monthly_cost": 1100.0, + "vs_current": "favorable", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + }, + { + "model": "openai.gpt-5.6-terra", + "monthly_cost": 8800.0, + "vs_current": "mixed", + "quality": "high", + "capabilities_match": true, + "missing_capabilities": [] + } + ], + "recommended_model": { + "model": "anthropic.claude-sonnet-4-5-v1:0", + "monthly_cost": 13200.0, + "breakdown": {"input": 9000.0, "output": 4200.0, "embeddings": 0.0}, + "rationale": "Primary chat tier routed through OpenRouter to Claude Sonnet; same-family Bedrock target is favorable vs. router pricing." + }, + "backup_model": { + "model": "anthropic.claude-haiku-4-5-v1:0", + "monthly_cost": 1100.0, + "rationale": "Low-cost tier backup within the same model family." + }, + "cost_comparison": { + "current_source_monthly": 2400.0, + "projected_bedrock_monthly": 23100.0, + "monthly_difference": 20700.0, + "annual_difference": 248400.0, + "percent_change": 862.5 + }, + "migration_cost_considerations": { + "categories": [], + "complexity_factors": ["integration.pattern = \"framework\" -> lower integration touch surface", "summary.total_models_detected > 3 -> multi-model coordination"], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": { + "monthly_cost_delta": 20700.0, + "annual_cost_delta": 248400.0, + "justification": "Direct per-model Bedrock pricing is favorable vs. the router's blended markup once volume is normalized across all three upstream models.", + "non_cost_benefits": ["model flexibility (100+ models)", "AWS ecosystem (Guardrails, Knowledge Bases, AgentCore)"] + }, + "optimization_opportunities": [ + { + "opportunity": "Multi-model tiered routing", + "type": "multi_model_routing", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "60-87%", + "commitment": "none", + "implementation_effort": "low", + "available": "day_1", + "description": "High-volume workload with a 3-tier routing strategy already recorded in the design artifact." + }, + { + "opportunity": "Intelligent Prompt Routing", + "type": "intelligent_prompt_routing", + "target_services": ["Bedrock"], + "potential_savings_monthly": null, + "potential_savings_percent": "up to 30%", + "commitment": "none", + "implementation_effort": "low", + "available": "after_2_weeks_production_traffic", + "description": "Anthropic family router is available for the Claude Sonnet/Haiku tiers once production traffic has run for 2+ weeks." + } + ], + "optimized_projection": { + "monthly_with_optimizations": 15000.0, + "vs_current": "favorable", + "note": "Best-case projection assuming multi-model tiered routing is fully adopted." + }, + "recommendation": { + "path": "migrate_optimized", + "path_label": "Migrate to Bedrock", + "migrate_if": "Direct per-model Bedrock pricing beats the router's blended markup at this volume.", + "stay_if": "Router flexibility across many providers is a higher priority than per-model cost.", + "confidence": "high", + "rationale": "All three router upstreams land on favorable or mixed Bedrock pricing, and the design assessment is strong_migrate across the board." + }, + "cost_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "monthly_cost": 13200.0, + "vs_source_percent": null + }, + { + "model": "claude-3-haiku-20240307", + "monthly_cost": 1100.0, + "vs_source_percent": null + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "monthly_cost": 8800.0, + "vs_source_percent": null + } + ] + } + ], + "services_not_estimated": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json b/plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json new file mode 100644 index 000000000..347fe40a6 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/generation-ai/multi-provider-direct/generation-ai.json @@ -0,0 +1,74 @@ +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "2026-09-08T04:00:00Z", + "migration_plan": { + "duration_drivers": ["multiple models (adapter + evaluation per model)", "framework integration"], + "approach": "direct_sdk_adapter", + "phases": [ + {"name": "Phase 1", "sequence": 1, "activities": ["Enable Bedrock access", "create IAM role", "develop provider adapter with feature flag", "unit test"]}, + {"name": "Phase 2", "sequence": 2, "activities": ["Deploy to staging", "run A/B comparison", "measure latency/quality/cost", "tune prompts"]}, + {"name": "Phase 3", "sequence": 3, "activities": ["Gradual rollout (10% -> 50% -> 100%)", "monitor", "disable source provider after 48h stable"]} + ], + "models_to_migrate": ["claude-3-5-sonnet-20240620", "gpt-4o"] + }, + "step_by_step_guide": { + "languages": ["python"], + "primary_pattern": "direct_sdk", + "files_to_modify": ["src/ai/anthropic_client.py", "src/ai/openai_client.py"], + "dependency_changes": "boto3 for the Anthropic client; openai>=2.45.0 with aws_bedrock_token_generator for the OpenAI client." + }, + "rollback_plan": { + "mechanism": "feature_flag", + "flag_name": "AI_PROVIDER", + "default_value": "anthropic_api", + "rollback_time": "instant", + "triggers": ["quality below threshold", "P95 latency > 2x baseline", "error rate > 1% for 5 min", "cost per request > 3x source"] + }, + "monitoring": { + "dashboards": ["Request volume by provider", "latency comparison (P50/P95/P99)", "error rates", "token usage", "cost tracking", "quality scores"], + "alerting_rules": [ + {"severity": "Critical", "condition": "Error rate > 5% for 2 min", "action": "auto-rollback"}, + {"severity": "High", "condition": "Latency P95 > 3x baseline for 5 min", "action": "page on-call"} + ] + }, + "production_readiness_checklist": [ + "Bedrock model access enabled", + "IAM role with bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream", + "Provider adapter deployed and tested in staging", + "A/B test with >= 100 representative prompts", + "Rollback procedure documented and tested" + ], + "success_criteria": { + "quality": {"category": "Quality", "criteria": "Response quality", "target": ">= 90% of source baseline"}, + "latency": {"category": "Latency", "criteria": "P95", "target": "Within 2x of source"}, + "cost": {"category": "Cost", "criteria": "Monthly", "target": "Within 20% of estimation-ai.json"} + }, + "recommendation": { + "approach": "Direct SDK adapter per source, rolled out gradually with a shared feature flag.", + "confidence": "high", + "key_risks": ["multi-model coordination across two simultaneous sources"] + }, + "artifacts_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "generated_files": ["src/ai/anthropic_client.py"] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "generated_files": ["src/ai/openai_client.py"] + } + ] + } + ] +} diff --git a/plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json b/plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json new file mode 100644 index 000000000..c9246f5cc --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/generation-ai/openrouter-gateway-multi-upstream/generation-ai.json @@ -0,0 +1,78 @@ +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "2026-09-08T04:00:00Z", + "migration_plan": { + "duration_drivers": ["multiple models (adapter + evaluation per model)", "custom inference pipeline"], + "approach": "gateway_config_swap", + "phases": [ + {"name": "Phase 1", "sequence": 1, "activities": ["Replace OpenRouter SDK call sites with per-provider Bedrock clients"]}, + {"name": "Phase 2", "sequence": 2, "activities": ["Deploy to staging", "run A/B comparison per tier"]}, + {"name": "Phase 3", "sequence": 3, "activities": ["Gradual rollout per tier", "monitor", "disable router after 48h stable"]} + ], + "models_to_migrate": ["claude-3-5-sonnet-20240620", "claude-3-haiku-20240307", "gpt-4o"] + }, + "step_by_step_guide": { + "languages": ["python"], + "primary_pattern": "framework", + "files_to_modify": ["src/ai/router_client.py"], + "dependency_changes": "Replace the OpenRouter SDK call sites with per-provider Bedrock clients (boto3 Converse for Anthropic, Mantle Responses for OpenAI)." + }, + "rollback_plan": { + "mechanism": "feature_flag", + "flag_name": "AI_PROVIDER", + "default_value": "openrouter", + "rollback_time": "instant", + "triggers": ["quality below threshold", "P95 latency > 2x baseline", "error rate > 1% for 5 min", "cost per request > 3x source"] + }, + "monitoring": { + "dashboards": ["Request volume by provider", "latency comparison (P50/P95/P99)", "error rates", "token usage", "cost tracking", "quality scores"], + "alerting_rules": [ + {"severity": "Critical", "condition": "Error rate > 5% for 2 min", "action": "auto-rollback"}, + {"severity": "Medium", "condition": "Daily cost > 2x projected", "action": "notify finance channel"} + ] + }, + "production_readiness_checklist": [ + "Bedrock model access enabled for all three tiers", + "IAM role with bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream", + "Provider adapters deployed and tested in staging for each tier", + "A/B test with >= 100 representative prompts per tier", + "Rollback procedure documented and tested" + ], + "success_criteria": { + "quality": {"category": "Quality", "criteria": "Response quality", "target": ">= 90% of source baseline"}, + "latency": {"category": "Latency", "criteria": "P95", "target": "Within 2x of source"}, + "cost": {"category": "Cost", "criteria": "Monthly", "target": "Within 20% of estimation-ai.json"} + }, + "recommendation": { + "approach": "Replace the OpenRouter gateway with direct per-provider Bedrock clients, keeping the existing 3-tier routing strategy.", + "confidence": "high", + "key_risks": ["multi-upstream gateway replacement touches three models across two providers simultaneously"] + }, + "artifacts_by_source": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", + "generated_files": ["src/ai/router_client.py"] + }, + { + "model": "claude-3-haiku-20240307", + "generated_files": ["src/ai/router_client.py"] + } + ] + }, + { + "provider": "openai", + "source_service": "openai_api", + "models": [ + { + "model": "gpt-4o", + "generated_files": ["src/ai/router_client.py"] + } + ] + } + ] +} diff --git a/plugins/aws-startup-advisor/scripts/artifact-contracts.json b/plugins/aws-startup-advisor/scripts/artifact-contracts.json index 9f8cbb8bf..cac36b77b 100644 --- a/plugins/aws-startup-advisor/scripts/artifact-contracts.json +++ b/plugins/aws-startup-advisor/scripts/artifact-contracts.json @@ -1,5 +1,5 @@ { - "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (exact or word-boundary prefix match on the heading text, backticks ignored — 'graviton' matches '`graviton` block …' but not '`graviton_profile` …'); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", + "_what_this_is": "Maps each migration artifact (by file name, fnmatch glob) to the contract document that defines it. Read by scripts/validate-artifacts.py. 'fixture_only' keeps a target contract available to neutral fixtures and self-checking without applying it to current named-skill runs. 'shape' points at a markdown document whose JSON/JSONC example block IS the contract; 'root_heading' selects which block (exact or word-boundary prefix match on the heading text, backticks ignored — 'graviton' matches '`graviton` block …' but not '`graviton_profile` …'); 'sections' maps sub-section headings to the key path they refine when the heading's own backticked token is not the path; 'json_schema' points at a real JSON Schema. Both may be present — the schema is authoritative where it speaks, the shape catches keys the schema leaves open.", "fixture_prefixes": { "gcp-": "gcp-to-aws", "heroku-": "heroku-to-aws", @@ -17,6 +17,14 @@ }, "aws-design-ai.json": { "json_schema": "scripts/contracts/aws-design-ai.schema.json" + }, + "estimation-ai.json": { + "json_schema": "scripts/contracts/estimation-ai.schema.json", + "fixture_only": true + }, + "generation-ai.json": { + "json_schema": "scripts/contracts/generation-ai.schema.json", + "fixture_only": true } }, "skills": { diff --git a/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json new file mode 100644 index 000000000..5f77f8045 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/estimation-ai.schema.json @@ -0,0 +1,183 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:estimation-ai", + "title": "estimation-ai.json (canonical, target-only estimate contract)", + "description": "Provider-neutral, target-only contract for the estimation-ai.json AI cost estimate artifact. Structural validity does NOT imply an implemented pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either skill's current estimate-ai.md actually emits this shape — this is an estimate-time target. cost_by_source[] carries forward the aws-design-ai.json metadata.ai_sources[]/gateways[] identity pattern so a per-source/per-model estimated cost traces back to the exact Design entry it was computed from.", + "type": "object", + "required": [ + "phase", + "timestamp", + "pricing_source", + "accuracy_confidence", + "current_costs", + "token_volume", + "model_comparison", + "recommended_model", + "backup_model", + "cost_comparison", + "migration_cost_considerations", + "roi_analysis", + "optimization_opportunities", + "optimized_projection", + "recommendation", + "cost_by_source", + "services_not_estimated" + ], + "additionalProperties": false, + "properties": { + "phase": {"const": "estimate"}, + "timestamp": {"type": "string", "minLength": 1}, + "pricing_source": {"enum": ["cached", "unavailable"]}, + "accuracy_confidence": {"type": "string", "minLength": 1}, + "current_costs": { + "type": "object", + "required": ["source", "monthly_ai_spend", "services"], + "additionalProperties": false, + "properties": { + "source": {"type": "string", "minLength": 1}, + "monthly_ai_spend": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "services": {"type": "array"} + } + }, + "token_volume": {"type": "object"}, + "model_comparison": {"type": "array"}, + "recommended_model": {"type": "object"}, + "backup_model": {"type": "object"}, + "embeddings": {"type": "object"}, + "cost_comparison": { + "type": "object", + "required": [ + "current_source_monthly", + "projected_bedrock_monthly", + "monthly_difference", + "annual_difference", + "percent_change" + ], + "additionalProperties": false, + "properties": { + "current_source_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "projected_bedrock_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "monthly_difference": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "annual_difference": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "percent_change": {"oneOf": [{"type": "null"}, {"type": "number"}]} + } + }, + "migration_cost_considerations": { + "type": "object", + "required": ["categories", "complexity_factors", "note"], + "additionalProperties": false, + "properties": { + "categories": {"type": "array", "maxItems": 0}, + "complexity_factors": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "note": {"type": "string", "minLength": 1} + } + }, + "roi_analysis": {"type": "object"}, + "optimization_opportunities": { + "type": "array", + "items": {"$ref": "#/definitions/optimizationOpportunity"} + }, + "optimized_projection": {"type": "object"}, + "recommendation": {"$ref": "#/definitions/recommendation"}, + "cost_by_source": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceCostEntry"} + }, + "services_not_estimated": { + "type": "array", + "items": {"$ref": "#/definitions/serviceNotEstimated"} + } + }, + "definitions": { + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "sourceCostModel": { + "type": "object", + "required": ["model", "monthly_cost", "vs_source_percent"], + "additionalProperties": false, + "properties": { + "model": {"$ref": "#/definitions/normalizedModel"}, + "monthly_cost": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "vs_source_percent": {"oneOf": [{"type": "null"}, {"type": "number"}]} + } + }, + "sourceCostEntry": { + "description": "A per-source, per-model estimated cost entry. (provider, source_service) MUST exist in the aws-design-ai.json metadata.ai_sources[]/gateways[].upstreams[] this estimate was built from, and every models[].model MUST be one of that source's models[] — see schema-estimate-ai.md's cost_by_source[] traceability rule.", + "type": "object", + "required": ["provider", "source_service", "models"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "models": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceCostModel"} + } + } + }, + "optimizationOpportunity": { + "type": "object", + "required": [ + "opportunity", + "type", + "target_services", + "commitment", + "potential_savings_monthly", + "potential_savings_percent", + "available", + "implementation_effort", + "description" + ], + "additionalProperties": false, + "properties": { + "opportunity": {"type": "string", "minLength": 1}, + "type": { + "enum": [ + "model_tiering", + "prompt_caching", + "batch_api", + "intelligent_prompt_routing", + "provisioned_throughput", + "input_token_reduction", + "multi_model_routing" + ] + }, + "target_services": {"type": "array", "items": {"const": "Bedrock"}, "minItems": 1}, + "commitment": {"type": "string", "minLength": 1}, + "potential_savings_monthly": {"oneOf": [{"type": "null"}, {"type": "number"}]}, + "potential_savings_percent": {"type": "string", "minLength": 1}, + "available": {"type": "string", "minLength": 1}, + "implementation_effort": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1} + } + }, + "recommendation": { + "type": "object", + "required": ["path", "path_label", "migrate_if", "stay_if", "confidence", "rationale"], + "additionalProperties": false, + "properties": { + "path": {"enum": ["migrate_optimized", "migrate_phased", "stay"]}, + "path_label": {"type": "string", "minLength": 1}, + "migrate_if": {"type": "string", "minLength": 1}, + "stay_if": {"type": "string", "minLength": 1}, + "confidence": {"enum": ["high", "medium", "low"]}, + "rationale": {"type": "string", "minLength": 1} + } + }, + "serviceNotEstimated": { + "type": "object", + "required": ["workload_id", "target_aws_service", "reason"], + "additionalProperties": false, + "properties": { + "workload_id": {"type": "string", "minLength": 1}, + "target_aws_service": { + "enum": ["textract", "rekognition", "transcribe", "comprehend", "translate", "polly", "sagemaker"] + }, + "reason": {"const": "not_token_priced"} + } + } + } +} diff --git a/plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json b/plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json new file mode 100644 index 000000000..dd9fdabc1 --- /dev/null +++ b/plugins/aws-startup-advisor/scripts/contracts/generation-ai.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "urn:awslabs:startups:migration:generation-ai", + "title": "generation-ai.json (canonical, target-only generate contract)", + "description": "Provider-neutral, target-only contract for the generation-ai.json AI migration-plan artifact. Structural validity does NOT imply an implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that either skill's current generate-ai.md/generate-artifacts-ai.md actually emits this shape — this is a generate-time target. artifacts_by_source[] carries the traceability chain one phase forward from estimation-ai.json's cost_by_source[] so a per-source/per-model generated-artifact reference traces back to the exact Estimate entry it was produced from.", + "type": "object", + "required": [ + "phase", + "generation_source", + "timestamp", + "migration_plan", + "step_by_step_guide", + "rollback_plan", + "monitoring", + "production_readiness_checklist", + "success_criteria", + "recommendation", + "artifacts_by_source" + ], + "additionalProperties": false, + "properties": { + "phase": {"const": "generate"}, + "generation_source": {"const": "ai"}, + "timestamp": {"type": "string", "minLength": 1}, + "migration_plan": { + "type": "object", + "required": ["duration_drivers", "approach", "phases", "models_to_migrate"], + "additionalProperties": false, + "properties": { + "duration_drivers": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "approach": {"type": "string", "minLength": 1}, + "phases": {"type": "array"}, + "models_to_migrate": {"type": "array", "items": {"$ref": "#/definitions/normalizedModel"}} + } + }, + "step_by_step_guide": { + "type": "object", + "required": ["languages", "primary_pattern", "files_to_modify", "dependency_changes"], + "additionalProperties": false, + "properties": { + "languages": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "primary_pattern": {"type": "string", "minLength": 1}, + "files_to_modify": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "dependency_changes": {"type": "string", "minLength": 1} + } + }, + "rollback_plan": { + "type": "object", + "required": ["mechanism", "flag_name", "default_value", "rollback_time", "triggers"], + "additionalProperties": false, + "properties": { + "mechanism": {"const": "feature_flag"}, + "flag_name": {"type": "string", "minLength": 1}, + "default_value": {"type": "string", "minLength": 1}, + "rollback_time": {"type": "string", "minLength": 1}, + "triggers": {"type": "array", "items": {"type": "string", "minLength": 1}, "minItems": 1} + } + }, + "monitoring": { + "type": "object", + "required": ["dashboards", "alerting_rules"], + "additionalProperties": false, + "properties": { + "dashboards": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "alerting_rules": { + "type": "array", + "items": { + "type": "object", + "required": ["severity", "condition", "action"], + "additionalProperties": false, + "properties": { + "severity": {"enum": ["Critical", "High", "Medium", "Low"]}, + "condition": {"type": "string", "minLength": 1}, + "action": {"type": "string", "minLength": 1} + } + } + } + } + }, + "production_readiness_checklist": { + "type": "array", + "minItems": 5, + "items": {"type": "string", "minLength": 1} + }, + "success_criteria": { + "type": "object", + "required": ["quality", "latency", "cost"], + "additionalProperties": false, + "properties": { + "quality": {"type": "object"}, + "latency": {"type": "object"}, + "cost": {"type": "object"} + } + }, + "recommendation": { + "type": "object", + "required": ["approach", "confidence", "key_risks"], + "additionalProperties": false, + "properties": { + "approach": {"type": "string", "minLength": 1}, + "confidence": {"enum": ["high", "medium", "low"]}, + "key_risks": {"type": "array", "items": {"type": "string", "minLength": 1}} + } + }, + "artifacts_by_source": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceArtifactEntry"} + } + }, + "definitions": { + "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, + "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, + "sourceArtifactModel": { + "type": "object", + "required": ["model", "generated_files"], + "additionalProperties": false, + "properties": { + "model": {"$ref": "#/definitions/normalizedModel"}, + "generated_files": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + }, + "sourceArtifactEntry": { + "description": "A per-source, per-model generated-artifact reference entry. (provider, source_service) MUST exist in the estimation-ai.json cost_by_source[] this plan was built from, and every models[].model MUST be one of that source's models[] — see schema-generate-ai.md's artifacts_by_source[] traceability rule.", + "type": "object", + "required": ["provider", "source_service", "models"], + "additionalProperties": false, + "properties": { + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "models": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/definitions/sourceArtifactModel"} + } + } + } + } +} diff --git a/plugins/aws-startup-advisor/scripts/validate-artifacts.py b/plugins/aws-startup-advisor/scripts/validate-artifacts.py index c23719aa3..42ecd6383 100644 --- a/plugins/aws-startup-advisor/scripts/validate-artifacts.py +++ b/plugins/aws-startup-advisor/scripts/validate-artifacts.py @@ -1019,9 +1019,17 @@ def _admit_schema_properties(node: Node, schema: Any, source: str) -> None: _admit_schema_properties(node.keys[k], sub, source) -def build_contracts(manifest: Dict[str, Any], skill: Optional[str], findings: List[Finding]) -> List[Contract]: +def build_contracts( + manifest: Dict[str, Any], + skill: Optional[str], + findings: List[Finding], + include_fixture_only: bool = False, +) -> List[Contract]: specs: Dict[str, Any] = {} - specs.update(manifest.get("shared", {})) + for glob_, spec in manifest.get("shared", {}).items(): + if skill is not None and spec.get("fixture_only") and not include_fixture_only: + continue + specs[glob_] = spec if skill is not None: specs.update(manifest.get("skills", {}).get(skill, {})) contracts: List[Contract] = [] @@ -1178,7 +1186,7 @@ def main(argv: Optional[List[str]] = None) -> int: if args.self_check: for skill in manifest.get("skills", {}): - build_contracts(manifest, skill, findings) + build_contracts(manifest, skill, findings, include_fixture_only=True) elif args.fixtures: fixtures_root = PLUGIN_ROOT / "fixtures" for d in sorted(p for p in fixtures_root.iterdir() if p.is_dir()): diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index c2781a4fd..b0980a9ba 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -43,6 +43,8 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | | `ai/sdk-capability-map.json` | `skills/shared/ai/sdk-capability-map.json` | | `ai/schema-design-aws-ai.md` | `skills/shared/ai/schema-design-aws-ai.md` | +| `ai/schema-estimate-ai.md` | `skills/shared/ai/schema-estimate-ai.md` | +| `ai/schema-generate-ai.md` | `skills/shared/ai/schema-generate-ai.md` | | `clarify/clarify-availability.md` | `skills/shared/clarify/clarify-availability.md` | | `clarify/clarify-compliance.md` | `skills/shared/clarify/clarify-compliance.md` | | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md new file mode 100644 index 000000000..1feda673d --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-estimate-ai.md @@ -0,0 +1,173 @@ +# Schema — estimation-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design-ai.json` had a prose-only contract that drifted between +> GCP and Azure until #416 canonicalized it (`schema-design-aws-ai.md`). `estimation-ai.json` is +> the same risk one phase over: `estimate-ai.md` produces it (both GCP's +> `skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +> `skills/azure-to-aws/references/phases/estimate/estimate-ai.md`), and `generate-ai.md` / +> `generate-artifacts-ai.md` consume it, with each skill's own "Schema — top-level fields" +> table carrying the same field set under GCP-/Azure-named keys (`gcp_monthly_ai_spend` vs +> `azure_monthly_ai_spend`, `current_gcp_monthly` vs `current_azure_monthly`). This file is the +> single, provider-neutral written contract, using neutral field names +> (`current_costs.monthly_ai_spend`, `cost_comparison.current_source_monthly`) so one JSON +> Schema (`scripts/contracts/estimation-ai.schema.json`) serves both skills without a GCP/Azure +> branch — the same provider-neutral framing `ai-workload-profile.schema.json` (#415) and +> `aws-design-ai.schema.json` (#416) already use. Each consuming skill carries a byte-identical +> vendored copy at `references/vendored/ai/schema-estimate-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410), +> `ai-workload-profile.schema.json` (#415), and `aws-design-ai.schema.json` (#416).** Structural +> validity against this document or its companion JSON Schema does NOT imply an implemented +> pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either +> skill's current `estimate-ai.md` actually emits this shape yet — NEITHER skill's `estimate-ai.md` +> currently emits the neutral field names (`current_costs.monthly_ai_spend`, +> `cost_comparison.current_source_monthly`, `cost_by_source[]`); current GCP/Azure output still +> uses the GCP-/Azure-named fields documented in each skill's own `estimate-ai.md`, unchanged by +> this document. No producer has been repointed to this canonical contract in this change — that +> is deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "estimate", + "timestamp": "", + "pricing_source": "cached", // cached | unavailable + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", // provenance label — see each skill's estimate-ai.md Part 1 + "monthly_ai_spend": 1200.0, // provider-neutral name for gcp_monthly_ai_spend / azure_monthly_ai_spend + "services": [] + }, + "token_volume": {}, + "model_comparison": [], + "recommended_model": {}, + "backup_model": {}, + "embeddings": {}, // present only when the workload uses embeddings + "cost_comparison": { + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly; number or null + "projected_bedrock_monthly": 1320.0, // number or null + "monthly_difference": 120.0, // number or null + "annual_difference": 1440.0, // number or null + "percent_change": 10.0 // number or null + }, + "migration_cost_considerations": { + "categories": [], // ALWAYS [] — human/pro-services costs are out of scope + "complexity_factors": [], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": {}, + "optimization_opportunities": [], // see § optimization_opportunities[] + "optimized_projection": {}, + "recommendation": {}, // see § recommendation + "cost_by_source": [], // many-valued — see § cost_by_source[] + "services_not_estimated": [] // ALWAYS present, [] when every workload is token-priced +} +``` + +## cost_by_source[] + +Carries the design-time identity chain forward into the estimate: each entry's +`(provider, source_service)` pair MUST exist in the `aws-design-ai.json` the estimate was built +from (either a direct `metadata.ai_sources[]` entry, or a `metadata.gateways[].upstreams[]` +entry), and every `models[].model` MUST be one of that source's `models[]` (direct) or +`model_families[].models` (gateway upstream). This is additive — it does not replace +`model_comparison[]`, which remains the per-model Bedrock-vs-source comparison for ALL viable +models regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source; null when its target rate is unavailable + "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `aws-design-ai.json`'s +`metadata.ai_sources[]` — a duplicate or overlapping pair is a contract violation, not a +presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted cost still +attributes to the fronted provider's `(provider, source_service)` pair, mirroring how +`aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. + +## Unknown costs and null propagation + +Unavailable monetary values are JSON `null`, never numeric zero. If the source baseline is +unknown, `current_costs.monthly_ai_spend` and `cost_comparison.current_source_monthly` are `null`, +and all differences derived from that baseline (`monthly_difference`, `annual_difference`, and +`percent_change`) are also `null`. If a target rate is unavailable, set +`cost_comparison.projected_bedrock_monthly` and the affected +`cost_by_source[].models[].monthly_cost` values to `null`; every difference that depends on that +unknown target is `null` as well. Numeric `0` is reserved for a cost that is known to be zero. + +## Design-decision: no `model_observations[]` here + +`model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in +`ai-workload-profile.schema.json` and `aws-design-ai.schema.json` to carry an identity that is +still being established at Discover/Design time. By Estimate, the identity is already resolved — +`aws-design-ai.json`'s `metadata.ai_sources[]`/`models[]` is the resolved, trusted identity list, +and this contract's `cost_by_source[]` entries key off `(provider, source_service, model)` triples +copied FROM that resolved list, not re-observed. Carrying `model_observations[]` forward into +`estimation-ai.json` would re-introduce an uncertainty concept that Design has already resolved. +This is intentionally dropped, not forgotten. + +## optimization_opportunities[] + +Each entry's `type` is REQUIRED and is one of `model_tiering`, `prompt_caching`, `batch_api`, +`intelligent_prompt_routing`, `provisioned_throughput`, `input_token_reduction`, +`multi_model_routing` (see each skill's `estimate-ai.md` Part 6 for the full vocabulary and +sequencing notes). `target_services` is always `["Bedrock"]`. `potential_savings_percent` is +REQUIRED on every entry — populated even when `potential_savings_monthly` is `null`, so a merged +report table is never blank. + +## recommendation + +`path` is REQUIRED and is one of `migrate_optimized`, `migrate_phased`, `stay` (see each skill's +`estimate-ai.md` Part 7 for the full decision logic). When the Design artifact's +`ai_architecture.honest_assessment` is `recommend_stay`, `recommendation.path` MUST be `stay` +regardless of cost numbers. + +## services_not_estimated[] + +ALWAYS present (`[]` when every workload is token-priced). Per entry: +`{ workload_id, target_aws_service, reason: "not_token_priced" }` — `design_blocks[]` rows with +`target_aws_service` set (Textract, Rekognition, Transcribe) are per-page/per-image/per-minute +priced, not token priced, and are listed here instead of silently omitted. + +## Validation Checklist + +- [ ] `cost_by_source[]` is non-empty whenever `aws-design-ai.json`'s `metadata.ai_sources[]` is + non-empty, and each entry's `(provider, source_service)` pair is unique. +- [ ] Every `cost_by_source[]` entry's `(provider, source_service)` pair exists in the matching + `aws-design-ai.json`'s `metadata.ai_sources[]` or a `metadata.gateways[].upstreams[]` entry. +- [ ] Every `cost_by_source[].models[].model` is one of that source's `models[]` (direct) or + `model_families[].models` (gateway upstream) in the matching `aws-design-ai.json`. +- [ ] No entry in `cost_by_source[]` has `provider: "openrouter"`. +- [ ] `recommendation` is present with non-empty `path`, `path_label`, `migrate_if`, `stay_if`, + and `rationale`; `path` is one of `migrate_optimized`, `migrate_phased`, `stay`. +- [ ] If Design `honest_assessment` is `recommend_stay`, `recommendation.path` is `stay`. +- [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, + `potential_savings_percent`, and `available`. +- [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] Unknown source or target costs and their dependent deltas are `null`; known zero-dollar + costs remain numeric `0`. +- [ ] `services_not_estimated[]` is present (`[]` when clean). + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. Both GCP's +`skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +`skills/azure-to-aws/references/phases/estimate/estimate-ai.md` still write against their own +"Schema — top-level fields" table with GCP-/Azure-named fields (`current_costs.gcp_monthly_ai_spend` +/ `current_costs.azure_monthly_ai_spend`, `cost_comparison.current_gcp_monthly` / +`cost_comparison.current_azure_monthly`), and neither emits `cost_by_source[]`. This document +(and its companion JSON Schema, `scripts/contracts/estimation-ai.schema.json`) is the TARGET +contract. Repointing either skill's `estimate-ai.md` to emit this shape is deferred +runtime-adoption work, named explicitly so a future PR does not silently assume today's runtime +output already satisfies this contract. `generate-ai.md` and `generate-artifacts-ai.md` are +unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md new file mode 100644 index 000000000..d8925c48c --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-generate-ai.md @@ -0,0 +1,131 @@ +# Schema — generation-ai.json (canonical, target-only) + +> **Why this file exists.** `estimation-ai.json` just gained a single, provider-neutral written +> contract (`schema-estimate-ai.md`, #389 follow-on). `generation-ai.json` is the same risk one +> phase further: GCP's `skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the +> `generation-ai.json` section of Azure's +> `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` each carry their own +> "Schema — top-level fields" table for the same artifact. This file is the single, +> provider-neutral written contract for `generation-ai.json`, matching +> `schema-estimate-ai.md`'s own canonicalization pattern so GCP and Azure share one definition +> instead of drifting independently. Each consuming skill carries a byte-identical vendored copy +> at `references/vendored/ai/schema-generate-ai.md`. +> +> **This is a target-only contract, like `schema-estimate-ai.md` and the rest of this +> contract family (#410, #415, #416).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/generation-ai.schema.json`) does NOT imply an +> implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that +> either skill's current `generate-ai.md` / `generate-artifacts-ai.md` actually emits this shape +> yet. No producer has been repointed to this canonical contract in this change — that is +> deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "", + "migration_plan": { + "duration_drivers": [], + "approach": "", + "phases": [], + "models_to_migrate": [] // normalizedModel[] — covers every model from aws-design-ai.json + }, + "step_by_step_guide": { + "languages": [], + "primary_pattern": "", + "files_to_modify": [], + "dependency_changes": "" + }, + "rollback_plan": { + "mechanism": "feature_flag", // MUST be this exact value + "flag_name": "", + "default_value": "", // free-form pre-cutover provider identifier, e.g. "vertex_ai" | "azure_openai" — not an enum + "rollback_time": "", + "triggers": [] + }, + "monitoring": { + "dashboards": [], + "alerting_rules": [] // each: { severity, condition, action } + }, + "production_readiness_checklist": [], // >= 5 string items + "success_criteria": { + "quality": {}, + "latency": {}, + "cost": {} + }, + "recommendation": { + "approach": "", + "confidence": "high | medium | low", + "key_risks": [] + }, + "artifacts_by_source": [] // many-valued — see § artifacts_by_source[] +} +``` + +## artifacts_by_source[] + +Carries the traceability chain one phase forward from `estimation-ai.json`'s `cost_by_source[]`: +each entry's `(provider, source_service)` pair MUST exist in the `estimation-ai.json` +`cost_by_source[]` the migration plan was built from, and every `models[].model` MUST be one of +that source's `models[]`. This is additive — it does not replace `migration_plan.models_to_migrate[]`, +which remains the flat list of every model being migrated regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching estimation-ai.json cost_by_source[] entry + "generated_files": ["src/ai/anthropic_client.py"] // files this migration plan touches for this specific model + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `estimation-ai.json`'s +`cost_by_source[]`. `provider` is never `"openrouter"` — an OpenRouter-fronted artifact still +attributes to the fronted provider's `(provider, source_service)` pair. + +## Design-decision: no `model_observations[]` here + +Same rationale as `schema-estimate-ai.md`: by Generate, the model identity has been resolved +twice over (Design, then Estimate). `artifacts_by_source[]` entries key off +`(provider, source_service, model)` triples copied FROM `estimation-ai.json`'s `cost_by_source[]`, +not re-observed. Carrying `model_observations[]` forward here would re-introduce an uncertainty +concept that was resolved two phases earlier. This is intentionally dropped, not forgotten. + +## rollback_plan + +`mechanism` MUST be the literal string `"feature_flag"` — this contract does not model any other +rollback mechanism. `default_value` is a free-form string naming the pre-cutover provider +identifier (e.g. GCP's `"vertex_ai"`, Azure's `"azure_openai"`); both skills already treat it as +a free-form string, not an enum, and this contract does not special-case the value. + +## Validation Checklist + +- [ ] `migration_plan.models_to_migrate` covers all models from the matching `aws-design-ai.json`. +- [ ] `artifacts_by_source[]` is non-empty whenever the matching `estimation-ai.json`'s + `cost_by_source[]` is non-empty, and each entry's `(provider, source_service)` pair is + unique. +- [ ] Every `artifacts_by_source[]` entry's `(provider, source_service)` pair exists in the + matching `estimation-ai.json`'s `cost_by_source[]`. +- [ ] Every `artifacts_by_source[].models[].model` is one of that source's `cost_by_source[]` + `models[].model` entries in the matching `estimation-ai.json`. +- [ ] No entry in `artifacts_by_source[]` has `provider: "openrouter"`. +- [ ] `rollback_plan.mechanism` is `"feature_flag"`. +- [ ] `production_readiness_checklist` has at least 5 items. +- [ ] `success_criteria` covers `quality`, `latency`, and `cost`. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. GCP's +`skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the `generation-ai.json` +section of Azure's `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` +still write against their own "Schema — top-level fields" tables, and neither emits +`artifacts_by_source[]`. This document (and its companion JSON Schema, +`scripts/contracts/generation-ai.schema.json`) is the TARGET contract. Repointing either skill's +generate phase to emit this shape is deferred runtime-adoption work, named explicitly so a +future PR does not silently assume today's runtime output already satisfies this contract. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index d1e496648..704879840 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -36,6 +36,8 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/design-ref-harness.md` | `skills/shared/ai/design-ref-harness.md` | | `ai/sdk-capability-map.json` | `skills/shared/ai/sdk-capability-map.json` | | `ai/schema-design-aws-ai.md` | `skills/shared/ai/schema-design-aws-ai.md` | +| `ai/schema-estimate-ai.md` | `skills/shared/ai/schema-estimate-ai.md` | +| `ai/schema-generate-ai.md` | `skills/shared/ai/schema-generate-ai.md` | | `clarify/clarify-availability.md` | `skills/shared/clarify/clarify-availability.md` | | `clarify/clarify-compliance.md` | `skills/shared/clarify/clarify-compliance.md` | | `clarify/clarify-cost-appetite.md` | `skills/shared/clarify/clarify-cost-appetite.md` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md new file mode 100644 index 000000000..1feda673d --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-estimate-ai.md @@ -0,0 +1,173 @@ +# Schema — estimation-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design-ai.json` had a prose-only contract that drifted between +> GCP and Azure until #416 canonicalized it (`schema-design-aws-ai.md`). `estimation-ai.json` is +> the same risk one phase over: `estimate-ai.md` produces it (both GCP's +> `skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +> `skills/azure-to-aws/references/phases/estimate/estimate-ai.md`), and `generate-ai.md` / +> `generate-artifacts-ai.md` consume it, with each skill's own "Schema — top-level fields" +> table carrying the same field set under GCP-/Azure-named keys (`gcp_monthly_ai_spend` vs +> `azure_monthly_ai_spend`, `current_gcp_monthly` vs `current_azure_monthly`). This file is the +> single, provider-neutral written contract, using neutral field names +> (`current_costs.monthly_ai_spend`, `cost_comparison.current_source_monthly`) so one JSON +> Schema (`scripts/contracts/estimation-ai.schema.json`) serves both skills without a GCP/Azure +> branch — the same provider-neutral framing `ai-workload-profile.schema.json` (#415) and +> `aws-design-ai.schema.json` (#416) already use. Each consuming skill carries a byte-identical +> vendored copy at `references/vendored/ai/schema-estimate-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410), +> `ai-workload-profile.schema.json` (#415), and `aws-design-ai.schema.json` (#416).** Structural +> validity against this document or its companion JSON Schema does NOT imply an implemented +> pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either +> skill's current `estimate-ai.md` actually emits this shape yet — NEITHER skill's `estimate-ai.md` +> currently emits the neutral field names (`current_costs.monthly_ai_spend`, +> `cost_comparison.current_source_monthly`, `cost_by_source[]`); current GCP/Azure output still +> uses the GCP-/Azure-named fields documented in each skill's own `estimate-ai.md`, unchanged by +> this document. No producer has been repointed to this canonical contract in this change — that +> is deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "estimate", + "timestamp": "", + "pricing_source": "cached", // cached | unavailable + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", // provenance label — see each skill's estimate-ai.md Part 1 + "monthly_ai_spend": 1200.0, // provider-neutral name for gcp_monthly_ai_spend / azure_monthly_ai_spend + "services": [] + }, + "token_volume": {}, + "model_comparison": [], + "recommended_model": {}, + "backup_model": {}, + "embeddings": {}, // present only when the workload uses embeddings + "cost_comparison": { + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly; number or null + "projected_bedrock_monthly": 1320.0, // number or null + "monthly_difference": 120.0, // number or null + "annual_difference": 1440.0, // number or null + "percent_change": 10.0 // number or null + }, + "migration_cost_considerations": { + "categories": [], // ALWAYS [] — human/pro-services costs are out of scope + "complexity_factors": [], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": {}, + "optimization_opportunities": [], // see § optimization_opportunities[] + "optimized_projection": {}, + "recommendation": {}, // see § recommendation + "cost_by_source": [], // many-valued — see § cost_by_source[] + "services_not_estimated": [] // ALWAYS present, [] when every workload is token-priced +} +``` + +## cost_by_source[] + +Carries the design-time identity chain forward into the estimate: each entry's +`(provider, source_service)` pair MUST exist in the `aws-design-ai.json` the estimate was built +from (either a direct `metadata.ai_sources[]` entry, or a `metadata.gateways[].upstreams[]` +entry), and every `models[].model` MUST be one of that source's `models[]` (direct) or +`model_families[].models` (gateway upstream). This is additive — it does not replace +`model_comparison[]`, which remains the per-model Bedrock-vs-source comparison for ALL viable +models regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source; null when its target rate is unavailable + "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `aws-design-ai.json`'s +`metadata.ai_sources[]` — a duplicate or overlapping pair is a contract violation, not a +presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted cost still +attributes to the fronted provider's `(provider, source_service)` pair, mirroring how +`aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. + +## Unknown costs and null propagation + +Unavailable monetary values are JSON `null`, never numeric zero. If the source baseline is +unknown, `current_costs.monthly_ai_spend` and `cost_comparison.current_source_monthly` are `null`, +and all differences derived from that baseline (`monthly_difference`, `annual_difference`, and +`percent_change`) are also `null`. If a target rate is unavailable, set +`cost_comparison.projected_bedrock_monthly` and the affected +`cost_by_source[].models[].monthly_cost` values to `null`; every difference that depends on that +unknown target is `null` as well. Numeric `0` is reserved for a cost that is known to be zero. + +## Design-decision: no `model_observations[]` here + +`model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in +`ai-workload-profile.schema.json` and `aws-design-ai.schema.json` to carry an identity that is +still being established at Discover/Design time. By Estimate, the identity is already resolved — +`aws-design-ai.json`'s `metadata.ai_sources[]`/`models[]` is the resolved, trusted identity list, +and this contract's `cost_by_source[]` entries key off `(provider, source_service, model)` triples +copied FROM that resolved list, not re-observed. Carrying `model_observations[]` forward into +`estimation-ai.json` would re-introduce an uncertainty concept that Design has already resolved. +This is intentionally dropped, not forgotten. + +## optimization_opportunities[] + +Each entry's `type` is REQUIRED and is one of `model_tiering`, `prompt_caching`, `batch_api`, +`intelligent_prompt_routing`, `provisioned_throughput`, `input_token_reduction`, +`multi_model_routing` (see each skill's `estimate-ai.md` Part 6 for the full vocabulary and +sequencing notes). `target_services` is always `["Bedrock"]`. `potential_savings_percent` is +REQUIRED on every entry — populated even when `potential_savings_monthly` is `null`, so a merged +report table is never blank. + +## recommendation + +`path` is REQUIRED and is one of `migrate_optimized`, `migrate_phased`, `stay` (see each skill's +`estimate-ai.md` Part 7 for the full decision logic). When the Design artifact's +`ai_architecture.honest_assessment` is `recommend_stay`, `recommendation.path` MUST be `stay` +regardless of cost numbers. + +## services_not_estimated[] + +ALWAYS present (`[]` when every workload is token-priced). Per entry: +`{ workload_id, target_aws_service, reason: "not_token_priced" }` — `design_blocks[]` rows with +`target_aws_service` set (Textract, Rekognition, Transcribe) are per-page/per-image/per-minute +priced, not token priced, and are listed here instead of silently omitted. + +## Validation Checklist + +- [ ] `cost_by_source[]` is non-empty whenever `aws-design-ai.json`'s `metadata.ai_sources[]` is + non-empty, and each entry's `(provider, source_service)` pair is unique. +- [ ] Every `cost_by_source[]` entry's `(provider, source_service)` pair exists in the matching + `aws-design-ai.json`'s `metadata.ai_sources[]` or a `metadata.gateways[].upstreams[]` entry. +- [ ] Every `cost_by_source[].models[].model` is one of that source's `models[]` (direct) or + `model_families[].models` (gateway upstream) in the matching `aws-design-ai.json`. +- [ ] No entry in `cost_by_source[]` has `provider: "openrouter"`. +- [ ] `recommendation` is present with non-empty `path`, `path_label`, `migrate_if`, `stay_if`, + and `rationale`; `path` is one of `migrate_optimized`, `migrate_phased`, `stay`. +- [ ] If Design `honest_assessment` is `recommend_stay`, `recommendation.path` is `stay`. +- [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, + `potential_savings_percent`, and `available`. +- [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] Unknown source or target costs and their dependent deltas are `null`; known zero-dollar + costs remain numeric `0`. +- [ ] `services_not_estimated[]` is present (`[]` when clean). + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. Both GCP's +`skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +`skills/azure-to-aws/references/phases/estimate/estimate-ai.md` still write against their own +"Schema — top-level fields" table with GCP-/Azure-named fields (`current_costs.gcp_monthly_ai_spend` +/ `current_costs.azure_monthly_ai_spend`, `cost_comparison.current_gcp_monthly` / +`cost_comparison.current_azure_monthly`), and neither emits `cost_by_source[]`. This document +(and its companion JSON Schema, `scripts/contracts/estimation-ai.schema.json`) is the TARGET +contract. Repointing either skill's `estimate-ai.md` to emit this shape is deferred +runtime-adoption work, named explicitly so a future PR does not silently assume today's runtime +output already satisfies this contract. `generate-ai.md` and `generate-artifacts-ai.md` are +unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md new file mode 100644 index 000000000..d8925c48c --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-generate-ai.md @@ -0,0 +1,131 @@ +# Schema — generation-ai.json (canonical, target-only) + +> **Why this file exists.** `estimation-ai.json` just gained a single, provider-neutral written +> contract (`schema-estimate-ai.md`, #389 follow-on). `generation-ai.json` is the same risk one +> phase further: GCP's `skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the +> `generation-ai.json` section of Azure's +> `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` each carry their own +> "Schema — top-level fields" table for the same artifact. This file is the single, +> provider-neutral written contract for `generation-ai.json`, matching +> `schema-estimate-ai.md`'s own canonicalization pattern so GCP and Azure share one definition +> instead of drifting independently. Each consuming skill carries a byte-identical vendored copy +> at `references/vendored/ai/schema-generate-ai.md`. +> +> **This is a target-only contract, like `schema-estimate-ai.md` and the rest of this +> contract family (#410, #415, #416).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/generation-ai.schema.json`) does NOT imply an +> implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that +> either skill's current `generate-ai.md` / `generate-artifacts-ai.md` actually emits this shape +> yet. No producer has been repointed to this canonical contract in this change — that is +> deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "", + "migration_plan": { + "duration_drivers": [], + "approach": "", + "phases": [], + "models_to_migrate": [] // normalizedModel[] — covers every model from aws-design-ai.json + }, + "step_by_step_guide": { + "languages": [], + "primary_pattern": "", + "files_to_modify": [], + "dependency_changes": "" + }, + "rollback_plan": { + "mechanism": "feature_flag", // MUST be this exact value + "flag_name": "", + "default_value": "", // free-form pre-cutover provider identifier, e.g. "vertex_ai" | "azure_openai" — not an enum + "rollback_time": "", + "triggers": [] + }, + "monitoring": { + "dashboards": [], + "alerting_rules": [] // each: { severity, condition, action } + }, + "production_readiness_checklist": [], // >= 5 string items + "success_criteria": { + "quality": {}, + "latency": {}, + "cost": {} + }, + "recommendation": { + "approach": "", + "confidence": "high | medium | low", + "key_risks": [] + }, + "artifacts_by_source": [] // many-valued — see § artifacts_by_source[] +} +``` + +## artifacts_by_source[] + +Carries the traceability chain one phase forward from `estimation-ai.json`'s `cost_by_source[]`: +each entry's `(provider, source_service)` pair MUST exist in the `estimation-ai.json` +`cost_by_source[]` the migration plan was built from, and every `models[].model` MUST be one of +that source's `models[]`. This is additive — it does not replace `migration_plan.models_to_migrate[]`, +which remains the flat list of every model being migrated regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching estimation-ai.json cost_by_source[] entry + "generated_files": ["src/ai/anthropic_client.py"] // files this migration plan touches for this specific model + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `estimation-ai.json`'s +`cost_by_source[]`. `provider` is never `"openrouter"` — an OpenRouter-fronted artifact still +attributes to the fronted provider's `(provider, source_service)` pair. + +## Design-decision: no `model_observations[]` here + +Same rationale as `schema-estimate-ai.md`: by Generate, the model identity has been resolved +twice over (Design, then Estimate). `artifacts_by_source[]` entries key off +`(provider, source_service, model)` triples copied FROM `estimation-ai.json`'s `cost_by_source[]`, +not re-observed. Carrying `model_observations[]` forward here would re-introduce an uncertainty +concept that was resolved two phases earlier. This is intentionally dropped, not forgotten. + +## rollback_plan + +`mechanism` MUST be the literal string `"feature_flag"` — this contract does not model any other +rollback mechanism. `default_value` is a free-form string naming the pre-cutover provider +identifier (e.g. GCP's `"vertex_ai"`, Azure's `"azure_openai"`); both skills already treat it as +a free-form string, not an enum, and this contract does not special-case the value. + +## Validation Checklist + +- [ ] `migration_plan.models_to_migrate` covers all models from the matching `aws-design-ai.json`. +- [ ] `artifacts_by_source[]` is non-empty whenever the matching `estimation-ai.json`'s + `cost_by_source[]` is non-empty, and each entry's `(provider, source_service)` pair is + unique. +- [ ] Every `artifacts_by_source[]` entry's `(provider, source_service)` pair exists in the + matching `estimation-ai.json`'s `cost_by_source[]`. +- [ ] Every `artifacts_by_source[].models[].model` is one of that source's `cost_by_source[]` + `models[].model` entries in the matching `estimation-ai.json`. +- [ ] No entry in `artifacts_by_source[]` has `provider: "openrouter"`. +- [ ] `rollback_plan.mechanism` is `"feature_flag"`. +- [ ] `production_readiness_checklist` has at least 5 items. +- [ ] `success_criteria` covers `quality`, `latency`, and `cost`. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. GCP's +`skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the `generation-ai.json` +section of Azure's `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` +still write against their own "Schema — top-level fields" tables, and neither emits +`artifacts_by_source[]`. This document (and its companion JSON Schema, +`scripts/contracts/generation-ai.schema.json`) is the TARGET contract. Repointing either skill's +generate phase to emit this shape is deferred runtime-adoption work, named explicitly so a +future PR does not silently assume today's runtime output already satisfies this contract. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/README.md b/plugins/aws-startup-advisor/skills/shared/ai/README.md index dccb94b9a..b0b909c69 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/README.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/README.md @@ -23,6 +23,8 @@ before opening a pull request. | `ai-capability-catalog.json` | Target-only source-provider/gateway capability catalog (schema_version, provider/gateway status, no implemented-adapter claim) | | `bedrock-pricing-cache.md` | Provider-neutral Bedrock model and service pricing | | `schema-design-aws-ai.md` | Target-only `aws-design-ai.json` contract; `metadata.ai_sources[]`/`gateways[]` mirror `ai-workload-profile.schema.json` exactly | +| `schema-estimate-ai.md` | Target-only `estimation-ai.json` contract; `cost_by_source[]` traces per-source/per-model cost back to `aws-design-ai.json` | +| `schema-generate-ai.md` | Target-only `generation-ai.json` contract; `artifacts_by_source[]` traces per-source/per-model generated artifacts back to `estimation-ai.json` | The Gemini and OpenRouter guides are source-cloud agnostic (the Bedrock target depends on the upstream model, not the source cloud), so they are canonical here diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md new file mode 100644 index 000000000..1feda673d --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-estimate-ai.md @@ -0,0 +1,173 @@ +# Schema — estimation-ai.json (canonical, target-only) + +> **Why this file exists.** `aws-design-ai.json` had a prose-only contract that drifted between +> GCP and Azure until #416 canonicalized it (`schema-design-aws-ai.md`). `estimation-ai.json` is +> the same risk one phase over: `estimate-ai.md` produces it (both GCP's +> `skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +> `skills/azure-to-aws/references/phases/estimate/estimate-ai.md`), and `generate-ai.md` / +> `generate-artifacts-ai.md` consume it, with each skill's own "Schema — top-level fields" +> table carrying the same field set under GCP-/Azure-named keys (`gcp_monthly_ai_spend` vs +> `azure_monthly_ai_spend`, `current_gcp_monthly` vs `current_azure_monthly`). This file is the +> single, provider-neutral written contract, using neutral field names +> (`current_costs.monthly_ai_spend`, `cost_comparison.current_source_monthly`) so one JSON +> Schema (`scripts/contracts/estimation-ai.schema.json`) serves both skills without a GCP/Azure +> branch — the same provider-neutral framing `ai-workload-profile.schema.json` (#415) and +> `aws-design-ai.schema.json` (#416) already use. Each consuming skill carries a byte-identical +> vendored copy at `references/vendored/ai/schema-estimate-ai.md`. +> +> **This is a target-only contract, like `ai-migration-scenario.schema.json` (#410), +> `ai-workload-profile.schema.json` (#415), and `aws-design-ai.schema.json` (#416).** Structural +> validity against this document or its companion JSON Schema does NOT imply an implemented +> pricing lookup, model mapping, or runtime-supported identity, and does NOT imply that either +> skill's current `estimate-ai.md` actually emits this shape yet — NEITHER skill's `estimate-ai.md` +> currently emits the neutral field names (`current_costs.monthly_ai_spend`, +> `cost_comparison.current_source_monthly`, `cost_by_source[]`); current GCP/Azure output still +> uses the GCP-/Azure-named fields documented in each skill's own `estimate-ai.md`, unchanged by +> this document. No producer has been repointed to this canonical contract in this change — that +> is deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "estimate", + "timestamp": "", + "pricing_source": "cached", // cached | unavailable + "accuracy_confidence": "±15-25%", + "current_costs": { + "source": "mixed", // provenance label — see each skill's estimate-ai.md Part 1 + "monthly_ai_spend": 1200.0, // provider-neutral name for gcp_monthly_ai_spend / azure_monthly_ai_spend + "services": [] + }, + "token_volume": {}, + "model_comparison": [], + "recommended_model": {}, + "backup_model": {}, + "embeddings": {}, // present only when the workload uses embeddings + "cost_comparison": { + "current_source_monthly": 1200.0, // provider-neutral name for current_gcp_monthly / current_azure_monthly; number or null + "projected_bedrock_monthly": 1320.0, // number or null + "monthly_difference": 120.0, // number or null + "annual_difference": 1440.0, // number or null + "percent_change": 10.0 // number or null + }, + "migration_cost_considerations": { + "categories": [], // ALWAYS [] — human/pro-services costs are out of scope + "complexity_factors": [], + "note": "Human and professional-services one-time migration costs are intentionally excluded from this advisor." + }, + "roi_analysis": {}, + "optimization_opportunities": [], // see § optimization_opportunities[] + "optimized_projection": {}, + "recommendation": {}, // see § recommendation + "cost_by_source": [], // many-valued — see § cost_by_source[] + "services_not_estimated": [] // ALWAYS present, [] when every workload is token-priced +} +``` + +## cost_by_source[] + +Carries the design-time identity chain forward into the estimate: each entry's +`(provider, source_service)` pair MUST exist in the `aws-design-ai.json` the estimate was built +from (either a direct `metadata.ai_sources[]` entry, or a `metadata.gateways[].upstreams[]` +entry), and every `models[].model` MUST be one of that source's `models[]` (direct) or +`model_families[].models` (gateway upstream). This is additive — it does not replace +`model_comparison[]`, which remains the per-model Bedrock-vs-source comparison for ALL viable +models regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching aws-design-ai.json source + "monthly_cost": 1320.0, // projected Bedrock monthly cost for this model under this source; null when its target rate is unavailable + "vs_source_percent": 10.0 // null when a source-side list rate is unavailable (e.g. an OpenRouter-only portion) + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `aws-design-ai.json`'s +`metadata.ai_sources[]` — a duplicate or overlapping pair is a contract violation, not a +presentation choice. `provider` is never `"openrouter"`; an OpenRouter-fronted cost still +attributes to the fronted provider's `(provider, source_service)` pair, mirroring how +`aws-design-ai.json`'s `metadata.gateways[].upstreams[]` represents gateway-fronted sources. + +## Unknown costs and null propagation + +Unavailable monetary values are JSON `null`, never numeric zero. If the source baseline is +unknown, `current_costs.monthly_ai_spend` and `cost_comparison.current_source_monthly` are `null`, +and all differences derived from that baseline (`monthly_difference`, `annual_difference`, and +`percent_change`) are also `null`. If a target rate is unavailable, set +`cost_comparison.projected_bedrock_monthly` and the affected +`cost_by_source[].models[].monthly_cost` values to `null`; every difference that depends on that +unknown target is `null` as well. Numeric `0` is reserved for a cost that is known to be zero. + +## Design-decision: no `model_observations[]` here + +`model_observations[]` (the raw/observed model string, provenance, and confidence pair) exists in +`ai-workload-profile.schema.json` and `aws-design-ai.schema.json` to carry an identity that is +still being established at Discover/Design time. By Estimate, the identity is already resolved — +`aws-design-ai.json`'s `metadata.ai_sources[]`/`models[]` is the resolved, trusted identity list, +and this contract's `cost_by_source[]` entries key off `(provider, source_service, model)` triples +copied FROM that resolved list, not re-observed. Carrying `model_observations[]` forward into +`estimation-ai.json` would re-introduce an uncertainty concept that Design has already resolved. +This is intentionally dropped, not forgotten. + +## optimization_opportunities[] + +Each entry's `type` is REQUIRED and is one of `model_tiering`, `prompt_caching`, `batch_api`, +`intelligent_prompt_routing`, `provisioned_throughput`, `input_token_reduction`, +`multi_model_routing` (see each skill's `estimate-ai.md` Part 6 for the full vocabulary and +sequencing notes). `target_services` is always `["Bedrock"]`. `potential_savings_percent` is +REQUIRED on every entry — populated even when `potential_savings_monthly` is `null`, so a merged +report table is never blank. + +## recommendation + +`path` is REQUIRED and is one of `migrate_optimized`, `migrate_phased`, `stay` (see each skill's +`estimate-ai.md` Part 7 for the full decision logic). When the Design artifact's +`ai_architecture.honest_assessment` is `recommend_stay`, `recommendation.path` MUST be `stay` +regardless of cost numbers. + +## services_not_estimated[] + +ALWAYS present (`[]` when every workload is token-priced). Per entry: +`{ workload_id, target_aws_service, reason: "not_token_priced" }` — `design_blocks[]` rows with +`target_aws_service` set (Textract, Rekognition, Transcribe) are per-page/per-image/per-minute +priced, not token priced, and are listed here instead of silently omitted. + +## Validation Checklist + +- [ ] `cost_by_source[]` is non-empty whenever `aws-design-ai.json`'s `metadata.ai_sources[]` is + non-empty, and each entry's `(provider, source_service)` pair is unique. +- [ ] Every `cost_by_source[]` entry's `(provider, source_service)` pair exists in the matching + `aws-design-ai.json`'s `metadata.ai_sources[]` or a `metadata.gateways[].upstreams[]` entry. +- [ ] Every `cost_by_source[].models[].model` is one of that source's `models[]` (direct) or + `model_families[].models` (gateway upstream) in the matching `aws-design-ai.json`. +- [ ] No entry in `cost_by_source[]` has `provider: "openrouter"`. +- [ ] `recommendation` is present with non-empty `path`, `path_label`, `migrate_if`, `stay_if`, + and `rationale`; `path` is one of `migrate_optimized`, `migrate_phased`, `stay`. +- [ ] If Design `honest_assessment` is `recommend_stay`, `recommendation.path` is `stay`. +- [ ] Every `optimization_opportunities[]` entry has `type`, `commitment`, `target_services`, + `potential_savings_percent`, and `available`. +- [ ] `migration_cost_considerations.categories` is `[]`. +- [ ] Unknown source or target costs and their dependent deltas are `null`; known zero-dollar + costs remain numeric `0`. +- [ ] `services_not_estimated[]` is present (`[]` when clean). + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. Both GCP's +`skills/gcp-to-aws/references/phases/estimate/estimate-ai.md` and Azure's +`skills/azure-to-aws/references/phases/estimate/estimate-ai.md` still write against their own +"Schema — top-level fields" table with GCP-/Azure-named fields (`current_costs.gcp_monthly_ai_spend` +/ `current_costs.azure_monthly_ai_spend`, `cost_comparison.current_gcp_monthly` / +`cost_comparison.current_azure_monthly`), and neither emits `cost_by_source[]`. This document +(and its companion JSON Schema, `scripts/contracts/estimation-ai.schema.json`) is the TARGET +contract. Repointing either skill's `estimate-ai.md` to emit this shape is deferred +runtime-adoption work, named explicitly so a future PR does not silently assume today's runtime +output already satisfies this contract. `generate-ai.md` and `generate-artifacts-ai.md` are +unchanged and still read today's artifact shape. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md new file mode 100644 index 000000000..d8925c48c --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-generate-ai.md @@ -0,0 +1,131 @@ +# Schema — generation-ai.json (canonical, target-only) + +> **Why this file exists.** `estimation-ai.json` just gained a single, provider-neutral written +> contract (`schema-estimate-ai.md`, #389 follow-on). `generation-ai.json` is the same risk one +> phase further: GCP's `skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the +> `generation-ai.json` section of Azure's +> `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` each carry their own +> "Schema — top-level fields" table for the same artifact. This file is the single, +> provider-neutral written contract for `generation-ai.json`, matching +> `schema-estimate-ai.md`'s own canonicalization pattern so GCP and Azure share one definition +> instead of drifting independently. Each consuming skill carries a byte-identical vendored copy +> at `references/vendored/ai/schema-generate-ai.md`. +> +> **This is a target-only contract, like `schema-estimate-ai.md` and the rest of this +> contract family (#410, #415, #416).** Structural validity against this document or its +> companion JSON Schema (`scripts/contracts/generation-ai.schema.json`) does NOT imply an +> implemented adapter, rollout mechanism, or runtime-supported identity, and does NOT imply that +> either skill's current `generate-ai.md` / `generate-artifacts-ai.md` actually emits this shape +> yet. No producer has been repointed to this canonical contract in this change — that is +> deferred runtime-adoption work. + +## Shape + +```jsonc +{ + "phase": "generate", + "generation_source": "ai", + "timestamp": "", + "migration_plan": { + "duration_drivers": [], + "approach": "", + "phases": [], + "models_to_migrate": [] // normalizedModel[] — covers every model from aws-design-ai.json + }, + "step_by_step_guide": { + "languages": [], + "primary_pattern": "", + "files_to_modify": [], + "dependency_changes": "" + }, + "rollback_plan": { + "mechanism": "feature_flag", // MUST be this exact value + "flag_name": "", + "default_value": "", // free-form pre-cutover provider identifier, e.g. "vertex_ai" | "azure_openai" — not an enum + "rollback_time": "", + "triggers": [] + }, + "monitoring": { + "dashboards": [], + "alerting_rules": [] // each: { severity, condition, action } + }, + "production_readiness_checklist": [], // >= 5 string items + "success_criteria": { + "quality": {}, + "latency": {}, + "cost": {} + }, + "recommendation": { + "approach": "", + "confidence": "high | medium | low", + "key_risks": [] + }, + "artifacts_by_source": [] // many-valued — see § artifacts_by_source[] +} +``` + +## artifacts_by_source[] + +Carries the traceability chain one phase forward from `estimation-ai.json`'s `cost_by_source[]`: +each entry's `(provider, source_service)` pair MUST exist in the `estimation-ai.json` +`cost_by_source[]` the migration plan was built from, and every `models[].model` MUST be one of +that source's `models[]`. This is additive — it does not replace `migration_plan.models_to_migrate[]`, +which remains the flat list of every model being migrated regardless of source attribution. + +```jsonc +{ + "provider": "anthropic", // extensibleIdentity — never "both" | "openrouter" | "unknown" + "source_service": "anthropic_api", + "models": [ + { + "model": "claude-3-5-sonnet-20240620", // normalizedModel — must trace to the matching estimation-ai.json cost_by_source[] entry + "generated_files": ["src/ai/anthropic_client.py"] // files this migration plan touches for this specific model + } + ] +} +``` + +Semantic uniqueness is the `(provider, source_service)` pair, same as `estimation-ai.json`'s +`cost_by_source[]`. `provider` is never `"openrouter"` — an OpenRouter-fronted artifact still +attributes to the fronted provider's `(provider, source_service)` pair. + +## Design-decision: no `model_observations[]` here + +Same rationale as `schema-estimate-ai.md`: by Generate, the model identity has been resolved +twice over (Design, then Estimate). `artifacts_by_source[]` entries key off +`(provider, source_service, model)` triples copied FROM `estimation-ai.json`'s `cost_by_source[]`, +not re-observed. Carrying `model_observations[]` forward here would re-introduce an uncertainty +concept that was resolved two phases earlier. This is intentionally dropped, not forgotten. + +## rollback_plan + +`mechanism` MUST be the literal string `"feature_flag"` — this contract does not model any other +rollback mechanism. `default_value` is a free-form string naming the pre-cutover provider +identifier (e.g. GCP's `"vertex_ai"`, Azure's `"azure_openai"`); both skills already treat it as +a free-form string, not an enum, and this contract does not special-case the value. + +## Validation Checklist + +- [ ] `migration_plan.models_to_migrate` covers all models from the matching `aws-design-ai.json`. +- [ ] `artifacts_by_source[]` is non-empty whenever the matching `estimation-ai.json`'s + `cost_by_source[]` is non-empty, and each entry's `(provider, source_service)` pair is + unique. +- [ ] Every `artifacts_by_source[]` entry's `(provider, source_service)` pair exists in the + matching `estimation-ai.json`'s `cost_by_source[]`. +- [ ] Every `artifacts_by_source[].models[].model` is one of that source's `cost_by_source[]` + `models[].model` entries in the matching `estimation-ai.json`. +- [ ] No entry in `artifacts_by_source[]` has `provider: "openrouter"`. +- [ ] `rollback_plan.mechanism` is `"feature_flag"`. +- [ ] `production_readiness_checklist` has at least 5 items. +- [ ] `success_criteria` covers `quality`, `latency`, and `cost`. + +## Status — target-only, no producer yet + +This canonical contract has no producer in this change. GCP's +`skills/gcp-to-aws/references/phases/generate/generate-ai.md` and the `generation-ai.json` +section of Azure's `skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md` +still write against their own "Schema — top-level fields" tables, and neither emits +`artifacts_by_source[]`. This document (and its companion JSON Schema, +`scripts/contracts/generation-ai.schema.json`) is the TARGET contract. Repointing either skill's +generate phase to emit this shape is deferred runtime-adoption work, named explicitly so a +future PR does not silently assume today's runtime output already satisfies this contract. diff --git a/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py new file mode 100644 index 000000000..1585f992d --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_estimation_ai_contracts.py @@ -0,0 +1,293 @@ +"""Contract tests for the target-only, canonical estimation-ai.json contract. + +Pins the schema's self-validity and proves that `cost_by_source[]` projects into +(is a subset of) the matching `aws-design-ai.json`'s `metadata.ai_sources[]` / +`metadata.gateways[].upstreams[]` it was built from, reusing #416's own validator helpers +as the oracle — the same way #416 reused #415's helpers. Target-only: no producer emits +this shape yet (`estimate-ai.md` is unchanged in both skills); no runtime behavior is +asserted. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "estimation-ai" +DESIGN_FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "aws-design-ai" +ESTIMATION_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "estimation-ai.schema.json" +VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +SYNC_VENDORED_PATH = PLUGIN_ROOT / "tools" / "sync-vendored.py" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +VALIDATOR = _load_module("estimation_ai__validator", VALIDATOR_PATH) + +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*/estimation-ai.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _estimation_ai_schema() -> dict: + return _load(ESTIMATION_AI_SCHEMA) + + +def _estimation_ai_findings(estimate: dict) -> list: + findings = [] + VALIDATOR.validate_json_schema(_estimation_ai_schema(), estimate, "", "estimate", str(ESTIMATION_AI_SCHEMA), findings) + return findings + + +def _base_estimate() -> dict: + return copy.deepcopy(_load(FIXTURE_ROOT / "multi-provider-direct" / "estimation-ai.json")) + + +def _matching_design(case: str) -> dict: + return _load(DESIGN_FIXTURE_ROOT / case / "aws-design-ai.json") + + +# --------------------------------------------------------------------------- schema self-validity + +def test_schema_has_no_unsupported_keywords() -> None: + assert VALIDATOR.unsupported_keywords(_estimation_ai_schema()) == [] + + +def test_schema_is_valid_json_and_draft07_shaped() -> None: + schema = _estimation_ai_schema() + assert schema["$schema"] == "http://json-schema.org/draft-07/schema#" + assert schema["type"] == "object" + + +# --------------------------------------------------------------------------- positive fixtures + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_validate_against_estimation_ai_schema(fixture_path: Path) -> None: + estimate = _load(fixture_path) + assert _estimation_ai_findings(estimate) == [] + + +def test_multi_provider_fixture_carries_two_simultaneous_direct_sources() -> None: + estimate = _load(FIXTURE_ROOT / "multi-provider-direct" / "estimation-ai.json") + providers = {s["provider"] for s in estimate["cost_by_source"]} + assert providers == {"anthropic", "openai"} + + +def test_gateway_fixture_carries_multi_upstream_openrouter_fronting_two_providers() -> None: + estimate = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "estimation-ai.json") + claude = next(s for s in estimate["cost_by_source"] if s["provider"] == "anthropic") + assert len(claude["models"]) == 2 + openai_source = next(s for s in estimate["cost_by_source"] if s["provider"] == "openai") + assert len(openai_source["models"]) == 1 + + +def test_unknown_source_baseline_propagates_null_to_dependent_deltas() -> None: + estimate = _base_estimate() + estimate["current_costs"]["monthly_ai_spend"] = None + comparison = estimate["cost_comparison"] + comparison["current_source_monthly"] = None + comparison["monthly_difference"] = None + comparison["annual_difference"] = None + comparison["percent_change"] = None + for source in estimate["cost_by_source"]: + for model in source["models"]: + model["vs_source_percent"] = None + + assert _estimation_ai_findings(estimate) == [] + assert comparison["projected_bedrock_monthly"] is not None + + comparison["projected_bedrock_monthly"] = 0 + assert comparison["projected_bedrock_monthly"] == 0 + assert comparison["projected_bedrock_monthly"] is not None + assert _estimation_ai_findings(estimate) == [] + + comparison["projected_bedrock_monthly"] = "0" + assert _estimation_ai_findings(estimate) != [] + + +def test_unavailable_target_pricing_preserves_unknown_costs_as_null() -> None: + estimate = _base_estimate() + estimate["pricing_source"] = "unavailable" + comparison = estimate["cost_comparison"] + comparison["projected_bedrock_monthly"] = None + comparison["monthly_difference"] = None + comparison["annual_difference"] = None + comparison["percent_change"] = None + for source in estimate["cost_by_source"]: + for model in source["models"]: + model["monthly_cost"] = None + model["vs_source_percent"] = None + + assert _estimation_ai_findings(estimate) == [] + + +# --------------------------------------------------------------------------- cross-phase projection + + +def _assert_cost_by_source_projects_into_design(estimate: dict, design: dict) -> None: + """Every `cost_by_source[]` entry's `(provider, source_service, models[])` MUST project + into (be a subset of) the matching `aws-design-ai.json`'s `metadata.ai_sources[]` / + `metadata.gateways[].upstreams[]` — same 'projects exactly/subset' style as + #416's `test_positive_fixtures_project_exactly_into_pr415_semantics`.""" + direct_models: dict[tuple, set] = {} + for source in design["metadata"]["ai_sources"]: + direct_models[(source["provider"], source["source_service"])] = set(source["models"]) + upstream_models: dict[tuple, set] = {} + for gateway in design["metadata"]["gateways"]: + for upstream in gateway["upstreams"]: + key = (upstream["provider"], upstream["source_service"]) + models = upstream_models.setdefault(key, set()) + for family in upstream["model_families"]: + models.update(family["models"]) + + for entry in estimate["cost_by_source"]: + key = (entry["provider"], entry["source_service"]) + allowed = direct_models.get(key) or upstream_models.get(key) + assert allowed is not None, ( + f"cost_by_source[] entry {key!r} traces to no aws-design-ai.json " + f"metadata.ai_sources[] entry or metadata.gateways[] upstream" + ) + entry_models = {m["model"] for m in entry["models"]} + assert entry_models <= allowed, ( + f"cost_by_source[] entry {key!r} names models {entry_models - allowed!r} " + f"not present in the matching aws-design-ai.json source" + ) + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_project_into_matching_design_source(fixture_path: Path) -> None: + estimate = _load(fixture_path) + design = _matching_design(fixture_path.parent.name) + _assert_cost_by_source_projects_into_design(estimate, design) + + +def test_untraceable_model_under_resolved_design_source_is_rejected() -> None: + """Mirrors #416's `test_design_block_naming_an_untraceable_model_is_rejected` one phase + forward: a cost_by_source[] entry naming a model not present in the matching + aws-design-ai.json's resolved ai_sources[]/gateways[] is a fabricated/untraceable model.""" + estimate = _base_estimate() + design = _matching_design("multi-provider-direct") + estimate["cost_by_source"][0]["models"][0]["model"] = "made-up-model" + with pytest.raises(AssertionError): + _assert_cost_by_source_projects_into_design(estimate, design) + + +def test_dropped_model_under_a_resolved_gateway_upstream_is_rejected() -> None: + """Mirrors #416's own test of the same name one phase forward: a model that is removed + from BOTH the gateway upstream and the direct source in aws-design-ai.json is no longer + traceable, so a cost_by_source[] entry still naming it must be rejected.""" + estimate = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "estimation-ai.json") + design = copy.deepcopy(_matching_design("openrouter-gateway-multi-upstream")) + anthropic_source = next(s for s in design["metadata"]["ai_sources"] if s["provider"] == "anthropic") + anthropic_source["models"].remove("claude-3-haiku-20240307") + anthropic_source["model_observations"] = [ + obs for obs in anthropic_source["model_observations"] if obs["normalized_model"] != "claude-3-haiku-20240307" + ] + for gateway in design["metadata"]["gateways"]: + for upstream in gateway["upstreams"]: + if upstream["provider"] == "anthropic": + for family in upstream["model_families"]: + if "claude-3-haiku-20240307" in family["models"]: + family["models"].remove("claude-3-haiku-20240307") + with pytest.raises(AssertionError): + _assert_cost_by_source_projects_into_design(estimate, design) + + +# --------------------------------------------------------------------------- negative mutations + + +def test_duplicate_cost_by_source_identity_is_rejected() -> None: + estimate = _base_estimate() + estimate["cost_by_source"].append(copy.deepcopy(estimate["cost_by_source"][0])) + assert _estimation_ai_findings(estimate) != [] + + +def _assert_cost_by_source_identity_is_unique(estimate: dict) -> None: + """uniqueItems on cost_by_source[] only rejects byte-identical duplicates; two entries + with the SAME (provider, source_service) pair but different payloads (an overlapping + identity, not an exact duplicate) need this semantic check instead, mirroring + #415's/#416's own key-based duplicate/overlap check for ai_sources[].""" + seen = set() + for entry in estimate["cost_by_source"]: + key = (entry["provider"], entry["source_service"]) + assert key not in seen, f"duplicate or overlapping cost_by_source identity: {key}" + seen.add(key) + + +def test_positive_fixtures_have_unique_cost_by_source_identity() -> None: + for fixture_path in POSITIVE_FIXTURES: + _assert_cost_by_source_identity_is_unique(_load(fixture_path)) + + +def test_overlapping_cost_by_source_identity_is_rejected() -> None: + """A second entry with the SAME (provider, source_service) pair but different model + content is still a duplicate/overlapping identity, not a legitimate second entry — + schema-level uniqueItems alone cannot catch this (see helper docstring), so the semantic + helper above is the lock.""" + estimate = _base_estimate() + overlapping = copy.deepcopy(estimate["cost_by_source"][0]) + overlapping["models"][0]["monthly_cost"] = 1.0 + estimate["cost_by_source"].append(overlapping) + with pytest.raises(AssertionError): + _assert_cost_by_source_identity_is_unique(estimate) + + +def test_openrouter_as_a_direct_cost_by_source_provider_is_rejected() -> None: + estimate = _base_estimate() + estimate["cost_by_source"][0]["provider"] = "openrouter" + assert _estimation_ai_findings(estimate) != [] + + +def test_malformed_cost_by_source_entry_missing_models_is_rejected() -> None: + estimate = _base_estimate() + del estimate["cost_by_source"][0]["models"] + assert _estimation_ai_findings(estimate) != [] + + +def test_recommendation_requires_a_valid_path_enum() -> None: + estimate = _base_estimate() + estimate["recommendation"]["path"] = "not_a_real_path" + assert _estimation_ai_findings(estimate) != [] + + +def test_optimization_opportunity_missing_type_is_rejected() -> None: + estimate = _base_estimate() + del estimate["optimization_opportunities"][0]["type"] + assert _estimation_ai_findings(estimate) != [] + + +def test_services_not_estimated_entry_requires_not_token_priced_reason() -> None: + estimate = _base_estimate() + estimate["services_not_estimated"] = [ + {"workload_id": "wl_doc1", "target_aws_service": "textract", "reason": "something_else"} + ] + assert _estimation_ai_findings(estimate) != [] + + +# --------------------------------------------------------------------------- vendored byte-identity + + +def test_vendored_copies_are_byte_identical() -> None: + """Reuses tools/sync-vendored.py --check (the dedicated CI gate) rather than + re-implementing filecmp logic inline, matching the project's existing pattern.""" + result = subprocess.run( + [sys.executable, str(SYNC_VENDORED_PATH), "--check"], + cwd=PLUGIN_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py b/plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py new file mode 100644 index 000000000..175f53168 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_generation_ai_contracts.py @@ -0,0 +1,237 @@ +"""Contract tests for the target-only, canonical generation-ai.json contract. + +Pins the schema's self-validity and proves that `artifacts_by_source[]` projects into +(is a subset of) the matching `estimation-ai.json`'s `cost_by_source[]` it was built from — +one phase forward from `test_estimation_ai_contracts.py`'s own projection into +`aws-design-ai.json`. Target-only: no producer emits this shape yet (`generate-ai.md` / +`generate-artifacts-ai.md` are unchanged in both skills); no runtime behavior is asserted. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "generation-ai" +ESTIMATION_FIXTURE_ROOT = PLUGIN_ROOT / "fixtures" / "estimation-ai" +GENERATION_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "generation-ai.schema.json" +VALIDATOR_PATH = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" +SYNC_VENDORED_PATH = PLUGIN_ROOT / "tools" / "sync-vendored.py" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +VALIDATOR = _load_module("generation_ai__validator", VALIDATOR_PATH) + +POSITIVE_FIXTURES = sorted(FIXTURE_ROOT.glob("*/generation-ai.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _generation_ai_schema() -> dict: + return _load(GENERATION_AI_SCHEMA) + + +def _generation_ai_findings(generation: dict) -> list: + findings = [] + VALIDATOR.validate_json_schema(_generation_ai_schema(), generation, "", "generation", str(GENERATION_AI_SCHEMA), findings) + return findings + + +def _base_generation() -> dict: + return copy.deepcopy(_load(FIXTURE_ROOT / "multi-provider-direct" / "generation-ai.json")) + + +def _matching_estimation(case: str) -> dict: + return _load(ESTIMATION_FIXTURE_ROOT / case / "estimation-ai.json") + + +# --------------------------------------------------------------------------- schema self-validity + +def test_schema_has_no_unsupported_keywords() -> None: + assert VALIDATOR.unsupported_keywords(_generation_ai_schema()) == [] + + +def test_schema_is_valid_json_and_draft07_shaped() -> None: + schema = _generation_ai_schema() + assert schema["$schema"] == "http://json-schema.org/draft-07/schema#" + assert schema["type"] == "object" + + +# --------------------------------------------------------------------------- positive fixtures + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_validate_against_generation_ai_schema(fixture_path: Path) -> None: + generation = _load(fixture_path) + assert _generation_ai_findings(generation) == [] + + +def test_multi_provider_fixture_carries_two_simultaneous_direct_sources() -> None: + generation = _load(FIXTURE_ROOT / "multi-provider-direct" / "generation-ai.json") + providers = {s["provider"] for s in generation["artifacts_by_source"]} + assert providers == {"anthropic", "openai"} + + +def test_gateway_fixture_carries_multi_upstream_openrouter_fronting_two_providers() -> None: + generation = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "generation-ai.json") + claude = next(s for s in generation["artifacts_by_source"] if s["provider"] == "anthropic") + assert len(claude["models"]) == 2 + openai_source = next(s for s in generation["artifacts_by_source"] if s["provider"] == "openai") + assert len(openai_source["models"]) == 1 + + +# --------------------------------------------------------------------------- cross-phase projection + + +def _assert_artifacts_by_source_projects_into_estimation(generation: dict, estimation: dict) -> None: + """Every `artifacts_by_source[]` entry's `(provider, source_service, models[])` MUST + project into (be a subset of) the matching `estimation-ai.json`'s `cost_by_source[]` — + one phase forward from the estimate-level projection into `aws-design-ai.json`.""" + allowed_models: dict[tuple, set] = {} + for entry in estimation["cost_by_source"]: + key = (entry["provider"], entry["source_service"]) + allowed_models[key] = {m["model"] for m in entry["models"]} + + for entry in generation["artifacts_by_source"]: + key = (entry["provider"], entry["source_service"]) + allowed = allowed_models.get(key) + assert allowed is not None, ( + f"artifacts_by_source[] entry {key!r} traces to no estimation-ai.json " + f"cost_by_source[] entry" + ) + entry_models = {m["model"] for m in entry["models"]} + assert entry_models <= allowed, ( + f"artifacts_by_source[] entry {key!r} names models {entry_models - allowed!r} " + f"not present in the matching estimation-ai.json cost_by_source[] entry" + ) + + +@pytest.mark.parametrize("fixture_path", POSITIVE_FIXTURES, ids=lambda p: p.parent.name) +def test_positive_fixtures_project_into_matching_estimation_source(fixture_path: Path) -> None: + generation = _load(fixture_path) + estimation = _matching_estimation(fixture_path.parent.name) + _assert_artifacts_by_source_projects_into_estimation(generation, estimation) + + +def test_untraceable_model_under_resolved_estimation_source_is_rejected() -> None: + """Mirrors the estimate-level untraceable-model negative one phase forward: an + artifacts_by_source[] entry naming a model not present in the matching + estimation-ai.json's cost_by_source[] is a fabricated/untraceable model.""" + generation = _base_generation() + estimation = _matching_estimation("multi-provider-direct") + generation["artifacts_by_source"][0]["models"][0]["model"] = "made-up-model" + with pytest.raises(AssertionError): + _assert_artifacts_by_source_projects_into_estimation(generation, estimation) + + +def test_dropped_model_under_a_resolved_gateway_upstream_is_rejected() -> None: + """Mirrors the estimate-level dropped-model negative one phase forward: a model removed + from the matching estimation-ai.json's cost_by_source[] entry is no longer traceable, so + an artifacts_by_source[] entry still naming it must be rejected.""" + generation = _load(FIXTURE_ROOT / "openrouter-gateway-multi-upstream" / "generation-ai.json") + estimation = copy.deepcopy(_matching_estimation("openrouter-gateway-multi-upstream")) + anthropic_entry = next(s for s in estimation["cost_by_source"] if s["provider"] == "anthropic") + anthropic_entry["models"] = [ + m for m in anthropic_entry["models"] if m["model"] != "claude-3-haiku-20240307" + ] + with pytest.raises(AssertionError): + _assert_artifacts_by_source_projects_into_estimation(generation, estimation) + + +# --------------------------------------------------------------------------- negative mutations + + +def test_duplicate_artifacts_by_source_identity_is_rejected() -> None: + generation = _base_generation() + generation["artifacts_by_source"].append(copy.deepcopy(generation["artifacts_by_source"][0])) + assert _generation_ai_findings(generation) != [] + + +def _assert_artifacts_by_source_identity_is_unique(generation: dict) -> None: + """uniqueItems on artifacts_by_source[] only rejects byte-identical duplicates; two + entries with the SAME (provider, source_service) pair but different payloads (an + overlapping identity, not an exact duplicate) need this semantic check instead, + mirroring the equivalent helper in test_estimation_ai_contracts.py.""" + seen = set() + for entry in generation["artifacts_by_source"]: + key = (entry["provider"], entry["source_service"]) + assert key not in seen, f"duplicate or overlapping artifacts_by_source identity: {key}" + seen.add(key) + + +def test_positive_fixtures_have_unique_artifacts_by_source_identity() -> None: + for fixture_path in POSITIVE_FIXTURES: + _assert_artifacts_by_source_identity_is_unique(_load(fixture_path)) + + +def test_overlapping_artifacts_by_source_identity_is_rejected() -> None: + """A second entry with the SAME (provider, source_service) pair but different generated + file content is still a duplicate/overlapping identity, not a legitimate second entry — + schema-level uniqueItems alone cannot catch this (see helper docstring), so the semantic + helper above is the lock.""" + generation = _base_generation() + overlapping = copy.deepcopy(generation["artifacts_by_source"][0]) + overlapping["models"][0]["generated_files"] = ["src/ai/other_client.py"] + generation["artifacts_by_source"].append(overlapping) + with pytest.raises(AssertionError): + _assert_artifacts_by_source_identity_is_unique(generation) + + +def test_openrouter_as_a_direct_artifacts_by_source_provider_is_rejected() -> None: + generation = _base_generation() + generation["artifacts_by_source"][0]["provider"] = "openrouter" + assert _generation_ai_findings(generation) != [] + + +def test_malformed_artifacts_by_source_entry_missing_models_is_rejected() -> None: + generation = _base_generation() + del generation["artifacts_by_source"][0]["models"] + assert _generation_ai_findings(generation) != [] + + +def test_rollback_plan_mechanism_must_be_feature_flag() -> None: + generation = _base_generation() + generation["rollback_plan"]["mechanism"] = "manual_cutover" + assert _generation_ai_findings(generation) != [] + + +def test_production_readiness_checklist_requires_at_least_five_items() -> None: + generation = _base_generation() + generation["production_readiness_checklist"] = generation["production_readiness_checklist"][:2] + assert _generation_ai_findings(generation) != [] + + +def test_generated_files_entry_requires_at_least_one_file() -> None: + generation = _base_generation() + generation["artifacts_by_source"][0]["models"][0]["generated_files"] = [] + assert _generation_ai_findings(generation) != [] + + +# --------------------------------------------------------------------------- vendored byte-identity + + +def test_vendored_copies_are_byte_identical() -> None: + """Reuses tools/sync-vendored.py --check (the dedicated CI gate) rather than + re-implementing filecmp logic inline, matching the project's existing pattern (and + test_estimation_ai_contracts.py's own choice to leave this check to the dedicated tool).""" + result = subprocess.run( + [sys.executable, str(SYNC_VENDORED_PATH), "--check"], + cwd=PLUGIN_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py index 20ce3c526..9ebb18a62 100644 --- a/plugins/aws-startup-advisor/tests/test_validate_artifacts.py +++ b/plugins/aws-startup-advisor/tests/test_validate_artifacts.py @@ -468,6 +468,53 @@ def test_neutral_fixture_skips_provider_artifacts_and_expected_snapshots(tmp_pat assert [p.name for p in va.find_artifacts(tmp_path)] == ["gcp-resource-inventory.json"] +@pytest.mark.parametrize("skill", ["gcp-to-aws", "azure-to-aws"]) +@pytest.mark.parametrize("artifact", ["estimation-ai.json", "generation-ai.json"]) +def test_target_only_ai_contracts_do_not_validate_current_skill_outputs( + tmp_path: Path, skill: str, artifact: str +): + """The canonical AI estimate/generate schemas are fixture-only until the producer + documents adopt their neutral fields and per-source traceability arrays.""" + source_prefix = "gcp" if skill == "gcp-to-aws" else "azure" + if artifact == "estimation-ai.json": + current = { + "phase": "estimate", + "timestamp": "2026-09-08T03:10:00Z", + "pricing_source": "cached", + "current_costs": { + "source": "billing", + f"{source_prefix}_monthly_ai_spend": 100.0, + "services": [], + }, + "cost_comparison": { + f"current_{source_prefix}_monthly": 100.0, + "projected_bedrock_monthly": 110.0, + "monthly_difference": 10.0, + "annual_difference": 120.0, + "percent_change": 10.0, + }, + } + else: + current = { + "phase": "generate", + "generation_source": "ai", + "timestamp": "2026-09-08T04:00:00Z", + "migration_plan": {"approach": "provider_adapter", "phases": [], "models_to_migrate": []}, + "step_by_step_guide": {}, + "rollback_plan": {"mechanism": "feature_flag", "flag_name": "AI_PROVIDER"}, + "monitoring": {}, + "production_readiness_checklist": [], + "success_criteria": {}, + "recommendation": {}, + } + + run = _write_run(tmp_path, f"{skill}-{artifact}", artifact, current) + findings = [] + manifest = va.load_manifest(PLUGIN_ROOT / "scripts" / "artifact-contracts.json") + assert va.validate_run_dir(run, skill, manifest, findings) == 0 + assert [(finding.code, finding.path) for finding in findings] == [("NO_CONTRACT", "")] + + def test_injected_off_contract_key_fails(tmp_path: Path): """The regression this tool exists for: a producer emits a key its contract never documented. Copy a golden run, plant the key, expect exit 1 naming the path."""