diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c8baee9d..78c703f2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,5 +1,11 @@ name: Benchmark +# Every scenario of scripts/compare_benchmarks.py (built-in rules only, built-in + +# Guard rule pack, built-in + custom Rego rule pack, and both packs) is a full +# engine × binding benchmark and always runs, in one job so every number in the +# report comes from the same runner. The per-template and aggregate reports stay +# on the runner; only the assembled comparison report is uploaded. + on: workflow_dispatch: inputs: @@ -33,6 +39,35 @@ concurrency: jobs: benchmark: runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v6 + + - name: List scenarios + id: plan + shell: bash + run: | + set -euo pipefail + # The matrix is the script's own scenario list so the workflow can never + # drift from the scenarios the script knows. + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json + import sys + + sys.path.insert(0, "scripts") + import compare_benchmarks as cb + + print(f"scenarios={json.dumps(cb.SCENARIO_IDS)}") + PY + + benchmark: + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + scenario: ${{ fromJSON(needs.plan.outputs.scenarios) }} steps: - uses: actions/checkout@v6 @@ -115,6 +150,7 @@ jobs: run: | set -euo pipefail + # Every scenario; the iteration count is applied to all of them. benchmark_command=(python3 ../scripts/compare_benchmarks.py) if [[ -n "$BENCHMARK_ITERATIONS" ]]; then benchmark_command+=(--iterations "$BENCHMARK_ITERATIONS") @@ -131,35 +167,22 @@ jobs: "${benchmark_command[@]}" - name: Validate aggregates - working-directory: ${{ env.WORKING_DIR }} shell: bash run: | set -euo pipefail - # Validate all 15 aggregate files (5 bindings × 3 engines): JSON parses, - # binding/engine labels match, and the enriched provenance/startup/memory - # contract produced by compare_benchmarks.py is present. + # Validate every aggregate the run must have produced (every scenario × binding + # × engine the scenario can run): JSON parses, scenario/binding/engine labels + # match, the rule pack and provenance metadata are present, and the + # startup/memory enrichment compare_benchmarks.py adds is in place. python3 - <<'PY' import json import sys from numbers import Real - EXPECTED = [ - ("native", "rego", "cfn-validate/reports/rego/aggregate_detailed.json"), - ("native", "cel", "cfn-validate/reports/cel/aggregate_detailed.json"), - ("native", "composite", "cfn-validate/reports/composite/aggregate_detailed.json"), - ("wasm", "rego", "bindings-wasm/reports/rego/aggregate_detailed.json"), - ("wasm", "cel", "bindings-wasm/reports/cel/aggregate_detailed.json"), - ("wasm", "composite", "bindings-wasm/reports/composite/aggregate_detailed.json"), - ("jvm", "rego", "bindings-jvm/reports/rego/aggregate_detailed.json"), - ("jvm", "cel", "bindings-jvm/reports/cel/aggregate_detailed.json"), - ("jvm", "composite", "bindings-jvm/reports/composite/aggregate_detailed.json"), - ("python", "rego", "bindings-python/reports/rego/aggregate_detailed.json"), - ("python", "cel", "bindings-python/reports/cel/aggregate_detailed.json"), - ("python", "composite", "bindings-python/reports/composite/aggregate_detailed.json"), - ("go", "rego", "bindings-go/reports/rego/aggregate_detailed.json"), - ("go", "cel", "bindings-go/reports/cel/aggregate_detailed.json"), - ("go", "composite", "bindings-go/reports/composite/aggregate_detailed.json"), - ] + sys.path.insert(0, "scripts") + import compare_benchmarks as cb + + expected = cb.expected_aggregate_files(cb.SCENARIOS, cb.ENGINES, cb.ALL_BINDINGS) def is_num(value): return isinstance(value, Real) and not isinstance(value, bool) @@ -173,17 +196,30 @@ jobs: return cur errors = [] - for exp_binding, exp_engine, fpath in EXPECTED: + for exp_scenario, exp_binding, exp_engine, fpath in expected: + scenario = cb.scenario_by_id(exp_scenario) try: with open(fpath) as handle: data = json.load(handle) except (OSError, json.JSONDecodeError) as exc: errors.append(f"{fpath}: cannot read/parse ({exc})") continue + if data.get("scenario") != exp_scenario: + errors.append(f"{fpath}: scenario={data.get('scenario')!r} expected {exp_scenario!r}") if data.get("binding") != exp_binding: errors.append(f"{fpath}: binding={data.get('binding')!r} expected {exp_binding!r}") if data.get("engine") != exp_engine: errors.append(f"{fpath}: engine={data.get('engine')!r} expected {exp_engine!r}") + if not isinstance(data.get("rules_fingerprint"), str) or not data.get("rules_fingerprint"): + errors.append(f"{fpath}: missing rules_fingerprint") + guard_files = dig(data, "custom_rules", "guard", "files") + rego_files = dig(data, "custom_rules", "rego", "files") + if not is_num(guard_files) or not is_num(rego_files): + errors.append(f"{fpath}: missing custom_rules.guard.files / custom_rules.rego.files") + elif scenario["guard"] and guard_files == 0: + errors.append(f"{fpath}: scenario loads the Guard pack but custom_rules.guard.files is 0") + elif scenario["rego"] and rego_files == 0: + errors.append(f"{fpath}: scenario loads the Rego pack but custom_rules.rego.files is 0") if not dig(data, "provenance", "cloudformation_validate"): errors.append(f"{fpath}: missing provenance.cloudformation_validate") if not is_num(dig(data, "performance", "startup", "consumer_init", "duration_ms")): @@ -203,8 +239,8 @@ jobs: for err in errors: print(f" - {err}") sys.exit(1) - print(f"All {len(EXPECTED)} aggregate files validated " - "(labels, provenance, startup, memory)") + print(f"All {len(expected)} aggregate files validated " + "(labels, rule pack, provenance, startup, memory)") PY - name: Upload comparison report diff --git a/.kiro/steering/structure.md b/.kiro/steering/structure.md index 19496cd4..3570d31c 100644 --- a/.kiro/steering/structure.md +++ b/.kiro/steering/structure.md @@ -73,7 +73,8 @@ src/ │ ├── public/ # Public example templates │ └── cdk/ # CDK-synthesized templates ├── expected/ # validation_reports*.json — numbered snapshot chunks (rego/cel/composite must agree) - ├── rules/ # Custom rule fixtures for testing (Rego, CEL, Guard) + ├── rules/ # Custom rule fixtures for testing (Rego, CEL, Guard); every .guard and .rego file + │ # here is also the benchmark's Guard / custom Rego rule pack (see resources/README.md) └── security/ # Security/stress fixtures (pathological conditions, deep nesting) ``` @@ -82,7 +83,7 @@ src/ - `.kiro/steering/` — persistent guidance recursively loaded by Kiro; tracked files contain shared project rules - `.kiro/steering/private/` — gitignored machine-local confidential agent context and skills; its filenames and contents must never be committed -- `scripts/` — Python comparison/audit scripts and their `snapshots/` data (see `tech.md` for usage) +- `scripts/` — Python comparison/audit/benchmark scripts and their `snapshots/` data (see `tech.md` for usage) - `.github/workflows/` — CI: format check, clippy, cargo audit, coverage tests on all supported OSes, JVM + WASM + Python + Go test jobs - `release-bin/` — prebuilt per-platform `cfn-validate` CLI binaries (committed); written by `cfn-validate/build.sh` diff --git a/.kiro/steering/tech.md b/.kiro/steering/tech.md index 0a2c59b1..fb527e2e 100644 --- a/.kiro/steering/tech.md +++ b/.kiro/steering/tech.md @@ -123,6 +123,12 @@ implementation and fix it there. cfn-lint checkout: `CFN_LINT_ROOT= python3 scripts/compare_cfnlint.py --engine rego|cel|composite`. First check whether cfn-lint is available on the machine (`cfn-lint --version`), then ask the user for the checkout path — never assume or hardcode one. +- `compare_benchmarks.py` — runs the benchmark harnesses of every binding (native `cfn-benchmark`, WASM, JVM, Python, + Go) for every scenario — `builtin` (built-in rules only), `guard` (every `.guard` file in `resources/rules` + loaded as a Guard rule pack), `rego` (every `.rego` file loaded as a custom Rego pack; Rego and composite engines + only), and `all` (both) — and writes the single `scripts/snapshots/benchmark_comparison.md` with a cross-scenario + rule-pack-cost summary plus the full engine × binding comparison per scenario. Needs GNU `/usr/bin/time` (or + `CFN_BENCHMARK_TIME_BIN`). The `benchmark` workflow runs every scenario in one job and uploads only that report. - `audit_rule_categorization.py` — audits rule registry for categorization consistency - `generate_licenses.py` — generates third-party license files diff --git a/scripts/compare_benchmarks.py b/scripts/compare_benchmarks.py index 1cc24512..36f5262d 100755 --- a/scripts/compare_benchmarks.py +++ b/scripts/compare_benchmarks.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 -"""Runs benchmarks for every engine × binding and writes a comparison report. +"""Runs benchmarks for every scenario × engine × binding and writes comparison reports. + +A *scenario* fixes the rules the engine evaluates: ``builtin`` measures the +built-in rules alone, ``guard`` layers every Guard rule file of +``src/resources/rules`` on top, ``rego`` layers every custom Rego rule file of +that directory (the CEL engine cannot evaluate Rego, so that scenario skips +CEL), and ``all`` layers both packs. The single report +(``scripts/snapshots/benchmark_comparison.md``) opens with a cross-scenario +summary of what each rule pack costs per engine and binding, then holds the +full engine × binding comparison for every scenario. The native benchmark builds ``cfn-benchmark`` from the workspace. The WASM, JVM, Python, and Go benchmarks consume the committed distribution artifacts that the @@ -7,6 +16,15 @@ Python wheels, and the Go module's static libraries), so they measure exactly what consumers install; only each binding's benchmark harness is built here. +Every harness accepts the same command line: ``[TEMPLATE|DIR] --engine E +--iterations N`` for a corpus run, ``--engine E --startup-probe`` for a startup +probe, and for a non-default scenario ``--scenario NAME`` plus +``--guard-rules DIR`` and/or ``--rego-rules DIR`` naming the rules directory +(a harness loads every ``.guard`` / ``.rego`` file below a directory argument). +Its aggregate and per-template reports carry ``scenario``, ``custom_rules``, and +``rules_fingerprint`` so this script can prove every binding measured the same +rules. + Subsequent distributions are per-template medians of iterations 2..N; throughput divides all timed ``validate()`` calls by the measured wall time. """ @@ -30,6 +48,47 @@ ENGINES = ["rego", "cel", "composite"] FORMATS = ["detailed"] + +# Every .guard file here is the Guard rule pack and every .rego file the custom Rego +# rule pack; harnesses receive the directory rather than a file list. +RULES_DIR = SRC_DIR / "resources" / "rules" + +# Recorded by a harness when no --scenario is given; its reports keep the historical +# reports// layout that other tooling reads. +DEFAULT_SCENARIO = "builtin" + +# The CEL engine cannot evaluate Rego, so Rego scenarios run Rego and composite only. +SCENARIOS = [ + { + "id": DEFAULT_SCENARIO, + "label": "Built-in rules only", + "engines": ENGINES, + "guard": False, + "rego": False, + }, + { + "id": "guard", + "label": "Built-in rules + Guard rule pack", + "engines": ENGINES, + "guard": True, + "rego": False, + }, + { + "id": "rego", + "label": "Built-in rules + custom Rego rule pack", + "engines": ["rego", "composite"], + "guard": False, + "rego": True, + }, + { + "id": "all", + "label": "Built-in rules + Guard rule pack + custom Rego rule pack", + "engines": ["rego", "composite"], + "guard": True, + "rego": True, + }, +] +SCENARIO_IDS = [scenario["id"] for scenario in SCENARIOS] ALL_BINDINGS = [ ("native", "Native Rust"), ("wasm", "WASM (Node.js)"), @@ -77,8 +136,9 @@ # External process timer used to measure startup and full-corpus memory. The # GNU coreutils build ("-v") and the macOS build ("-l") report different -# formats and different RSS units, handled by the two parsers below. -TIME_BIN = "/usr/bin/time" +# formats and different RSS units, handled by the two parsers below. The +# environment variable points at a GNU time built elsewhere on hosts without it. +TIME_BIN = os.environ.get("CFN_BENCHMARK_TIME_BIN", "/usr/bin/time") NATIVE_BENCH_BIN = SRC_DIR / "target" / "release" / "cfn-benchmark" WASM_BENCH_DIR = SRC_DIR / "bindings-wasm" / "bench" @@ -116,7 +176,7 @@ def parse_args(argv=None): parser = argparse.ArgumentParser( - description="Run benchmarks for every engine × binding and write a comparison report.", + description="Run benchmarks for every scenario × engine × binding and write comparison reports.", ) parser.add_argument( "--skip-build", @@ -129,6 +189,13 @@ def parse_args(argv=None): action="store_true", help="Generate report from existing aggregate files without running benchmarks.", ) + parser.add_argument( + "--scenarios", + nargs="+", + choices=SCENARIO_IDS, + default=None, + help="Subset of scenarios to benchmark, in canonical order (default: all).", + ) parser.add_argument( "--iterations", type=int, @@ -184,82 +251,144 @@ def parse_args(argv=None): return args -def build_run_plan(engines, bindings): - """Pair selected engines per binding and alternate canonical AB/BA order. +def scenario_by_id(scenario_id): + for scenario in SCENARIOS: + if scenario["id"] == scenario_id: + return scenario + raise ValueError(f"unknown scenario: {scenario_id}") + + +def select_scenarios(scenario_ids): + if not scenario_ids: + return list(SCENARIOS) + selected = set(scenario_ids) + return [scenario for scenario in SCENARIOS if scenario["id"] in selected] + + +def scenario_engines(scenario, engines): + return [engine for engine in engines if engine in scenario["engines"]] + + +def guard_pack_files(): + return sorted(RULES_DIR.glob("*.guard")) + + +def rego_pack_files(): + return sorted(RULES_DIR.glob("*.rego")) + + +def scenario_rule_files(scenario): + return ( + guard_pack_files() if scenario["guard"] else [], + rego_pack_files() if scenario["rego"] else [], + ) + + +def scenario_harness_args(scenario): + """The default scenario adds nothing so its invocation stays byte-for-byte the + historical one.""" + if scenario["id"] == DEFAULT_SCENARIO: + return [] + guard_files, rego_files = scenario_rule_files(scenario) + if scenario["guard"] and not guard_files: + sys.exit(f"scenario {scenario['id']}: no .guard files found in {RULES_DIR}") + if scenario["rego"] and not rego_files: + sys.exit(f"scenario {scenario['id']}: no .rego files found in {RULES_DIR}") + extra = ["--scenario", scenario["id"]] + if scenario["guard"]: + extra += ["--guard-rules", str(RULES_DIR)] + if scenario["rego"]: + extra += ["--rego-rules", str(RULES_DIR)] + return extra + + +def build_run_plan(scenarios, engines, bindings): + """Order runs scenario by scenario; within a scenario pair the engines it can + run per binding and alternate canonical AB/BA order. Alternation is based on each binding's position in ``ALL_BINDINGS``, not its position in a filtered subset. Thus WASM remains BA even when it is the only selected binding, and repeated subset runs retain the same positional bias mitigation as the full run. """ - if not engines or not bindings: + if not scenarios or not engines or not bindings: return [] canonical_positions = {binding: index for index, (binding, _) in enumerate(ALL_BINDINGS)} plan = [] - for selected_index, (binding, _label) in enumerate(bindings): - position = canonical_positions.get(binding, selected_index) - ordered_engines = engines if position % 2 == 0 else reversed(engines) - plan.extend((binding, engine) for engine in ordered_engines) + for scenario in scenarios: + runnable = scenario_engines(scenario, engines) + for selected_index, (binding, _label) in enumerate(bindings): + position = canonical_positions.get(binding, selected_index) + ordered_engines = runnable if position % 2 == 0 else list(reversed(runnable)) + plan.extend((scenario["id"], binding, engine) for engine in ordered_engines) return plan -def corpus_command(binding, engine, iterations, template_dir): +def corpus_command(binding, engine, iterations, template_dir, scenario=None): """The native binary keeps ``--format detailed`` so its invocation stays identical to the one ``compare_cfnlint.py`` relies on. The FFI harnesses hardcode DETAILED - and reject ``--format``, so it is passed to native only. + and reject ``--format``, so it is passed to native only. A non-default scenario + appends the same ``--scenario``/rule-file flags to every harness. """ template = str(template_dir) + scenario_args = scenario_harness_args(scenario) if scenario else [] if binding == "native": return ( [str(NATIVE_BENCH_BIN), template, "--engine", engine, - "--format", "detailed", "--iterations", str(iterations)], + "--format", "detailed", "--iterations", str(iterations)] + scenario_args, SRC_DIR, ) if binding == "wasm": return ( ["node", str(WASM_BENCH_JS), template, "--engine", engine, - "--iterations", str(iterations)], + "--iterations", str(iterations)] + scenario_args, WASM_BENCH_DIR, ) if binding == "jvm": return ( [str(JVM_BENCH_BIN), template, "--engine", engine, - "--iterations", str(iterations)], + "--iterations", str(iterations)] + scenario_args, JVM_BENCH_DIR, ) if binding == "python": return ( [str(PYTHON_VENV_PYTHON), str(PYTHON_BENCH_SCRIPT), template, "--engine", engine, - "--iterations", str(iterations)], + "--iterations", str(iterations)] + scenario_args, PYTHON_BENCH_DIR, ) if binding == "go": return ( [str(GO_BENCH_BIN), template, "--engine", engine, - "--iterations", str(iterations)], + "--iterations", str(iterations)] + scenario_args, GO_BENCH_DIR, ) raise ValueError(f"unknown binding: {binding}") -def probe_command(binding, engine): +def probe_command(binding, engine, scenario=None): + scenario_args = scenario_harness_args(scenario) if scenario else [] if binding == "native": - return ([str(NATIVE_BENCH_BIN), "--engine", engine, "--startup-probe"], SRC_DIR) + return ([str(NATIVE_BENCH_BIN), "--engine", engine, "--startup-probe"] + scenario_args, SRC_DIR) if binding == "wasm": - return (["node", str(WASM_BENCH_JS), "--engine", engine, "--startup-probe"], WASM_BENCH_DIR) + return (["node", str(WASM_BENCH_JS), "--engine", engine, "--startup-probe"] + scenario_args, WASM_BENCH_DIR) if binding == "jvm": - return ([str(JVM_BENCH_BIN), "--engine", engine, "--startup-probe"], JVM_BENCH_DIR) + return ([str(JVM_BENCH_BIN), "--engine", engine, "--startup-probe"] + scenario_args, JVM_BENCH_DIR) if binding == "python": return ( - [str(PYTHON_VENV_PYTHON), str(PYTHON_BENCH_SCRIPT), "--engine", engine, "--startup-probe"], + [str(PYTHON_VENV_PYTHON), str(PYTHON_BENCH_SCRIPT), "--engine", engine, "--startup-probe"] + + scenario_args, PYTHON_BENCH_DIR, ) if binding == "go": - return ([str(GO_BENCH_BIN), "--engine", engine, "--startup-probe"], GO_BENCH_DIR) + return ([str(GO_BENCH_BIN), "--engine", engine, "--startup-probe"] + scenario_args, GO_BENCH_DIR) raise ValueError(f"unknown binding: {binding}") +def display_command(cmd): + return " ".join(str(c) for c in cmd) + + def executable_path(binding): return { "native": NATIVE_BENCH_BIN, @@ -271,7 +400,7 @@ def executable_path(binding): def run_cmd(cmd, cwd, label): - print(f" $ {' '.join(str(c) for c in cmd)}", file=sys.stderr) + print(f" $ {display_command(cmd)}", file=sys.stderr) result = subprocess.run([str(c) for c in cmd], cwd=str(cwd)) if result.returncode != 0: sys.exit(f"{label} failed (exit {result.returncode})") @@ -494,54 +623,65 @@ def run_with_time(cmd, cwd, env, flavor): return proc, wall_ms, rss_bytes -def _parse_probe_json(stdout, binding, engine): +def _parse_probe_json(stdout, binding, engine, scenario_id=DEFAULT_SCENARIO): lines = [ln for ln in stdout.splitlines() if ln.strip()] if not lines: - sys.exit(f"startup probe for {binding}/{engine} produced no JSON on stdout") + sys.exit(f"startup probe for {scenario_id}/{binding}/{engine} produced no JSON on stdout") try: data = json.loads(lines[-1]) except json.JSONDecodeError as exc: - sys.exit(f"startup probe for {binding}/{engine} emitted invalid JSON: {exc}") + sys.exit(f"startup probe for {scenario_id}/{binding}/{engine} emitted invalid JSON: {exc}") if not isinstance(data, dict): - sys.exit(f"startup probe for {binding}/{engine} JSON is not an object") + sys.exit(f"startup probe for {scenario_id}/{binding}/{engine} JSON is not an object") probe_binding = data.get("binding") if probe_binding is not None and probe_binding != binding: sys.exit(f"startup probe binding mismatch: expected '{binding}', got {probe_binding!r}") probe_engine = data.get("engine") if probe_engine is not None and probe_engine != engine: sys.exit(f"startup probe engine mismatch: expected '{engine}', got {probe_engine!r}") + # A rule-pack scenario must prove the harness understood the flags; only the + # default scenario tolerates a probe without the label. + probe_scenario = data.get("scenario") + if probe_scenario != scenario_id and not (probe_scenario is None and scenario_id == DEFAULT_SCENARIO): + sys.exit(f"startup probe scenario mismatch: expected '{scenario_id}', got {probe_scenario!r}") return data -def run_startup_probes(binding, engine, samples, env, flavor): - cmd, cwd = probe_command(binding, engine) - print(f"=== {binding} startup probe (engine={engine}, samples={samples}) ===", file=sys.stderr) +def run_startup_probes(binding, engine, samples, env, flavor, scenario): + cmd, cwd = probe_command(binding, engine, scenario) + print( + f"=== {scenario['id']}/{binding} startup probe (engine={engine}, samples={samples}) ===", + file=sys.stderr, + ) collected = [] for index in range(samples): proc, wall_ms, rss_bytes = run_with_time(cmd, cwd, env, flavor) if proc.returncode != 0: sys.exit( - f"startup probe failed for {binding}/{engine} " + f"startup probe failed for {scenario['id']}/{binding}/{engine} " f"(sample {index + 1}/{samples}, exit {proc.returncode}):\n{proc.stderr.strip()}" ) if wall_ms is None or rss_bytes is None: - sys.exit(f"could not parse /usr/bin/time output for {binding}/{engine} startup probe") - data = _parse_probe_json(proc.stdout, binding, engine) + sys.exit(f"could not parse {TIME_BIN} output for {scenario['id']}/{binding}/{engine} startup probe") + data = _parse_probe_json(proc.stdout, binding, engine, scenario["id"]) collected.append({"json": data, "wall_ms": wall_ms, "rss_bytes": rss_bytes}) return collected -def run_corpus(binding, engine, iterations, template_dir, env, flavor): - cmd, cwd = corpus_command(binding, engine, iterations, template_dir) - print(f"=== {binding} corpus benchmark (engine={engine}) ===", file=sys.stderr) - print(f" $ {' '.join(cmd)}", file=sys.stderr) +def run_corpus(binding, engine, iterations, template_dir, env, flavor, scenario): + cmd, cwd = corpus_command(binding, engine, iterations, template_dir, scenario) + print(f"=== {scenario['id']}/{binding} corpus benchmark (engine={engine}) ===", file=sys.stderr) + print(f" $ {display_command(cmd)}", file=sys.stderr) proc, wall_ms, rss_bytes = run_with_time(cmd, cwd, env, flavor) if proc.stderr: sys.stderr.write(proc.stderr) if proc.returncode != 0: - sys.exit(f"{binding} corpus benchmark failed (engine={engine}, exit {proc.returncode})") + sys.exit( + f"{binding} corpus benchmark failed (scenario={scenario['id']}, engine={engine}, " + f"exit {proc.returncode})" + ) if rss_bytes is None: - sys.exit(f"could not parse /usr/bin/time RSS for {binding}/{engine} corpus run") + sys.exit(f"could not parse {TIME_BIN} RSS for {scenario['id']}/{binding}/{engine} corpus run") return wall_ms, rss_bytes @@ -634,10 +774,27 @@ def enrich_aggregate(path, process_startup, corpus_rss_bytes): os.replace(str(tmp_path), str(path)) -def aggregate_path(engine, fmt, binding): - if binding == "native": - return SRC_DIR / "cfn-validate" / "reports" / engine / f"aggregate_{fmt}.json" - return SRC_DIR / f"bindings-{binding}" / "reports" / engine / f"aggregate_{fmt}.json" +def reports_dir(engine, binding, scenario_id=DEFAULT_SCENARIO): + crate_dir = SRC_DIR / ("cfn-validate" if binding == "native" else f"bindings-{binding}") + if scenario_id == DEFAULT_SCENARIO: + return crate_dir / "reports" / engine + return crate_dir / "reports" / "scenarios" / scenario_id / engine + + +def aggregate_path(engine, fmt, binding, scenario_id=DEFAULT_SCENARIO): + return reports_dir(engine, binding, scenario_id) / f"aggregate_{fmt}.json" + + +def expected_aggregate_files(scenarios, engines, bindings): + """``(scenario_id, binding, engine, path)`` per run; the workflow validates exactly this list.""" + expected = [] + for scenario in scenarios: + for binding, _label in bindings: + for engine in scenario_engines(scenario, engines): + expected.append( + (scenario["id"], binding, engine, aggregate_path(engine, FORMATS[0], binding, scenario["id"])) + ) + return expected def _is_finite_number(val): @@ -679,6 +836,24 @@ def _validate_aggregate_structure(data, path): if not isinstance(fp, str) or not fp: sys.exit(f"aggregate {path}: missing or empty 'corpus_fingerprint'") + scenario = data.get("scenario") + if not isinstance(scenario, str) or scenario not in SCENARIO_IDS: + sys.exit(f"aggregate {path}: 'scenario' must be one of {SCENARIO_IDS} (got {scenario!r})") + rules_fp = data.get("rules_fingerprint") + if not isinstance(rules_fp, str) or not rules_fp: + sys.exit(f"aggregate {path}: missing or empty 'rules_fingerprint'") + custom_rules = data.get("custom_rules") + if not isinstance(custom_rules, dict): + sys.exit(f"aggregate {path}: missing 'custom_rules' object") + for kind, fields in (("guard", ("files", "rules", "bytes")), ("rego", ("files", "bytes"))): + section = custom_rules.get(kind) + if not isinstance(section, dict): + sys.exit(f"aggregate {path}: missing 'custom_rules.{kind}' object") + for field in fields: + val = section.get(field) + if isinstance(val, bool) or not isinstance(val, int) or val < 0: + sys.exit(f"aggregate {path}: custom_rules.{kind}.{field} must be a non-negative integer (got {val!r})") + provenance = data.get("provenance") if not isinstance(provenance, dict): sys.exit(f"aggregate {path}: missing 'provenance' object") @@ -715,7 +890,7 @@ def _validate_aggregate_structure(data, path): sys.exit(f"aggregate {path}: memory.full_corpus_peak_rss_bytes is not a finite number") -def load_aggregate(path, run_start_epoch): +def load_aggregate(path, run_start_epoch, scenario_id=DEFAULT_SCENARIO): if not path.exists(): sys.exit(f"expected aggregate not found: {path}") if run_start_epoch > 0: @@ -725,28 +900,36 @@ def load_aggregate(path, run_start_epoch): with open(path) as f: data = json.load(f) _validate_aggregate_structure(data, path) + if data.get("scenario") != scenario_id: + sys.exit(f"aggregate {path}: scenario={data.get('scenario')!r} but it was loaded for scenario {scenario_id!r}") return data -def enforce_corpus_parity(all_loaded, bindings): - """Every binding of every engine must have scanned the same bytes.""" - fps = {} - for engine, by_binding in all_loaded.items(): - for binding, agg in by_binding.items(): - fp = agg.get("corpus_fingerprint") - if not fp: - sys.exit(f"{engine}/{binding}: aggregate missing corpus_fingerprint. " - f"Rebuild + rerun benchmarks against current harness.") - fps.setdefault(fp, []).append(f"{engine}/{binding}") - if len(fps) > 1: - lines = [f" {fp}: {', '.join(who)}" for fp, who in fps.items()] - sys.exit("corpus fingerprint mismatch across bindings - cannot compare:\n" - + "\n".join(lines)) - - -def enforce_run_metadata_parity(all_loaded, bindings): - """Every selected run must agree on iteration count, detail level, corpus totals, - and failure lists.""" +def enforce_corpus_parity(all_loaded, bindings, scenario_id=DEFAULT_SCENARIO): + """Every binding of every engine must have scanned the same template bytes and + loaded the same rule files.""" + for field, what in (("corpus_fingerprint", "corpus"), ("rules_fingerprint", "rules")): + fps = {} + for engine, by_binding in all_loaded.items(): + for binding, agg in by_binding.items(): + fp = agg.get(field) + if not fp: + sys.exit(f"{scenario_id}/{engine}/{binding}: aggregate missing {field}. " + f"Rebuild + rerun benchmarks against current harness.") + fps.setdefault(fp, []).append(f"{engine}/{binding}") + if len(fps) > 1: + lines = [f" {fp}: {', '.join(who)}" for fp, who in fps.items()] + sys.exit(f"{what} fingerprint mismatch across bindings in scenario {scenario_id} - cannot compare:\n" + + "\n".join(lines)) + + +def failure_set(agg): + return sorted((f.get("file"), f.get("status")) for f in (agg.get("failures") or [])) + + +def enforce_run_metadata_parity(all_loaded, bindings, scenario_id=DEFAULT_SCENARIO): + """Every selected run of a scenario must agree on iteration count, detail level, + corpus totals, the rule pack it loaded, and failure lists.""" reference_key = None reference_meta = None for engine, by_binding in all_loaded.items(): @@ -755,14 +938,15 @@ def enforce_run_metadata_parity(all_loaded, bindings): "iterations_per_template": agg.get("iterations_per_template"), "detail_level": agg.get("detail_level"), "corpus_fingerprint": agg.get("corpus_fingerprint"), + "scenario": agg.get("scenario"), + "rules_fingerprint": agg.get("rules_fingerprint"), + "custom_rules": agg.get("custom_rules"), "templates_total": agg.get("templates_total"), "templates_ok": agg.get("templates_ok"), "templates_failed": agg.get("templates_failed"), - "failures": sorted( - [(f.get("file"), f.get("status")) for f in (agg.get("failures") or [])], - ), + "failures": failure_set(agg), } - key = f"{engine}/{binding}" + key = f"{scenario_id}/{engine}/{binding}" if reference_meta is None: reference_meta = meta reference_key = key @@ -780,18 +964,40 @@ def enforce_run_metadata_parity(all_loaded, bindings): ) -def _per_template_dir(engine, binding): - if binding == "native": - return SRC_DIR / "cfn-validate" / "reports" / engine / "json_detailed" - return SRC_DIR / f"bindings-{binding}" / "reports" / engine / "json_detailed" +def scenario_failure_differences(loaded_by_scenario): + """Run-metadata parity already guarantees every binding of a scenario agrees, so + one binding's aggregate per engine is representative. Templates a rule pack + cannot evaluate (a Guard type block against an empty ``Resources`` section) + drop out of that scenario's timings, so the report lists them.""" + baseline = loaded_by_scenario.get(DEFAULT_SCENARIO) + if not baseline: + return {} + reference = set(failure_set(next(iter(next(iter(baseline.values())).values())))) + differences = {} + for scenario_id, by_engine in loaded_by_scenario.items(): + if scenario_id == DEFAULT_SCENARIO: + continue + for engine, by_binding in by_engine.items(): + failures = set(failure_set(next(iter(by_binding.values())))) + introduced = sorted(failures - reference) + removed = sorted(reference - failures) + if introduced or removed: + differences.setdefault(scenario_id, {})[engine] = {"introduced": introduced, "removed": removed} + return differences + +def _per_template_dir(engine, binding, scenario_id=DEFAULT_SCENARIO): + return reports_dir(engine, binding, scenario_id) / "json_detailed" -def load_and_validate_detailed_reports(engines, bindings): - """Load per-template detailed-level JSON reports for all engine×binding pairs. + +def load_and_validate_detailed_reports(engines, bindings, scenario_id=DEFAULT_SCENARIO): + """Load per-template detailed-level JSON reports for all engine×binding pairs of + one scenario. Each report is loaded exactly once and indexed by filePath. Validation rules: 1. Directory must exist and be nonempty. - 2. Root must be a JSON object with engine/binding labels matching the expected pair. + 2. Root must be a JSON object with engine/binding labels matching the expected pair + and, for a rule-pack scenario, the scenario label. 3. filePath must be a nonempty string, unique within each engine×binding directory. 4. benchmarkMetrics.subsequent is canonical: sampleCount is a non-negative integer; zero requires all REQUIRED_SUBSEQUENT_METRICS null, positive requires them finite. @@ -806,8 +1012,8 @@ def load_and_validate_detailed_reports(engines, bindings): for engine in engines: all_detailed[engine] = {} for binding, label in bindings: - d = _per_template_dir(engine, binding) - key = f"{engine}/{label}" + d = _per_template_dir(engine, binding, scenario_id) + key = f"{scenario_id}/{engine}/{label}" # Rule 1: nonempty directory if not d.exists() or not d.is_dir(): @@ -845,6 +1051,11 @@ def load_and_validate_detailed_reports(engines, bindings): errors.append( f"{key}: {json_file.name} binding='{file_binding}' expected '{binding}'" ) + file_scenario = data.get("scenario") + if file_scenario != scenario_id and not (file_scenario is None and scenario_id == DEFAULT_SCENARIO): + errors.append( + f"{key}: {json_file.name} scenario={file_scenario!r} expected '{scenario_id}'" + ) # Rule 3: unique nonempty string filePath file_path = data.get("filePath") @@ -1499,7 +1710,7 @@ def _field_diff(a, b): return {k: (a.get(k, ""), b.get(k, "")) for k in keys if a.get(k) != b.get(k)} -def diagnostics_parity(all_loaded, engine, bindings, all_detailed=None): +def diagnostics_parity(all_loaded, engine, bindings, all_detailed=None, scenario_id=DEFAULT_SCENARIO): """Full parity check across all binding pairs. If all_detailed is provided (already loaded per-template reports keyed by @@ -1571,7 +1782,7 @@ def diagnostics_parity(all_loaded, engine, bindings, all_detailed=None): examples)) else: # Fallback: read from disk - dirs = {b: _per_template_dir(engine, b) for b, _ in bindings} + dirs = {b: _per_template_dir(engine, b, scenario_id) for b, _ in bindings} missing_dirs = [] for b, lbl in bindings: @@ -1703,16 +1914,45 @@ def diagnostics_parity(all_loaded, engine, bindings, all_detailed=None): return lines, False -def data_sources_section(all_loaded, engines, bindings): +def data_sources_section(all_loaded, engines, bindings, scenario_id=DEFAULT_SCENARIO): lines = ["## Data Sources", ""] for engine in engines: for b, lbl in bindings: - p = aggregate_path(engine, FORMATS[0], b) + p = aggregate_path(engine, FORMATS[0], b, scenario_id) lines.append(f"- {engine}/{lbl}: `{p.relative_to(PROJECT_ROOT)}`") lines.append("") return lines +def rule_pack_section(scenario, agg): + """Counts come from the aggregate so they describe what was measured, not what + this checkout would load.""" + rules = agg.get("custom_rules") or {} + guard = rules.get("guard") or {} + rego = rules.get("rego") or {} + lines = [ + "## Rule Pack", "", + f"Scenario `{scenario['id']}` - {scenario['label']}.", "", + f"- **Guard rules**: {guard.get('files', 0)} file(s), {guard.get('rules', 0)} distinct rule name(s), " + f"{fmt_bytes(guard.get('bytes', 0))}", + f"- **Custom Rego rules**: {rego.get('files', 0)} file(s), {fmt_bytes(rego.get('bytes', 0))}", + f"- **rules fingerprint**: `{agg.get('rules_fingerprint', 'unknown')}` (identical across every binding " + "of this scenario, checked before comparing)", + "", + ] + guard_files, rego_files = scenario_rule_files(scenario) + if guard_files or rego_files: + lines += ["Pack files (from `src/resources/rules/`):", ""] + lines += [f"- `{path.name}`" for path in guard_files + rego_files] + lines.append("") + if scenario["rego"]: + lines += [ + "The CEL engine cannot evaluate Rego, so this scenario compares the Rego and composite " + "engines only.", "", + ] + return lines + + def host_metadata(): return { "os": f"{platform.system()} {platform.release()}", @@ -1721,46 +1961,40 @@ def host_metadata(): } -def run_all_benchmarks(engines, bindings, args, flavor): +def run_all_benchmarks(scenarios, engines, bindings, args, flavor): env = benchmark_env() - plan = build_run_plan(engines, bindings) - for binding, engine in plan: + plan = build_run_plan(scenarios, engines, bindings) + for scenario_id, binding, engine in plan: + scenario = scenario_by_id(scenario_id) probes = run_startup_probes( - binding, engine, args.startup_samples, env, flavor + binding, engine, args.startup_samples, env, flavor, scenario ) _corpus_wall_ms, corpus_rss_bytes = run_corpus( - binding, engine, args.iterations, args.template_dir, env, flavor + binding, engine, args.iterations, args.template_dir, env, flavor, scenario ) process_startup = aggregate_process_startup(probes) - enrich_aggregate(aggregate_path(engine, FORMATS[0], binding), process_startup, corpus_rss_bytes) - - -def build_report(all_loaded, all_detailed, engines, bindings, args, corpus_fp, corpus_file_count): - host = host_metadata() - lines = [ - "# Benchmark Comparison", - "", - f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}", - "", - "## Host", "", - *[f"- **{k}**: {v}" for k, v in host.items()], - f"- **iterations/template**: {args.iterations}", - f"- **startup samples/binding**: {args.startup_samples} (1 cold + " - f"{args.startup_samples - 1} warm)", - f"- **corpus fingerprint**: `{corpus_fp}` ({corpus_file_count} files)", - f"- **bindings**: {', '.join(lbl for _, lbl in bindings)} ({len(bindings)} total)", - f"- **engines**: {', '.join(e.upper() for e in engines)}", - "", - ] + enrich_aggregate( + aggregate_path(engine, FORMATS[0], binding, scenario_id), process_startup, corpus_rss_bytes + ) - lines += provenance_section(all_loaded, engines, bindings) - lines += [ +def methodology_section(): + return [ "## Methodology Notes", "", + "### Scenarios - rule packs loaded into the engine", "", + "Every scenario is a complete engine × binding run over the same corpus with a different rule " + f"set loaded into the engine. `{DEFAULT_SCENARIO}` evaluates the built-in rules alone; the other " + "scenarios load the Guard rule pack and/or the custom Rego rule pack of `src/resources/rules` on " + "top (the CEL engine cannot evaluate Rego, so Rego scenarios compare the Rego and composite " + "engines). Each harness records the scenario label, the rule pack it loaded, and a fingerprint of " + "the rule files; every binding of a scenario must report the same fingerprint before it is " + "compared. The cross-scenario table reads the rule-evaluation medians as the most robust " + "measure of a pack's cost because they exclude process startup and host I/O.", "", "### Process startup (cold vs warm) - externally measured", "", "Startup is measured by launching independent OS processes of each binding's " "benchmark harness in `--startup-probe` mode, each wrapped with `/usr/bin/time`. " - "A probe constructs the real consumer validation setup (schema validator + engine) " + "A probe constructs the real consumer validation setup (schema validator + engine, " + "including the scenario's rule pack) " "and performs the first `validate()` call on a single small template, printing a " "JSON object with the in-process init and first-validation timings. `/usr/bin/time` " "reports that process's external wall time and peak RSS (GNU `-v` reports RSS in " @@ -1810,53 +2044,205 @@ def build_report(all_loaded, all_detailed, engines, bindings, args, corpus_fp, c "are more useful than cross-run absolute numbers. The corpus run pairs engines per " "binding and alternates run order (AB/BA) across bindings to distribute warm-up and " "load drift; results should be read as directional indicators, not precise " - "measurements.", "", + "measurements. Scenario runs may execute as separate CI jobs on different runners, so " + "cross-scenario ratios are directional as well.", "", ] - # Table of contents - engine_anchors = [f"- [{e.upper()} Engine](#{e}-engine)" for e in engines] - toc_items = [ - "- [Provenance](#provenance)", - "- [Methodology Notes](#methodology-notes)", - "- [Latency & Memory Summary](#latency--memory-summary)", - *engine_anchors, + +def _ratio(value, base): + if not _is_finite_number(value) or not _is_finite_number(base) or base <= 0: + return "-" + return f"{float(value) / float(base):.2f}×" + + +def scenario_overview_section(loaded_by_scenario, scenarios, engines): + lines = ["## Scenarios", ""] + header = ["Scenario", "Description", "Engines", "Guard files", "Guard rules", "Rego files", "Rules fingerprint"] + rows = [] + for scenario in scenarios: + by_engine = loaded_by_scenario.get(scenario["id"], {}) + sample = next((agg for by_binding in by_engine.values() for agg in by_binding.values()), None) + rules = (sample or {}).get("custom_rules") or {} + rows.append([ + f"[`{scenario['id']}`](#{scenario_anchor(scenario)})", scenario["label"], + ", ".join(e.upper() for e in scenario_engines(scenario, engines)), + str(get(rules, "guard", "files", default="-")), str(get(rules, "guard", "rules", default="-")), + str(get(rules, "rego", "files", default="-")), + f"`{(sample or {}).get('rules_fingerprint', '-')[:16]}…`" if sample else "-", + ]) + return lines + table(header, rows) + [""] + + +def rule_pack_cost_section(loaded_by_scenario, scenarios, engines, bindings): + lines = [ + "## Rule Pack Cost per Engine × Binding", "", + "Columns: engine init and first validation from the cold startup probe; rule evaluation and " + "wall clock are the subsequent per-template medians (iterations 2..N) with p99 in parentheses; " + "throughput is ok × iterations / measured validation wall time; RSS is the corpus process peak. " + f"Ratios (×) are relative to the `{DEFAULT_SCENARIO}` scenario of the same engine and binding. " + "Templates that fail to evaluate under a rule pack are excluded from its timings (see the " + "failure list below the table when there are any).", "", ] - toc_items.append( - f"- [Top-{args.top_slowest} Slowest Templates](#top-{args.top_slowest}-slowest-templates-subsequent-wall-clock)" - ) - if "rego" in engines and "cel" in engines: - toc_items.append( - "- [Paired Engine Comparison](#paired-engine-comparison)" - ) - toc_items.append("- [Data Sources](#data-sources)") + header = ["Scenario", "Templates ok", "Engine init (ms)", "First validation (ms)", + "Rule eval median (p99) ms", "Rule eval ×", "Wall median (p99) ms", "Wall ×", + "Throughput (val/s)", "Corpus RSS", "Diagnostics (F/E/W/I)"] + baseline = loaded_by_scenario.get(DEFAULT_SCENARIO, {}) + for engine in engines: + for binding, label in bindings: + rows = [] + base = get(baseline, engine, binding) + for scenario in scenarios: + agg = get(loaded_by_scenario, scenario["id"], engine, binding) + if agg is None: + continue + rule_eval = get(agg, "performance", "rule_evaluation_ms", default={}) + wall = get(agg, "performance", "subsequent_wall_clock_ms", default={}) + base_rule = get(base, "performance", "rule_evaluation_ms", "median") if base else None + base_wall = get(base, "performance", "subsequent_wall_clock_ms", "median") if base else None + diags = agg.get("diagnostics") or {} + rows.append([ + f"`{scenario['id']}`", + str(agg.get("templates_ok", "-")), + ms(*_present(get(agg, "process_startup", "cold", "engine_init_ms"))), + ms(*_present(get(agg, "process_startup", "cold", "first_validation_host_ms"))), + f"{ms(*_stat_present(rule_eval, 'median'))} ({ms(*_stat_present(rule_eval, 'p99'))})", + _ratio(_stat_value(rule_eval, "median"), base_rule), + f"{ms(*_stat_present(wall, 'median'))} ({ms(*_stat_present(wall, 'p99'))})", + _ratio(_stat_value(wall, "median"), base_wall), + ms(recomputed_throughput(agg), True, 2), + fmt_bytes(get(agg, "memory", "full_corpus_peak_rss_bytes")), + "/".join(str(diags.get(k, "-")) for k in + ("total_fatal", "total_errors", "total_warnings", "total_informational")), + ]) + if rows: + lines += [f"### {engine.upper()} - {label}", ""] + table(header, rows) + [""] + return lines - lines += ["## Table of Contents", "", *toc_items, ""] - lines += latency_memory_summary(all_loaded, engines, bindings) - # Track parity results - parity_all_passed = True +def failure_difference_section(differences): + if not differences: + return [] + lines = [ + "## Templates Failing Under a Rule Pack", "", + f"Templates whose validation fails in a scenario but not in `{DEFAULT_SCENARIO}` (or the reverse). " + "Every binding of the scenario agrees on this list. A failing template gets a report with no " + "diagnostics and zero timings, so it is excluded from that scenario's latency and throughput " + "figures. The usual cause is a Guard type block, which the Guard evaluator cannot apply to a " + "template whose `Resources` section is empty.", "", + ] + grouped = {} + for scenario_id, by_engine in differences.items(): + for engine, diff in by_engine.items(): + key = (tuple(diff["introduced"]), tuple(diff["removed"])) + grouped.setdefault(key, []).append(f"`{scenario_id}` / {engine.upper()}") + for (introduced, removed), labels in grouped.items(): + where = ", ".join(labels) + if introduced: + lines.append(f"- {where}: {len(introduced)} template(s) fail only here") + lines += [f" - `{file}` ({status})" for file, status in introduced] + if removed: + lines.append(f"- {where}: {len(removed)} template(s) fail only in `{DEFAULT_SCENARIO}`") + lines += [f" - `{file}` ({status})" for file, status in removed] + lines.append("") + return lines + + +def scenario_anchor(scenario): + return f"scenario-{scenario['id']}" + + +def demote_headings(lines): + return [f"#{line}" if line.startswith("#") else line for line in lines] + + +def scenario_section(all_loaded, all_detailed, engines, bindings, args, scenario): + scenario_id = scenario["id"] + corpus_fp = all_loaded[engines[0]][bindings[0][0]].get("corpus_fingerprint") + corpus_file_count = all_loaded[engines[0]][bindings[0][0]].get("corpus_file_count") + body = [ + f"- **corpus fingerprint**: `{corpus_fp}` ({corpus_file_count} files)", + f"- **engines**: {', '.join(e.upper() for e in engines)}", + "", + ] + body += rule_pack_section(scenario, all_loaded[engines[0]][bindings[0][0]]) + body += provenance_section(all_loaded, engines, bindings) + body += latency_memory_summary(all_loaded, engines, bindings) + parity_all_passed = True for engine in engines: - lines += [f"## {engine.upper()} Engine", ""] - lines += model_section(all_loaded, engine, bindings) - lines += headline_section(all_loaded, engine, bindings) - lines += phase_table(all_loaded, engine, bindings) - lines += overhead_table(all_loaded, engine, bindings) + body += [f"## {engine.upper()} Engine", ""] + body += model_section(all_loaded, engine, bindings) + body += headline_section(all_loaded, engine, bindings) + body += phase_table(all_loaded, engine, bindings) + body += overhead_table(all_loaded, engine, bindings) parity_lines, parity_passed = diagnostics_parity( - all_loaded, engine, bindings, all_detailed=all_detailed + all_loaded, engine, bindings, all_detailed=all_detailed, scenario_id=scenario_id ) - lines += parity_lines + body += parity_lines if not parity_passed: parity_all_passed = False - lines += top_slowest_section(all_detailed, engines, bindings, args.top_slowest) - if "rego" in engines and "cel" in engines: - lines += paired_engine_comparison(all_detailed, bindings) + body += top_slowest_section(all_detailed, engines, bindings, args.top_slowest) + if len(engines) >= 2: + body += paired_engine_comparison(all_detailed, bindings) + body += data_sources_section(all_loaded, engines, bindings, scenario_id) - lines += data_sources_section(all_loaded, engines, bindings) + heading = [f"## Scenario: `{scenario_id}` - {scenario['label']} ", ""] + return heading + demote_headings(body), parity_all_passed + + +def build_report(loaded_by_scenario, detailed_by_scenario, scenarios, engines, bindings, args): + host = host_metadata() + first = scenarios[0] + first_loaded = loaded_by_scenario[first["id"]] + first_engines = scenario_engines(first, engines) + sample = first_loaded[first_engines[0]][bindings[0][0]] + startup_samples = int(get(sample, "process_startup", "samples", default=args.startup_samples)) + lines = [ + "# Benchmark Comparison", + "", + f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}", + "", + "## Host", "", + *[f"- **{k}**: {v}" for k, v in host.items()], + f"- **iterations/template**: {sample.get('iterations_per_template')}", + f"- **startup samples/binding**: {startup_samples} (1 cold + {startup_samples - 1} warm)", + f"- **corpus fingerprint**: `{sample.get('corpus_fingerprint')}` ({sample.get('corpus_file_count')} files)", + f"- **bindings**: {', '.join(lbl for _, lbl in bindings)} ({len(bindings)} total)", + f"- **engines**: {', '.join(e.upper() for e in engines)}", + f"- **scenarios**: {', '.join(s['id'] for s in scenarios)} ({len(scenarios)} total)", + "", + "## Table of Contents", "", + "- [Scenarios](#scenarios)", + "- [Rule Pack Cost per Engine × Binding](#rule-pack-cost-per-engine--binding)", + "- [Methodology Notes](#methodology-notes)", + *[f"- [Scenario: {s['id']} - {s['label']}](#{scenario_anchor(s)})" for s in scenarios], + "", + ] + lines += scenario_overview_section(loaded_by_scenario, scenarios, engines) + lines += rule_pack_cost_section(loaded_by_scenario, scenarios, engines, bindings) + lines += failure_difference_section(scenario_failure_differences(loaded_by_scenario)) + lines += methodology_section() + + parity_all_passed = True + for scenario in scenarios: + section, parity_passed = scenario_section( + loaded_by_scenario[scenario["id"]], + detailed_by_scenario[scenario["id"]], + scenario_engines(scenario, engines), + bindings, + args, + scenario, + ) + lines += section + if not parity_passed: + parity_all_passed = False return lines, parity_all_passed +REPORT_PATH = SCRIPT_DIR / "snapshots" / "benchmark_comparison.md" + + def main(argv=None): args = parse_args(argv) @@ -1866,6 +2252,9 @@ def main(argv=None): if args.bindings else ALL_BINDINGS ) + scenarios = [s for s in select_scenarios(args.scenarios) if scenario_engines(s, engines)] + if not scenarios: + sys.exit(f"no selected scenario can run with engines {engines}") if args.report_only: print("Report-only mode - using existing aggregate files", file=sys.stderr) @@ -1881,36 +2270,36 @@ def main(argv=None): if flavor is None: sys.exit( f"{TIME_BIN} (GNU '-v' or macOS '-l') is required to measure process startup and " - f"memory but is unavailable. Install it (Linux: 'time' package) or use " - f"--report-only against existing aggregates." + f"memory but is unavailable. Install it (Linux: 'time' package), point " + f"CFN_BENCHMARK_TIME_BIN at a GNU time binary, or use --report-only against " + f"existing aggregates." ) run_start_epoch = time.time() - run_all_benchmarks(engines, bindings, args, flavor) - - all_loaded = { - e: {b: load_aggregate(aggregate_path(e, FORMATS[0], b), run_start_epoch) - for b, _ in bindings} - for e in engines - } - - enforce_corpus_parity(all_loaded, bindings) - enforce_run_metadata_parity(all_loaded, bindings) - corpus_fp = all_loaded[engines[0]][bindings[0][0]].get("corpus_fingerprint") - corpus_file_count = all_loaded[engines[0]][bindings[0][0]].get("corpus_file_count") - - all_detailed = load_and_validate_detailed_reports(engines, bindings) - validate_detailed_counts(all_detailed, all_loaded, engines, bindings) + run_all_benchmarks(scenarios, engines, bindings, args, flavor) + + loaded_by_scenario = {} + detailed_by_scenario = {} + for scenario in scenarios: + runnable = scenario_engines(scenario, engines) + all_loaded = { + e: {b: load_aggregate(aggregate_path(e, FORMATS[0], b, scenario["id"]), run_start_epoch, scenario["id"]) + for b, _ in bindings} + for e in runnable + } + enforce_corpus_parity(all_loaded, bindings, scenario["id"]) + enforce_run_metadata_parity(all_loaded, bindings, scenario["id"]) + all_detailed = load_and_validate_detailed_reports(runnable, bindings, scenario["id"]) + validate_detailed_counts(all_detailed, all_loaded, runnable, bindings) + loaded_by_scenario[scenario["id"]] = all_loaded + detailed_by_scenario[scenario["id"]] = all_detailed lines, parity_all_passed = build_report( - all_loaded, all_detailed, engines, bindings, args, corpus_fp, corpus_file_count + loaded_by_scenario, detailed_by_scenario, scenarios, engines, bindings, args ) - - out_dir = SCRIPT_DIR / "snapshots" - out_dir.mkdir(parents=True, exist_ok=True) - output_path = out_dir / "benchmark_comparison.md" - output_path.write_text("\n".join(lines) + "\n") - print(f"\nComparison written to {output_path}", file=sys.stderr) + REPORT_PATH.parent.mkdir(parents=True, exist_ok=True) + REPORT_PATH.write_text("\n".join(lines) + "\n") + print(f"\nComparison written to {REPORT_PATH}", file=sys.stderr) if not parity_all_passed: print( diff --git a/src/bindings-go/bench/main.go b/src/bindings-go/bench/main.go index 26d5fb75..6da2b4f9 100644 --- a/src/bindings-go/bench/main.go +++ b/src/bindings-go/bench/main.go @@ -5,6 +5,13 @@ // Usage: // // go run . [TEMPLATE|DIR] --engine rego|cel|composite --iterations N +// go run . --engine rego|cel|composite --startup-probe +// +// Either form accepts the shared scenario flags: --guard-rules PATH and +// --rego-rules PATH (repeatable; a file or a directory of .guard / .rego files) +// load a rule pack into the engine (Rego rules are rejected for --engine cel), +// and --scenario NAME labels the run and moves its reports from +// reports/{engine}/ to reports/scenarios/NAME/{engine}/. // // The default corpus is src/resources/templates (relative to the workspace // root). Reports are written to src/bindings-go/reports/{engine}/. @@ -19,6 +26,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "sort" "strings" @@ -36,6 +44,10 @@ const ( defaultStartupTemplate = "good/minimal.yaml" + // Recorded when no --scenario is given; its reports keep the historical + // reports/{engine}/ layout that other tooling reads. + defaultScenario = "builtin" + goBindingModulePath = "github.com/aws-cloudformation/cloudformation-validate/src/bindings-go/go" cargoVersionEnv = "BENCHMARK_CARGO_VERSION" @@ -56,6 +68,7 @@ func run() error { args := os.Args[1:] if hasFlag(args, "-h") || hasFlag(args, "--help") { fmt.Fprintln(os.Stderr, "Usage: bench [TEMPLATE|DIR] --engine rego|cel|composite --iterations N [--startup-probe]") + fmt.Fprintln(os.Stderr, " [--guard-rules PATH]... [--rego-rules PATH]... [--scenario NAME]") return usageError("help requested") } @@ -76,19 +89,175 @@ func run() error { return err } + scenario, err := requiredFlagValue(args, "--scenario", defaultScenario) + if err != nil { + return err + } + if !scenarioNamePattern.MatchString(scenario) { + return usageError(fmt.Sprintf("--scenario must be a lowercase name of letters, digits, '-' or '_' (max 64), got %q", scenario)) + } + guardPaths := flagValues(args, "--guard-rules") + regoPaths := flagValues(args, "--rego-rules") + if engineFlag == "cel" && len(regoPaths) > 0 { + return usageError("--rego-rules cannot be loaded into the CEL engine; use --engine rego or composite") + } + // Read before any timer starts so only engine construction is measured. + pack, err := loadRulePack(guardPaths, regoPaths) + if err != nil { + return usageError(err.Error()) + } + factory := engineFactory{engine: engineFlag, pack: pack} + validateConfig := &cfnvalidate.ValidateConfig{ SeverityLevel: cfnvalidate.SeverityDebug, DetailLevel: cfnvalidate.DetailLevelDetailed, } if hasFlag(args, "--startup-probe") { - return runStartupProbe(engineFlag, validateConfig) + return runStartupProbe(factory, scenario, validateConfig) + } + + return runBenchmark(factory, scenario, iterations, validateConfig, positionalArg(args)) +} + +var scenarioNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,63}$`) + +// rulePack's fingerprint format - one "\t\t\n" line per file, sorted, hashed - is shared by every harness so the +// comparison can prove all bindings loaded the same rules. +type rulePack struct { + guard []cfnvalidate.ExternalRuleSource + rego []cfnvalidate.ExternalRuleSource + guardBytes int + regoBytes int + entries []string +} + +func loadRulePack(guardPaths, regoPaths []string) (*rulePack, error) { + pack := &rulePack{} + if err := pack.load("guard", guardPaths); err != nil { + return nil, err + } + if err := pack.load("rego", regoPaths); err != nil { + return nil, err + } + sort.Strings(pack.entries) + return pack, nil +} + +func (p *rulePack) load(kind string, rawPaths []string) error { + for _, rawPath := range rawPaths { + info, err := os.Stat(rawPath) + if err != nil { + return fmt.Errorf("--%s-rules path not found: %s", kind, rawPath) + } + files, err := collectRuleFiles(rawPath, info.IsDir(), "."+kind) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("--%s-rules path contains no .%s files: %s", kind, kind, rawPath) + } + for _, file := range files { + content, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("reading %s rule file %q: %w", kind, file, err) + } + relative := filepath.Base(file) + if info.IsDir() { + relative = filepath.ToSlash(relativePath(rawPath, file)) + } + digest := sha256.Sum256(content) + p.entries = append(p.entries, fmt.Sprintf("%s\t%s\t%s\n", kind, relative, hex.EncodeToString(digest[:]))) + source := cfnvalidate.ExternalRuleSource{Name: file, Content: string(content)} + if kind == "guard" { + p.guard = append(p.guard, source) + p.guardBytes += len(content) + } else { + p.rego = append(p.rego, source) + p.regoBytes += len(content) + } + } + } + return nil +} + +func collectRuleFiles(root string, isDir bool, extension string) ([]string, error) { + if !isDir { + return []string{root}, nil + } + var files []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), extension) { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking %s: %w", root, err) + } + sort.Strings(files) + return files, nil +} + +func (p *rulePack) fingerprint() string { + h := sha256.Sum256([]byte(strings.Join(p.entries, ""))) + return hex.EncodeToString(h[:]) +} + +func (p *rulePack) json(guardRuleCount int) map[string]interface{} { + return map[string]interface{}{ + "guard": map[string]interface{}{"files": len(p.guard), "rules": guardRuleCount, "bytes": p.guardBytes}, + "rego": map[string]interface{}{"files": len(p.rego), "bytes": p.regoBytes}, } +} - return runBenchmark(engineFlag, iterations, validateConfig, positionalArg(args)) +// guardRuleCount counts distinct names: a rule name shared by two files counts once. +func guardRuleCount(engine *cfnvalidate.Engine) (int, error) { + rules, err := engine.ListRules() + if err != nil { + return 0, fmt.Errorf("listing rules: %w", err) + } + count := 0 + for _, rule := range rules { + if rule.Origin == cfnvalidate.RuleOriginGuard { + count++ + } + } + return count, nil } -func runStartupProbe(engineFlag string, validateConfig *cfnvalidate.ValidateConfig) error { +type engineFactory struct { + engine string + pack *rulePack +} + +func (f engineFactory) newEngine() (*cfnvalidate.Engine, error) { + switch f.engine { + case "cel": + return cfnvalidate.NewCelEngine(&cfnvalidate.EngineConfig{GuardRules: f.pack.guard}) + case "composite": + return cfnvalidate.NewCompositeEngine(&cfnvalidate.CompositeEngineConfig{RegoRules: f.pack.rego, GuardRules: f.pack.guard}) + default: + return cfnvalidate.NewRegoEngine(&cfnvalidate.EngineConfig{CustomRules: f.pack.rego, GuardRules: f.pack.guard}) + } +} + +func flagValues(args []string, flag string) []string { + var values []string + for i := 0; i < len(args); i++ { + if args[i] == flag && i+1 < len(args) { + values = append(values, args[i+1]) + i++ + } + } + return values +} + +func runStartupProbe(factory engineFactory, scenario string, validateConfig *cfnvalidate.ValidateConfig) error { defaultCorpus, err := resolveDefaultCorpus() if err != nil { return fmt.Errorf("resolving default corpus: %w", err) @@ -101,13 +270,20 @@ func runStartupProbe(engineFlag string, validateConfig *cfnvalidate.ValidateConf } startupLabel := filepath.Base(startupPath) - engine, startup, err := measureStartup(engineFlag, startupBytes, startupLabel, validateConfig) + engine, startup, err := measureStartup(factory, startupBytes, startupLabel, validateConfig) if err != nil { return err } defer engine.Destroy() + guardRules, err := guardRuleCount(engine) + if err != nil { + return err + } probe := startupProbeJSON(startup, engine.EngineName()) + probe["scenario"] = scenario + probe["custom_rules"] = factory.pack.json(guardRules) + probe["rules_fingerprint"] = factory.pack.fingerprint() serialized, err := json.Marshal(probe) if err != nil { return fmt.Errorf("serializing startup probe: %w", err) @@ -116,7 +292,8 @@ func runStartupProbe(engineFlag string, validateConfig *cfnvalidate.ValidateConf return nil } -func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate.ValidateConfig, positional string) error { +func runBenchmark(factory engineFactory, scenario string, iterations int, validateConfig *cfnvalidate.ValidateConfig, positional string) error { + engineFlag := factory.engine defaultTemplateDir, err := resolveDefaultCorpus() if err != nil { return fmt.Errorf("resolving default corpus: %w", err) @@ -144,19 +321,28 @@ func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate if err != nil { return fmt.Errorf("reading startup template %q: %w", templates[0], err) } - engine, startup, err := measureStartup(engineFlag, startupBytes, startupLabel, validateConfig) + engine, startup, err := measureStartup(factory, startupBytes, startupLabel, validateConfig) if err != nil { return err } defer engine.Destroy() + guardRules, err := guardRuleCount(engine) + if err != nil { + return err + } + rulesFingerprint := factory.pack.fingerprint() + customRules := factory.pack.json(guardRules) + fmt.Fprintf(os.Stderr, "Scenario %q: %d Guard file(s), %d Rego file(s), rules fingerprint %s\n", + scenario, len(factory.pack.guard), len(factory.pack.rego), rulesFingerprint) + engineInitSamples := []float64{startup.EngineInitMs} initSamples := []float64{startup.EngineInitMs} coldInitMs := moduleLoadMs + initSamples[0] subsequentInitSamples := []float64{} schemaInitSamples := []float64{} - reportDir, err := resolveReportDir(engineFlag) + reportDir, err := resolveReportDir(engineFlag, scenario) if err != nil { return fmt.Errorf("resolving report dir: %w", err) } @@ -205,7 +391,7 @@ func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate return fmt.Errorf("creating parse-failure report for %s: %w", rel, reportErr) } normalizeParseFailureReport(parseFailureReport) - payload, marshalErr := buildPerTemplatePayload(parseFailureReport, rel, engineFlag, zeroBenchmarkMetrics()) + payload, marshalErr := buildPerTemplatePayload(parseFailureReport, rel, engineFlag, scenario, zeroBenchmarkMetrics()) if marshalErr != nil { return fmt.Errorf("marshaling parse-failure payload for %s: %w", rel, marshalErr) } @@ -224,6 +410,11 @@ func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate hostValidateMs := elapsed(t0) if valErr != nil { results = append(results, errorResult(rel, "error", valErr.Error())) + // Every attempted template gets a report so the comparison script can pair + // the same template set across bindings and scenarios. + if writeErr := writePerTemplateReport(jsonPath, failedTemplatePayload(rel, engineFlag, scenario, valErr.Error())); writeErr != nil { + return writeErr + } failed = true break } @@ -258,7 +449,7 @@ func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate benchmarkMetrics := perTemplateMetricsJSON(iterations, iterHostModel, iterModelBuild, iterSchemaValidate, iterRuleEval, iterFinalize, iterEngineInternal, iterHostValidate, bindingOverheadMs) - payload, marshalErr := buildPerTemplatePayload(report, rel, engineFlag, benchmarkMetrics) + payload, marshalErr := buildPerTemplatePayload(report, rel, engineFlag, scenario, benchmarkMetrics) if marshalErr != nil { return fmt.Errorf("marshaling per-template payload for %s: %w", rel, marshalErr) } @@ -323,7 +514,7 @@ func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate if fpErr != nil { return fmt.Errorf("computing corpus fingerprint: %w", fpErr) } - runFingerprint := computeRunFingerprint(corpusFingerprint, engineFlag, detailLevelName, iterations) + runFingerprint := computeRunFingerprint(corpusFingerprint, rulesFingerprint, scenario, engineFlag, detailLevelName, iterations) provenance := provenanceJSON() @@ -354,6 +545,9 @@ func runBenchmark(engineFlag string, iterations int, validateConfig *cfnvalidate "iterations_per_template": iterations, "corpus_fingerprint": corpusFingerprint, "corpus_file_count": corpusFileCount, + "scenario": scenario, + "custom_rules": customRules, + "rules_fingerprint": rulesFingerprint, "run_fingerprint": runFingerprint, "performance": performance, "diagnostics": buildDiagnosticsBlock(ok), @@ -397,11 +591,11 @@ type startupMeasurement struct { InternalTimeToFirstResultMs float64 } -func measureStartup(engineFlag string, startupBytes []byte, startupLabel string, validateConfig *cfnvalidate.ValidateConfig) (*cfnvalidate.Engine, startupMeasurement, error) { +func measureStartup(factory engineFactory, startupBytes []byte, startupLabel string, validateConfig *cfnvalidate.ValidateConfig) (*cfnvalidate.Engine, startupMeasurement, error) { const moduleLoadMs = 0.0 engineStart := time.Now() - engine, err := newEngine(engineFlag) + engine, err := factory.newEngine() if err != nil { return nil, startupMeasurement{}, fmt.Errorf("engine init failed: %w", err) } @@ -510,12 +704,22 @@ func queryToolVersion(tool string) string { } var knownFlags = map[string]bool{ - "-h": true, "--help": true, - "--engine": true, "--iterations": true, "--startup-probe": true, + "-h": true, + "--help": true, + "--engine": true, + "--iterations": true, + "--startup-probe": true, + "--guard-rules": true, + "--rego-rules": true, + "--scenario": true, } var flagsWithValues = map[string]bool{ - "--engine": true, "--iterations": true, + "--engine": true, + "--iterations": true, + "--guard-rules": true, + "--rego-rules": true, + "--scenario": true, } func validateFlags(args []string) error { @@ -605,17 +809,6 @@ func isUsageError(err error) bool { return ok } -func newEngine(name string) (*cfnvalidate.Engine, error) { - switch name { - case "cel": - return cfnvalidate.NewCelEngine(nil) - case "composite": - return cfnvalidate.NewCompositeEngine(nil) - default: - return cfnvalidate.NewRegoEngine(nil) - } -} - var templateExtensions = map[string]bool{ ".yaml": true, ".yml": true, @@ -702,12 +895,15 @@ func resolveDefaultCorpus() (string, error) { return abs, nil } -func resolveReportDir(engine string) (string, error) { +func resolveReportDir(engine, scenario string) (string, error) { dir, err := sourceFileDir() if err != nil { return "", err } reportDir := filepath.Join(dir, "..", "reports", engine) + if scenario != defaultScenario { + reportDir = filepath.Join(dir, "..", "reports", "scenarios", scenario, engine) + } abs, err := filepath.Abs(reportDir) if err != nil { return "", fmt.Errorf("resolving absolute path for report dir: %w", err) @@ -828,7 +1024,7 @@ func normalizeParseFailureReport(report *cfnvalidate.ValidationReport) { report.Diagnostics = []cfnvalidate.Diagnostic{} } -func buildPerTemplatePayload(report *cfnvalidate.ValidationReport, rel, engine string, benchmarkMetrics map[string]interface{}) (map[string]interface{}, error) { +func buildPerTemplatePayload(report *cfnvalidate.ValidationReport, rel, engine, scenario string, benchmarkMetrics map[string]interface{}) (map[string]interface{}, error) { data, err := json.Marshal(report) if err != nil { return nil, fmt.Errorf("marshaling report: %w", err) @@ -840,11 +1036,28 @@ func buildPerTemplatePayload(report *cfnvalidate.ValidationReport, rel, engine s payload["engine"] = engine payload["binding"] = bindingName payload["detailLevel"] = detailLevelName + payload["scenario"] = scenario payload["filePath"] = rel payload["benchmarkMetrics"] = benchmarkMetrics return payload, nil } +// failedTemplatePayload keeps the envelope of a successful report so consumers can +// tell a failed template from a clean one without a second schema. +func failedTemplatePayload(rel, engine, scenario, message string) map[string]interface{} { + return map[string]interface{}{ + "filePath": rel, + "status": "ERROR", + "error": message, + "diagnostics": []interface{}{}, + "engine": engine, + "binding": bindingName, + "detailLevel": detailLevelName, + "scenario": scenario, + "benchmarkMetrics": zeroBenchmarkMetrics(), + } +} + func writePerTemplateReport(path string, payload map[string]interface{}) error { data, marshalErr := json.MarshalIndent(payload, "", " ") if marshalErr != nil { @@ -975,8 +1188,8 @@ func computeCorpusFingerprint(root string) (string, int, error) { return hex.EncodeToString(outer.Sum(nil)), len(files), nil } -func computeRunFingerprint(corpusFP, engine, format string, iterations int) string { - h := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%s|%d", corpusFP, engine, format, iterations))) +func computeRunFingerprint(corpusFP, rulesFP, scenario, engine, format string, iterations int) string { + h := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%s|%s|%s|%d", corpusFP, rulesFP, scenario, engine, format, iterations))) return hex.EncodeToString(h[:]) } diff --git a/src/bindings-jvm/bench/src/main/kotlin/Benchmark.kt b/src/bindings-jvm/bench/src/main/kotlin/Benchmark.kt index 720b82d3..7ac62f5e 100644 --- a/src/bindings-jvm/bench/src/main/kotlin/Benchmark.kt +++ b/src/bindings-jvm/bench/src/main/kotlin/Benchmark.kt @@ -13,8 +13,10 @@ import software.amazon.cloudformation.validate.ValidateConfig import software.amazon.cloudformation.validate.diagnostics.ValidationReport import software.amazon.cloudformation.validate.engine.CompositeEngineConfig import software.amazon.cloudformation.validate.engine.EngineConfig +import software.amazon.cloudformation.validate.engine.ExternalRuleSource import software.amazon.cloudformation.validate.gson.buildBindingsGson import software.amazon.cloudformation.validate.rules.RuleFilterConfig +import software.amazon.cloudformation.validate.rules.RuleOrigin import software.amazon.cloudformation.validate.rules.Severity import software.amazon.cloudformation.validate.templatemodel.PseudoParameterOverrides import software.amazon.cloudformation.validate.version @@ -30,11 +32,119 @@ import kotlin.system.exitProcess const val DEFAULT_STARTUP_TEMPLATE = "good/minimal.yaml" +// Recorded when no --scenario is given; its reports keep the historical reports// layout +// that other tooling reads. +const val DEFAULT_SCENARIO = "builtin" +private val SCENARIO_NAME_PATTERN = Regex("^[a-z0-9][a-z0-9_-]{0,63}$") + +/** + * The fingerprint format - one "\t\t\n" line + * per file, sorted, hashed - is shared by every harness so the comparison can prove all bindings + * loaded the same rules. + */ +class RulePack { + val guard = mutableListOf() + val rego = mutableListOf() + var guardBytes = 0L + var regoBytes = 0L + private val entries = mutableListOf() + + fun load( + kind: String, + rawPaths: List, + ) { + for (rawPath in rawPaths) { + val root = File(rawPath) + if (!root.exists()) { + System.err.println("Error: --$kind-rules path not found: $rawPath") + exitProcess(2) + } + val files = collectRuleFiles(root, kind) + if (files.isEmpty()) { + System.err.println("Error: --$kind-rules path contains no .$kind files: $rawPath") + exitProcess(2) + } + for (file in files) { + val bytes = file.readBytes() + val content = bytes.toString(Charsets.UTF_8) + val relative = + if (root.isFile) file.name else root.toPath().relativize(file.toPath()).toString().replace('\\', '/') + entries.add("$kind\t$relative\t${sha256Hex(bytes)}\n") + val source = ExternalRuleSource(file.path, content) + if (kind == "guard") { + guard.add(source) + guardBytes += bytes.size + } else { + rego.add(source) + regoBytes += bytes.size + } + } + } + } + + fun fingerprint(): String = sha256Hex(entries.sorted().joinToString("")) + + fun json(guardRuleCount: Int): JsonObject = + JsonObject().apply { + add( + "guard", + JsonObject().apply { + addProperty("files", guard.size) + addProperty("rules", guardRuleCount) + addProperty("bytes", guardBytes) + }, + ) + add( + "rego", + JsonObject().apply { + addProperty("files", rego.size) + addProperty("bytes", regoBytes) + }, + ) + } +} + +private fun collectRuleFiles( + root: File, + extension: String, +): List { + if (root.isFile) return listOf(root) + return root.walkTopDown().filter { it.isFile && it.name.endsWith(".$extension") }.sortedBy { it.path }.toList() +} + +private fun flagValues( + args: Array, + flag: String, +): List { + val values = mutableListOf() + var i = 0 + while (i < args.size) { + if (args[i] == flag) { + val value = args.getOrNull(i + 1) + if (value == null || value.startsWith("-")) { + System.err.println("Error: $flag requires a path value") + exitProcess(2) + } + values.add(value) + i += 2 + } else { + i++ + } + } + return values +} + fun main(args: Array) { if (args.any { it == "-h" || it == "--help" }) { System.err.println( "Usage: gradle run --args=\"[TEMPLATE|DIR] [--engine rego|cel|composite] [--iterations N]\"\n" + - " gradle run --args=\"--startup-probe [--engine rego|cel|composite]\"", + " gradle run --args=\"--startup-probe [--engine rego|cel|composite]\"\n" + + "\n" + + "Either form accepts the shared scenario flags:\n" + + " --guard-rules PATH Load a Guard (.guard) rule file or directory into the engine; repeatable\n" + + " --rego-rules PATH Load a custom Rego (.rego) rule file or directory; repeatable, not valid with --engine cel\n" + + " --scenario NAME Label the run and write reports to reports/scenarios/NAME// instead of\n" + + " reports// (default scenario: $DEFAULT_SCENARIO)", ) return } @@ -74,9 +184,35 @@ fun main(args: Array) { parsed } + val scenario = + run { + val idx = args.indexOf("--scenario") + if (idx < 0) return@run DEFAULT_SCENARIO + val value = args.getOrNull(idx + 1) + if (value == null) { + System.err.println("Error: --scenario requires a value") + exitProcess(2) + } + if (!SCENARIO_NAME_PATTERN.matches(value)) { + System.err.println("Error: --scenario must be a lowercase name of letters, digits, '-' or '_' (max 64), got '$value'") + exitProcess(2) + } + value + } + val guardRulePaths = flagValues(args, "--guard-rules") + val regoRulePaths = flagValues(args, "--rego-rules") + if (engineFlag == "cel" && regoRulePaths.isNotEmpty()) { + System.err.println("Error: --rego-rules cannot be loaded into the CEL engine; use --engine rego or composite") + exitProcess(2) + } + // Read before any timer starts so only engine construction is measured. + val rulePack = RulePack() + rulePack.load("guard", guardRulePaths) + rulePack.load("rego", regoRulePaths) + val positionalArg: String? = run { - val flagsWithValues = setOf("--engine", "--iterations") + val flagsWithValues = setOf("--engine", "--iterations", "--guard-rules", "--rego-rules", "--scenario") var i = 0 while (i < args.size) { if (flagsWithValues.contains(args[i])) { @@ -100,7 +236,7 @@ fun main(args: Array) { val benchValidateConfig = validateConfig() if (startupProbe) { - exitProcess(runStartupProbe(engineFlag, coreVersion, moduleLoadMs, defaultTemplateDir, benchValidateConfig)) + exitProcess(runStartupProbe(engineFlag, rulePack, scenario, coreVersion, moduleLoadMs, defaultTemplateDir, benchValidateConfig)) } val templateDir = positionalArg ?: defaultTemplateDir @@ -122,8 +258,13 @@ fun main(args: Array) { .toString() .replace('\\', '/') .ifEmpty { startupTemplate.name } - val startup = measureStartup(engineFlag, moduleLoadMs, startupBytes, startupLabel, benchValidateConfig) + val startup = measureStartup(engineFlag, rulePack, moduleLoadMs, startupBytes, startupLabel, benchValidateConfig) val engine: Any = startup.engine + val rulesFingerprint = rulePack.fingerprint() + val customRules = rulePack.json(guardRuleCount(engine)) + System.err.println( + "Scenario '$scenario': ${rulePack.guard.size} Guard file(s), ${rulePack.rego.size} Rego file(s), rules fingerprint $rulesFingerprint", + ) val schemaInitSamples = emptyList() val engineInitSamples = listOf(startup.engineInitMs) @@ -131,7 +272,10 @@ fun main(args: Array) { val coldInitMs = moduleLoadMs + initSamples[0] val subsequentInitSamples = emptyList() - val reportDir = File(System.getProperty("user.dir")).resolve("../reports/$engineFlag").also { it.mkdirs() } + val reportDir = + File(System.getProperty("user.dir")) + .resolve(if (scenario == DEFAULT_SCENARIO) "../reports/$engineFlag" else "../reports/scenarios/$scenario/$engineFlag") + .also { it.mkdirs() } val jsonDir = reportDir.resolve("json_$formatDir").also { dir -> // Clean previous output so stale reports from dropped/renamed templates are not left behind. @@ -199,6 +343,7 @@ fun main(args: Array) { outputGson, parseFailureReport, engineFlag, + scenario, zeroBenchmarkMetrics(), normalizeParseFailure = true, ) @@ -219,7 +364,11 @@ fun main(args: Array) { iterWallClock.add(wallMs) if (i == iterations - 1) lastReport = report } catch (e: Exception) { - results.add(errorResult(rel, "error", e.message ?: "unknown")) + val message = e.message ?: "unknown" + results.add(errorResult(rel, "error", message)) + // Every attempted template gets a report so the comparison script can pair the same + // template set across bindings and scenarios. + writeFailedReportJson(jsonPath, outputGson, rel, engineFlag, scenario, message) failed = true } } @@ -252,7 +401,7 @@ fun main(args: Array) { iterWallClock, bindingOverheadMs, ) - writeReportJson(jsonPath, treeGson, outputGson, report, engineFlag, metrics, normalizeParseFailure = false) + writeReportJson(jsonPath, treeGson, outputGson, report, engineFlag, scenario, metrics, normalizeParseFailure = false) val tr = TemplateResult( @@ -325,7 +474,7 @@ fun main(args: Array) { val measuredValidationWallMs = ok.sumOf { it.wallClockTotalMs } val (corpusFingerprint, corpusFileCount) = computeCorpusFingerprint(File(templateDir)) - val runFingerprint = sha256Hex("$corpusFingerprint|$engineFlag|$formatFlag|$iterations") + val runFingerprint = sha256Hex("$corpusFingerprint|$rulesFingerprint|$scenario|$engineFlag|$formatFlag|$iterations") // Provenance is assembled after all timed work so its cargo/rustc subprocess spawns never // contaminate the measurements. @@ -402,6 +551,9 @@ fun main(args: Array) { addProperty("iterations_per_template", iterations) addProperty("corpus_fingerprint", corpusFingerprint) addProperty("corpus_file_count", corpusFileCount) + addProperty("scenario", scenario) + add("custom_rules", customRules) + addProperty("rules_fingerprint", rulesFingerprint) addProperty("run_fingerprint", runFingerprint) add("performance", perfObj) add("diagnostics", diagObj) @@ -502,13 +654,14 @@ private data class StartupMeasurement( private fun measureStartup( engineFlag: String, + rulePack: RulePack, moduleLoadMs: Double, startupBytes: ByteArray, startupLabel: String, benchmarkConfig: ValidateConfig, ): StartupMeasurement { val engineStart = System.nanoTime() - val engine = newEngine(engineFlag) + val engine = newEngine(engineFlag, rulePack) val engineInitMs = (System.nanoTime() - engineStart) / 1_000_000.0 // The JVM engine constructor embeds a SchemaValidator, so consumer init is engine-only. val consumerInitMs = engineInitMs @@ -574,6 +727,8 @@ private fun startupSectionJson(startup: StartupMeasurement): JsonObject = private fun runStartupProbe( engineFlag: String, + rulePack: RulePack, + scenario: String, coreVersion: String, moduleLoadMs: Double, defaultTemplateDir: String, @@ -589,12 +744,15 @@ private fun runStartupProbe( } val startupLabel = startupFile.name - val startup = measureStartup(engineFlag, moduleLoadMs, startupBytes, startupLabel, benchmarkConfig) + val startup = measureStartup(engineFlag, rulePack, moduleLoadMs, startupBytes, startupLabel, benchmarkConfig) val engineName = engineName(startup.engine) val probe = startupSectionJson(startup) probe.addProperty("binding", "jvm") probe.addProperty("engine", engineName) + probe.addProperty("scenario", scenario) + probe.add("custom_rules", rulePack.json(guardRuleCount(startup.engine))) + probe.addProperty("rules_fingerprint", rulePack.fingerprint()) probe.add("versions", provenanceJson(coreVersion)) println(GsonBuilder().serializeNulls().create().toJson(probe)) @@ -693,13 +851,25 @@ private fun queryToolVersion(tool: String): String = "unknown" } -private fun newEngine(engineFlag: String): Any = +private fun newEngine( + engineFlag: String, + rulePack: RulePack, +): Any = when (engineFlag) { - "cel" -> JvmCelEngine(engineConfig()) - "composite" -> JvmCompositeEngine(compositeEngineConfig()) - else -> JvmRegoEngine(engineConfig()) + "cel" -> JvmCelEngine(engineConfig(rulePack)) + "composite" -> JvmCompositeEngine(compositeEngineConfig(rulePack)) + else -> JvmRegoEngine(EngineConfig(customRules = rulePack.rego.toList(), guardRules = rulePack.guard.toList())) } +/** Distinct names: a rule name shared by two files counts once. */ +private fun guardRuleCount(engine: Any): Int = + when (engine) { + is JvmCelEngine -> engine.listRules() + is JvmCompositeEngine -> engine.listRules() + is JvmRegoEngine -> engine.listRules() + else -> throw IllegalArgumentException("Unknown engine type") + }.count { it.origin == RuleOrigin.GUARD } + private fun engineName(engine: Any): String = when (engine) { is JvmCelEngine -> engine.engineName() @@ -744,9 +914,9 @@ private fun validateTemplate( else -> throw IllegalArgumentException("Unknown engine type") } -fun engineConfig() = EngineConfig(customRules = listOf(), guardRules = listOf()) +fun engineConfig(rulePack: RulePack) = EngineConfig(customRules = listOf(), guardRules = rulePack.guard.toList()) -fun compositeEngineConfig() = CompositeEngineConfig(regoRules = listOf(), guardRules = listOf()) +fun compositeEngineConfig(rulePack: RulePack) = CompositeEngineConfig(regoRules = rulePack.rego.toList(), guardRules = rulePack.guard.toList()) fun validateConfig() = ValidateConfig( @@ -919,6 +1089,7 @@ private fun writeReportJson( outputGson: Gson, report: ValidationReport, engineFlag: String, + scenario: String, metrics: JsonObject, normalizeParseFailure: Boolean, ) { @@ -927,10 +1098,38 @@ private fun writeReportJson( reportElement.addProperty("engine", engineFlag) reportElement.addProperty("binding", "jvm") reportElement.addProperty("detailLevel", formatFlag) + reportElement.addProperty("scenario", scenario) reportElement.add("benchmarkMetrics", metrics) dest.writeText(outputGson.toJson(reportElement)) } +/** + * Keeps the envelope of a successful report so consumers can tell a failed template from a clean one + * without a second schema. + */ +private fun writeFailedReportJson( + dest: File, + outputGson: Gson, + rel: String, + engineFlag: String, + scenario: String, + message: String, +) { + val reportElement = + JsonObject().apply { + addProperty("filePath", rel) + addProperty("status", "ERROR") + addProperty("error", message) + add("diagnostics", JsonArray()) + addProperty("engine", engineFlag) + addProperty("binding", "jvm") + addProperty("detailLevel", formatFlag) + addProperty("scenario", scenario) + add("benchmarkMetrics", zeroBenchmarkMetrics()) + } + dest.writeText(outputGson.toJson(reportElement)) +} + private fun minOf(vals: List): Double = vals.minOrNull() ?: 0.0 private fun maxOf(vals: List): Double = vals.maxOrNull() ?: 0.0 diff --git a/src/bindings-python/bench/benchmark.py b/src/bindings-python/bench/benchmark.py index 9cb90b09..d241dec0 100644 --- a/src/bindings-python/bench/benchmark.py +++ b/src/bindings-python/bench/benchmark.py @@ -8,6 +8,12 @@ Usage: python -m bench.benchmark [TEMPLATE|DIR] --engine rego|cel|composite --iterations N python -m bench.benchmark --engine rego|cel|composite --startup-probe + +Either form accepts the shared scenario flags: ``--guard-rules PATH`` and +``--rego-rules PATH`` (repeatable; a file or a directory of ``.guard`` / +``.rego`` files) load a rule pack into the engine (Rego rules are rejected for +``--engine cel``), and ``--scenario NAME`` labels the run and moves its reports +from ``reports//`` to ``reports/scenarios/NAME//``. """ from __future__ import annotations @@ -37,6 +43,72 @@ _DEFAULT_STARTUP_TEMPLATE = "good/minimal.yaml" +# Recorded when no --scenario is given; its reports keep the historical +# reports// layout that other tooling reads. +_DEFAULT_SCENARIO = "builtin" +_SCENARIO_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") + + +def _collect_rule_files(path: Path, extension: str) -> List[Path]: + if path.is_file(): + return [path] + return sorted(p for p in path.rglob(f"*.{extension}") if p.is_file()) + + +class RulePack: + """The fingerprint format - one ``\t\t`` line per file, sorted, hashed - is shared by every harness so the + comparison can prove all bindings loaded the same rules.""" + + def __init__(self) -> None: + self.guard: List[Tuple[str, str]] = [] + self.rego: List[Tuple[str, str]] = [] + self.guard_bytes = 0 + self.rego_bytes = 0 + self._entries: List[str] = [] + + def load(self, kind: str, raw_paths: List[str]) -> None: + for raw_path in raw_paths: + root = Path(raw_path) + if not root.exists(): + print(f"Error: --{kind}-rules path not found: {raw_path}", file=sys.stderr) + sys.exit(2) + files = _collect_rule_files(root, kind) + if not files: + print(f"Error: --{kind}-rules path contains no .{kind} files: {raw_path}", file=sys.stderr) + sys.exit(2) + for file in files: + content = file.read_text(encoding="utf-8") + relative = file.name if root.is_file() else file.relative_to(root).as_posix() + self._entries.append(f"{kind}\t{relative}\t{_sha256_hex(content.encode('utf-8'))}\n") + if kind == "guard": + self.guard.append((str(file), content)) + self.guard_bytes += len(content.encode("utf-8")) + else: + self.rego.append((str(file), content)) + self.rego_bytes += len(content.encode("utf-8")) + + def fingerprint(self) -> str: + return _sha256_hex("".join(sorted(self._entries)).encode("utf-8")) + + def json(self, guard_rule_count: int) -> Dict[str, Any]: + return { + "guard": {"files": len(self.guard), "rules": guard_rule_count, "bytes": self.guard_bytes}, + "rego": {"files": len(self.rego), "bytes": self.rego_bytes}, + } + + +def _guard_rule_count(engine: Any) -> int: + """Distinct names: a rule name shared by two files counts once.""" + return sum(1 for rule in engine._inner.list_rules() if getattr(rule.origin, "name", str(rule.origin)) == "GUARD") + + +def _report_output_dir(engine_name: str, scenario: str) -> Path: + if scenario == _DEFAULT_SCENARIO: + return _BINDINGS_DIR / "reports" / engine_name + return _BINDINGS_DIR / "reports" / "scenarios" / scenario / engine_name + + _CONSUMER_INIT_SCOPE = "engine_includes_schema_validator" _CARGO_VERSION_ENV = "BENCHMARK_CARGO_VERSION" @@ -313,14 +385,14 @@ def _startup_section(startup: StartupMeasurement) -> Dict[str, Any]: def _measure_startup( - engine_class: Any, + engine_factory: Any, startup_bytes: bytes, startup_label: str, benchmark_config: Any, module_load_ms: float, ) -> Tuple[Any, StartupMeasurement]: engine_start = time.perf_counter() - engine = engine_class() + engine = engine_factory() engine_init_ms = (time.perf_counter() - engine_start) * 1000.0 consumer_init_ms = engine_init_ms @@ -449,8 +521,8 @@ def _compute_corpus_fingerprint(root: Path, files: List[Path]) -> Tuple[str, int return outer.hexdigest(), len(relative_and_absolute) -def _run_fingerprint(corpus_fp: str, engine: str, fmt: str, iterations: int) -> str: - data = f"{corpus_fp}|{engine}|{fmt}|{iterations}" +def _run_fingerprint(corpus_fp: str, rules_fp: str, scenario: str, engine: str, fmt: str, iterations: int) -> str: + data = f"{corpus_fp}|{rules_fp}|{scenario}|{engine}|{fmt}|{iterations}" return _sha256_hex(data.encode()) @@ -561,12 +633,14 @@ def _write_template_report( report: Any, benchmark_metrics: Dict[str, Any], engine_name: str, + scenario: str, ) -> None: try: template_json = to_jsonable(report) template_json["engine"] = engine_name template_json["binding"] = "python" template_json["detailLevel"] = "DETAILED" + template_json["scenario"] = scenario template_json["benchmarkMetrics"] = benchmark_metrics serialized = json.dumps(template_json, indent=2) with open(json_path, "w", encoding="utf-8") as f: @@ -580,6 +654,30 @@ def _write_template_report( sys.exit(1) +def _write_failed_template_report( + json_path: Path, rel_path: str, message: str, engine_name: str, scenario: str +) -> None: + """Keeps the envelope of a successful report so consumers can tell a failed + template from a clean one without a second schema.""" + template_json = { + "filePath": rel_path, + "status": "ERROR", + "error": message, + "diagnostics": [], + "engine": engine_name, + "binding": "python", + "detailLevel": "DETAILED", + "scenario": scenario, + "benchmarkMetrics": _zero_benchmark_metrics(), + } + try: + with open(json_path, "w", encoding="utf-8") as f: + f.write(json.dumps(template_json, indent=2)) + except OSError as exc: + print(f"ERROR: failed to write per-template report {json_path} (template: {rel_path}): {exc}", file=sys.stderr) + sys.exit(1) + + def _collect_files(root: Path) -> List[Path]: if root.is_file(): return [root] @@ -631,17 +729,45 @@ def _parse_args() -> argparse.Namespace: "raw-byte validation), print one JSON object, and exit." ), ) + parser.add_argument( + "--guard-rules", + action="append", + default=[], + metavar="PATH", + help="Guard (.guard) rule file or directory to load into the engine; repeatable.", + ) + parser.add_argument( + "--rego-rules", + action="append", + default=[], + metavar="PATH", + help="Custom Rego (.rego) rule file or directory to load; repeatable, not valid with --engine cel.", + ) + parser.add_argument( + "--scenario", + default=_DEFAULT_SCENARIO, + help=( + "Label the run and write reports to reports/scenarios/NAME// instead of " + f"reports// (default: {_DEFAULT_SCENARIO})." + ), + ) args = parser.parse_args() if args.iterations is not None and args.iterations < 1: parser.error("--iterations must be a positive integer") if not args.startup_probe and args.iterations is None: parser.error("--iterations is required") + if not _SCENARIO_NAME_PATTERN.match(args.scenario): + parser.error("--scenario must be a lowercase name of letters, digits, '-' or '_' (max 64)") + if args.engine == "cel" and args.rego_rules: + parser.error("--rego-rules cannot be loaded into the CEL engine; use --engine rego or composite") return args def _run_startup_probe( engine_name: str, - engine_class: Any, + engine_factory: Any, + pack: RulePack, + scenario: str, version_fn: Any, benchmark_config: Any, module_load_ms: float, @@ -654,13 +780,16 @@ def _run_startup_probe( sys.exit(1) startup_label = startup_path.name - _engine, startup = _measure_startup( - engine_class, startup_bytes, startup_label, benchmark_config, module_load_ms + engine, startup = _measure_startup( + engine_factory, startup_bytes, startup_label, benchmark_config, module_load_ms ) probe = _startup_section(startup) probe["binding"] = "python" probe["engine"] = engine_name + probe["scenario"] = scenario + probe["custom_rules"] = pack.json(_guard_rule_count(engine)) + probe["rules_fingerprint"] = pack.fingerprint() probe["versions"] = _provenance(version_fn) print(json.dumps(probe)) @@ -677,8 +806,11 @@ def main() -> None: from cloudformation_validate import ( # noqa: E402 CelEngine, CompositeEngine, + CompositeEngineConfig, DetailLevel, + EngineConfig, EntityType, + ExternalRuleSource, JsonValue, RegoEngine, Severity, @@ -693,14 +825,29 @@ def main() -> None: _JsonValue = JsonValue _EntityType = EntityType - engine_class = {"rego": RegoEngine, "cel": CelEngine, "composite": CompositeEngine}[engine_name] + # Read before any timer starts so only engine construction is measured. + scenario: str = args.scenario + pack = RulePack() + pack.load("guard", args.guard_rules) + pack.load("rego", args.rego_rules) + guard_sources = [ExternalRuleSource(name=name, content=content) for name, content in pack.guard] + rego_sources = [ExternalRuleSource(name=name, content=content) for name, content in pack.rego] + + def engine_factory() -> Any: + if engine_name == "rego": + return RegoEngine(EngineConfig(custom_rules=rego_sources, guard_rules=guard_sources)) + if engine_name == "cel": + return CelEngine(EngineConfig(guard_rules=guard_sources)) + return CompositeEngine(CompositeEngineConfig(rego_rules=rego_sources, guard_rules=guard_sources)) benchmark_config = ValidateConfig(severity_level=Severity.DEBUG, detail_level=DetailLevel.DETAILED) if startup_probe: _run_startup_probe( engine_name, - engine_class, + engine_factory, + pack, + scenario, version, benchmark_config, import_elapsed_ms, @@ -712,7 +859,7 @@ def main() -> None: Path(args.template_dir).resolve() if args.template_dir is not None else _DEFAULT_TEMPLATE_DIR.resolve() ) - output_dir = _BINDINGS_DIR / "reports" / engine_name + output_dir = _report_output_dir(engine_name, scenario) output_dir.mkdir(parents=True, exist_ok=True) json_dir = output_dir / "json_detailed" @@ -743,11 +890,11 @@ def main() -> None: if template_data: startup_label, startup_bytes = template_data[0] engine, startup = _measure_startup( - engine_class, startup_bytes, startup_label, benchmark_config, import_elapsed_ms + engine_factory, startup_bytes, startup_label, benchmark_config, import_elapsed_ms ) else: engine_start = time.perf_counter() - engine = engine_class() + engine = engine_factory() engine_init_ms = (time.perf_counter() - engine_start) * 1000.0 startup = StartupMeasurement( startup_template="", @@ -801,6 +948,7 @@ def main() -> None: _normalize_parse_failure_report(parse_failure_report), _zero_benchmark_metrics(), engine_name, + scenario, ) del parse_failure_report results.append(_error_result(rel_path, 0, "parse_error", str(exc))) @@ -819,6 +967,7 @@ def main() -> None: except Exception as exc: print(f" FAILED: {exc}", file=sys.stderr) results.append(_error_result(rel_path, size_bytes, "error", str(exc))) + _write_failed_template_report(json_path, rel_path, str(exc), engine_name, scenario) failed = True break host_validate_ms = (time.perf_counter() - t0) * 1000.0 @@ -887,7 +1036,7 @@ def main() -> None: binding_overhead_ms, ) - _write_template_report(json_path, rel_path, report, benchmark_metrics, engine_name) + _write_template_report(json_path, rel_path, report, benchmark_metrics, engine_name, scenario) template_result = { "file": rel_path, @@ -1000,7 +1149,9 @@ def main() -> None: binding_overhead_vec = [r["binding_overhead_ms"] for r in successful_results] corpus_fingerprint, corpus_file_count = _compute_corpus_fingerprint(template_dir, templates) - run_fp = _run_fingerprint(corpus_fingerprint, engine_name, "DETAILED", iterations) + rules_fingerprint = pack.fingerprint() + custom_rules = pack.json(_guard_rule_count(engine)) + run_fp = _run_fingerprint(corpus_fingerprint, rules_fingerprint, scenario, engine_name, "DETAILED", iterations) # Provenance is built after all timed work so the cargo/rustc spawns never # contaminate a measurement. @@ -1021,6 +1172,9 @@ def main() -> None: "iterations_per_template": iterations, "corpus_fingerprint": corpus_fingerprint, "corpus_file_count": corpus_file_count, + "scenario": scenario, + "custom_rules": custom_rules, + "rules_fingerprint": rules_fingerprint, "run_fingerprint": run_fp, "performance": { "module_load_ms": _round4(import_elapsed_ms), diff --git a/src/bindings-python/bench/test_cli.py b/src/bindings-python/bench/test_cli.py index a48e3003..fa5228e7 100644 --- a/src/bindings-python/bench/test_cli.py +++ b/src/bindings-python/bench/test_cli.py @@ -50,6 +50,21 @@ def main(): if code == 0: errors.append("zero iterations should fail, got exit 0") + # Rego rules cannot be loaded into the CEL engine + code, out, err = run(["--engine", "cel", "--iterations", "1", "--rego-rules", "rules.rego"]) + if code == 0 or "--rego-rules" not in err: + errors.append(f"--rego-rules with --engine cel should fail naming the flag, got exit {code}: {err.strip()}") + + # A scenario name is a single lowercase path component + code, out, err = run(["--engine", "rego", "--iterations", "1", "--scenario", "Bad Name"]) + if code == 0 or "--scenario" not in err: + errors.append(f"invalid --scenario should fail naming the flag, got exit {code}: {err.strip()}") + + # --guard-rules / --rego-rules require a value + code, out, err = run(["--engine", "rego", "--iterations", "1", "--guard-rules"]) + if code == 0: + errors.append("--guard-rules without a value should fail, got exit 0") + if errors: print("FAILURES:") for e in errors: diff --git a/src/bindings-wasm/bench/benchmark.ts b/src/bindings-wasm/bench/benchmark.ts index 82203c05..ec114556 100644 --- a/src/bindings-wasm/bench/benchmark.ts +++ b/src/bindings-wasm/bench/benchmark.ts @@ -28,11 +28,32 @@ const args = process.argv.slice(2); if (args.includes('-h') || args.includes('--help')) { console.error( 'Usage: npx ts-node benchmark.ts [TEMPLATE|DIR] [--engine rego|cel|composite] [--iterations N]\n' + - ' npx ts-node benchmark.ts --startup-probe [--engine rego|cel|composite]', + ' npx ts-node benchmark.ts --startup-probe [--engine rego|cel|composite]\n' + + '\n' + + 'Either form accepts the shared scenario flags:\n' + + ' --guard-rules PATH Load a Guard (.guard) rule file or directory into the engine; repeatable\n' + + ' --rego-rules PATH Load a custom Rego (.rego) rule file or directory; repeatable, not valid with --engine cel\n' + + ' --scenario NAME Label the run and write reports to reports/scenarios/NAME// instead of\n' + + ' reports// (default scenario: builtin)', ); process.exit(2); } +function flagValues(flag: string): string[] { + const values: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] !== flag) continue; + const value = args[i + 1]; + if (value === undefined || value.startsWith('-')) { + console.error(`Error: ${flag} requires a path value`); + process.exit(2); + } + values.push(value); + i++; + } + return values; +} + function argValue(flag: string): string | undefined { const idx = args.indexOf(flag); return idx >= 0 ? args[idx + 1] : undefined; @@ -43,7 +64,10 @@ function argValue(flag: string): string | undefined { const benchDir = path.basename(__dirname) === 'build' ? path.dirname(__dirname) : __dirname; const DEFAULT_STARTUP_TEMPLATE = path.join('good', 'minimal.yaml'); const DEFAULT_TEMPLATE_DIR = path.resolve(benchDir, '../../resources/templates'); -const FLAGS_WITH_VALUES = new Set(['--engine', '--iterations']); +const FLAGS_WITH_VALUES = new Set(['--engine', '--iterations', '--guard-rules', '--rego-rules', '--scenario']); +// Recorded when no --scenario is given; its reports keep the historical reports// +// layout that other tooling reads. +const DEFAULT_SCENARIO = 'builtin'; const startupProbe = args.includes('--startup-probe'); const positionalArg = (() => { for (let i = 0; i < args.length; i++) { @@ -89,6 +113,112 @@ const iterations: number = (() => { return parsed; })(); +const scenario: string = (() => { + if (!args.includes('--scenario')) return DEFAULT_SCENARIO; + const value = argValue('--scenario'); + if (value === undefined) { + console.error('Error: --scenario requires a value'); + process.exit(2); + } + if (!/^[a-z0-9][a-z0-9_-]{0,63}$/.test(value)) { + console.error( + `Error: --scenario must be a lowercase name of letters, digits, '-' or '_' (max 64), got '${value}'`, + ); + process.exit(2); + } + return value; +})(); +const guardRulePaths = flagValues('--guard-rules'); +const regoRulePaths = flagValues('--rego-rules'); +if (engineFlag === 'cel' && regoRulePaths.length > 0) { + console.error('Error: --rego-rules cannot be loaded into the CEL engine; use --engine rego or composite'); + process.exit(2); +} + +function collectRuleFiles(root: string, extension: string): string[] { + if (fs.statSync(root).isFile()) return [root]; + const found: string[] = []; + const walk = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.isFile() && entry.name.endsWith(`.${extension}`)) found.push(full); + } + }; + walk(root); + return found.sort(); +} + +/** + * The fingerprint format - one `\t\t` + * line per file, sorted, hashed - is shared by every harness so the comparison can prove all + * bindings loaded the same rules. + */ +class RulePack { + readonly guard: { name: string; content: string }[] = []; + readonly rego: { name: string; content: string }[] = []; + guardBytes = 0; + regoBytes = 0; + private readonly entries: string[] = []; + + load(kind: 'guard' | 'rego', rawPaths: string[]): void { + for (const rawPath of rawPaths) { + if (!fs.existsSync(rawPath)) { + console.error(`Error: --${kind}-rules path not found: ${rawPath}`); + process.exit(2); + } + const files = collectRuleFiles(rawPath, kind); + if (files.length === 0) { + console.error(`Error: --${kind}-rules path contains no .${kind} files: ${rawPath}`); + process.exit(2); + } + const isFile = fs.statSync(rawPath).isFile(); + for (const file of files) { + const content = fs.readFileSync(file, 'utf8'); + const relative = isFile ? path.basename(file) : path.relative(rawPath, file).replace(/\\/g, '/'); + const digest = crypto.createHash('sha256').update(content, 'utf8').digest('hex'); + this.entries.push(`${kind}\t${relative}\t${digest}\n`); + const bytes = Buffer.byteLength(content, 'utf8'); + if (kind === 'guard') { + this.guard.push({ name: file, content }); + this.guardBytes += bytes; + } else { + this.rego.push({ name: file, content }); + this.regoBytes += bytes; + } + } + } + } + + fingerprint(): string { + return crypto.createHash('sha256').update(this.entries.slice().sort().join(''), 'utf8').digest('hex'); + } + + json(guardRuleCount: number): Record { + return { + guard: { files: this.guard.length, rules: guardRuleCount, bytes: this.guardBytes }, + rego: { files: this.rego.length, bytes: this.regoBytes }, + }; + } +} + +// Read before any timer starts so only engine construction is measured. +const rulePack = new RulePack(); +rulePack.load('guard', guardRulePaths); +rulePack.load('rego', regoRulePaths); + +/** Distinct names: a rule name shared by two files counts once. */ +function guardRuleCount(engine: WasmEngine): number { + const rules: { origin?: string }[] = engine.listRules(); + return rules.filter((rule) => rule.origin === 'GUARD').length; +} + +function reportOutputDir(engineName: string): string { + return scenario === DEFAULT_SCENARIO + ? path.resolve(benchDir, `../reports/${engineName}`) + : path.resolve(benchDir, `../reports/scenarios/${scenario}/${engineName}`); +} + function round4(v: number): number { return Math.round(v * 10000) / 10000; } @@ -391,15 +521,15 @@ function reportPath(jsonDir: string, relativePath: string): string { } const engineConfig: EngineConfig = { - customRules: [], - guardRules: [], + customRules: engineFlag === 'rego' ? rulePack.rego : [], + guardRules: rulePack.guard, }; // CompositeEngine takes a CompositeEngineConfig (regoRules + guardRules) rather than an // EngineConfig. It is constructed through the raw binding namespace, which shares the // same structural instance shape as the Rego and CEL engines. const compositeEngineConfig = { - regoRules: [], - guardRules: [], + regoRules: rulePack.rego, + guardRules: rulePack.guard, }; function newEngine(): WasmEngine { if (engineFlag === 'composite') return new wasmRaw.WasmCompositeEngine(compositeEngineConfig); @@ -503,7 +633,13 @@ function measureStartup( const consumerInitMs = engineInitMs; const validateStart = performance.now(); - const report: ValidationReport = engine.validateTemplate(startupBytes, validateConfig, startupLabel); + let report: ValidationReport; + try { + report = engine.validateTemplate(startupBytes, validateConfig, startupLabel); + } catch (e: any) { + console.error(`Error: startup first validation failed on '${startupLabel}': ${e?.message ?? e}`); + process.exit(1); + } const hostMs = performance.now() - validateStart; const perf = report.performance; @@ -566,6 +702,7 @@ function runStartupProbe(): void { const startupLabel = path.basename(startupPath); const { engine, startup } = measureStartup(startupBytes, startupLabel); const engineName = engine.engineName(); + const customRules = rulePack.json(guardRuleCount(engine)); try { engine.free(); } catch {} @@ -573,6 +710,9 @@ function runStartupProbe(): void { ...startupSectionJson(startup), binding: 'wasm', engine: engineName, + scenario, + custom_rules: customRules, + rules_fingerprint: rulePack.fingerprint(), versions: provenanceJson(), }; process.stdout.write(`${JSON.stringify(probe)}\n`); @@ -610,7 +750,14 @@ const initSamples = engineInitSamples.slice(); const coldInitMs = moduleLoadMs + initSamples[0]; const subsequentInitSamples: number[] = []; -const reportDir = path.resolve(benchDir, `../reports/${engineFlag}`); +const rulesFingerprint = rulePack.fingerprint(); +const customRules = rulePack.json(guardRuleCount(engine)); +console.error( + `Scenario '${scenario}': ${rulePack.guard.length} Guard file(s), ${rulePack.rego.length} Rego file(s), ` + + `rules fingerprint ${rulesFingerprint}`, +); + +const reportDir = reportOutputDir(engineFlag); const jsonDir = path.join(reportDir, `json_${formatDir}`); // Clean previous output so stale reports from dropped/renamed templates are not left behind. if (fs.existsSync(jsonDir)) { @@ -663,6 +810,7 @@ for (const tpl of templates) { engine: engineFlag, binding: 'wasm', detailLevel: formatFlag, + scenario, benchmarkMetrics: zeroBenchmarkMetrics(), }, null, @@ -691,7 +839,28 @@ for (const tpl of templates) { iterWallClock.push(wallMs); if (i === iterations - 1) lastReport = report; } catch (e: any) { - results.push(errorResult(rel, 'error', e.message ?? String(e))); + const message = e.message ?? String(e); + results.push(errorResult(rel, 'error', message)); + // Every attempted template gets a report so the comparison script can pair the + // same template set across bindings and scenarios. + fs.writeFileSync( + jsonPath, + JSON.stringify( + { + filePath: rel, + status: 'ERROR', + error: message, + diagnostics: [], + engine: engineFlag, + binding: 'wasm', + detailLevel: formatFlag, + scenario, + benchmarkMetrics: zeroBenchmarkMetrics(), + }, + null, + 2, + ), + ); failed = true; break; } @@ -720,6 +889,7 @@ for (const tpl of templates) { engine: engineFlag, binding: 'wasm', detailLevel: formatFlag, + scenario, benchmarkMetrics: perTemplateMetricsJson( iterations, iterHostModel, @@ -808,7 +978,7 @@ const throughputPerSec = const { fingerprint: corpusFingerprint, fileCount: corpusFileCount } = computeCorpusFingerprint(templateDir); const runFingerprint = crypto .createHash('sha256') - .update(`${corpusFingerprint}|${engineFlag}|${formatFlag}|${iterations}`) + .update(`${corpusFingerprint}|${rulesFingerprint}|${scenario}|${engineFlag}|${formatFlag}|${iterations}`) .digest('hex'); // Provenance is queried only after every timed measurement above so the @@ -828,6 +998,9 @@ const aggregate = { iterations_per_template: iterations, corpus_fingerprint: corpusFingerprint, corpus_file_count: corpusFileCount, + scenario, + custom_rules: customRules, + rules_fingerprint: rulesFingerprint, run_fingerprint: runFingerprint, performance: { module_load_ms: round4(moduleLoadMs), diff --git a/src/cfn-validate/src/benchmark.rs b/src/cfn-validate/src/benchmark.rs index 3cf5f815..d1dfc648 100644 --- a/src/cfn-validate/src/benchmark.rs +++ b/src/cfn-validate/src/benchmark.rs @@ -10,16 +10,132 @@ use composite_engine::CompositeEngine; use diagnostics::{DetailLevel, ValidationReport}; use log::{error, info}; use rego_engine::RegoEngine; -use rules::Severity; +use rules::{RuleOrigin, Severity}; use schema_validator::SchemaValidator; use sha2::{Digest, Sha256}; use template_model::SemanticModel; use validation_engine::{ - CompositeEngineConfig, EngineConfig, EngineType, ValidateConfig, ValidationEngine, validate_bytes_with_path, + CompositeEngineConfig, EngineConfig, EngineType, ExternalRuleSource, ValidateConfig, ValidationEngine, + validate_bytes_with_path, }; const DEFAULT_STARTUP_TEMPLATE: &str = "good/minimal.yaml"; +/// Recorded when no `--scenario` is given. Its reports keep the historical +/// `reports//` layout that other tooling reads. +const DEFAULT_SCENARIO: &str = "builtin"; + +const GUARD_RULE_EXTENSION: &str = "guard"; +const REGO_RULE_EXTENSION: &str = "rego"; + +/// The rule sources a scenario loads, with the provenance every harness records so +/// the comparison can prove all bindings measured the same rules. +struct RulePack { + guard: Vec, + rego: Vec, + guard_bytes: usize, + rego_bytes: usize, + /// One `\t\t` line per loaded file; the + /// format is shared by every harness so fingerprints compare byte-for-byte. + fingerprint_entries: Vec, +} + +impl RulePack { + fn load(guard_paths: &[String], rego_paths: &[String]) -> Result { + let mut fingerprint_entries = Vec::new(); + let (guard, guard_bytes) = load_rule_sources(GUARD_RULE_EXTENSION, guard_paths, &mut fingerprint_entries)?; + let (rego, rego_bytes) = load_rule_sources(REGO_RULE_EXTENSION, rego_paths, &mut fingerprint_entries)?; + fingerprint_entries.sort(); + Ok(Self { guard, rego, guard_bytes, rego_bytes, fingerprint_entries }) + } + + /// Independent of where the files are checked out, because each path is + /// relative to the argument that named it. + fn fingerprint(&self) -> String { + let mut hasher = Sha256::new(); + for entry in &self.fingerprint_entries { + hasher.update(entry.as_bytes()); + } + to_hex(hasher.finalize()) + } + + /// Only Guard has an engine-side rule count; a Rego pack's rules are discovered + /// during evaluation, so it reports files and bytes alone. + fn json(&self, guard_rule_count: usize) -> serde_json::Value { + serde_json::json!({ + "guard": {"files": self.guard.len(), "rules": guard_rule_count, "bytes": self.guard_bytes}, + "rego": {"files": self.rego.len(), "bytes": self.rego_bytes}, + }) + } +} + +fn load_rule_sources( + kind: &str, + paths: &[String], + fingerprint_entries: &mut Vec, +) -> Result<(Vec, usize), String> { + let mut sources = Vec::new(); + let mut total_bytes = 0; + for raw_path in paths { + let root = Path::new(raw_path); + if !root.exists() { + return Err(format!("--{kind}-rules path not found: {raw_path}")); + } + let files = cfn_validate::collect_files_with_extensions(root, &[kind]); + if files.is_empty() { + return Err(format!("--{kind}-rules path contains no .{kind} files: {raw_path}")); + } + for file in files { + let content = fs::read_to_string(&file) + .map_err(|e| format!("failed to read {kind} rule file '{}': {e}", file.display()))?; + let relative = relative_template_key(raw_path, &file)?; + let mut hasher = Sha256::new(); + hasher.update(content.as_bytes()); + fingerprint_entries.push(format!("{kind}\t{relative}\t{}\n", to_hex(hasher.finalize()))); + total_bytes += content.len(); + sources.push(ExternalRuleSource { name: file.display().to_string(), content }); + } + } + Ok((sources, total_bytes)) +} + +fn guard_rule_count(engine: &dyn ValidationEngine) -> usize { + engine.list_rules().iter().filter(|rule| rule.origin == RuleOrigin::Guard).count() +} + +fn report_output_dir(manifest_dir: &Path, engine_name: &str, scenario: &str) -> PathBuf { + let reports = manifest_dir.join("reports"); + if scenario == DEFAULT_SCENARIO { + reports.join(engine_name) + } else { + reports.join("scenarios").join(scenario).join(engine_name) + } +} + +fn valid_scenario_name(name: &str) -> bool { + let mut chars = name.chars(); + matches!(chars.next(), Some(first) if first.is_ascii_lowercase() || first.is_ascii_digit()) + && chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') + && name.len() <= 64 +} + +fn flag_values(args: &[String], flag: &str) -> Result, String> { + let mut values = Vec::new(); + let mut i = 0; + while i < args.len() { + if args[i] == flag { + match args.get(i + 1) { + Some(value) if !value.starts_with('-') => values.push(value.clone()), + _ => return Err(format!("{flag} requires a path value")), + } + i += 2; + } else { + i += 1; + } + } + Ok(values) +} + /// Replaces the file-extension suffix of a path string, leaving interior occurrences untouched. /// Only the trailing `suffix` is replaced; if the string does not end with `suffix`, it is /// returned unchanged. @@ -38,18 +154,29 @@ fn main() { } } -fn build_engine(engine_type: EngineType, config: &EngineConfig) -> Result, String> { +fn build_engine(engine_type: EngineType, pack: &RulePack) -> Result, String> { match engine_type { EngineType::Cel => { - Ok(Box::new(CelEngine::new(config.clone()).map_err(|e| format!("CEL engine initialization failed: {e}"))?)) + if !pack.rego.is_empty() { + return Err("--rego-rules cannot be loaded into the CEL engine; use --engine rego or composite".into()); + } + let config = EngineConfig::new().with_guard_rules(pack.guard.iter().cloned()); + Ok(Box::new(CelEngine::new(config).map_err(|e| format!("CEL engine initialization failed: {e}"))?)) + } + EngineType::Rego => { + let config = EngineConfig::new() + .with_custom_rules(pack.rego.iter().cloned()) + .with_guard_rules(pack.guard.iter().cloned()); + Ok(Box::new(RegoEngine::new(config).map_err(|e| format!("Rego engine initialization failed: {e}"))?)) + } + EngineType::Composite => { + let config = CompositeEngineConfig::new() + .with_rego_rules(pack.rego.iter().cloned()) + .with_guard_rules(pack.guard.iter().cloned()); + Ok(Box::new( + CompositeEngine::new(config).map_err(|e| format!("Composite engine initialization failed: {e}"))?, + )) } - EngineType::Rego => Ok(Box::new( - RegoEngine::new(config.clone()).map_err(|e| format!("Rego engine initialization failed: {e}"))?, - )), - EngineType::Composite => Ok(Box::new( - CompositeEngine::new(CompositeEngineConfig::default()) - .map_err(|e| format!("Composite engine initialization failed: {e}"))?, - )), } } @@ -75,7 +202,7 @@ struct StartupMeasurement { fn measure_startup( engine_type: EngineType, - config: &EngineConfig, + pack: &RulePack, startup_bytes: &[u8], startup_label: &str, benchmark_config: &ValidateConfig, @@ -87,7 +214,7 @@ fn measure_startup( let schema_init_ms = schema_start.elapsed().as_secs_f64() * 1000.0; let engine_start = Instant::now(); - let engine = build_engine(engine_type, config)?; + let engine = build_engine(engine_type, pack)?; let engine_init_ms = engine_start.elapsed().as_secs_f64() * 1000.0; let consumer_init_ms = schema_init_ms + engine_init_ms; @@ -196,7 +323,13 @@ fn run() -> Result<(), String> { let args: Vec = env::args().collect(); if args.iter().any(|a| a == "-h" || a == "--help") { eprintln!( - "Usage: cfn-benchmark [TEMPLATE|DIR] [--engine rego|cel|composite] [--iterations N] [--startup-probe]" + "Usage: cfn-benchmark [TEMPLATE|DIR] [--engine rego|cel|composite] [--iterations N] [--startup-probe]\n\ + \x20 [--guard-rules PATH]... [--rego-rules PATH]... [--scenario NAME]\n\ + \n\ + \x20 --guard-rules PATH Load a Guard (.guard) rule file or directory into the engine; repeatable\n\ + \x20 --rego-rules PATH Load a custom Rego (.rego) rule file or directory; repeatable, not valid with --engine cel\n\ + \x20 --scenario NAME Label the run and write reports to reports/scenarios/NAME// instead of\n\ + \x20 reports// (default scenario: {DEFAULT_SCENARIO})" ); process::exit(2); } @@ -240,16 +373,50 @@ fn run() -> Result<(), String> { None => 20, }; + let scenario: String = match args.iter().position(|a| a == "--scenario") { + Some(i) => match args.get(i + 1) { + Some(name) if valid_scenario_name(name) => name.clone(), + Some(name) => { + eprintln!( + "Error: --scenario must be a lowercase name of letters, digits, '-' or '_' (max 64), got '{}'", + name + ); + process::exit(2); + } + None => { + eprintln!("Error: --scenario requires a value"); + process::exit(2); + } + }, + None => DEFAULT_SCENARIO.to_string(), + }; + + let guard_paths = flag_values(&args, "--guard-rules").unwrap_or_else(|e| { + eprintln!("Error: {e}"); + process::exit(2); + }); + let rego_paths = flag_values(&args, "--rego-rules").unwrap_or_else(|e| { + eprintln!("Error: {e}"); + process::exit(2); + }); + if engine_type == EngineType::Cel && !rego_paths.is_empty() { + eprintln!("Error: --rego-rules cannot be loaded into the CEL engine; use --engine rego or composite"); + process::exit(2); + } + let pack = RulePack::load(&guard_paths, ®o_paths).unwrap_or_else(|e| { + eprintln!("Error: {e}"); + process::exit(2); + }); + // Hardcoded: benchmarks always use DETAILED format and DEBUG severity to capture // all diagnostics, so all five binding harnesses (native/wasm/jvm/python/go) measure the same work. let detail_level = DetailLevel::Detailed; let severity_level = Severity::Debug; let format_str = "detailed"; let benchmark_config = ValidateConfig { detail_level: detail_level.clone(), severity_level, ..Default::default() }; - let config = EngineConfig::default(); if startup_probe { - return run_startup_probe(engine_type, &config, &benchmark_config, &default_template_dir); + return run_startup_probe(engine_type, &pack, &scenario, &benchmark_config, &default_template_dir); } let template_dir = match positional.as_deref() { @@ -278,8 +445,17 @@ fn run() -> Result<(), String> { let startup_label = relative_template_key(&template_dir, startup_template_path)?; let (schema_validator, engine, startup) = - measure_startup(engine_type, &config, &startup_bytes, &startup_label, &benchmark_config)?; + measure_startup(engine_type, &pack, &startup_bytes, &startup_label, &benchmark_config)?; let engine_name = engine.engine_name(); + let rules_fingerprint = pack.fingerprint(); + let custom_rules = pack.json(guard_rule_count(engine.as_ref())); + info!( + "Scenario '{}': {} Guard file(s), {} Rego file(s), rules fingerprint {}", + scenario, + pack.guard.len(), + pack.rego.len(), + rules_fingerprint + ); let schema_init_samples_ms: Vec = vec![startup.schema_init_ms.unwrap_or(0.0)]; let engine_init_samples_ms: Vec = vec![startup.engine_init_ms]; @@ -289,7 +465,7 @@ fn run() -> Result<(), String> { let subsequent_init_samples_ms: Vec = Vec::new(); let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let output_dir = manifest.join("reports").join(engine_name); + let output_dir = report_output_dir(&manifest, engine_name, &scenario); let json_dir = output_dir.join(format!("json_{}", format_str)); // Clean previous output so stale reports from dropped/renamed templates are not left behind. @@ -338,6 +514,7 @@ fn run() -> Result<(), String> { let mut iter_host_validate_ms: Vec = Vec::with_capacity(iterations); let mut last_report = None; let mut parse_failure_report: Option = None; + let mut evaluation_failure: Option = None; let mut failed = false; for i in 0..iterations { @@ -390,12 +567,14 @@ fn run() -> Result<(), String> { Ok(Ok(r)) => r, Ok(Err(e)) => { results.push(TemplateResult::error(&relative_path, "error", &e.to_string())); + evaluation_failure = Some(e.to_string()); failed = true; break; } Err(_) => { error!("{} panicked during validation", relative_path); results.push(TemplateResult::error(&relative_path, "panic", "panic during validate")); + evaluation_failure = Some("panic during validate".to_string()); failed = true; break; } @@ -414,8 +593,19 @@ fn run() -> Result<(), String> { } } if failed { + // Every attempted template gets a report so the comparison script can pair + // the same template set across bindings and scenarios. if let Some(report) = parse_failure_report { - write_template_report(&json_path, &relative_path, &report, zero_benchmark_metrics(), engine_name)?; + write_template_report( + &json_path, + &relative_path, + &report, + zero_benchmark_metrics(), + engine_name, + &scenario, + )?; + } else if let Some(message) = evaluation_failure { + write_failed_template_report(&json_path, &relative_path, &message, engine_name, &scenario)?; } continue; } @@ -453,7 +643,7 @@ fn run() -> Result<(), String> { &iter_host_validate_ms, binding_overhead_ms, ); - write_template_report(&json_path, &relative_path, &report, benchmark_metrics, engine_name)?; + write_template_report(&json_path, &relative_path, &report, benchmark_metrics, engine_name, &scenario)?; drop(report); let template_result = TemplateResult { @@ -512,7 +702,8 @@ fn run() -> Result<(), String> { }; let (corpus_fingerprint, fingerprint_file_count) = compute_corpus_fingerprint(input_path)?; - let run_fingerprint = run_fingerprint(&corpus_fingerprint, engine_name, "DETAILED", iterations); + let run_fingerprint = + run_fingerprint(&corpus_fingerprint, &rules_fingerprint, &scenario, engine_name, "DETAILED", iterations); let provenance = provenance_json(); let aggregate_stats = serde_json::json!({ @@ -528,6 +719,9 @@ fn run() -> Result<(), String> { "iterations_per_template": iterations, "corpus_fingerprint": corpus_fingerprint, "corpus_file_count": fingerprint_file_count, + "scenario": scenario, + "custom_rules": custom_rules, + "rules_fingerprint": rules_fingerprint, "run_fingerprint": run_fingerprint, "performance": { "module_load_ms": round4(startup.module_load_ms), @@ -592,6 +786,9 @@ fn run() -> Result<(), String> { iterations, corpus_fingerprint: &corpus_fingerprint, corpus_file_count: fingerprint_file_count, + scenario: &scenario, + rules_fingerprint: &rules_fingerprint, + custom_rules: &custom_rules, }); let report_path = output_dir.join(format!("report_{}.md", format_str)); fs::write(&report_path, &report_markdown) @@ -625,7 +822,8 @@ fn run() -> Result<(), String> { fn run_startup_probe( engine_type: EngineType, - config: &EngineConfig, + pack: &RulePack, + scenario: &str, benchmark_config: &ValidateConfig, default_template_dir: &Path, ) -> Result<(), String> { @@ -638,12 +836,15 @@ fn run_startup_probe( .unwrap_or_else(|| startup_path.display().to_string()); let (_schema_validator, engine, startup) = - measure_startup(engine_type, config, &startup_bytes, &startup_label, benchmark_config)?; + measure_startup(engine_type, pack, &startup_bytes, &startup_label, benchmark_config)?; let engine_name = engine.engine_name(); let mut probe = startup_section_json(&startup); probe["binding"] = serde_json::json!("native"); probe["engine"] = serde_json::json!(engine_name); + probe["scenario"] = serde_json::json!(scenario); + probe["custom_rules"] = pack.json(guard_rule_count(engine.as_ref())); + probe["rules_fingerprint"] = serde_json::json!(pack.fingerprint()); probe["versions"] = provenance_json(); let serialized = serde_json::to_string(&probe).map_err(|e| format!("failed to serialize startup probe: {e}"))?; println!("{serialized}"); @@ -770,6 +971,7 @@ fn write_template_report( report: &ValidationReport, benchmark_metrics: serde_json::Value, engine_name: &str, + scenario: &str, ) -> Result<(), String> { let detailed = report.to_report(DetailLevel::Detailed); let mut template_json = serde_json::to_value(&detailed) @@ -777,6 +979,7 @@ fn write_template_report( template_json["engine"] = serde_json::json!(engine_name); template_json["binding"] = serde_json::json!("native"); template_json["detailLevel"] = serde_json::json!("DETAILED"); + template_json["scenario"] = serde_json::json!(scenario); template_json["benchmarkMetrics"] = benchmark_metrics; let mut f = fs::File::create(json_path) .map_err(|e| format!("failed to create report file '{}': {e}", json_path.display()))?; @@ -787,6 +990,31 @@ fn write_template_report( Ok(()) } +/// Keeps the envelope of a successful report so consumers can tell a failed +/// template from a clean one without a second schema. +fn write_failed_template_report( + json_path: &Path, + relative_path: &str, + message: &str, + engine_name: &str, + scenario: &str, +) -> Result<(), String> { + let template_json = serde_json::json!({ + "filePath": relative_path, + "status": "ERROR", + "error": message, + "diagnostics": [], + "engine": engine_name, + "binding": "native", + "detailLevel": "DETAILED", + "scenario": scenario, + "benchmarkMetrics": zero_benchmark_metrics(), + }); + let json_bytes = serde_json::to_string_pretty(&template_json) + .map_err(|e| format!("failed to serialize failure report for '{relative_path}': {e}"))?; + fs::write(json_path, json_bytes).map_err(|e| format!("failed to write report file '{}': {e}", json_path.display())) +} + fn zero_benchmark_metrics() -> serde_json::Value { let zero_iteration = serde_json::json!({ "hostModelMs": 0.0, @@ -931,6 +1159,9 @@ struct MarkdownInput<'a> { iterations: usize, corpus_fingerprint: &'a str, corpus_file_count: usize, + scenario: &'a str, + rules_fingerprint: &'a str, + custom_rules: &'a serde_json::Value, } fn provenance_str<'a>(provenance: &'a serde_json::Value, key: &str) -> &'a str { @@ -946,6 +1177,14 @@ fn generate_markdown(input: &MarkdownInput) -> String { "Corpus fingerprint: `{}` ({} files)\n\n", input.corpus_fingerprint, input.corpus_file_count )); + report_markdown.push_str(&format!( + "Scenario: `{}` - {} Guard file(s) ({} rules), {} custom Rego file(s); rules fingerprint `{}`\n\n", + input.scenario, + input.custom_rules["guard"]["files"], + input.custom_rules["guard"]["rules"], + input.custom_rules["rego"]["files"], + input.rules_fingerprint + )); report_markdown.push_str("## Provenance\n\n"); report_markdown.push_str("| Field | Value |\n|---|---|\n"); @@ -1226,9 +1465,146 @@ fn compute_corpus_fingerprint(root: &Path) -> Result<(String, usize), String> { Ok((to_hex(outer.finalize()), count)) } -/// Deterministic across bindings for the same (corpus, engine, format, iterations) tuple. -fn run_fingerprint(corpus_fp: &str, engine: &str, format: &str, iterations: usize) -> String { +/// Deterministic across bindings for the same (corpus, rules, scenario, engine, format, iterations) tuple. +fn run_fingerprint( + corpus_fp: &str, + rules_fp: &str, + scenario: &str, + engine: &str, + format: &str, + iterations: usize, +) -> String { let mut h = Sha256::new(); - h.update(format!("{}|{}|{}|{}", corpus_fp, engine, format, iterations).as_bytes()); + h.update(format!("{corpus_fp}|{rules_fp}|{scenario}|{engine}|{format}|{iterations}").as_bytes()); to_hex(h.finalize()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn strings(values: &[&str]) -> Vec { + values.iter().map(|v| v.to_string()).collect() + } + + #[test] + fn scenario_names_are_single_lowercase_path_components() { + for valid in ["builtin", "guard", "rego-pack", "big_rules_2"] { + assert!(valid_scenario_name(valid), "{valid} must be accepted"); + } + for invalid in ["", "Guard", "with space", "../escape", "a/b", "-leading", &"x".repeat(65)] { + assert!(!valid_scenario_name(invalid), "{invalid:?} must be rejected"); + } + } + + #[test] + fn default_scenario_keeps_the_historical_report_layout() { + let manifest = Path::new("/crate"); + assert_eq!(report_output_dir(manifest, "rego", DEFAULT_SCENARIO), PathBuf::from("/crate/reports/rego")); + assert_eq!( + report_output_dir(manifest, "cel", "guard"), + PathBuf::from("/crate/reports/scenarios/guard/cel"), + "a named scenario must never overwrite the default reports" + ); + } + + #[test] + fn flag_values_collects_every_occurrence_in_order() { + let args = strings(&["cfn-benchmark", "--guard-rules", "a.guard", "--engine", "cel", "--guard-rules", "dir"]); + assert_eq!(flag_values(&args, "--guard-rules").unwrap(), strings(&["a.guard", "dir"])); + assert!(flag_values(&args, "--rego-rules").unwrap().is_empty()); + } + + #[test] + fn flag_values_rejects_a_missing_or_flag_like_value() { + let trailing = strings(&["cfn-benchmark", "--rego-rules"]); + assert!(flag_values(&trailing, "--rego-rules").is_err()); + let flag_as_value = strings(&["cfn-benchmark", "--rego-rules", "--engine", "rego"]); + assert!(flag_values(&flag_as_value, "--rego-rules").is_err()); + } + + #[test] + fn rule_pack_loads_directories_and_files_and_fingerprints_relative_paths() { + let dir = tempfile::tempdir().unwrap(); + let guard_dir = dir.path().join("guard"); + fs::create_dir(&guard_dir).unwrap(); + fs::write(guard_dir.join("b.guard"), "rule b { true }").unwrap(); + fs::write(guard_dir.join("a.guard"), "rule a { true }").unwrap(); + fs::write(guard_dir.join("ignored.txt"), "not a rule").unwrap(); + let rego_file = dir.path().join("custom.rego"); + fs::write(®o_file, "package p\n").unwrap(); + + let pack = + RulePack::load(&[guard_dir.to_string_lossy().into_owned()], &[rego_file.to_string_lossy().into_owned()]) + .expect("pack loads"); + + assert_eq!(pack.guard.len(), 2); + assert_eq!(pack.rego.len(), 1); + assert_eq!(pack.guard_bytes, "rule b { true }".len() * 2); + assert_eq!(pack.rego_bytes, "package p\n".len()); + let relative: Vec<&str> = pack.fingerprint_entries.iter().map(|e| e.split('\t').nth(1).unwrap()).collect(); + assert_eq!(relative, vec!["a.guard", "b.guard", "custom.rego"], "sorted, relative to the argument"); + + // The same files checked out elsewhere must fingerprint identically. + let copy = tempfile::tempdir().unwrap(); + let copy_guard = copy.path().join("elsewhere"); + fs::create_dir(©_guard).unwrap(); + fs::write(copy_guard.join("a.guard"), "rule a { true }").unwrap(); + fs::write(copy_guard.join("b.guard"), "rule b { true }").unwrap(); + fs::write(copy.path().join("custom.rego"), "package p\n").unwrap(); + let relocated = RulePack::load( + &[copy_guard.to_string_lossy().into_owned()], + &[copy.path().join("custom.rego").to_string_lossy().into_owned()], + ) + .expect("pack loads"); + assert_eq!(relocated.fingerprint(), pack.fingerprint()); + + let changed = RulePack::load(&[copy_guard.to_string_lossy().into_owned()], &[]).expect("pack loads"); + assert_ne!(changed.fingerprint(), pack.fingerprint(), "dropping a file changes the fingerprint"); + } + + #[test] + fn empty_rule_pack_has_a_stable_fingerprint_and_zero_counts() { + let pack = RulePack::load(&[], &[]).expect("empty pack loads"); + assert_eq!(pack.fingerprint(), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + assert_eq!(pack.json(0)["guard"]["files"], 0); + assert_eq!(pack.json(0)["rego"]["bytes"], 0); + } + + #[test] + fn rule_pack_rejects_missing_paths_and_directories_without_rule_files() { + let dir = tempfile::tempdir().unwrap(); + let error = RulePack::load(&["/nonexistent/rules".into()], &[]).err().expect("missing path is rejected"); + assert!(error.contains("not found"), "got: {error}"); + let error = RulePack::load(&[], &[dir.path().to_string_lossy().into_owned()]) + .err() + .expect("a directory without rule files is rejected"); + assert!(error.contains("no .rego files"), "got: {error}"); + } + + #[test] + fn cel_engine_rejects_rego_rules_but_accepts_guard_rules() { + let pack = RulePack { + guard: vec![], + rego: vec![ExternalRuleSource { name: "x.rego".into(), content: "package x\n".into() }], + guard_bytes: 0, + rego_bytes: 10, + fingerprint_entries: vec![], + }; + let error = build_engine(EngineType::Cel, &pack).err().expect("CEL cannot load Rego"); + assert!(error.contains("--rego-rules"), "got: {error}"); + + let guard_only = RulePack { + guard: vec![ExternalRuleSource { + name: "s3.guard".into(), + content: "rule bucket_name { AWS::S3::Bucket { Properties.BucketName EXISTS } }".into(), + }], + rego: vec![], + guard_bytes: 0, + rego_bytes: 0, + fingerprint_entries: vec![], + }; + let engine = build_engine(EngineType::Cel, &guard_only).expect("CEL loads Guard rules"); + assert_eq!(guard_rule_count(engine.as_ref()), 1); + } +} diff --git a/src/cfn-validate/src/lib.rs b/src/cfn-validate/src/lib.rs index e982fe07..4768e922 100644 --- a/src/cfn-validate/src/lib.rs +++ b/src/cfn-validate/src/lib.rs @@ -58,29 +58,38 @@ fn read_schema_file(path: &Path) -> Result Vec { + collect_files_with_extensions(path, TEMPLATE_EXTENSIONS) +} + +/// An explicitly named file is returned regardless of its extension; a directory +/// is filtered recursively and sorted so fingerprints and reports are +/// deterministic. +pub fn collect_files_with_extensions(path: &Path, extensions: &[&str]) -> Vec { if path.is_file() { return vec![path.to_path_buf()]; } let mut files = Vec::new(); - collect_files_recursive(path, &mut files); + collect_files_recursive(path, extensions, &mut files); files.sort(); files } -fn is_template_file(path: &Path) -> bool { - matches!(path.extension().and_then(|s| s.to_str()), Some("yaml" | "yml" | "json")) +fn has_extension(path: &Path, extensions: &[&str]) -> bool { + path.extension().and_then(|s| s.to_str()).is_some_and(|ext| extensions.contains(&ext)) } -fn collect_files_recursive(dir: &Path, files: &mut Vec) { +fn collect_files_recursive(dir: &Path, extensions: &[&str], files: &mut Vec) { let Ok(entries) = fs::read_dir(dir) else { return; }; for entry in entries.flatten() { let path = entry.path(); if path.is_dir() { - collect_files_recursive(&path, files); - } else if path.is_file() && is_template_file(&path) { + collect_files_recursive(&path, extensions, files); + } else if path.is_file() && has_extension(&path, extensions) { files.push(path); } } @@ -282,4 +291,34 @@ mod tests { let names: Vec<_> = result.iter().map(|p| p.file_name().unwrap().to_str().unwrap()).collect(); assert_eq!(names, vec!["a.yaml", "b.yml", "c.json"], "only .yaml, .yml, .json files should be collected"); } + + #[test] + fn collect_files_with_extensions_filters_a_directory_by_the_given_extensions() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("nested"); + fs::create_dir(&nested).unwrap(); + fs::write(dir.path().join("b.rego"), "b").unwrap(); + fs::write(nested.join("a.rego"), "a").unwrap(); + fs::write(dir.path().join("policy.guard"), "g").unwrap(); + fs::write(dir.path().join("template.yaml"), "t").unwrap(); + + let result = collect_files_with_extensions(dir.path(), &["rego"]); + let names: Vec<_> = result.iter().map(|p| p.file_name().unwrap().to_str().unwrap()).collect(); + assert_eq!(names, vec!["b.rego", "a.rego"], "sorted by full path: the root file precedes nested/"); + assert!(result[1].ends_with("nested/a.rego")); + } + + #[test] + fn collect_files_with_extensions_returns_an_explicit_file_regardless_of_extension() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("rules.txt"); + fs::write(&file, "content").unwrap(); + + assert_eq!(collect_files_with_extensions(&file, &["guard"]), vec![file]); + } + + #[test] + fn collect_files_with_extensions_returns_empty_for_a_missing_path() { + assert!(collect_files_with_extensions(Path::new("/nonexistent/rules"), &["guard"]).is_empty()); + } } diff --git a/src/resources/README.md b/src/resources/README.md index d212cd0f..a9f21dd7 100644 --- a/src/resources/README.md +++ b/src/resources/README.md @@ -8,10 +8,46 @@ and snapshot tests, and exposes fixture paths plus discovery used by snapshot ge | Directory | Contents | |--------------|------------------------------------------------------------------------------------------------| | `templates/` | CloudFormation templates grouped by intent (`bad/`, `good/`, `cdk/`, `public/`, …) | -| `rules/` | Custom-rule fixtures loaded by rule tests | +| `rules/` | Custom-rule fixtures loaded by rule tests, and the benchmark rule packs (see below) | | `security/` | Security-scenario fixtures used by security tests and snapshot generation | | `expected/` | `validation_reports1.json`, `validation_reports2.json`, … numbered chunk snapshots | +## Benchmark rule packs + +`scripts/compare_benchmarks.py` measures every engine under load from custom rules as well as with the built-in +rules alone. Its `guard` scenario loads every `.guard` file in `rules/` into the engine, its `rego` scenario every +`.rego` file, and its `all` scenario both; the harnesses receive the directory and load the files themselves. The +packs are therefore exactly the `.guard` and `.rego` files of this directory - a new fixture joins the next benchmark +automatically, and one that fails to evaluate on a corpus template shows up in the report's "Templates Failing Under +a Rule Pack" list rather than silently. + +- **Guard pack** (`guard_*.guard`, 19 files, 39 rule declarations under 34 distinct names, which is the count the report shows). Three are the fixtures the rule tests use + (`guard_encryption.guard`, `guard_multi.guard`, `guard_semantics.guard`). The other 16 are the security and + compliance rules of [cloudformation-guard](https://github.com/aws-cloudformation/cloudformation-guard) at commit + `814bd00a4e6d761e8b5c9f615dd510b1a2a7c374`, copied verbatim and renamed `guard_.guard`: every + CloudFormation example under `guard-examples/` (security policies, encryption, deployment safety, cross-account + access, tagging, network reachability) plus the compliance rules among the `guard/resources/validate/` fixtures + (`workshop`, `db_param_port_rule`, and the three `s3_bucket_*` rules). The remaining upstream fixtures exercise + evaluator semantics and built-in functions rather than check anything (`a_first`...`g_seventh`, `count`, `join`, + `substring`, ...) and are left out: every Guard file costs roughly the same per template regardless of what it + checks (about 0.5 ms on the corpus average), so the pack is limited to rules worth paying for and the benchmark + job stays well inside the runner's time limit. Guard type blocks (used by the three test fixtures) are an + evaluation error on the 26 corpus templates whose `Resources` section is empty; those templates are reported as + failed in the Guard scenarios and excluded from their timings. +- **Custom Rego pack** (`rego_*.rego`, 10 files). Three are the rule-test fixtures; the other seven are authored + here for the benchmark and hold 50 rule IDs across seven packages (`custom_iam`, `custom_network`, + `custom_encryption`, `custom_s3`, `custom_compute`, `custom_data`, `custom_graph`). They are deliberately + expensive and realistic: policy-document decomposition over every policy-bearing resource type, a whole-template + credential scan over serialized properties, table-driven encryption checks resolved per condition scenario, + pairwise subnet CIDR overlap under compatible conditions, service-role trust chains followed through references, + pairwise duplicate-definition detection, and transitive dependency hubs computed with the graph builtins. Every + rule evaluates cleanly on every corpus template. Keep new rules deterministic (no wall-clock or random builtins) + and check `cfn-validate resources/templates --engine rego --rule-source ` reports no + `Custom rule package ... failed to evaluate` errors before adding one. + +Both engines see the same packs, so the Guard scenario also verifies that Rego, CEL, and composite report identical +Guard findings under load. + ## Snapshot generation `expected/validation_reports*.json` are the recorded `cfn-validate --format detailed` output for the regular template diff --git a/src/resources/rules/guard_apigateway_restapi.guard b/src/resources/rules/guard_apigateway_restapi.guard new file mode 100644 index 00000000..80dd42bd --- /dev/null +++ b/src/resources/rules/guard_apigateway_restapi.guard @@ -0,0 +1,48 @@ +# +# Select from Resources section of the template all ApiGateway resources +# present in the template. Sample template, the filter below will select +# /Resources/apiGw/ from the sample template below +# +# Resources: +# apiGw: +# Type: 'AWS::ApiGateway::RestApi' +# Properties: +# EndpointConfiguration: ["PRIVATE"] +# Policy: +# Statement: +# - Action: '*' +# Effect: Allow +# Resource: ['*', "aws:"] +# Condition: +# Bool: +# 'aws:IsSecure': true +# +# +let api_gws = Resources.*[ Type == 'AWS::ApiGateway::RestApi' ] + +# +# Rule intent +# a) All ApiGateway instances deployed must be private +# b) All ApiGateway instances must have atleast one IAM policy condition key to allow access from a VPC +# +# Expectations: +# 1) SKIP when there are not API Gateway instances in the template +# 2) PASS when ALL ApiGateway instances MUST be "PRIVATE" and +# ALL ApiGateway instances MUST have one IAM Condition key with aws:sourceVpc or aws:SourceVpc +# 3) FAIL otherwise +# +rule check_rest_api_is_private_and_has_access when %api_gws !empty { + %api_gws.Properties { + # + # ALL ApiGateways must be PRIVATE, checking with EndpointConfiguration + # + EndpointConfiguration == ["PRIVATE"] + + # ALL ApiGateways must have atleast one IAM statement that has Condition keys with + # aws:sourceVpc + # + some Policy.Statement[*] { + Condition.*[ keys == /aws:[sS]ource(Vpc|VPC|Vpce|VPCE)/ ] !empty + } + } +} diff --git a/src/resources/rules/guard_check_tags_present.guard b/src/resources/rules/guard_check_tags_present.guard new file mode 100644 index 00000000..12b74c26 --- /dev/null +++ b/src/resources/rules/guard_check_tags_present.guard @@ -0,0 +1,43 @@ +# +# This is an exclusion list for resources that do not support Tags. We +# will skip these resources when we select them from the CFN template +# +let excluded_resources = [ + /AWS::AmazonBroker/, + /AWS::App*/ +] + +# +# Here is a sample template with resources, one exempt and other selected +# this would PASS the rule assert_all_resources_have_non_empty_tags +# +# Resources: +# skipped: +# Type: 'AWS::AmazonBroker::Service' # this is skipped +# compliant: +# Type: Consoto::Network::VPC +# Properties: +# Tags: [ +# { +# Key: "Hi", +# Value: "Accepted!" +# } +# ] +# +# This is a filter, sub-selects resources from a list. We select all values +# for Resources section, and from the list of values we select those that +# have a Type attribute that does not match excluded_resources +# +let resources = Resources.*[ + Type not in %excluded_resources +] + +# +# This rule will return +# 1) SKIP if there are no resources that were selected, protected by the guard clause !empty +# 2) FAIL if any one resource did have empty tags or did not have tags specified at all +# 3) PASS when ALL resource do have non-empty tags +# +rule assert_all_resources_have_non_empty_tags when %resources !empty { + %resources.Properties.Tags !empty +} diff --git a/src/resources/rules/guard_db_param_port_rule.guard b/src/resources/rules/guard_db_param_port_rule.guard new file mode 100644 index 00000000..2229327c --- /dev/null +++ b/src/resources/rules/guard_db_param_port_rule.guard @@ -0,0 +1,31 @@ +let redshift_clusters = Resources.*[ Type == 'AWS::Redshift::Cluster'] + +rule DB_PORT_COMPLIANT when %redshift_clusters !empty { + Parameters.DBPort exists + Parameters.DBPort.Default exists + Parameters.DBPort.Default == 3306 + << + Violation: DB Port Default should be 3306. + Fix: Set Default DB Port to 3306 + >> +} + +rule DB_NAME_COMPLIANT when %redshift_clusters !empty { + %redshift_clusters.Properties.DBName exists + %redshift_clusters.Properties.DBName == "mydb" + << + Violation: DB Port Name should be "mydb". + Fix: Set DBName to "mydb" + >> +} + +rule DB_METADATA_EXISTS when %redshift_clusters !empty { + Metadata.Instances exists + Metadata.Instances.Description exists + Metadata.Databases exists + Metadata.Databases.Description exists + << + Violation: DB Metadata is incomplete. Need description for instances and databases. + Fix: Add metadata properties for description of Instances and Databases. + >> +} \ No newline at end of file diff --git a/src/resources/rules/guard_dynamodb_table_sse.guard b/src/resources/rules/guard_dynamodb_table_sse.guard new file mode 100644 index 00000000..be94b3e0 --- /dev/null +++ b/src/resources/rules/guard_dynamodb_table_sse.guard @@ -0,0 +1,97 @@ +# +# Common rule, all resources must have Tags present on them +# +rule assert_all_resources_have_non_empty_tags { + Resources.*.Properties.Tags !empty +} + +# +# Select all DDB resources from the incoming template (payload) +# +let ddb = Resources.*[ Type == 'AWS::DynamoDB::Table' ] + +# +# Run this DDB rule when there are DDB table present and +# we PASSED the check that all resources did have tags in them +# +# Rule Intent: ALL DDB Table must have encryption at rest turned +# on. +# +# Expectations: +# a) SKIP, when there are not DDB tables present or assert_all_resources_have_non_empty_tags FAILED +# b) PASS when all DDB Tables do have encryption turned on +# c) FAIL if wasn't set for them +# +rule dynamo_db_sse_on when %ddb !empty + assert_all_resources_have_non_empty_tags +{ + # + # Ensure ALL DynamoDB Tables have encryption at rest turned on + # + %ddb.Properties.SSESpecification.SSEEnabled == true +} + +# +# We need a differing set of constraints for DynamoDB Tables that are in PROD. +# For these table we have the following additional constraints +# a) The allowed encryption at rest key must be KMS and not server-side-encryption +# b) The table has "delete" protection on for these tables +# +# All DynamoDB Tables intended for PROD have a Tag Key == /PROD/ and a Value with App prefixed +# +# Expectations: +# a) PASS if PROD does only allow KMS keys for encryption +# b) SKIP is there are no DDB tables present or if SSE was not turned on +# c) FAIL if PROD ones do not use KMS +# +# + +# +# Only valid keys that are allowed are KMS +# +let allowed_algorithms = [ 'KMS' ] + +rule dynamo_db_sse_on_for_prod_only when dynamo_db_sse_on +{ + + # + # From the set of DynamoDB Tables that had SSE on (dependent rule dynamo_db_sse_on), + # check the ones that are targeted for PRODuction based on + # Key containing /PROD/ and Value starting with /^App/ + # + let only_prod_ddb = %ddb[ + # + # At least one Tag exists that contains Key and Value + # needed on PROD DynamoDB Table + # + some Properties.Tags[*] { + # + # contains at-least-one key with PROD + # + Key == /PROD/ + + # + # Value that starts with App + # + Value == /^App/ + } + ] + + # + # Skip the evaluation if there were no such DDB Tables + # + when %only_prod_ddb !empty { + %only_prod_ddb { + # + # Only permit allowed ones (currently just KMS) + # + Properties.SSESpecification.SSEType == %allowed_algorithms + + # + # Prod DDB Table must have retain + # + DeletionPolicy == 'Retain' + } + } +} + diff --git a/src/resources/rules/guard_ec2_instance_eip.guard b/src/resources/rules/guard_ec2_instance_eip.guard new file mode 100644 index 00000000..a79892c3 --- /dev/null +++ b/src/resources/rules/guard_ec2_instance_eip.guard @@ -0,0 +1,34 @@ +# +# For all EC2 Instances specified in the Template that have network interfaces +# associated with them +# +let nifs = Resources.*[ + Type == 'AWS::EC2::Instance' + Properties.NetworkInterfaces[*] !empty +] + +# +# Rule Intent +# ---- +# +# Ensure that all EC2 instance resources that do have NetworkInterfaces +# does not allow public IP address to be associated at launch. +# +rule prevent_ec2_with_public_ip when %nifs !empty { + %nifs.AssociatePublicIpAddress == false +} + +let eip_resources = Resources.*[ Type == 'AWS::EC2::EIP' ] + +# +# Rule Intent +# ---- +# +# For all EIPs specified in the template, ensure that it does not +# have an instance ID associated with it, or in other words no EIP +# has been assigned to any EC2 instance +# +rule prevent_eip_associations_with_ec2 when %eip_resources !empty { + %eip_resources.Properties.InstanceId !exists +} + diff --git a/src/resources/rules/guard_ec2_secgroup_inbound_outbound_access.guard b/src/resources/rules/guard_ec2_secgroup_inbound_outbound_access.guard new file mode 100644 index 00000000..3cf9eadb --- /dev/null +++ b/src/resources/rules/guard_ec2_secgroup_inbound_outbound_access.guard @@ -0,0 +1,29 @@ +let sg_resources = Resources.*[ + Type == "AWS::EC2::SecurityGroup" +] + +rule prevent_outbound_access_to_any_ip when %sg_resources !empty { + # select egress rules that are strings + let egress = %sg_resources.Properties.SecurityGroupEgress[ + CidrIp is_string or + CidrIpv6 is_string + ] + + when %egress !empty { + %egress.CidrIp != '0.0.0.0/0' <> or + %egress.CidrIpv6 != '::/0' <> + } +} + +rule prevent_inbound_access_to_any_ip when %sg_resources !empty { + let ingress = %sg_resources.Properties.SecurityGroupIngress[ + CidrIp is_string or + CidrIpv6 is_string + ] + + when %ingress !empty { + %ingress.CidrIp != '0.0.0.0/0' <> or + %ingress.CidrIpv6 != '::/0' <> + } +} + diff --git a/src/resources/rules/guard_ecs_taskdef.guard b/src/resources/rules/guard_ecs_taskdef.guard new file mode 100644 index 00000000..ae2792a4 --- /dev/null +++ b/src/resources/rules/guard_ecs_taskdef.guard @@ -0,0 +1,118 @@ +# +# Select as ECS TaskDefinitions from the template +# +let ecs_tasks = Resources.*[ + Type == 'AWS::ECS::TaskDefinition' +] + +# +# Select a subset of TaskDefinitions whose TaskRoleArn is a Fn::Gett Ref +# +let task_role_refs = some %ecs_tasks.Properties.TaskRoleArn.'Fn::GetAtt'[0] + +# +# Select subset of TaskDefinitions that has a direct reference (a string) +# to an arn +# +let task_role_shared = %ecs_tasks[ + Properties.TaskRoleArn is_string +] + +# +# Select a subset of TaskDefinitions whose ExecutionRoleArn is a Fn::Gett Ref +# +let execution_role_refs = some %ecs_tasks.Properties.ExecutionRoleArn.'Fn::GetAtt'[0] + +# +# Select subset of TaskDefinitions that has a direct reference (a string) +# to an arn +# +let execution_role_shared = %ecs_tasks[ + Properties.ExecutionRoleArn is_string +] + +# +# Rule Intent +# ---- +# +# ALL ECS Task Definition must have both TaskRoleArn and Execution Role Arn +# specified +# +rule all_ecs_tasks_must_have_task_end_execution_roles + when %ecs_tasks !empty +{ + %ecs_tasks.Properties { + TaskRoleArn exists + ExecutionRoleArn exists + } +} + +# +# Rule Intent +# ---- +# +# when all_ecs_tasks_must_have_task_end_execution_roles == PASS +# a) ALL TaskRoleArn that have an Fn::Get, ensure that they are defined in the same stack +# b) That they are of Type IAM::Role +# c) A permissions boundary does exist for these roles +# +rule check_ecs_task_role_refs_are_local + when all_ecs_tasks_must_have_task_end_execution_roles + %task_role_refs !empty +{ + let iam_references = Resources.%task_role_refs + %iam_references { + Type == 'AWS::IAM::Role' + Properties.PermissionsBoundary exists + } +} + +# +# Rule Intent +# ---- +# +# when all_ecs_tasks_must_have_task_end_execution_roles == PASS +# a) ALL ExecutionRoleArn that have an Fn::Get, ensure that they are defined in the same stack +# b) That they are of type IAM::Role +# c) A permissions boundary does exist for these roles +# +rule check_ecs_execution_role_refs_are_local + when all_ecs_tasks_must_have_task_end_execution_roles + %execution_role_refs !empty +{ + let iam_references = Resources.%execution_role_refs + %iam_references { + Type == 'AWS::IAM::Role' + Properties.PermissionsBoundary exists + } +} + +# +# Rule Intent +# ---- +# +# when all_ecs_tasks_must_have_task_end_execution_roles == PASS +# a) ALL task definitions that have a direct arn reference, must +# be allowed only for SharedExecutionRoles +# +rule check_ecs_task_role_refs_are_shared + when all_ecs_tasks_must_have_task_end_execution_roles + %task_role_shared !empty +{ + %task_role_shared.Metadata.SharedExecutionRole exists +} + +# +# Rule Intent +# ---- +# +# when all_ecs_tasks_must_have_task_end_execution_roles == PASS +# a) ALL task definitions that have a direct arn reference, must +# be allowed only for SharedExecutionRoles for Execution role +# +rule check_ecs_execution_role_refs_are_shared + when all_ecs_tasks_must_have_task_end_execution_roles + %execution_role_shared !empty +{ + %execution_role_shared.Metadata.SharedExecutionRole exists +} diff --git a/src/resources/rules/guard_elb_listener.guard b/src/resources/rules/guard_elb_listener.guard new file mode 100644 index 00000000..ea54bac7 --- /dev/null +++ b/src/resources/rules/guard_elb_listener.guard @@ -0,0 +1,41 @@ +# +# Allowed valid protocols for ELB +# +let allowed_protocols = [ "HTTPS", "TLS" ] + +# +# Select ALL ELBs in the template +# +let elbs = Resources.*[ Type == 'AWS::ElasticLoadBalancingV2::Listener' ] + +# +# Rule Intent +# ---- +# +# If there ELBs present, ensure that ELBs have protocols specified from the +# allows list and the Certificates are not empty +# +# Outcome: +# SKIP: when there are no ELBs +# FAIL: when protocols or certificates don't match +# PASS: when they do +# +# +rule ensure_all_elbs_are_secure when %elbs !empty { + %elbs.Properties { + Protocol in %allowed_protocols + Certificates !empty + } +} + +# +# Rule Intent +# ---- +# +# In addition to secure settings, ensure that ELBs are only private +# +# +rule ensure_elbs_are_internal when %elbs !empty { + ensure_all_elbs_are_secure + %elbs.Properties.Scheme == 'internal' +} diff --git a/src/resources/rules/guard_redshift_clustersubnetgroup.guard b/src/resources/rules/guard_redshift_clustersubnetgroup.guard new file mode 100644 index 00000000..11237784 --- /dev/null +++ b/src/resources/rules/guard_redshift_clustersubnetgroup.guard @@ -0,0 +1,29 @@ +let local_subnet_refs = some Resources.*[ Type == /Redshift::ClusterSubnetGroup/ ].Properties.SubnetIds[*].Ref +let subnets = Resources.%local_subnet_refs + +rule redshift_is_not_internet_accessible when %local_subnet_refs !empty { + + # check that local references where indeed subnet type. FAIL otherwise + %subnets.Type == 'AWS::EC2::Subnet' + + # find all route tables associated with the subnet + let route_tables = some Resources.*[ + Type == 'AWS::EC2::SubnetRouteTableAssociation' + Properties.SubnetId.Ref in %local_subnet_refs + ].Properties.RouteTableId.Ref + + # let rts = Resources.%route_tables + Resources.%route_tables.Type == 'AWS::EC2::RouteTable' + + # find all routes that have a gateway associated with it + let gws_ids = some Resources.*[ + Type == 'AWS::EC2::Route' + Properties.GatewayId.Ref exists + Properties.RouteTableId.Ref in %route_tables + ].Properties.GatewayId.Ref + + let gws = Resources.%gws_ids + + %gws.Type != 'AWS::EC2::InternetGateway' + +} diff --git a/src/resources/rules/guard_s3_bucket_logging_enabled.guard b/src/resources/rules/guard_s3_bucket_logging_enabled.guard new file mode 100644 index 00000000..60677340 --- /dev/null +++ b/src/resources/rules/guard_s3_bucket_logging_enabled.guard @@ -0,0 +1,41 @@ +# +##################################### +## Gherkin ## +##################################### +# Rule Identifier: +# S3_BUCKET_LOGGING_ENABLED +# +# Description: +# Checks whether logging is enabled for your S3 buckets. +# +# Reports on: +# AWS::S3::Bucket +# +# Evaluates: +# AWS CloudFormation +# +# Rule Parameters: +# NA +# +# Scenarios: +# a) SKIP: when there are no S3 resource present +# b) PASS: when all S3 resources Logging Configuration exists +# c) FAIL: when all S3 resources have Logging Configuration is not set +# d) SKIP: when metadata includes the suppression for rule S3_BUCKET_LOGGING_ENABLED + +# +# Select all S3 resources from incoming template (payload) +# + +let s3_buckets_bucket_logging_enabled = Resources.*[ Type == 'AWS::S3::Bucket' + Metadata.guard.SuppressedRules not exists or + Metadata.guard.SuppressedRules.* != "S3_BUCKET_LOGGING_ENABLED" +] + +rule S3_BUCKET_LOGGING_ENABLED when %s3_buckets_bucket_logging_enabled !empty { + %s3_buckets_bucket_logging_enabled.Properties.LoggingConfiguration exists + << + Violation: S3 Bucket Logging needs to be configured to enable logging. + Fix: Set the S3 Bucket property LoggingConfiguration to start logging into S3 bucket. + >> +} \ No newline at end of file diff --git a/src/resources/rules/guard_s3_bucket_public_read_prohibited.guard b/src/resources/rules/guard_s3_bucket_public_read_prohibited.guard new file mode 100644 index 00000000..e508e5f9 --- /dev/null +++ b/src/resources/rules/guard_s3_bucket_public_read_prohibited.guard @@ -0,0 +1,13 @@ +let s3_bucket_public_read_prohibited = Resources.*[ Type == 'AWS::S3::Bucket'] + +rule S3_BUCKET_PUBLIC_READ_PROHIBITED when %s3_bucket_public_read_prohibited !empty { + %s3_bucket_public_read_prohibited.Properties.PublicAccessBlockConfiguration exists + %s3_bucket_public_read_prohibited.Properties.PublicAccessBlockConfiguration.BlockPublicAcls == true + %s3_bucket_public_read_prohibited.Properties.PublicAccessBlockConfiguration.BlockPublicPolicy == true + %s3_bucket_public_read_prohibited.Properties.PublicAccessBlockConfiguration.IgnorePublicAcls == true + %s3_bucket_public_read_prohibited.Properties.PublicAccessBlockConfiguration.RestrictPublicBuckets == true + << + Violation: S3 Bucket Public Write Access controls need to be restricted. + Fix: Set S3 Bucket PublicAccessBlockConfiguration properties for BlockPublicAcls, BlockPublicPolicy, IgnorePublicAcls, RestrictPublicBuckets parameters to true. + >> +} \ No newline at end of file diff --git a/src/resources/rules/guard_s3_bucket_server_side_encryption_enabled.guard b/src/resources/rules/guard_s3_bucket_server_side_encryption_enabled.guard new file mode 100644 index 00000000..5219c718 --- /dev/null +++ b/src/resources/rules/guard_s3_bucket_server_side_encryption_enabled.guard @@ -0,0 +1,13 @@ +let s3_buckets_server_side_encryption = Resources.*[ Type == 'AWS::S3::Bucket' + Metadata.guard.SuppressedRules not exists or + Metadata.guard.SuppressedRules.* != "S3_BUCKET_SERVER_SIDE_ENCRYPTION_ENABLED" +] + +rule S3_BUCKET_SERVER_SIDE_ENCRYPTION_ENABLED when %s3_buckets_server_side_encryption !empty { + %s3_buckets_server_side_encryption.Properties.BucketEncryption exists + %s3_buckets_server_side_encryption.Properties.BucketEncryption.ServerSideEncryptionConfiguration[*].ServerSideEncryptionByDefault.SSEAlgorithm in ["aws:kms","AES256"] + << + Violation: S3 Bucket must enable server-side encryption. + Fix: Set the S3 Bucket property BucketEncryption.ServerSideEncryptionConfiguration.ServerSideEncryptionByDefault.SSEAlgorithm to either "aws:kms" or "AES256" + >> +} diff --git a/src/resources/rules/guard_s3_bucket_sse.guard b/src/resources/rules/guard_s3_bucket_sse.guard new file mode 100644 index 00000000..1f0a903d --- /dev/null +++ b/src/resources/rules/guard_s3_bucket_sse.guard @@ -0,0 +1,9 @@ +let s3_buckets = Resources.*[ Type == 'AWS::S3::Bucket' ] +let allowed_algos = ["aws:kms"] + +rule s3_buckets_allowed_sse_algorithm when %s3_buckets !empty { + let encryption = %s3_buckets.Properties.BucketEncryption + %encryption exists + %encryption.ServerSideEncryptionConfiguration[*].ServerSideEncryptionByDefault.SSEAlgorithm in %allowed_algos + +} diff --git a/src/resources/rules/guard_sns_cross_account.guard b/src/resources/rules/guard_sns_cross_account.guard new file mode 100644 index 00000000..00835a41 --- /dev/null +++ b/src/resources/rules/guard_sns_cross_account.guard @@ -0,0 +1,61 @@ +let allowed_accounts = [ # we use regex over string, so that we can match inside ARNs as well + /111122223333/, + /444455556666/ +] + +rule sns_cross_account_only_allowed_accounts { + Resources.* + { + when Type == 'AWS::SNS::TopicPolicy' + { + Properties.PolicyDocument.Statement[*] + { + when Effect == "Allow" + { + Action[*] == /(sns|SNS):Publish/ # ensure that we do not sneak in other permission other than sns:Publish + + # + # Check for different combinations. + # + # We are not using OR disjunction as all of these + # can ALL occur within the same statement and must be checked + # + when Principal is_string { + Principal in %allowed_accounts # when used directly as a string for principal + } + + when Principal.AWS exists { + Principal.AWS[*] in %allowed_accounts # when used for providing access from other accounts + } + + # + # when accessed via any AWS service, one MUST specify a Condition with sourceAccount|Owner + # Arn. + # + when Principal.Service exists { + # + # when accessed via any AWS services, ensure that source account is only from allowed lists + # We want to check StringEquals, StringLike, ArnEquals, ArnLike checks + # + let expected_conditions = Condition[ keys == /String(Equals|Like)|Arn(Equals|Like)/ ] + + # + # Ensure that these are specified, else it is an error + # + %expected_conditions not empty + + # + # Then extract values against these (aws|AWS):[sS]ourceAccount, (aws|AWS):[sS]ourceOwner, (aws|AWS):[sS]ource(Arn|ARN) + # + let source_accounts = %expected_conditions[ keys == /(aws|AWS):[sS]ource(Account|Owner|Arn|ARN)/ ] + + # + # It is an error to not specify this. Ensure the ones specified match allowed accounts + # + %source_accounts in %allowed_accounts + } + } + } + } + } +} diff --git a/src/resources/rules/guard_sns_cross_account_t_parameterized.guard b/src/resources/rules/guard_sns_cross_account_t_parameterized.guard new file mode 100644 index 00000000..06f76678 --- /dev/null +++ b/src/resources/rules/guard_sns_cross_account_t_parameterized.guard @@ -0,0 +1,39 @@ +let allowed = [ + /111122223333/, + /444455556666/ +] + +rule check_direct_principals(principals) { + %principals in %allowed +} + +rule check_aws_specified(principals) { + %principals.AWS in %allowed +} + +rule check_via_aws_service(statement) { + when %statement.Principal.Service exists { + %statement.Condition[ keys == /String(Equals|Like)|Arn(Equals|Like)/ ] not empty { + let source_accounts = this[ keys == /(aws|AWS):[sS]ource(Account|Owner|Arn|ARN)/ ] + %source_accounts in %allowed + } + } +} + +rule check_only_allowed_aws_accounts(statement) { + %statement + { + when Effect == 'Allow' + { + check_direct_principals(Principal) or + check_aws_specified(Principal) or + check_via_aws_service(this) + } + } +} + +rule check_sns_topic_cross_account { + Resources[ Type == 'AWS::SNS::TopicPolicy' ] { + check_only_allowed_aws_accounts(Properties.PolicyDocument.Statement[*]) + } +} diff --git a/src/resources/rules/guard_workshop.guard b/src/resources/rules/guard_workshop.guard new file mode 100644 index 00000000..874e2dd9 --- /dev/null +++ b/src/resources/rules/guard_workshop.guard @@ -0,0 +1,62 @@ +# This rule will return +# 1) FAIL if any IAM policy has a wildcard action +# 2) PASS if there are no IAM policies or IAM policies have no wildcard actions +# +rule assert_no_wildcard_actions { + Roles[*] empty or + Roles[*] { + # if there are no policies - PASS + Policies[*] empty or + Policies[*].Policy.Statement[*] { + Action[*] != '*' + } + } + + # if there are no users - PASS + Users[*] empty or + Users[*] { + # if there are no policies - PASS + Policies[*] empty or + Policies[*].Policy.Statement[*] { + Action[*] != '*' + } + } + + # if there are no groups - PASS + Groups[*] empty or + Groups[*] { + # if there are no policies - PASS + Policies[*] empty or + Policies[*].Policy.Statement[*] { + Action[*] != '*' + } + } + + # if there are no resources - PASS + Resources[*] empty or + Resources[*] { + # resources only have a single policy + Policy.Policy.Statement[*] { + Action[*] != '*' + } + } + + # if there are no permission sets - PASS + PermissionSets[*] empty or + PermissionSets[*] { + # if there are no policies - PASS + Policies[*] empty or + Policies[*].Policy.Statement[*] { + Action[*] != '*' + } + } + + # if there are no orphaned policies - PASS + OrphanedPolicies[*] empty or + OrphanedPolicies[*] { + # orphaned policies have direct policy elements, no need to traverse to Policies[*] + Policy.Statement[*] { + Action[*] != '*' + } + } +} diff --git a/src/resources/rules/rego_compute.rego b/src/resources/rules/rego_compute.rego new file mode 100644 index 00000000..4b0b1208 --- /dev/null +++ b/src/resources/rules/rego_compute.rego @@ -0,0 +1,200 @@ +# Compute hardening: Lambda runtimes, secrets in environment blocks, log groups, service-role trust chains resolved through references, EC2 launch settings and block devices, ECS container definitions, and hardcoded Availability Zones on every resource. +package custom_compute + +import rego.v1 + +_end_of_life_runtimes := { + "python2.7", "python3.6", "python3.7", "python3.8", + "nodejs", "nodejs4.3", "nodejs4.3-edge", "nodejs6.10", "nodejs8.10", "nodejs10.x", "nodejs12.x", "nodejs14.x", "nodejs16.x", + "ruby2.5", "ruby2.7", + "go1.x", + "dotnetcore1.0", "dotnetcore2.0", "dotnetcore2.1", "dotnetcore3.1", "dotnet5.0", "dotnet7", + "java8", + "provided", +} + +_secret_key_pattern := `(?i)(secret|passw(or)?d|token|api[_-]?key|private[_-]?key|credential)` + +# Resources that hand a role to a service principal: the role's trust policy must name that principal. +_service_roles := [ + ["AWS::Lambda::Function", "Properties.Role", "lambda.amazonaws.com"], + ["AWS::ECS::TaskDefinition", "Properties.ExecutionRoleArn", "ecs-tasks.amazonaws.com"], + ["AWS::ECS::TaskDefinition", "Properties.TaskRoleArn", "ecs-tasks.amazonaws.com"], + ["AWS::CodeBuild::Project", "Properties.ServiceRole", "codebuild.amazonaws.com"], + ["AWS::StepFunctions::StateMachine", "Properties.RoleArn", "states.amazonaws.com"], + ["AWS::Glue::Job", "Properties.Role", "glue.amazonaws.com"], + ["AWS::Glue::Crawler", "Properties.Role", "glue.amazonaws.com"], + ["AWS::Events::Rule", "Properties.RoleArn", "events.amazonaws.com"], + ["AWS::Scheduler::Schedule", "Properties.Target.RoleArn", "scheduler.amazonaws.com"], + ["AWS::CodePipeline::Pipeline", "Properties.RoleArn", "codepipeline.amazonaws.com"], + ["AWS::CodeDeploy::DeploymentGroup", "Properties.ServiceRoleArn", "codedeploy.amazonaws.com"], + ["AWS::Config::ConfigurationRecorder", "Properties.RoleARN", "config.amazonaws.com"], + ["AWS::EKS::Cluster", "Properties.RoleArn", "eks.amazonaws.com"], + ["AWS::SageMaker::Model", "Properties.ExecutionRoleArn", "sagemaker.amazonaws.com"], + ["AWS::SageMaker::NotebookInstance", "Properties.RoleArn", "sagemaker.amazonaws.com"], + ["AWS::KinesisFirehose::DeliveryStream", "Properties.S3DestinationConfiguration.RoleARN", "firehose.amazonaws.com"], + ["AWS::ApiGateway::Account", "Properties.CloudWatchRoleArn", "apigateway.amazonaws.com"], + ["AWS::AppSync::GraphQLApi", "Properties.LogConfig.CloudWatchLogsRoleArn", "appsync.amazonaws.com"], + ["AWS::Batch::ComputeEnvironment", "Properties.ServiceRole", "batch.amazonaws.com"], + ["AWS::CloudFormation::Stack", "Properties.RoleARN", "cloudformation.amazonaws.com"], +] + +_imds_paths := [ + ["AWS::EC2::Instance", "Properties.MetadataOptions.HttpTokens"], + ["AWS::EC2::LaunchTemplate", "Properties.LaunchTemplateData.MetadataOptions.HttpTokens"], + ["AWS::AutoScaling::LaunchConfiguration", "Properties.MetadataOptions.HttpTokens"], +] + +_network_interface_lists := [ + ["AWS::EC2::Instance", "Properties.NetworkInterfaces"], + ["AWS::EC2::LaunchTemplate", "Properties.LaunchTemplateData.NetworkInterfaces"], +] + +_block_device_lists := [ + ["AWS::EC2::Instance", "Properties.BlockDeviceMappings"], + ["AWS::EC2::LaunchTemplate", "Properties.LaunchTemplateData.BlockDeviceMappings"], + ["AWS::AutoScaling::LaunchConfiguration", "Properties.BlockDeviceMappings"], +] + +_true_like(value) if value == true + +_true_like(value) if value == "true" + +_trusts_service(role, service) if { + some statement in ensure_list(input.resources[role].properties.AssumeRolePolicyDocument.Statement) + statement.Effect == "Allow" + some principal in ensure_list(statement.Principal.Service) + _matches_service(principal, service) +} + +_matches_service(principal, service) if principal == service + +# Legacy regional principals such as states.us-east-1.amazonaws.com name the same service. +_matches_service(principal, service) if { + is_string(principal) + startswith(principal, sprintf("%s.", [split(service, ".")[0]])) + endswith(principal, ".amazonaws.com") +} + +# A trust policy supplied through a parameter or another opaque value cannot be judged. +_trusts_service(role, _) if is_dynamic(role, "Properties.AssumeRolePolicyDocument") + +_has_log_group(function) if { + some source in ref_sources(function) + input.resources[source].resourceType == "AWS::Logs::LogGroup" +} + +_has_log_group(function) if { + some _, res in input.resources + res.resourceType == "AWS::Logs::LogGroup" + is_string(res.properties.LogGroupName) + some function_name in resolve_all(function, "Properties.FunctionName") + res.properties.LogGroupName == sprintf("/aws/lambda/%s", [function_name]) +} + +_requires_tokens(name, path) if { + some value in resolve_all(name, path) + value == "required" +} + +_requires_tokens(name, path) if is_dynamic(name, path) + +_mutable_tag(image) if endswith(image, ":latest") + +_mutable_tag(image) if { + not contains(image, "@sha256:") + not regex.match(`:[A-Za-z0-9_][A-Za-z0-9_.-]*$`, image) +} + +containers contains {"resource": name, "index": i, "container": container} if { + some name in resources_of_type("AWS::ECS::TaskDefinition") + definitions := input.resources[name].properties.ContainerDefinitions + is_array(definitions) + some i, container in definitions + is_object(container) +} + +violation contains make_diag_at("compute.lambda-runtime-end-of-life", "WARN", name, "Properties.Runtime", sprintf("runtime %s no longer receives security updates; migrate to a supported runtime", [runtime])) if { + some name in resources_of_type("AWS::Lambda::Function") + some runtime in resolve_all(name, "Properties.Runtime") + _end_of_life_runtimes[runtime] +} + +violation contains make_diag_at("compute.role-not-assumable-by-service", "ERROR", name, path, sprintf("role %s does not trust %s, so %s cannot assume it", [role, service, type])) if { + some [type, path, service] in _service_roles + some name in resources_of_type(type) + role := follow_ref(name, path) + input.resources[role].resourceType == "AWS::IAM::Role" + not _trusts_service(role, service) +} + +violation contains make_diag_at("compute.lambda-secret-in-environment", "WARN", name, path, sprintf("environment variable %s looks like a secret stored in plain text; read it from Secrets Manager or SSM at runtime", [key])) if { + some name in resources_of_type("AWS::Lambda::Function") + variables := input.resources[name].properties.Environment.Variables + is_object(variables) + some key, value in variables + regex.match(_secret_key_pattern, key) + is_string(value) + count(value) > 0 + not startswith(value, "{{resolve:") + path := sprintf("Properties.Environment.Variables.%s", [key]) + not is_from_parameter(name, path) +} + +violation contains make_diag("compute.lambda-log-group-missing", "INFO", name, "no AWS::Logs::LogGroup declares this function's log group, so logs are retained forever") if { + some name in resources_of_type("AWS::Lambda::Function") + not has_property(name, "LoggingConfig") + not _has_log_group(name) +} + +violation contains make_diag_at("compute.imdsv1-allowed", "WARN", name, path, "instance metadata service allows IMDSv1 because HttpTokens is not 'required'") if { + some [type, path] in _imds_paths + some name in resources_of_type(type) + not _requires_tokens(name, path) +} + +violation contains make_diag_at("compute.public-ip-on-launch", "WARN", name, path, "network interface requests a public IP address at launch") if { + some [type, base] in _network_interface_lists + some name in resources_of_type(type) + some item in flatten_list(name, base) + _true_like(item.value.AssociatePublicIpAddress) + path := sprintf("%s.%d.AssociatePublicIpAddress", [base, item.index]) +} + +violation contains make_diag_at("compute.block-device-unencrypted", "ERROR", name, path, "EBS volume created at launch is not encrypted") if { + some [type, base] in _block_device_lists + some name in resources_of_type(type) + some item in flatten_list(name, base) + is_object(item.value.Ebs) + not _true_like(item.value.Ebs.Encrypted) + path := sprintf("%s.%d.Ebs.Encrypted", [base, item.index]) + not is_dynamic(name, path) +} + +violation contains make_diag_at("compute.ecs-container-privileged", "ERROR", c.resource, sprintf("Properties.ContainerDefinitions.%d.Privileged", [c.index]), sprintf("container %v runs in privileged mode", [object.get(c.container, "Name", c.index)])) if { + some c in containers + _true_like(c.container.Privileged) +} + +violation contains make_diag_at("compute.ecs-container-image-tag-mutable", "WARN", c.resource, sprintf("Properties.ContainerDefinitions.%d.Image", [c.index]), sprintf("image %s uses a mutable tag; pin a version tag or digest", [image])) if { + some c in containers + image := c.container.Image + is_string(image) + _mutable_tag(image) +} + +violation contains make_diag_at("compute.ecs-container-secret-in-environment", "WARN", c.resource, sprintf("Properties.ContainerDefinitions.%d.Environment.%d", [c.index, j]), sprintf("environment variable %s looks like a secret stored in plain text; use the Secrets property", [variable.Name])) if { + some c in containers + environment := c.container.Environment + is_array(environment) + some j, variable in environment + is_string(variable.Name) + regex.match(_secret_key_pattern, variable.Name) + is_string(variable.Value) + count(variable.Value) > 0 +} + +violation contains make_diag_at("compute.hardcoded-availability-zone", "INFO", name, hit.path, sprintf("availability zone %s is hardcoded; derive it from Fn::GetAZs or a parameter", [hit.zone])) if { + some name, res in input.resources + some hit in hardcoded_azs(name, res.resourceType) +} diff --git a/src/resources/rules/rego_data.rego b/src/resources/rules/rego_data.rego new file mode 100644 index 00000000..4983942f --- /dev/null +++ b/src/resources/rules/rego_data.rego @@ -0,0 +1,116 @@ +# Data stores: public exposure and backup posture of RDS, DocumentDB, Neptune, Redshift, and DMS, plus DynamoDB recovery and key-schema consistency. +package custom_data + +import rego.v1 + +_public_flags := [ + ["AWS::RDS::DBInstance", "Properties.PubliclyAccessible"], + ["AWS::RDS::DBCluster", "Properties.PubliclyAccessible"], + ["AWS::Redshift::Cluster", "Properties.PubliclyAccessible"], + ["AWS::DMS::ReplicationInstance", "Properties.PubliclyAccessible"], +] + +_backed_up_types := ["AWS::RDS::DBInstance", "AWS::RDS::DBCluster", "AWS::DocDB::DBCluster", "AWS::Neptune::DBCluster"] + +_protected_types := ["AWS::RDS::DBInstance", "AWS::RDS::DBCluster", "AWS::DocDB::DBCluster", "AWS::Neptune::DBCluster", "AWS::DynamoDB::Table"] + +_retaining_policies := {"Snapshot", "Retain", "RetainExceptOnCreate"} + +_minimum_retention_days := 7 + +_true_like(value) if value == true + +_true_like(value) if value == "true" + +_conditions(scenario) := object.get(scenario, "conditions", {}) + +_scenario_diag(rule_id, severity, name, path, message, scenario) := make_diag_conditional(rule_id, severity, name, path, message, _conditions(scenario)) if { + count(_conditions(scenario)) > 0 +} + +_scenario_diag(rule_id, severity, name, path, message, scenario) := make_diag_at(rule_id, severity, name, path, message) if { + count(_conditions(scenario)) == 0 +} + +# Replicas and cluster members inherit their backup settings from the source or cluster. +_inherits_backups(name) if has_property(name, "SourceDBInstanceIdentifier") + +_inherits_backups(name) if has_property(name, "DBClusterIdentifier") + +_inherits_backups(name) if has_property(name, "SourceDBClusterIdentifier") + +_deletion_protected(name) if { + some value in resolve_all(name, "Properties.DeletionProtection") + _true_like(value) +} + +_deletion_protected(name) if { + some value in resolve_all(name, "Properties.DeletionProtectionEnabled") + _true_like(value) +} + +_retained(name) if _retaining_policies[input.resources[name].deletionPolicy] + +_pitr_enabled(name) if { + some value in resolve_all(name, "Properties.PointInTimeRecoverySpecification.PointInTimeRecoveryEnabled") + _true_like(value) +} + +_pitr_enabled(name) if is_dynamic(name, "Properties.PointInTimeRecoverySpecification") + +_key_attributes(table) := array.concat(primary, indexes) if { + primary := [[sprintf("Properties.KeySchema.%d.AttributeName", [i]), key.AttributeName] | + some i, key in table.KeySchema + is_string(key.AttributeName) + ] + indexes := [[sprintf("Properties.%s.%d.KeySchema.%d.AttributeName", [field, i, j]), key.AttributeName] | + some field in ["GlobalSecondaryIndexes", "LocalSecondaryIndexes"] + some i, index in table[field] + some j, key in index.KeySchema + is_string(key.AttributeName) + ] +} + +violation contains _scenario_diag("data.publicly-accessible", "ERROR", name, path, sprintf("%s is reachable from the public internet", [type]), scenario) if { + some [type, path] in _public_flags + some name in resources_of_type(type) + some scenario in resolve_scenarios(name, path) + _true_like(scenario.value) +} + +violation contains make_diag_at("data.backup-retention-short", "WARN", name, "Properties.BackupRetentionPeriod", sprintf("automated backups are kept for %d days; keep at least %d", [days, _minimum_retention_days])) if { + some type in _backed_up_types + some name in resources_of_type(type) + not _inherits_backups(name) + some value in resolve_all(name, "Properties.BackupRetentionPeriod") + days := coerce_to_integer(value) + days < _minimum_retention_days +} + +violation contains make_diag_at("data.backup-retention-short", "WARN", name, "Properties.BackupRetentionPeriod", "BackupRetentionPeriod is not set; the default keeps automated backups for a single day") if { + some type in _backed_up_types + some name in resources_of_type(type) + not _inherits_backups(name) + not has_property(name, "BackupRetentionPeriod") +} + +violation contains make_diag("data.deletion-protection-missing", "WARN", name, "data store is deleted with the stack; enable deletion protection or set DeletionPolicy to Snapshot or Retain") if { + some type in _protected_types + some name in resources_of_type(type) + not _deletion_protected(name) + not _retained(name) +} + +violation contains make_diag_at("data.dynamodb-point-in-time-recovery-disabled", "INFO", name, "Properties.PointInTimeRecoverySpecification", "table does not enable point-in-time recovery") if { + some name in resources_of_type("AWS::DynamoDB::Table") + not _pitr_enabled(name) +} + +violation contains make_diag_at("data.dynamodb-key-attribute-undefined", "ERROR", name, path, sprintf("key attribute %s is not declared in AttributeDefinitions", [attribute])) if { + some name in resources_of_type("AWS::DynamoDB::Table") + table := input.resources[name].properties + defined := {definition.AttributeName | some definition in table.AttributeDefinitions; is_string(definition.AttributeName)} + count(defined) > 0 + some [path, attribute] in _key_attributes(table) + not defined[attribute] +} diff --git a/src/resources/rules/rego_encryption.rego b/src/resources/rules/rego_encryption.rego new file mode 100644 index 00000000..86d444f6 --- /dev/null +++ b/src/resources/rules/rego_encryption.rego @@ -0,0 +1,166 @@ +# Encryption posture: table-driven at-rest and in-transit checks over two dozen resource types, resolved per condition scenario, plus KMS key hygiene and S3 transport policies. +package custom_encryption + +import rego.v1 + +# Resource types whose at-rest encryption is a boolean property that defaults to off. +_at_rest_flags := [ + ["AWS::RDS::DBInstance", "Properties.StorageEncrypted"], + ["AWS::RDS::DBCluster", "Properties.StorageEncrypted"], + ["AWS::EC2::Volume", "Properties.Encrypted"], + ["AWS::EFS::FileSystem", "Properties.Encrypted"], + ["AWS::DynamoDB::Table", "Properties.SSESpecification.SSEEnabled"], + ["AWS::Redshift::Cluster", "Properties.Encrypted"], + ["AWS::ElastiCache::ReplicationGroup", "Properties.AtRestEncryptionEnabled"], + ["AWS::OpenSearchService::Domain", "Properties.EncryptionAtRestOptions.Enabled"], + ["AWS::Elasticsearch::Domain", "Properties.EncryptionAtRestOptions.Enabled"], + ["AWS::DocDB::DBCluster", "Properties.StorageEncrypted"], + ["AWS::Neptune::DBCluster", "Properties.StorageEncrypted"], + ["AWS::DAX::Cluster", "Properties.SSESpecification.SSEEnabled"], + ["AWS::WorkSpaces::Workspace", "Properties.RootVolumeEncryptionEnabled"], + ["AWS::WorkSpaces::Workspace", "Properties.UserVolumeEncryptionEnabled"], + ["AWS::EC2::LaunchTemplate", "Properties.LaunchTemplateData.BlockDeviceMappings.{}.Ebs.Encrypted"], +] + +# Resource types whose at-rest encryption is an object or key property that is absent by default. +_at_rest_settings := [ + ["AWS::Kinesis::Stream", "Properties.StreamEncryption"], + ["AWS::EKS::Cluster", "Properties.EncryptionConfig"], + ["AWS::SNS::Topic", "Properties.KmsMasterKeyId"], + ["AWS::CloudTrail::Trail", "Properties.KMSKeyId"], + ["AWS::Logs::LogGroup", "Properties.KmsKeyId"], + ["AWS::SageMaker::NotebookInstance", "Properties.KmsKeyId"], + ["AWS::Glue::SecurityConfiguration", "Properties.EncryptionConfiguration"], + ["AWS::Athena::WorkGroup", "Properties.WorkGroupConfiguration.ResultConfiguration.EncryptionConfiguration"], + ["AWS::CodeBuild::Project", "Properties.EncryptionKey"], +] + +# Absence counts as a finding: every listed setting defaults to the insecure value. +_in_transit_required := [ + ["AWS::OpenSearchService::Domain", "Properties.DomainEndpointOptions.EnforceHTTPS", {true, "true"}], + ["AWS::OpenSearchService::Domain", "Properties.NodeToNodeEncryptionOptions.Enabled", {true, "true"}], + ["AWS::Elasticsearch::Domain", "Properties.DomainEndpointOptions.EnforceHTTPS", {true, "true"}], + ["AWS::Elasticsearch::Domain", "Properties.NodeToNodeEncryptionOptions.Enabled", {true, "true"}], + ["AWS::ElastiCache::ReplicationGroup", "Properties.TransitEncryptionEnabled", {true, "true"}], + ["AWS::MSK::Cluster", "Properties.EncryptionInfo.EncryptionInTransit.ClientBroker", {"TLS"}], + ["AWS::ApiGateway::DomainName", "Properties.SecurityPolicy", {"TLS_1_2"}], + ["AWS::ApiGatewayV2::DomainName", "Properties.DomainNameConfigurations.{}.SecurityPolicy", {"TLS_1_2"}], + ["AWS::CloudFront::Distribution", "Properties.DistributionConfig.DefaultCacheBehavior.ViewerProtocolPolicy", {"https-only", "redirect-to-https"}], + ["AWS::CloudFront::Distribution", "Properties.DistributionConfig.CacheBehaviors.{}.ViewerProtocolPolicy", {"https-only", "redirect-to-https"}], + ["AWS::ElasticLoadBalancingV2::Listener", "Properties.Protocol", {"HTTPS", "TLS", "TCP", "UDP", "TCP_UDP", "GENEVE"}], + ["AWS::ElasticLoadBalancing::LoadBalancer", "Properties.Listeners.{}.Protocol", {"HTTPS", "SSL", "TCP"}], +] + +_disabled(value) if value == false + +_disabled(value) if value == "false" + +_true_like(value) if value == true + +_true_like(value) if value == "true" + +_conditions(scenario) := object.get(scenario, "conditions", {}) + +_scenario_path(scenario, fallback) := object.get(scenario, "path", fallback) + +_false_like_values(value) := [item | some item in ensure_list(value); item in {false, "false"}] + +_scenario_diag(rule_id, severity, name, path, message, scenario) := make_diag_conditional(rule_id, severity, name, path, message, _conditions(scenario)) if { + count(_conditions(scenario)) > 0 +} + +_scenario_diag(rule_id, severity, name, path, message, scenario) := make_diag_at(rule_id, severity, name, path, message) if { + count(_conditions(scenario)) == 0 +} + +_symmetric_key(name) if not has_property(name, "KeySpec") + +_symmetric_key(name) if { + some spec in resolve_all(name, "Properties.KeySpec") + spec == "SYMMETRIC_DEFAULT" +} + +_rotation_enabled(name) if { + some value in resolve_all(name, "Properties.EnableKeyRotation") + _true_like(value) +} + +_grants_root_admin(document) if { + some statement in ensure_list(document.Statement) + statement.Effect == "Allow" + some action in ensure_list(statement.Action) + action in {"kms:*", "*"} + some principal in ensure_list(statement.Principal.AWS) + is_string(principal) + arn_matches(principal, "arn:*:iam::*:root") +} + +_targets_bucket(value, bucket) if value.__ref == bucket + +_targets_bucket(value, bucket) if { + is_string(value) + some bucket_name in resolve_all(bucket, "Properties.BucketName") + value == bucket_name +} + +_denies_insecure_transport(bucket) if { + some _, res in input.resources + res.resourceType == "AWS::S3::BucketPolicy" + _targets_bucket(res.properties.Bucket, bucket) + some statement in ensure_list(res.properties.PolicyDocument.Statement) + statement.Effect == "Deny" + some operator, operands in statement.Condition + lower(operator) == "bool" + some key, value in operands + lower(key) == "aws:securetransport" + count(_false_like_values(value)) > 0 +} + +violation contains _scenario_diag("encryption.at-rest-disabled", "ERROR", name, _scenario_path(scenario, path), sprintf("%s resolves to %v, so data at rest is stored unencrypted", [_scenario_path(scenario, path), scenario.value]), scenario) if { + some [type, path] in _at_rest_flags + some name in resources_of_type(type) + some scenario in resolve_scenarios(name, path) + _disabled(scenario.value) +} + +violation contains make_diag_at("encryption.at-rest-not-configured", "WARN", name, path, sprintf("%s is not set, so %s is created without encryption at rest", [path, type])) if { + some [type, path] in array.concat(_at_rest_flags, _at_rest_settings) + not contains(path, "{}") + some name in resources_of_type(type) + property_can_be_absent(name, path) + not is_dynamic(name, path) +} + +violation contains _scenario_diag("encryption.in-transit-not-enforced", "WARN", name, _scenario_path(scenario, path), sprintf("%s resolves to %v; expected one of %v", [_scenario_path(scenario, path), scenario.value, allowed]), scenario) if { + some [type, path, allowed] in _in_transit_required + some name in resources_of_type(type) + some scenario in resolve_scenarios(name, path) + not allowed[scenario.value] + is_string(scenario.value) +} + +violation contains _scenario_diag("encryption.in-transit-not-enforced", "WARN", name, _scenario_path(scenario, path), sprintf("%s resolves to %v; expected one of %v", [_scenario_path(scenario, path), scenario.value, allowed]), scenario) if { + some [type, path, allowed] in _in_transit_required + some name in resources_of_type(type) + some scenario in resolve_scenarios(name, path) + not allowed[scenario.value] + is_boolean(scenario.value) +} + +violation contains make_diag_at("encryption.kms-key-rotation-disabled", "WARN", name, "Properties.EnableKeyRotation", "symmetric KMS key does not enable automatic key rotation") if { + some name in resources_of_type("AWS::KMS::Key") + _symmetric_key(name) + not _rotation_enabled(name) +} + +violation contains make_diag_at("encryption.kms-key-policy-no-root-admin", "WARN", name, "Properties.KeyPolicy", "key policy grants kms:* to no account root principal, so the key can become unmanageable") if { + some name in resources_of_type("AWS::KMS::Key") + document := input.resources[name].properties.KeyPolicy + is_object(document) + not _grants_root_admin(document) +} + +violation contains make_diag("encryption.s3-secure-transport-policy-missing", "INFO", name, "bucket has no bucket policy denying requests over plain HTTP (aws:SecureTransport)") if { + some name in resources_of_type("AWS::S3::Bucket") + not _denies_insecure_transport(name) +} diff --git a/src/resources/rules/rego_graph.rego b/src/resources/rules/rego_graph.rego new file mode 100644 index 00000000..e0398a55 --- /dev/null +++ b/src/resources/rules/rego_graph.rego @@ -0,0 +1,121 @@ +# Reference-graph analysis: unreferenced resources, redundant DependsOn, references whose condition is not implied by the target's condition (SAT-checked per edge), unused parameters, transitive dependency hubs computed pairwise, and pairwise duplicate resource definitions. +package custom_graph + +import rego.v1 + +# Resource types that act on other resources and are legitimately never referenced themselves. +_terminal_types := { + "AWS::ApiGateway::Account", "AWS::ApiGateway::BasePathMapping", "AWS::ApiGateway::Deployment", "AWS::ApiGateway::Method", + "AWS::ApiGateway::Stage", "AWS::ApplicationAutoScaling::ScalingPolicy", "AWS::AutoScaling::AutoScalingGroup", + "AWS::AutoScaling::ScalingPolicy", "AWS::AutoScaling::ScheduledAction", "AWS::Backup::BackupSelection", "AWS::Budgets::Budget", + "AWS::CloudFormation::CustomResource", "AWS::CloudFormation::Macro", "AWS::CloudFormation::Stack", "AWS::CloudFormation::WaitCondition", + "AWS::CloudFront::Distribution", "AWS::CloudTrail::Trail", "AWS::CloudWatch::Alarm", "AWS::CloudWatch::Dashboard", + "AWS::CodeBuild::Project", "AWS::CodeDeploy::DeploymentGroup", "AWS::CodePipeline::Pipeline", "AWS::Config::ConfigRule", + "AWS::Config::ConfigurationRecorder", "AWS::Config::DeliveryChannel", "AWS::EC2::EIPAssociation", "AWS::EC2::Instance", + "AWS::EC2::NetworkAclEntry", "AWS::EC2::Route", "AWS::EC2::SecurityGroupEgress", "AWS::EC2::SecurityGroupIngress", + "AWS::EC2::SubnetNetworkAclAssociation", "AWS::EC2::SubnetRouteTableAssociation", "AWS::EC2::VPCGatewayAttachment", + "AWS::EC2::VolumeAttachment", "AWS::ECR::ReplicationConfiguration", "AWS::ECS::Service", "AWS::ElasticLoadBalancingV2::Listener", + "AWS::ElasticLoadBalancingV2::ListenerRule", "AWS::Events::EventBusPolicy", "AWS::Events::Rule", "AWS::Glue::Crawler", + "AWS::Glue::Trigger", "AWS::GuardDuty::Detector", "AWS::IAM::GroupPolicy", "AWS::IAM::ManagedPolicy", "AWS::IAM::Policy", + "AWS::IAM::RolePolicy", "AWS::IAM::UserPolicy", "AWS::Lambda::EventSourceMapping", "AWS::Lambda::Permission", + "AWS::Logs::MetricFilter", "AWS::Logs::ResourcePolicy", "AWS::Logs::SubscriptionFilter", "AWS::Route53::RecordSet", + "AWS::Route53::RecordSetGroup", "AWS::S3::BucketPolicy", "AWS::SNS::Subscription", "AWS::SNS::TopicPolicy", + "AWS::SQS::QueuePolicy", "AWS::SSM::Association", "AWS::SSM::Parameter", "AWS::Scheduler::Schedule", "AWS::SecurityHub::Hub", + "AWS::StepFunctions::StateMachine", "AWS::WAFv2::WebACLAssociation", +} + +_hub_minimum_resources := 10 + +_hub_minimum_direct_dependents := 1 + +_referenced_targets := {edge.target | some edge in input.edges} + +_referenced_by_output(name) if { + some _, output in input.outputs + contains(json.marshal(output.value), sprintf("\"__ref\":\"%s\"", [name])) +} + +_referenced_by_output(name) if { + some _, output in input.outputs + some reference in output.getattRefs + reference.resource == name +} + +_referenced_parameter(name) if _referenced_targets[name] + +_referenced_parameter(name) if name in input.conditionParamRefs + +_referenced_parameter(name) if name in input.globalsParamRefs + +_referenced_parameter(name) if name in input.paramsReferencedInDefinitions + +_referenced_parameter(name) if { + some rule in input.parsedRules + contains(json.marshal(rule), sprintf("\"%s\"", [name])) +} + +_referenced_parameter(name) if { + some _, output in input.outputs + contains(json.marshal(output.value), sprintf("\"%s\"", [name])) +} + +_transitive_dependents(name) := {other | + some other, _ in input.resources + other != name + depends_on(other, name) +} + +violation contains make_diag("graph.unreferenced-resource", "INFO", name, sprintf("%s is neither referenced by another resource nor exposed through an output", [name])) if { + some name, res in input.resources + not _terminal_types[res.resourceType] + not startswith(res.resourceType, "Custom::") + count(ref_sources(name)) == 0 + not _referenced_by_output(name) +} + +violation contains make_diag_at("graph.redundant-depends-on", "INFO", name, "DependsOn", sprintf("DependsOn %s is implied by an existing reference at %s", [target, edge.sourcePath])) if { + some name, res in input.resources + some target in res.dependsOn + some edge in edges_from(name) + edge.target == target + edge.kind != "DependsOn" +} + +violation contains make_diag_at("graph.reference-to-conditional-resource", "ERROR", edge.source, edge.sourcePath, sprintf("%s references %s, which exists only when %s is true, but %v does not imply that", [edge.source, edge.target, target_condition, source_condition])) if { + some edge in input.edges + edge.kind != "DependsOn" + input.resources[edge.source] + input.resources[edge.target] + target_condition := resource_condition(edge.target) + target_condition != null + source_condition := resource_condition(edge.source) + not conjunction_implies(source_condition, object.get(edge, "conditionContext", null), target_condition) +} + +violation contains make_diag("graph.parameter-unused", "INFO", "", sprintf("parameter %s is never referenced", [name])) if { + some name, _ in input.parameters + not _referenced_parameter(name) +} + +violation contains make_diag("graph.dependency-hub", "INFO", name, sprintf("%d of %d resources transitively depend on %s; a change to it ripples through most of the stack", [count(dependents), total, name])) if { + total := count(input.resources) + total >= _hub_minimum_resources + some name, _ in input.resources + count(ref_sources(name)) >= _hub_minimum_direct_dependents + dependents := _transitive_dependents(name) + count(dependents) * 2 > total +} + +_duplicates_of(name) := {other | + some other, candidate in input.resources + name < other + candidate.resourceType == input.resources[name].resourceType + candidate.properties == input.resources[name].properties +} + +violation contains make_diag("graph.duplicate-resource-definition", "INFO", name, sprintf("%d later resource(s) such as %s have the same type and identical properties as %s", [count(duplicates), min(duplicates), name])) if { + some name, res in input.resources + count(res.properties) > 0 + duplicates := _duplicates_of(name) + count(duplicates) > 0 +} diff --git a/src/resources/rules/rego_iam.rego b/src/resources/rules/rego_iam.rego new file mode 100644 index 00000000..7059cf3e --- /dev/null +++ b/src/resources/rules/rego_iam.rego @@ -0,0 +1,198 @@ +# IAM and credentials: every identity, trust, and resource policy document in the template is decomposed into statements and checked for over-broad grants, and every resource's serialized properties are scanned for hardcoded credentials. +package custom_iam + +import rego.v1 + +_identity_types := {"AWS::IAM::Role", "AWS::IAM::User", "AWS::IAM::Group"} + +_standalone_policy_types := { + "AWS::IAM::Policy", + "AWS::IAM::ManagedPolicy", + "AWS::IAM::RolePolicy", + "AWS::IAM::UserPolicy", + "AWS::IAM::GroupPolicy", +} + +_resource_policy_properties := { + "AWS::S3::BucketPolicy": "PolicyDocument", + "AWS::SQS::QueuePolicy": "PolicyDocument", + "AWS::SNS::TopicPolicy": "PolicyDocument", + "AWS::KMS::Key": "KeyPolicy", + "AWS::SecretsManager::ResourcePolicy": "ResourcePolicy", + "AWS::ECR::Repository": "RepositoryPolicyText", + "AWS::EFS::FileSystem": "FileSystemPolicy", + "AWS::Backup::BackupVault": "AccessPolicy", + "AWS::OpenSearchService::Domain": "AccessPolicies", + "AWS::Elasticsearch::Domain": "AccessPolicies", + "AWS::ApiGateway::RestApi": "Policy", + "AWS::S3::AccessPoint": "Policy", + "AWS::CodeArtifact::Domain": "PermissionsPolicyDocument", + "AWS::CodeArtifact::Repository": "PermissionsPolicyDocument", + "AWS::IoT::Policy": "PolicyDocument", + "AWS::EventSchemas::RegistryPolicy": "Policy", + "AWS::Glacier::VaultLockPolicy": "Policy", + "AWS::MediaStore::ContainerPolicy": "Policy", + "AWS::Lambda::LayerVersionPermission": "Policy", +} + +_credential_patterns := [ + ["AWS access key ID", `(A3T[A-Z0-9]|AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA)[A-Z0-9]{16}`], + ["private key block", `-----BEGIN (RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY( BLOCK)?-----`], + ["password literal", `(?i)"[a-z_-]*(password|passwd|secret)[a-z_-]*":"[^"{}\[\]]{8,}"`], +] + +# The size limit CloudFormation and IAM apply to the aggregate inline policies of a principal. +_inline_policy_limits := {"AWS::IAM::Role": 10240, "AWS::IAM::User": 2048, "AWS::IAM::Group": 5120} + +policy_documents contains {"resource": name, "path": sprintf("Properties.Policies.%d.PolicyDocument", [i]), "kind": "identity", "document": document} if { + some name, res in input.resources + _identity_types[res.resourceType] + is_array(res.properties.Policies) + some i, policy in res.properties.Policies + document := policy.PolicyDocument + is_object(document) +} + +policy_documents contains {"resource": name, "path": "Properties.PolicyDocument", "kind": "identity", "document": document} if { + some name, res in input.resources + _standalone_policy_types[res.resourceType] + document := res.properties.PolicyDocument + is_object(document) +} + +policy_documents contains {"resource": name, "path": sprintf("Properties.%s", [property]), "kind": "resource", "document": document} if { + some name, res in input.resources + property := _resource_policy_properties[res.resourceType] + document := res.properties[property] + is_object(document) +} + +trust_documents contains {"resource": name, "path": "Properties.AssumeRolePolicyDocument", "document": document} if { + some name, res in input.resources + res.resourceType == "AWS::IAM::Role" + document := res.properties.AssumeRolePolicyDocument + is_object(document) +} + +# A lone statement object is valid IAM and is treated as a one-element list. +statements contains {"resource": p.resource, "path": sprintf("%s.Statement.%d", [p.path, i]), "kind": p.kind, "statement": statement} if { + some p in policy_documents + some i, statement in ensure_list(p.document.Statement) + is_object(statement) +} + +_strings(value) := [item | some item in ensure_list(value); is_string(item)] + +_allows(statement) if statement.Effect == "Allow" + +_full_wildcard(action) if action == "*" + +_full_wildcard(action) if action == "*:*" + +_service_wildcard(action) if { + not _full_wildcard(action) + regex.match(`^[A-Za-z0-9-]+:\*$`, action) +} + +_all_resources(statement) if { + some resource in _strings(statement.Resource) + resource == "*" +} + +_has_condition(statement) if { + is_object(statement.Condition) + count(statement.Condition) > 0 +} + +_any_principal(principal) if principal == "*" + +_any_principal(principal) if { + is_object(principal) + some aws in _strings(principal.AWS) + aws == "*" +} + +_passed_to_service_condition(statement) if { + some _, operands in statement.Condition + is_object(operands) + some key, _ in operands + lower(key) == "iam:passedtoservice" +} + +violation contains make_diag_at("iam.allow-all-actions-all-resources", "ERROR", s.resource, s.path, sprintf("statement allows every action (%s) on every resource without a condition", [action])) if { + some s in statements + _allows(s.statement) + some action in _strings(s.statement.Action) + _full_wildcard(action) + _all_resources(s.statement) + not _has_condition(s.statement) +} + +violation contains make_diag_at("iam.service-wildcard-action", "WARN", s.resource, s.path, sprintf("statement allows %s on every resource; list the actions and scope the resources", [action])) if { + some s in statements + s.kind == "identity" + _allows(s.statement) + some action in _strings(s.statement.Action) + _service_wildcard(action) + _all_resources(s.statement) +} + +violation contains make_diag_full("iam.passrole-unrestricted", "WARN", s.resource, s.path, "iam:PassRole is allowed on every role without an iam:PassedToService condition", "Restrict Resource to the roles that may be passed or add an iam:PassedToService condition", "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_passrole.html") if { + some s in statements + s.kind == "identity" + _allows(s.statement) + some action in _strings(s.statement.Action) + lower(action) in {"iam:passrole", "iam:*", "iam:pass*"} + _all_resources(s.statement) + not _passed_to_service_condition(s.statement) +} + +violation contains make_diag_at("iam.allow-with-negated-matcher", "WARN", s.resource, s.path, sprintf("Allow statement uses %s, which grants everything that is not listed", [field])) if { + some s in statements + _allows(s.statement) + some field in ["NotAction", "NotResource"] + object.get(s.statement, field, null) != null +} + +violation contains make_diag_at("iam.trust-policy-any-principal", "ERROR", t.resource, sprintf("%s.Statement.%d", [t.path, i]), "role can be assumed by any AWS principal because the trust statement has no condition") if { + some t in trust_documents + some i, statement in ensure_list(t.document.Statement) + is_object(statement) + _allows(statement) + _any_principal(statement.Principal) + not _has_condition(statement) +} + +violation contains make_diag_at("iam.inline-policy-size-limit", "WARN", name, "Properties.Policies", sprintf("inline policies total %d characters; %s allows %d", [total, res.resourceType, limit])) if { + some name, res in input.resources + limit := _inline_policy_limits[res.resourceType] + is_array(res.properties.Policies) + total := sum([count(json.marshal(policy.PolicyDocument)) | some policy in res.properties.Policies; is_object(policy.PolicyDocument)]) + total > limit +} + +violation contains make_diag_at("iam.administrator-access-attached", "WARN", name, sprintf("Properties.ManagedPolicyArns.%d", [i]), "the AdministratorAccess managed policy is attached; grant the specific permissions the principal needs") if { + some name, res in input.resources + _identity_types[res.resourceType] + is_array(res.properties.ManagedPolicyArns) + some i, arn in res.properties.ManagedPolicyArns + is_string(arn) + arn_matches(arn, "arn:*:iam::aws:policy/AdministratorAccess") +} + +violation contains make_diag_at("iam.resource-policy-public-principal", "ERROR", s.resource, s.path, sprintf("resource policy statement grants %v to any principal without a condition", [_strings(s.statement.Action)])) if { + some s in statements + s.kind == "resource" + _allows(s.statement) + _any_principal(s.statement.Principal) + not _has_condition(s.statement) +} + +_redact(match) := sprintf("%s...", [substring(match, 0, 24)]) + +violation contains make_diag("iam.hardcoded-credential", "ERROR", name, sprintf("%s found in the resource properties: %s", [label, _redact(match)])) if { + some name, res in input.resources + serialized := json.marshal(res.properties) + some [label, pattern] in _credential_patterns + some match in regex.find_n(pattern, serialized, -1) +} diff --git a/src/resources/rules/rego_network.rego b/src/resources/rules/rego_network.rego new file mode 100644 index 00000000..636e312b --- /dev/null +++ b/src/resources/rules/rego_network.rego @@ -0,0 +1,122 @@ +# Network exposure: security group rules from every declaration form, subnet CIDR consistency with the owning VPC (pairwise overlap under compatible conditions), network ACL entries, and VPC flow logs. +package custom_network + +import rego.v1 + +_world := {"0.0.0.0/0", "::/0"} + +_sensitive_ports := {21, 22, 23, 25, 1433, 1521, 2049, 3306, 3389, 5432, 5601, 6379, 8020, 9200, 9300, 11211, 27017, 50070} + +ingress_rules contains {"resource": name, "path": sprintf("Properties.SecurityGroupIngress.%d", [i]), "rule": rule} if { + some name, res in input.resources + res.resourceType == "AWS::EC2::SecurityGroup" + is_array(res.properties.SecurityGroupIngress) + some i, rule in res.properties.SecurityGroupIngress + is_object(rule) +} + +ingress_rules contains {"resource": name, "path": "Properties", "rule": res.properties} if { + some name, res in input.resources + res.resourceType == "AWS::EC2::SecurityGroupIngress" + is_object(res.properties) +} + +_open_to_world(rule) if _world[rule.CidrIp] + +_open_to_world(rule) if _world[rule.CidrIpv6] + +_all_protocols(rule) if rule.IpProtocol in {"-1", -1} + +_port_range(rule) := [0, 65535] if _all_protocols(rule) + +_port_range(rule) := [from, to] if { + not _all_protocols(rule) + from := coerce_to_integer(rule.FromPort) + to := coerce_to_integer(rule.ToPort) +} + +subnets contains {"resource": name, "cidr": cidr, "vpc": vpc} if { + some name, res in input.resources + res.resourceType == "AWS::EC2::Subnet" + cidr := res.properties.CidrBlock + is_string(cidr) + is_valid_cidr_strict(cidr) + vpc := _vpc_key(res.properties.VpcId) +} + +_vpc_key(value) := value.__ref if value.__kind == "resource" + +_vpc_key(value) := value if is_string(value) + +_egress(entry) if entry.Egress == true + +_egress(entry) if entry.Egress == "true" + +_entry_open_to_world(entry) if _world[entry.CidrBlock] + +_entry_open_to_world(entry) if _world[entry.Ipv6CidrBlock] + +_has_flow_log(vpc) if { + some _, res in input.resources + res.resourceType == "AWS::EC2::FlowLog" + res.properties.ResourceId.__ref == vpc +} + +violation contains make_diag_at("network.security-group-open-to-world-all-traffic", "ERROR", r.resource, r.path, "ingress rule allows all traffic on every port from any address") if { + some r in ingress_rules + _open_to_world(r.rule) + _all_protocols(r.rule) +} + +violation contains make_diag_at("network.security-group-open-to-world-sensitive-port", "ERROR", r.resource, r.path, sprintf("ingress rule exposes port %d to any address", [port])) if { + some r in ingress_rules + _open_to_world(r.rule) + not _all_protocols(r.rule) + [from, to] := _port_range(r.rule) + some port in _sensitive_ports + port >= from + port <= to +} + +violation contains make_diag_at("network.security-group-open-to-world-wide-range", "WARN", r.resource, r.path, sprintf("ingress rule opens %d ports (%d-%d) to any address", [(to - from) + 1, from, to])) if { + some r in ingress_rules + _open_to_world(r.rule) + not _all_protocols(r.rule) + [from, to] := _port_range(r.rule) + to - from >= 1000 +} + +violation contains make_diag_at("network.subnet-outside-vpc-cidr", "ERROR", s.resource, "Properties.CidrBlock", sprintf("subnet CIDR %s is not inside the CIDR %s of VPC %s", [s.cidr, vpc_cidr, s.vpc])) if { + some s in subnets + vpc := input.resources[s.vpc] + vpc.resourceType == "AWS::EC2::VPC" + vpc_cidr := vpc.properties.CidrBlock + is_string(vpc_cidr) + is_valid_cidr_strict(vpc_cidr) + not ip_subnet_of(s.cidr, vpc_cidr) +} + +violation contains make_diag_at("network.subnet-cidr-overlap", "ERROR", a.resource, "Properties.CidrBlock", sprintf("subnet CIDR %s overlaps %s (%s) in VPC %s", [a.cidr, b.resource, b.cidr, a.vpc])) if { + some a in subnets + some b in subnets + a.vpc == b.vpc + a.resource < b.resource + ip_overlaps(a.cidr, b.cidr) + conditions_compatible(a.resource, b.resource) +} + +violation contains make_diag("network.network-acl-allows-all-ingress", "WARN", name, "network ACL entry allows all inbound traffic from any address") if { + some name, res in input.resources + res.resourceType == "AWS::EC2::NetworkAclEntry" + entry := res.properties + lower(entry.RuleAction) == "allow" + entry.Protocol in {"-1", -1} + not _egress(entry) + _entry_open_to_world(entry) +} + +violation contains make_diag("network.vpc-flow-logs-missing", "INFO", name, "VPC has no AWS::EC2::FlowLog resource capturing its traffic") if { + some name, res in input.resources + res.resourceType == "AWS::EC2::VPC" + not _has_flow_log(name) +} diff --git a/src/resources/rules/rego_s3.rego b/src/resources/rules/rego_s3.rego new file mode 100644 index 00000000..f0c22965 --- /dev/null +++ b/src/resources/rules/rego_s3.rego @@ -0,0 +1,127 @@ +# S3 bucket hygiene: public access block per condition scenario, bucket-name grammar, self-logging, lifecycle transition ordering, versioning retention, and Lambda notification permissions. +package custom_s3 + +import rego.v1 + +_public_access_flags := ["BlockPublicAcls", "BlockPublicPolicy", "IgnorePublicAcls", "RestrictPublicBuckets"] + +# Colder storage classes rank higher; a transition must never move objects to a lower rank later in time. +_storage_class_rank := { + "STANDARD_IA": 1, + "ONEZONE_IA": 2, + "INTELLIGENT_TIERING": 3, + "GLACIER_IR": 4, + "GLACIER": 5, + "DEEP_ARCHIVE": 6, +} + +_bucket_name_checks := [ + ["shape", "must be 3-63 lowercase letters, digits, dots, or hyphens, starting and ending with a letter or digit"], + ["dots", "must not contain consecutive dots"], + ["ip", "must not be formatted like an IP address"], + ["prefix", "must not start with xn--, sthree-, or amzn-s3-demo-"], + ["suffix", "must not end with -s3alias, --ol-s3, .mrap, or --x-s3"], +] + +_true_like(value) if value == true + +_true_like(value) if value == "true" + +_conditions(scenario) := object.get(scenario, "conditions", {}) + +_scenario_diag(rule_id, severity, name, path, message, scenario) := make_diag_conditional(rule_id, severity, name, path, message, _conditions(scenario)) if { + count(_conditions(scenario)) > 0 +} + +_scenario_diag(rule_id, severity, name, path, message, scenario) := make_diag_at(rule_id, severity, name, path, message) if { + count(_conditions(scenario)) == 0 +} + +_missing_public_access_flags(properties) := [flag | + some flag in _public_access_flags + not _true_like(object.get(properties, ["PublicAccessBlockConfiguration", flag], null)) +] + +_bucket_name_violates("shape", bucket_name) if not regex.match(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`, bucket_name) + +_bucket_name_violates("dots", bucket_name) if contains(bucket_name, "..") + +_bucket_name_violates("ip", bucket_name) if regex.match(`^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$`, bucket_name) + +_bucket_name_violates("prefix", bucket_name) if strings.any_prefix_match(bucket_name, ["xn--", "sthree-", "amzn-s3-demo-"]) + +_bucket_name_violates("suffix", bucket_name) if strings.any_suffix_match(bucket_name, ["-s3alias", "--ol-s3", ".mrap", "--x-s3"]) + +_bucket_name_problems(bucket_name) := {message | + some [check, message] in _bucket_name_checks + _bucket_name_violates(check, bucket_name) +} + +_expires_noncurrent_versions(name) if { + some rule in input.resources[name].properties.LifecycleConfiguration.Rules + rule.Status == "Enabled" + some field in ["NoncurrentVersionExpiration", "NoncurrentVersionExpirationInDays"] + object.get(rule, field, null) != null +} + +_s3_invoke_permission(function) if { + some _, res in input.resources + res.resourceType == "AWS::Lambda::Permission" + res.properties.Principal == "s3.amazonaws.com" + res.properties.FunctionName.__ref == function +} + +violation contains _scenario_diag("s3.public-access-block-incomplete", "ERROR", name, "Properties.PublicAccessBlockConfiguration", sprintf("public access block does not enable %v", [missing]), scenario) if { + some name in resources_of_type("AWS::S3::Bucket") + not is_dynamic(name, "Properties.PublicAccessBlockConfiguration") + some scenario in properties_scenarios(name, ["PublicAccessBlockConfiguration"]) + missing := _missing_public_access_flags(object.get(scenario, "properties", {})) + count(missing) > 0 +} + +violation contains make_diag_at("s3.bucket-name-invalid", "ERROR", name, "Properties.BucketName", sprintf("bucket name '%s' %s", [bucket_name, problem])) if { + some name in resources_of_type("AWS::S3::Bucket") + some bucket_name in resolve_all(name, "Properties.BucketName") + is_string(bucket_name) + not contains(bucket_name, "${") + some problem in _bucket_name_problems(bucket_name) +} + +violation contains make_diag_at("s3.logging-to-self", "WARN", name, "Properties.LoggingConfiguration.DestinationBucketName", "bucket delivers its own access logs to itself, which recursively generates log objects") if { + some name in resources_of_type("AWS::S3::Bucket") + follow_ref(name, "Properties.LoggingConfiguration.DestinationBucketName") == name +} + +violation contains make_diag_at("s3.versioning-without-noncurrent-expiration", "INFO", name, "Properties.LifecycleConfiguration", "versioned bucket has no enabled lifecycle rule expiring noncurrent object versions") if { + some name in resources_of_type("AWS::S3::Bucket") + some status in resolve_all(name, "Properties.VersioningConfiguration.Status") + status == "Enabled" + not _expires_noncurrent_versions(name) +} + +violation contains make_diag_at("s3.lifecycle-transition-order", "WARN", name, sprintf("Properties.LifecycleConfiguration.Rules.%d.Transitions", [i]), sprintf("transition to %s after %d days is followed by the warmer class %s after %d days", [colder.StorageClass, colder_days, warmer.StorageClass, warmer_days])) if { + some name in resources_of_type("AWS::S3::Bucket") + rules := input.resources[name].properties.LifecycleConfiguration.Rules + is_array(rules) + some i, rule in rules + transitions := rule.Transitions + is_array(transitions) + some j, colder in transitions + some k, warmer in transitions + j != k + colder_days := coerce_to_integer(colder.TransitionInDays) + warmer_days := coerce_to_integer(warmer.TransitionInDays) + colder_days < warmer_days + _storage_class_rank[colder.StorageClass] > _storage_class_rank[warmer.StorageClass] +} + +violation contains make_diag_at("s3.lambda-notification-without-permission", "WARN", name, sprintf("Properties.NotificationConfiguration.LambdaConfigurations.%d.Function", [i]), sprintf("no AWS::Lambda::Permission lets s3.amazonaws.com invoke %s, so the notification configuration will fail to deploy", [target])) if { + some name in resources_of_type("AWS::S3::Bucket") + configurations := input.resources[name].properties.NotificationConfiguration.LambdaConfigurations + is_array(configurations) + some i, configuration in configurations + target := configuration.Function.__ref + is_string(target) + input.resources[target].resourceType == "AWS::Lambda::Function" + not _s3_invoke_permission(target) +}