From 374b65eb2fc335e520f630f54f0d506f1fd52229 Mon Sep 17 00:00:00 2001 From: Satyaki Ghosh Date: Tue, 22 Sep 2026 12:12:28 -0400 Subject: [PATCH] Reduce benchmark scenarios --- .github/workflows/benchmark.yml | 2 +- .kiro/steering/tech.md | 8 ++--- scripts/compare_benchmarks.py | 58 ++++++++++++--------------------- src/resources/README.md | 12 +++---- 4 files changed, 32 insertions(+), 48 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ecf93bb4..e7bbc166 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,7 @@ name: Benchmark # Every scenario of scripts/compare_benchmarks.py (built-in rules only, built-in + -# Guard rule pack, built-in + custom Rego rule pack, and both packs) is a full +# custom Rego rule pack, and built-in + Guard rule pack) is a full Rego/composite # engine × binding benchmark and always runs, in one job so every number in the # report comes from the same runner. The per-template and aggregate reports stay # on the runner; only the assembled comparison report is uploaded. diff --git a/.kiro/steering/tech.md b/.kiro/steering/tech.md index fb527e2e..736dfc11 100644 --- a/.kiro/steering/tech.md +++ b/.kiro/steering/tech.md @@ -124,10 +124,10 @@ implementation and fix it there. cfn-lint is available on the machine (`cfn-lint --version`), then ask the user for the checkout path — never assume or hardcode one. - `compare_benchmarks.py` — runs the benchmark harnesses of every binding (native `cfn-benchmark`, WASM, JVM, Python, - Go) for every scenario — `builtin` (built-in rules only), `guard` (every `.guard` file in `resources/rules` - loaded as a Guard rule pack), `rego` (every `.rego` file loaded as a custom Rego pack; Rego and composite engines - only), and `all` (both) — and writes the single `scripts/snapshots/benchmark_comparison.md` with a cross-scenario - rule-pack-cost summary plus the full engine × binding comparison per scenario. Needs GNU `/usr/bin/time` (or + Go) with the Rego and composite engines for three scenarios — `builtin` (built-in rules only), `custom` (every + `.rego` file in `resources/rules` loaded as a custom Rego pack), and `guard` (every `.guard` file loaded as a Guard + rule pack) — and writes the single `scripts/snapshots/benchmark_comparison.md` with a cross-scenario rule-pack-cost + summary plus the full engine × binding comparison per scenario. Needs GNU `/usr/bin/time` (or `CFN_BENCHMARK_TIME_BIN`). The `benchmark` workflow runs every scenario in one job and uploads only that report. - `audit_rule_categorization.py` — audits rule registry for categorization consistency - `generate_licenses.py` — generates third-party license files diff --git a/scripts/compare_benchmarks.py b/scripts/compare_benchmarks.py index 36f5262d..5f5f2b0d 100755 --- a/scripts/compare_benchmarks.py +++ b/scripts/compare_benchmarks.py @@ -2,13 +2,11 @@ """Runs benchmarks for every scenario × engine × binding and writes comparison reports. A *scenario* fixes the rules the engine evaluates: ``builtin`` measures the -built-in rules alone, ``guard`` layers every Guard rule file of -``src/resources/rules`` on top, ``rego`` layers every custom Rego rule file of -that directory (the CEL engine cannot evaluate Rego, so that scenario skips -CEL), and ``all`` layers both packs. The single report -(``scripts/snapshots/benchmark_comparison.md``) opens with a cross-scenario -summary of what each rule pack costs per engine and binding, then holds the -full engine × binding comparison for every scenario. +built-in rules alone, ``custom`` layers every custom Rego rule file of +``src/resources/rules`` on top, and ``guard`` layers every Guard rule file of +that directory. The single report (``scripts/snapshots/benchmark_comparison.md``) +opens with a cross-scenario summary of what each rule pack costs per engine and +binding, then holds the full engine × binding comparison for every scenario. The native benchmark builds ``cfn-benchmark`` from the workspace. The WASM, JVM, Python, and Go benchmarks consume the committed distribution artifacts that the @@ -46,7 +44,9 @@ PROJECT_ROOT = SCRIPT_DIR.parent SRC_DIR = PROJECT_ROOT / "src" -ENGINES = ["rego", "cel", "composite"] +# Composite uses the CEL built-in implementation and also exercises custom Rego +# composition, so a separate CEL benchmark would measure no additional path. +ENGINES = ["rego", "composite"] FORMATS = ["detailed"] # Every .guard file here is the Guard rule pack and every .rego file the custom Rego @@ -57,7 +57,6 @@ # reports// layout that other tooling reads. DEFAULT_SCENARIO = "builtin" -# The CEL engine cannot evaluate Rego, so Rego scenarios run Rego and composite only. SCENARIOS = [ { "id": DEFAULT_SCENARIO, @@ -67,25 +66,18 @@ "rego": False, }, { - "id": "guard", - "label": "Built-in rules + Guard rule pack", - "engines": ENGINES, - "guard": True, - "rego": False, - }, - { - "id": "rego", + "id": "custom", "label": "Built-in rules + custom Rego rule pack", - "engines": ["rego", "composite"], + "engines": ENGINES, "guard": False, "rego": True, }, { - "id": "all", - "label": "Built-in rules + Guard rule pack + custom Rego rule pack", - "engines": ["rego", "composite"], + "id": "guard", + "label": "Built-in rules + Guard rule pack", + "engines": ENGINES, "guard": True, - "rego": True, + "rego": False, }, ] SCENARIO_IDS = [scenario["id"] for scenario in SCENARIOS] @@ -132,7 +124,7 @@ VALID_BINDINGS = {"native", "wasm", "jvm", "python", "go"} # Valid engine labels. -VALID_ENGINES = {"rego", "cel", "composite"} +VALID_ENGINES = set(ENGINES) # External process timer used to measure startup and full-corpus memory. The # GNU coreutils build ("-v") and the macOS build ("-l") report different @@ -1358,20 +1350,18 @@ def top_slowest_section(all_detailed, engines, bindings, top_n): def engine_display_name(engine): """Human-readable engine label for report headings and table cells.""" - return {"rego": "Rego", "cel": "CEL", "composite": "Composite"}.get(engine, engine) + return {"rego": "Rego", "composite": "Composite"}.get(engine, engine) def paired_engine_pairs(all_detailed): """Every unordered pair of engines present in the loaded reports, in canonical - ENGINES order, so the report always compares e.g. Rego vs CEL, Rego vs - Composite, and CEL vs Composite when all three ran.""" + ENGINES order.""" present = [engine for engine in ENGINES if engine in all_detailed] return list(itertools.combinations(present, 2)) def paired_engine_comparison(all_detailed, bindings): - """Paired engine-vs-engine analysis per binding, for every pair of engines - that ran (Rego vs CEL, Rego vs Composite, CEL vs Composite). + """Paired engine-vs-engine analysis per binding for Rego vs Composite. For each engine pair and binding, computes: - Representative corpus-pass sums (sum of per-template subsequent wallClockMs @@ -1945,11 +1935,6 @@ def rule_pack_section(scenario, agg): lines += ["Pack files (from `src/resources/rules/`):", ""] lines += [f"- `{path.name}`" for path in guard_files + rego_files] lines.append("") - if scenario["rego"]: - lines += [ - "The CEL engine cannot evaluate Rego, so this scenario compares the Rego and composite " - "engines only.", "", - ] return lines @@ -1983,10 +1968,9 @@ def methodology_section(): "## Methodology Notes", "", "### Scenarios - rule packs loaded into the engine", "", "Every scenario is a complete engine × binding run over the same corpus with a different rule " - f"set loaded into the engine. `{DEFAULT_SCENARIO}` evaluates the built-in rules alone; the other " - "scenarios load the Guard rule pack and/or the custom Rego rule pack of `src/resources/rules` on " - "top (the CEL engine cannot evaluate Rego, so Rego scenarios compare the Rego and composite " - "engines). Each harness records the scenario label, the rule pack it loaded, and a fingerprint of " + f"set loaded into the engine. `{DEFAULT_SCENARIO}` evaluates the built-in rules alone; `custom` " + "loads the custom Rego rule pack and `guard` loads the Guard rule pack of `src/resources/rules`. " + "Each harness records the scenario label, the rule pack it loaded, and a fingerprint of " "the rule files; every binding of a scenario must report the same fingerprint before it is " "compared. The cross-scenario table reads the rule-evaluation medians as the most robust " "measure of a pack's cost because they exclude process startup and host I/O.", "", diff --git a/src/resources/README.md b/src/resources/README.md index a9f21dd7..c76aecd7 100644 --- a/src/resources/README.md +++ b/src/resources/README.md @@ -14,10 +14,10 @@ and snapshot tests, and exposes fixture paths plus discovery used by snapshot ge ## Benchmark rule packs -`scripts/compare_benchmarks.py` measures every engine under load from custom rules as well as with the built-in -rules alone. Its `guard` scenario loads every `.guard` file in `rules/` into the engine, its `rego` scenario every -`.rego` file, and its `all` scenario both; the harnesses receive the directory and load the files themselves. The -packs are therefore exactly the `.guard` and `.rego` files of this directory - a new fixture joins the next benchmark +`scripts/compare_benchmarks.py` measures the Rego and composite engines with the built-in rules alone and under +separate custom-rule loads. Its `custom` scenario loads every `.rego` file in `rules/`, while its `guard` scenario +loads every `.guard` file; the harnesses receive the directory and load the files themselves. The packs are therefore +exactly the `.guard` and `.rego` files of this directory - a new fixture joins the next benchmark automatically, and one that fails to evaluate on a corpus template shows up in the report's "Templates Failing Under a Rule Pack" list rather than silently. @@ -45,8 +45,8 @@ a Rule Pack" list rather than silently. and check `cfn-validate resources/templates --engine rego --rule-source ` reports no `Custom rule package ... failed to evaluate` errors before adding one. -Both engines see the same packs, so the Guard scenario also verifies that Rego, CEL, and composite report identical -Guard findings under load. +Both benchmarked engines see the same packs, so the Guard scenario also verifies that Rego and composite report +identical Guard findings under load. ## Snapshot generation