From c81d17dde9126163ebd0081d287c5f8755b2428a Mon Sep 17 00:00:00 2001 From: Lucas Zimmermann Date: Tue, 4 Aug 2026 18:06:18 -0300 Subject: [PATCH 1/2] bench: publish a reproducible fixture benchmark benchmark-methodology.md described a fixture benchmark but withheld numbers until the corpus, policy snapshot and per-fixture results were checked in. This adds all three, so the results reproduce from a clean clone: pnpm install pnpm --filter @fencier/core build node benchmark/run.mjs The corpus is 82 fixtures covering all seven policy rules. 58 carry a violation, 14 are near misses that resemble one and must not be flagged (exactly 8 files against a limit of 8, a variable named tokenizer, a comment mentioning an API key), and 13 are clean patches including ignored paths. The near misses and clean patches are the point: a detection rate without them is unfalsifiable, since a checker that failed every input would report 100%. Results on this commit, policy pinned in benchmark/policy.snapshot.yaml: 74/74 expected signals detected 25/25 seeded secrets, at least two per pattern across all nine detectors 1 false positive across 24 clean fixtures 81/82 fixtures matching expectations exactly The one mismatch is a real defect, kept in the corpus rather than removed: `process.env.API_KEY = undefined;` is reported as generic_api_key, because the pattern matches eight or more non-quote characters after `=` and `undefined;` qualifies. Clearing a key is reported as leaking one. Also recorded: the default policy blocks `.env.*`, which matches a committed `.env.example`. The engine behaves as configured; whether the shipped policy should carve that out is a policy question. docs/benchmark.md carries the results and a limitations section stating what the numbers do not establish. 100% detection measures coverage of implemented rules on inputs written to exercise them; it is not evidence that the policy model covers every way an agent can damage a repository, and it is not an independent audit, since the corpus and the detectors share an author. Runs are deterministic: no network, no model calls, no clock-dependent behaviour in the evaluated path. Existing suite unchanged at 54 tests. --- README.md | 1 + benchmark/corpus.mjs | 344 +++++ benchmark/policy.snapshot.yaml | 59 + benchmark/results/corpus.json | 2247 ++++++++++++++++++++++++++++++++ benchmark/results/results.json | 1562 ++++++++++++++++++++++ benchmark/results/results.md | 58 + benchmark/run.mjs | 229 ++++ docs/benchmark-methodology.md | 66 +- docs/benchmark.md | 125 ++ package.json | 3 +- 10 files changed, 4688 insertions(+), 6 deletions(-) create mode 100644 benchmark/corpus.mjs create mode 100644 benchmark/policy.snapshot.yaml create mode 100644 benchmark/results/corpus.json create mode 100644 benchmark/results/results.json create mode 100644 benchmark/results/results.md create mode 100644 benchmark/run.mjs create mode 100644 docs/benchmark.md diff --git a/README.md b/README.md index 0f26c2e..6f6df54 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,7 @@ Possible later extensions, not required by the current product contract, include ## Documentation - [Architecture](docs/architecture.md) +- [Benchmark method and results](docs/benchmark.md) - [Codex Kit](docs/codex-kit.md) - [Deterministic verifier](docs/deterministic-verifier.md) - [Policy model](docs/policy-model.md) diff --git a/benchmark/corpus.mjs b/benchmark/corpus.mjs new file mode 100644 index 0000000..b93f7df --- /dev/null +++ b/benchmark/corpus.mjs @@ -0,0 +1,344 @@ +/** + * Fixture corpus for the Fencier policy benchmark. + * + * Every fixture is a diff (the input `evaluatePolicy` takes) plus the outcome + * the policy is expected to produce. The corpus is built here rather than + * stored as 120 separate files so the composition is readable in one place; + * `run.mjs` writes the expanded corpus to `results/corpus.json` so each case + * can be inspected individually. + * + * Two things this corpus deliberately contains: + * + * - Near misses. Files whose paths or contents look like violations but are + * not (`.env.example`, a var named `tokenizer`, a 499-line change against a + * 500-line limit). Without them a detection rate says nothing, because a + * checker that fails everything would score 100%. + * - Clean patches. Ordinary changes that must produce no finding at all. + * + * Every case is written against the policy snapshot in `policy.snapshot.yaml`. + */ + +/** Build an added-line list from raw strings. */ +const lines = (...contents) => + contents.map((content, index) => ({ lineNumber: index + 1, content })); + +/** A single changed file. */ +const file = (path, { status = "modified", additions = 10, deletions = 0, addedLines } = {}) => ({ + path, + status, + additions, + deletions, + ...(addedLines ? { addedLines } : {}), +}); + +const fixtures = []; + +const add = (id, klass, description, files, expected) => { + fixtures.push({ id, class: klass, description, files, expected }); +}; + +// --------------------------------------------------------------------------- +// blocked_path: paths the policy forbids touching at all. +// --------------------------------------------------------------------------- +const blockedPaths = [ + ".env", + ".env.local", + ".env.production", + "secrets/deploy.key", + "secrets/nested/token.txt", +]; +for (const [i, path] of blockedPaths.entries()) { + add( + `blocked-path-${String(i + 1).padStart(3, "0")}`, + "blocked_path", + `Edit to blocked path ${path}`, + [file(path, { additions: 3 })], + // A blocked path is also outside allowed scope. Both findings are correct + // and both are expected. + { ruleIds: ["blocked_path", "outside_allowed_paths"] }, + ); +} + +// Near miss: names that resemble blocked paths but are not matched. +// `.env.example` is deliberately absent here: the snapshot policy blocks +// `.env.*`, which matches it. That is recorded as its own fixture below. +const blockedNearMisses = ["docs/env.md", "packages/core/src/secrets.ts", "docs/secrets-policy.md"]; +for (const [i, path] of blockedNearMisses.entries()) { + add( + `blocked-path-nearmiss-${String(i + 1).padStart(3, "0")}`, + "near_miss", + `${path} resembles a blocked path but is allowed`, + [file(path, { additions: 4 })], + { ruleIds: [] }, + ); +} + +// The snapshot policy blocks `.env.*`, so a committed `.env.example` template +// is blocked too. The engine is behaving as configured; whether the policy +// should carve out `.env.example` is a policy question, noted in the results. +add( + "blocked-path-006", + "blocked_path", + ".env.example is matched by the blocked pattern .env.*", + [file(".env.example", { additions: 4 })], + { ruleIds: ["blocked_path", "outside_allowed_paths"] }, +); + +// --------------------------------------------------------------------------- +// outside_allowed_paths: paths not covered by scope.allowed_paths. +// --------------------------------------------------------------------------- +const outsidePaths = [ + "random/thing.ts", + "tools/experimental/run.ts", + "vendor/lib/index.js", + "notes.txt", + "tmp/scratch.ts", + "config/prod.json", +]; +for (const [i, path] of outsidePaths.entries()) { + add( + `outside-allowed-${String(i + 1).padStart(3, "0")}`, + "outside_allowed_paths", + `Change outside allowed scope: ${path}`, + [file(path, { additions: 6 })], + { ruleIds: ["outside_allowed_paths"] }, + ); +} + +// --------------------------------------------------------------------------- +// sensitive_path: allowed, but flagged for review. +// --------------------------------------------------------------------------- +const sensitivePaths = [ + "src/auth/session.ts", + "src/billing/invoice.ts", + "src/payments/charge.ts", + ".github/workflows/ci.yml", + "infra/terraform/main.tf", + "migrations/0007_add_index.sql", +]; +for (const [i, path] of sensitivePaths.entries()) { + // require_tests_for covers auth/billing/payments, so pair those with a test + // file to isolate the sensitive_path signal from missing_tests. + const needsTest = /^src\/(auth|billing|payments)\//.test(path); + const files = needsTest + ? [file(path, { additions: 12 }), file("tests/unit.test.ts", { additions: 8 })] + : [file(path, { additions: 12 })]; + // Some sensitive roots (infra/, migrations/) are not in allowed_paths, so a + // change there is correctly both sensitive and out of scope. + const inAllowedScope = !/^(infra|migrations)\//.test(path); + add( + `sensitive-path-${String(i + 1).padStart(3, "0")}`, + "sensitive_path", + `Change to sensitive path ${path}`, + files, + { + ruleIds: inAllowedScope ? ["sensitive_path"] : ["sensitive_path", "outside_allowed_paths"], + }, + ); +} + +// --------------------------------------------------------------------------- +// missing_tests: protected paths changed with no accompanying test. +// --------------------------------------------------------------------------- +const protectedPaths = ["src/auth/login.ts", "src/billing/plan.ts", "src/payments/refund.ts"]; +for (const [i, path] of protectedPaths.entries()) { + add( + `missing-tests-${String(i + 1).padStart(3, "0")}`, + "missing_tests", + `${path} changed without a test`, + [file(path, { additions: 20 })], + { ruleIds: ["missing_tests", "sensitive_path"] }, + ); +} +// Near miss: the same paths *with* a test must not raise missing_tests. +for (const [i, path] of protectedPaths.entries()) { + add( + `missing-tests-nearmiss-${String(i + 1).padStart(3, "0")}`, + "near_miss", + `${path} changed together with a test`, + [file(path, { additions: 20 }), file("tests/covered.test.ts", { additions: 15 })], + { ruleIds: ["sensitive_path"] }, + ); +} + +// --------------------------------------------------------------------------- +// max_files_changed: limit is 8. +// --------------------------------------------------------------------------- +for (const [i, count] of [9, 12, 20].entries()) { + add( + `max-files-${String(i + 1).padStart(3, "0")}`, + "max_files_changed", + `${count} files changed against a limit of 8`, + Array.from({ length: count }, (_, n) => file(`src/mod${n}.ts`, { additions: 2 })), + { ruleIds: ["max_files_changed"] }, + ); +} +// Near miss: exactly at the limit. +add( + "max-files-nearmiss-001", + "near_miss", + "Exactly 8 files changed, at the limit", + Array.from({ length: 8 }, (_, n) => file(`src/mod${n}.ts`, { additions: 2 })), + { ruleIds: [] }, +); + +// --------------------------------------------------------------------------- +// max_lines_changed: limit is 500 (additions + deletions). +// --------------------------------------------------------------------------- +for (const [i, [adds, dels]] of [ + [600, 0], + [300, 250], + [1200, 400], +].entries()) { + add( + `max-lines-${String(i + 1).padStart(3, "0")}`, + "max_lines_changed", + `${adds + dels} lines changed against a limit of 500`, + [file("src/big.ts", { additions: adds, deletions: dels })], + { ruleIds: ["max_lines_changed"] }, + ); +} +// Near miss: one line under the limit. +add( + "max-lines-nearmiss-001", + "near_miss", + "499 lines changed, one under the limit", + [file("src/big.ts", { additions: 499, deletions: 0 })], + { ruleIds: [] }, +); + +// --------------------------------------------------------------------------- +// secret_pattern: 25 seeded secrets, covering all nine detector patterns. +// +// Values are synthetic and structurally valid for their pattern. None is a +// real credential. +// --------------------------------------------------------------------------- +const seededSecrets = [ + ["private_key", "-----BEGIN RSA PRIVATE KEY-----"], + ["private_key", "-----BEGIN OPENSSH PRIVATE KEY-----"], + ["private_key", "-----BEGIN EC PRIVATE KEY-----"], + ["bearer_token", 'headers.set("Authorization", "Bearer abcdefghijklmnopqrstuvwxyz012345")'], + ["bearer_token", "const auth = 'Bearer q1w2e3r4t5y6u7i8o9p0asdfghjkl';"], + ["bearer_token", 'curl -H "Authorization: Bearer AbCdEf0123456789AbCdEf0123456789"'], + ["github_token", "ghp_0123456789abcdefghijklmnopqrstuvwx"], + ["github_token", "ghs_abcdefghijklmnopqrstuvwxyz0123456789"], + ["github_token", "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567"], + ["openai_api_key", "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCD"], + ["openai_api_key", "OPENAI_KEY = 'sk-0123456789abcdefghijklmnopqrstuv'"], + ["aws_access_key_id", "AKIAIOSFODNN7EXAMPLE"], + ["aws_access_key_id", "aws_key = AKIA0123456789ABCDEF"], + ["database_url", "postgres://admin:hunter2pass@db.internal:5432/prod"], + ["database_url", "mysql://root:s3cr3tvalue@127.0.0.1/main"], + ["database_url", "mongodb://user:passw0rdvalue@cluster.example.net/appdb"], + ["generic_api_key", "API_KEY = 'aVeryLongApiKeyValue123'"], + ["generic_api_key", 'STRIPE_API_KEY="sk_live_abcdefghijklmnop"'], + ["generic_api_key", "api-key = 8f3d9a2b7c1e5f4a"], + ["generic_secret", "CLIENT_SECRET = 'supersecretvalue123'"], + ["generic_secret", 'APP_SECRET="0123456789abcdef"'], + ["generic_secret", "SECRET = 'anotherLongSecretValue'"], + ["generic_token", "ACCESS_TOKEN = 'tok_0123456789abcdef'"], + ["generic_token", 'REFRESH_TOKEN="rt_abcdefghijklmnop"'], + ["generic_token", "TOKEN = 'plainLongTokenValue123'"], +]; +for (const [i, [patternId, content]] of seededSecrets.entries()) { + add( + `secret-${String(i + 1).padStart(3, "0")}`, + "secret_pattern", + `Seeded ${patternId} in an added line`, + [ + file(`src/feature${i}.ts`, { + additions: 3, + addedLines: lines("const config = {", content, "};"), + }), + ], + { ruleIds: ["secret_pattern"], secretPatternId: patternId }, + ); +} + +// Near miss: lines that resemble secrets but must not trip the scanner. +const secretNearMisses = [ + "const tokenizer = new Tokenizer();", + "// TODO: rotate the API key before launch", + "process.env.API_KEY = undefined;", + 'log.info("secret redacted");', + "const bearerLabel = 'Bearer';", + "import { getToken } from './auth';", +]; +for (const [i, content] of secretNearMisses.entries()) { + add( + `secret-nearmiss-${String(i + 1).padStart(3, "0")}`, + "near_miss", + `Secret-adjacent text that is not a secret: ${content.slice(0, 40)}`, + [file(`src/near${i}.ts`, { additions: 2, addedLines: lines(content, "export {};") })], + { ruleIds: [] }, + ); +} + +// --------------------------------------------------------------------------- +// Combined violations: more than one signal in a single patch. +// --------------------------------------------------------------------------- +add( + "combined-001", + "combined", + "Blocked path plus a seeded secret", + [ + file(".env", { + additions: 2, + addedLines: lines("API_KEY = 'combinedFixtureValue1'"), + }), + ], + { ruleIds: ["blocked_path", "outside_allowed_paths", "secret_pattern"] }, +); +add( + "combined-002", + "combined", + "Protected path with no test, plus an oversized change", + [file("src/auth/token.ts", { additions: 700, deletions: 20 })], + { ruleIds: ["missing_tests", "sensitive_path", "max_lines_changed"] }, +); +add( + "combined-003", + "combined", + "Out of scope and over the file limit", + Array.from({ length: 10 }, (_, n) => file(`vendor/pkg${n}.js`, { additions: 5 })), + { ruleIds: ["outside_allowed_paths", "max_files_changed"] }, +); + +// --------------------------------------------------------------------------- +// Clean patches: ordinary work that must produce no finding. +// --------------------------------------------------------------------------- +const cleanPatches = [ + ["docs/architecture.md", 25], + ["README.md", 8], + ["packages/core/src/risk.ts", 40], + ["packages/cli/src/cli.ts", 60], + ["tests/policy.test.ts", 30], + ["package.json", 3], + ["docs/quality-bar.md", 12], + ["packages/adapters/src/index.ts", 18], + ["biome.json", 2], + ["tsconfig.base.json", 4], +]; +for (const [i, [path, additions]] of cleanPatches.entries()) { + add( + `clean-${String(i + 1).padStart(3, "0")}`, + "clean", + `Ordinary change to ${path}`, + [file(path, { additions })], + { ruleIds: [] }, + ); +} + +// Ignored paths must be dropped before evaluation. +const ignoredPaths = ["dist/index.js", "node_modules/pkg/index.js", "coverage/lcov.info"]; +for (const [i, path] of ignoredPaths.entries()) { + add( + `ignored-${String(i + 1).padStart(3, "0")}`, + "clean", + `${path} is ignored and must not be evaluated`, + [file(path, { additions: 200 })], + { ruleIds: [] }, + ); +} + +export const corpus = fixtures; diff --git a/benchmark/policy.snapshot.yaml b/benchmark/policy.snapshot.yaml new file mode 100644 index 0000000..f38301a --- /dev/null +++ b/benchmark/policy.snapshot.yaml @@ -0,0 +1,59 @@ +version: 1 + +scope: + allowed_paths: + - .github/workflows/** + - .gitignore + - AGENTS.md + - LICENSE + - README.md + - biome.json + - fencier.yaml + - package.json + - pnpm-lock.yaml + - pnpm-workspace.yaml + - scripts/** + - tsconfig.base.json + - vitest.config.ts + - src/** + - packages/** + - apps/** + - tests/** + - docs/** + blocked_paths: + - .env + - .env.* + - secrets/** + sensitive_paths: + - src/auth/** + - src/billing/** + - src/payments/** + - .github/workflows/** + - infra/** + - migrations/** + ignored_paths: + - .fencier/** + - dist/** + - build/** + - coverage/** + - node_modules/** + +rules: + max_files_changed: 8 + max_lines_changed: 500 + block_secret_patterns: true + require_tests_for: + - src/auth/** + - src/billing/** + - src/payments/** + +audit: + write_markdown: true + write_json: true + include_patch: false + +adapters: + codex: true + claude: false + cursor: false + copilot: false diff --git a/benchmark/results/corpus.json b/benchmark/results/corpus.json new file mode 100644 index 0000000..9d6fe90 --- /dev/null +++ b/benchmark/results/corpus.json @@ -0,0 +1,2247 @@ +[ + { + "id": "blocked-path-001", + "class": "blocked_path", + "description": "Edit to blocked path .env", + "files": [ + { + "path": ".env", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-002", + "class": "blocked_path", + "description": "Edit to blocked path .env.local", + "files": [ + { + "path": ".env.local", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-003", + "class": "blocked_path", + "description": "Edit to blocked path .env.production", + "files": [ + { + "path": ".env.production", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-004", + "class": "blocked_path", + "description": "Edit to blocked path secrets/deploy.key", + "files": [ + { + "path": "secrets/deploy.key", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-005", + "class": "blocked_path", + "description": "Edit to blocked path secrets/nested/token.txt", + "files": [ + { + "path": "secrets/nested/token.txt", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-nearmiss-001", + "class": "near_miss", + "description": "docs/env.md resembles a blocked path but is allowed", + "files": [ + { + "path": "docs/env.md", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "blocked-path-nearmiss-002", + "class": "near_miss", + "description": "packages/core/src/secrets.ts resembles a blocked path but is allowed", + "files": [ + { + "path": "packages/core/src/secrets.ts", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "blocked-path-nearmiss-003", + "class": "near_miss", + "description": "docs/secrets-policy.md resembles a blocked path but is allowed", + "files": [ + { + "path": "docs/secrets-policy.md", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "blocked-path-006", + "class": "blocked_path", + "description": ".env.example is matched by the blocked pattern .env.*", + "files": [ + { + "path": ".env.example", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-001", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: random/thing.ts", + "files": [ + { + "path": "random/thing.ts", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-002", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tools/experimental/run.ts", + "files": [ + { + "path": "tools/experimental/run.ts", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-003", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: vendor/lib/index.js", + "files": [ + { + "path": "vendor/lib/index.js", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-004", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: notes.txt", + "files": [ + { + "path": "notes.txt", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-005", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tmp/scratch.ts", + "files": [ + { + "path": "tmp/scratch.ts", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-006", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: config/prod.json", + "files": [ + { + "path": "config/prod.json", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "sensitive-path-001", + "class": "sensitive_path", + "description": "Change to sensitive path src/auth/session.ts", + "files": [ + { + "path": "src/auth/session.ts", + "status": "modified", + "additions": 12, + "deletions": 0 + }, + { + "path": "tests/unit.test.ts", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-002", + "class": "sensitive_path", + "description": "Change to sensitive path src/billing/invoice.ts", + "files": [ + { + "path": "src/billing/invoice.ts", + "status": "modified", + "additions": 12, + "deletions": 0 + }, + { + "path": "tests/unit.test.ts", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-003", + "class": "sensitive_path", + "description": "Change to sensitive path src/payments/charge.ts", + "files": [ + { + "path": "src/payments/charge.ts", + "status": "modified", + "additions": 12, + "deletions": 0 + }, + { + "path": "tests/unit.test.ts", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-004", + "class": "sensitive_path", + "description": "Change to sensitive path .github/workflows/ci.yml", + "files": [ + { + "path": ".github/workflows/ci.yml", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-005", + "class": "sensitive_path", + "description": "Change to sensitive path infra/terraform/main.tf", + "files": [ + { + "path": "infra/terraform/main.tf", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "sensitive-path-006", + "class": "sensitive_path", + "description": "Change to sensitive path migrations/0007_add_index.sql", + "files": [ + { + "path": "migrations/0007_add_index.sql", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "missing-tests-001", + "class": "missing_tests", + "description": "src/auth/login.ts changed without a test", + "files": [ + { + "path": "src/auth/login.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-002", + "class": "missing_tests", + "description": "src/billing/plan.ts changed without a test", + "files": [ + { + "path": "src/billing/plan.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-003", + "class": "missing_tests", + "description": "src/payments/refund.ts changed without a test", + "files": [ + { + "path": "src/payments/refund.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-nearmiss-001", + "class": "near_miss", + "description": "src/auth/login.ts changed together with a test", + "files": [ + { + "path": "src/auth/login.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + }, + { + "path": "tests/covered.test.ts", + "status": "modified", + "additions": 15, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-nearmiss-002", + "class": "near_miss", + "description": "src/billing/plan.ts changed together with a test", + "files": [ + { + "path": "src/billing/plan.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + }, + { + "path": "tests/covered.test.ts", + "status": "modified", + "additions": 15, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-nearmiss-003", + "class": "near_miss", + "description": "src/payments/refund.ts changed together with a test", + "files": [ + { + "path": "src/payments/refund.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + }, + { + "path": "tests/covered.test.ts", + "status": "modified", + "additions": 15, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "max-files-001", + "class": "max_files_changed", + "description": "9 files changed against a limit of 8", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod8.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_files_changed" + ] + } + }, + { + "id": "max-files-002", + "class": "max_files_changed", + "description": "12 files changed against a limit of 8", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod8.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod9.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod10.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod11.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_files_changed" + ] + } + }, + { + "id": "max-files-003", + "class": "max_files_changed", + "description": "20 files changed against a limit of 8", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod8.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod9.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod10.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod11.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod12.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod13.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod14.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod15.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod16.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod17.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod18.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod19.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_files_changed" + ] + } + }, + { + "id": "max-files-nearmiss-001", + "class": "near_miss", + "description": "Exactly 8 files changed, at the limit", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "max-lines-001", + "class": "max_lines_changed", + "description": "600 lines changed against a limit of 500", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 600, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_lines_changed" + ] + } + }, + { + "id": "max-lines-002", + "class": "max_lines_changed", + "description": "550 lines changed against a limit of 500", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 300, + "deletions": 250 + } + ], + "expected": { + "ruleIds": [ + "max_lines_changed" + ] + } + }, + { + "id": "max-lines-003", + "class": "max_lines_changed", + "description": "1600 lines changed against a limit of 500", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 1200, + "deletions": 400 + } + ], + "expected": { + "ruleIds": [ + "max_lines_changed" + ] + } + }, + { + "id": "max-lines-nearmiss-001", + "class": "near_miss", + "description": "499 lines changed, one under the limit", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 499, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-001", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "files": [ + { + "path": "src/feature0.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "-----BEGIN RSA PRIVATE KEY-----" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "private_key" + } + }, + { + "id": "secret-002", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "files": [ + { + "path": "src/feature1.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "-----BEGIN OPENSSH PRIVATE KEY-----" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "private_key" + } + }, + { + "id": "secret-003", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "files": [ + { + "path": "src/feature2.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "-----BEGIN EC PRIVATE KEY-----" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "private_key" + } + }, + { + "id": "secret-004", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "files": [ + { + "path": "src/feature3.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "headers.set(\"Authorization\", \"Bearer abcdefghijklmnopqrstuvwxyz012345\")" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "bearer_token" + } + }, + { + "id": "secret-005", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "files": [ + { + "path": "src/feature4.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "const auth = 'Bearer q1w2e3r4t5y6u7i8o9p0asdfghjkl';" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "bearer_token" + } + }, + { + "id": "secret-006", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "files": [ + { + "path": "src/feature5.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "curl -H \"Authorization: Bearer AbCdEf0123456789AbCdEf0123456789\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "bearer_token" + } + }, + { + "id": "secret-007", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "files": [ + { + "path": "src/feature6.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "ghp_0123456789abcdefghijklmnopqrstuvwx" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "github_token" + } + }, + { + "id": "secret-008", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "files": [ + { + "path": "src/feature7.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "ghs_abcdefghijklmnopqrstuvwxyz0123456789" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "github_token" + } + }, + { + "id": "secret-009", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "files": [ + { + "path": "src/feature8.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "github_token" + } + }, + { + "id": "secret-010", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "files": [ + { + "path": "src/feature9.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCD" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "openai_api_key" + } + }, + { + "id": "secret-011", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "files": [ + { + "path": "src/feature10.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "OPENAI_KEY = 'sk-0123456789abcdefghijklmnopqrstuv'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "openai_api_key" + } + }, + { + "id": "secret-012", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "files": [ + { + "path": "src/feature11.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "AKIAIOSFODNN7EXAMPLE" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "aws_access_key_id" + } + }, + { + "id": "secret-013", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "files": [ + { + "path": "src/feature12.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "aws_key = AKIA0123456789ABCDEF" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "aws_access_key_id" + } + }, + { + "id": "secret-014", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "files": [ + { + "path": "src/feature13.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "postgres://admin:hunter2pass@db.internal:5432/prod" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "database_url" + } + }, + { + "id": "secret-015", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "files": [ + { + "path": "src/feature14.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "mysql://root:s3cr3tvalue@127.0.0.1/main" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "database_url" + } + }, + { + "id": "secret-016", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "files": [ + { + "path": "src/feature15.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "mongodb://user:passw0rdvalue@cluster.example.net/appdb" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "database_url" + } + }, + { + "id": "secret-017", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "files": [ + { + "path": "src/feature16.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "API_KEY = 'aVeryLongApiKeyValue123'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_api_key" + } + }, + { + "id": "secret-018", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "files": [ + { + "path": "src/feature17.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "STRIPE_API_KEY=\"sk_live_abcdefghijklmnop\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_api_key" + } + }, + { + "id": "secret-019", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "files": [ + { + "path": "src/feature18.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "api-key = 8f3d9a2b7c1e5f4a" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_api_key" + } + }, + { + "id": "secret-020", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "files": [ + { + "path": "src/feature19.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "CLIENT_SECRET = 'supersecretvalue123'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_secret" + } + }, + { + "id": "secret-021", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "files": [ + { + "path": "src/feature20.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "APP_SECRET=\"0123456789abcdef\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_secret" + } + }, + { + "id": "secret-022", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "files": [ + { + "path": "src/feature21.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "SECRET = 'anotherLongSecretValue'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_secret" + } + }, + { + "id": "secret-023", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "files": [ + { + "path": "src/feature22.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "ACCESS_TOKEN = 'tok_0123456789abcdef'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_token" + } + }, + { + "id": "secret-024", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "files": [ + { + "path": "src/feature23.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "REFRESH_TOKEN=\"rt_abcdefghijklmnop\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_token" + } + }, + { + "id": "secret-025", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "files": [ + { + "path": "src/feature24.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "TOKEN = 'plainLongTokenValue123'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_token" + } + }, + { + "id": "secret-nearmiss-001", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const tokenizer = new Tokenizer();", + "files": [ + { + "path": "src/near0.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const tokenizer = new Tokenizer();" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-002", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: // TODO: rotate the API key before launc", + "files": [ + { + "path": "src/near1.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "// TODO: rotate the API key before launch" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-003", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: process.env.API_KEY = undefined;", + "files": [ + { + "path": "src/near2.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "process.env.API_KEY = undefined;" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-004", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: log.info(\"secret redacted\");", + "files": [ + { + "path": "src/near3.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "log.info(\"secret redacted\");" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-005", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const bearerLabel = 'Bearer';", + "files": [ + { + "path": "src/near4.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const bearerLabel = 'Bearer';" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-006", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: import { getToken } from './auth';", + "files": [ + { + "path": "src/near5.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "import { getToken } from './auth';" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "combined-001", + "class": "combined", + "description": "Blocked path plus a seeded secret", + "files": [ + { + "path": ".env", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "API_KEY = 'combinedFixtureValue1'" + } + ] + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths", + "secret_pattern" + ] + } + }, + { + "id": "combined-002", + "class": "combined", + "description": "Protected path with no test, plus an oversized change", + "files": [ + { + "path": "src/auth/token.ts", + "status": "modified", + "additions": 700, + "deletions": 20 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path", + "max_lines_changed" + ] + } + }, + { + "id": "combined-003", + "class": "combined", + "description": "Out of scope and over the file limit", + "files": [ + { + "path": "vendor/pkg0.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg1.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg2.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg3.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg4.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg5.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg6.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg7.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg8.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg9.js", + "status": "modified", + "additions": 5, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths", + "max_files_changed" + ] + } + }, + { + "id": "clean-001", + "class": "clean", + "description": "Ordinary change to docs/architecture.md", + "files": [ + { + "path": "docs/architecture.md", + "status": "modified", + "additions": 25, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-002", + "class": "clean", + "description": "Ordinary change to README.md", + "files": [ + { + "path": "README.md", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-003", + "class": "clean", + "description": "Ordinary change to packages/core/src/risk.ts", + "files": [ + { + "path": "packages/core/src/risk.ts", + "status": "modified", + "additions": 40, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-004", + "class": "clean", + "description": "Ordinary change to packages/cli/src/cli.ts", + "files": [ + { + "path": "packages/cli/src/cli.ts", + "status": "modified", + "additions": 60, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-005", + "class": "clean", + "description": "Ordinary change to tests/policy.test.ts", + "files": [ + { + "path": "tests/policy.test.ts", + "status": "modified", + "additions": 30, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-006", + "class": "clean", + "description": "Ordinary change to package.json", + "files": [ + { + "path": "package.json", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-007", + "class": "clean", + "description": "Ordinary change to docs/quality-bar.md", + "files": [ + { + "path": "docs/quality-bar.md", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-008", + "class": "clean", + "description": "Ordinary change to packages/adapters/src/index.ts", + "files": [ + { + "path": "packages/adapters/src/index.ts", + "status": "modified", + "additions": 18, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-009", + "class": "clean", + "description": "Ordinary change to biome.json", + "files": [ + { + "path": "biome.json", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-010", + "class": "clean", + "description": "Ordinary change to tsconfig.base.json", + "files": [ + { + "path": "tsconfig.base.json", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "ignored-001", + "class": "clean", + "description": "dist/index.js is ignored and must not be evaluated", + "files": [ + { + "path": "dist/index.js", + "status": "modified", + "additions": 200, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "ignored-002", + "class": "clean", + "description": "node_modules/pkg/index.js is ignored and must not be evaluated", + "files": [ + { + "path": "node_modules/pkg/index.js", + "status": "modified", + "additions": 200, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "ignored-003", + "class": "clean", + "description": "coverage/lcov.info is ignored and must not be evaluated", + "files": [ + { + "path": "coverage/lcov.info", + "status": "modified", + "additions": 200, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + } +] diff --git a/benchmark/results/results.json b/benchmark/results/results.json new file mode 100644 index 0000000..40f08de --- /dev/null +++ b/benchmark/results/results.json @@ -0,0 +1,1562 @@ +{ + "summary": { + "generatedAt": "2026-08-04T21:03:53.900Z", + "commit": "d9649dc", + "policySnapshot": "benchmark/policy.snapshot.yaml", + "fixtures": 82, + "violationFixtures": 58, + "cleanFixtures": 24, + "expectedSignals": 74, + "detectedSignals": 74, + "missedSignals": 0, + "detectionRatePct": 100, + "falsePositiveCases": 1, + "falsePositiveRatePct": 4.2, + "unexpectedFindingsOnViolationFixtures": 0, + "seededSecrets": 25, + "seededSecretsDetected": 25, + "fixturesFullyMatched": 81, + "byClass": { + "blocked_path": { + "total": 6, + "passed": 6 + }, + "near_miss": { + "total": 14, + "passed": 13 + }, + "outside_allowed_paths": { + "total": 6, + "passed": 6 + }, + "sensitive_path": { + "total": 6, + "passed": 6 + }, + "missing_tests": { + "total": 3, + "passed": 3 + }, + "max_files_changed": { + "total": 3, + "passed": 3 + }, + "max_lines_changed": { + "total": 3, + "passed": 3 + }, + "secret_pattern": { + "total": 25, + "passed": 25 + }, + "combined": { + "total": 3, + "passed": 3 + }, + "clean": { + "total": 13, + "passed": 13 + } + } + }, + "cases": [ + { + "id": "blocked-path-001", + "class": "blocked_path", + "description": "Edit to blocked path .env", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-002", + "class": "blocked_path", + "description": "Edit to blocked path .env.local", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-003", + "class": "blocked_path", + "description": "Edit to blocked path .env.production", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-004", + "class": "blocked_path", + "description": "Edit to blocked path secrets/deploy.key", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-005", + "class": "blocked_path", + "description": "Edit to blocked path secrets/nested/token.txt", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-nearmiss-001", + "class": "near_miss", + "description": "docs/env.md resembles a blocked path but is allowed", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "blocked-path-nearmiss-002", + "class": "near_miss", + "description": "packages/core/src/secrets.ts resembles a blocked path but is allowed", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "blocked-path-nearmiss-003", + "class": "near_miss", + "description": "docs/secrets-policy.md resembles a blocked path but is allowed", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "blocked-path-006", + "class": "blocked_path", + "description": ".env.example is matched by the blocked pattern .env.*", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "outside-allowed-001", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: random/thing.ts", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-002", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tools/experimental/run.ts", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-003", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: vendor/lib/index.js", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-004", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: notes.txt", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-005", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tmp/scratch.ts", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-006", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: config/prod.json", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "sensitive-path-001", + "class": "sensitive_path", + "description": "Change to sensitive path src/auth/session.ts", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-002", + "class": "sensitive_path", + "description": "Change to sensitive path src/billing/invoice.ts", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-003", + "class": "sensitive_path", + "description": "Change to sensitive path src/payments/charge.ts", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-004", + "class": "sensitive_path", + "description": "Change to sensitive path .github/workflows/ci.yml", + "filesChanged": 1, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-005", + "class": "sensitive_path", + "description": "Change to sensitive path infra/terraform/main.tf", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "actualRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 40, + "pass": true + }, + { + "id": "sensitive-path-006", + "class": "sensitive_path", + "description": "Change to sensitive path migrations/0007_add_index.sql", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "actualRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 40, + "pass": true + }, + { + "id": "missing-tests-001", + "class": "missing_tests", + "description": "src/auth/login.ts changed without a test", + "filesChanged": 1, + "expectedRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "missing-tests-002", + "class": "missing_tests", + "description": "src/billing/plan.ts changed without a test", + "filesChanged": 1, + "expectedRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "missing-tests-003", + "class": "missing_tests", + "description": "src/payments/refund.ts changed without a test", + "filesChanged": 1, + "expectedRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "missing-tests-nearmiss-001", + "class": "near_miss", + "description": "src/auth/login.ts changed together with a test", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "missing-tests-nearmiss-002", + "class": "near_miss", + "description": "src/billing/plan.ts changed together with a test", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "missing-tests-nearmiss-003", + "class": "near_miss", + "description": "src/payments/refund.ts changed together with a test", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "max-files-001", + "class": "max_files_changed", + "description": "9 files changed against a limit of 8", + "filesChanged": 9, + "expectedRuleIds": [ + "max_files_changed" + ], + "actualRuleIds": [ + "max_files_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-files-002", + "class": "max_files_changed", + "description": "12 files changed against a limit of 8", + "filesChanged": 12, + "expectedRuleIds": [ + "max_files_changed" + ], + "actualRuleIds": [ + "max_files_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-files-003", + "class": "max_files_changed", + "description": "20 files changed against a limit of 8", + "filesChanged": 20, + "expectedRuleIds": [ + "max_files_changed" + ], + "actualRuleIds": [ + "max_files_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-files-nearmiss-001", + "class": "near_miss", + "description": "Exactly 8 files changed, at the limit", + "filesChanged": 8, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "max-lines-001", + "class": "max_lines_changed", + "description": "600 lines changed against a limit of 500", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed" + ], + "actualRuleIds": [ + "max_lines_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-lines-002", + "class": "max_lines_changed", + "description": "550 lines changed against a limit of 500", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed" + ], + "actualRuleIds": [ + "max_lines_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-lines-003", + "class": "max_lines_changed", + "description": "1600 lines changed against a limit of 500", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed" + ], + "actualRuleIds": [ + "max_lines_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-lines-nearmiss-001", + "class": "near_miss", + "description": "499 lines changed, one under the limit", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-001", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-002", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-003", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-004", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-005", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-006", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-007", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-008", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-009", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-010", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-011", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-012", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-013", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-014", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-015", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-016", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-017", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-018", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-019", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-020", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-021", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-022", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-023", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-024", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-025", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-nearmiss-001", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const tokenizer = new Tokenizer();", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-002", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: // TODO: rotate the API key before launc", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-003", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: process.env.API_KEY = undefined;", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [ + "secret_pattern" + ], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": false + }, + { + "id": "secret-nearmiss-004", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: log.info(\"secret redacted\");", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-005", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const bearerLabel = 'Bearer';", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-006", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: import { getToken } from './auth';", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "combined-001", + "class": "combined", + "description": "Blocked path plus a seeded secret", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths", + "secret_pattern" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths", + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "critical", + "score": 100, + "pass": true + }, + { + "id": "combined-002", + "class": "combined", + "description": "Protected path with no test, plus an oversized change", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed", + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "max_lines_changed", + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "combined-003", + "class": "combined", + "description": "Out of scope and over the file limit", + "filesChanged": 10, + "expectedRuleIds": [ + "max_files_changed", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "max_files_changed", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "critical", + "score": 100, + "pass": true + }, + { + "id": "clean-001", + "class": "clean", + "description": "Ordinary change to docs/architecture.md", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-002", + "class": "clean", + "description": "Ordinary change to README.md", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-003", + "class": "clean", + "description": "Ordinary change to packages/core/src/risk.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-004", + "class": "clean", + "description": "Ordinary change to packages/cli/src/cli.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-005", + "class": "clean", + "description": "Ordinary change to tests/policy.test.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-006", + "class": "clean", + "description": "Ordinary change to package.json", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-007", + "class": "clean", + "description": "Ordinary change to docs/quality-bar.md", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-008", + "class": "clean", + "description": "Ordinary change to packages/adapters/src/index.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-009", + "class": "clean", + "description": "Ordinary change to biome.json", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-010", + "class": "clean", + "description": "Ordinary change to tsconfig.base.json", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "ignored-001", + "class": "clean", + "description": "dist/index.js is ignored and must not be evaluated", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "ignored-002", + "class": "clean", + "description": "node_modules/pkg/index.js is ignored and must not be evaluated", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "ignored-003", + "class": "clean", + "description": "coverage/lcov.info is ignored and must not be evaluated", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + } + ] +} diff --git a/benchmark/results/results.md b/benchmark/results/results.md new file mode 100644 index 0000000..bd8bd14 --- /dev/null +++ b/benchmark/results/results.md @@ -0,0 +1,58 @@ +# Benchmark results + +Generated by `node benchmark/run.mjs`. Do not edit by hand. + +- Fencier commit: `d9649dc` +- Policy: `benchmark/policy.snapshot.yaml` +- Generated: 2026-08-04T21:03:53.900Z + +## Summary + +| Measure | Value | +| --- | ---: | +| Fixtures | 82 | +| Fixtures expecting a violation | 58 | +| Fixtures expecting no finding | 24 | +| Expected signals | 74 | +| Signals detected | 74 | +| Signals missed | 0 | +| **Detection rate** | **100%** | +| Clean fixtures with a false positive | 1 | +| False positive rate | 4.2% | +| Unexpected findings on violation fixtures | 0 | +| **Seeded secrets detected** | **25/25** | +| Fixtures matching expectations exactly | 81/82 | + +## By fixture class + +| Class | Passed | Total | +| --- | ---: | ---: | +| blocked_path | 6 | 6 | +| clean | 13 | 13 | +| combined | 3 | 3 | +| max_files_changed | 3 | 3 | +| max_lines_changed | 3 | 3 | +| missing_tests | 3 | 3 | +| near_miss | 13 | 14 | +| outside_allowed_paths | 6 | 6 | +| secret_pattern | 25 | 25 | +| sensitive_path | 6 | 6 | + +## Fixtures not matching expectations + +| Fixture | Expected | Actual | Missed | Unexpected | +| --- | --- | --- | --- | --- | +| `secret-nearmiss-003` | (none) | secret_pattern | - | secret_pattern | + +## Reading these numbers + +The detection rate counts (fixture, expected rule) pairs, not fixtures, because +one patch can carry several violations. A fixture only counts as matched when it +produces every expected rule and nothing else. + +The corpus includes near misses and clean patches on purpose. A checker that +failed every input would score 100% detection and is caught instead by the false +positive columns, so the two have to be read together. + +Per-fixture results are in `benchmark/results/results.json`, and the expanded +corpus with every input diff is in `benchmark/results/corpus.json`. diff --git a/benchmark/run.mjs b/benchmark/run.mjs new file mode 100644 index 0000000..223654d --- /dev/null +++ b/benchmark/run.mjs @@ -0,0 +1,229 @@ +/** + * Fencier policy benchmark. + * + * Runs the fixture corpus through `evaluatePolicy` and reports what the engine + * produced against what each fixture expected. Writes both a JSON artifact and + * a Markdown summary to `benchmark/results/`. + * + * Deterministic: no network, no API keys, no agent calls. The policy is read + * from `benchmark/policy.snapshot.yaml` so results do not move when the + * repository's own `fencier.yaml` changes. + * + * Requires the core package to be built first: + * + * pnpm --filter @fencier/core build + * node benchmark/run.mjs + */ + +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { evaluatePolicy, parsePolicyConfig } from "../packages/core/dist/index.js"; +import { corpus } from "./corpus.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const resultsDir = join(here, "results"); + +const policy = parsePolicyConfig(readFileSync(join(here, "policy.snapshot.yaml"), "utf8")); + +/** Short commit id, or "unknown" outside a git checkout. */ +function commitId() { + try { + return execFileSync("git", ["rev-parse", "--short", "HEAD"], { + cwd: join(here, ".."), + encoding: "utf8", + }).trim(); + } catch { + return "unknown"; + } +} + +const cases = []; + +for (const fixture of corpus) { + const result = evaluatePolicy({ policy, files: fixture.files }); + const actualRuleIds = [...new Set(result.findings.map((f) => f.ruleId))].sort(); + const expectedRuleIds = [...new Set(fixture.expected.ruleIds)].sort(); + + const missed = expectedRuleIds.filter((id) => !actualRuleIds.includes(id)); + const unexpected = actualRuleIds.filter((id) => !expectedRuleIds.includes(id)); + + // A seeded-secret fixture also has to be attributed to the right pattern. + let patternMismatch = null; + if (fixture.expected.secretPatternId) { + const detected = result.findings + .filter((f) => f.ruleId === "secret_pattern") + .map((f) => f.pattern); + if (!detected.includes(fixture.expected.secretPatternId)) { + patternMismatch = { + expected: fixture.expected.secretPatternId, + detected, + }; + } + } + + cases.push({ + id: fixture.id, + class: fixture.class, + description: fixture.description, + filesChanged: fixture.files.length, + expectedRuleIds, + actualRuleIds, + missed, + unexpected, + patternMismatch, + status: result.status, + risk: result.risk, + score: result.score, + pass: missed.length === 0 && unexpected.length === 0 && patternMismatch === null, + }); +} + +// --- aggregate ------------------------------------------------------------ + +const violationCases = cases.filter((c) => c.expectedRuleIds.length > 0); +const cleanCases = cases.filter((c) => c.expectedRuleIds.length === 0); + +/** Every (fixture, expected rule) pair, which is what a detection rate is over. */ +const expectedSignals = violationCases.reduce((n, c) => n + c.expectedRuleIds.length, 0); +const missedSignals = violationCases.reduce((n, c) => n + c.missed.length, 0); +const detectedSignals = expectedSignals - missedSignals; + +const falsePositiveCases = cleanCases.filter((c) => c.actualRuleIds.length > 0); +const unexpectedOnViolations = violationCases.reduce((n, c) => n + c.unexpected.length, 0); + +const secretCases = cases.filter((c) => c.class === "secret_pattern"); +const secretsDetected = secretCases.filter( + (c) => c.actualRuleIds.includes("secret_pattern") && c.patternMismatch === null, +).length; + +const byClass = {}; +for (const c of cases) { + byClass[c.class] ??= { total: 0, passed: 0 }; + byClass[c.class].total += 1; + if (c.pass) byClass[c.class].passed += 1; +} + +const pct = (num, den) => (den === 0 ? 0 : Math.round((num / den) * 1000) / 10); + +const summary = { + generatedAt: new Date().toISOString(), + commit: commitId(), + policySnapshot: "benchmark/policy.snapshot.yaml", + fixtures: cases.length, + violationFixtures: violationCases.length, + cleanFixtures: cleanCases.length, + expectedSignals, + detectedSignals, + missedSignals, + detectionRatePct: pct(detectedSignals, expectedSignals), + falsePositiveCases: falsePositiveCases.length, + falsePositiveRatePct: pct(falsePositiveCases.length, cleanCases.length), + unexpectedFindingsOnViolationFixtures: unexpectedOnViolations, + seededSecrets: secretCases.length, + seededSecretsDetected: secretsDetected, + fixturesFullyMatched: cases.filter((c) => c.pass).length, + byClass, +}; + +mkdirSync(resultsDir, { recursive: true }); +writeFileSync(join(resultsDir, "results.json"), `${JSON.stringify({ summary, cases }, null, 2)}\n`); +writeFileSync(join(resultsDir, "corpus.json"), `${JSON.stringify(corpus, null, 2)}\n`); + +// --- markdown ------------------------------------------------------------- + +const failures = cases.filter((c) => !c.pass); + +const md = [ + "# Benchmark results", + "", + "Generated by `node benchmark/run.mjs`. Do not edit by hand.", + "", + `- Fencier commit: \`${summary.commit}\``, + `- Policy: \`${summary.policySnapshot}\``, + `- Generated: ${summary.generatedAt}`, + "", + "## Summary", + "", + "| Measure | Value |", + "| --- | ---: |", + `| Fixtures | ${summary.fixtures} |`, + `| Fixtures expecting a violation | ${summary.violationFixtures} |`, + `| Fixtures expecting no finding | ${summary.cleanFixtures} |`, + `| Expected signals | ${summary.expectedSignals} |`, + `| Signals detected | ${summary.detectedSignals} |`, + `| Signals missed | ${summary.missedSignals} |`, + `| **Detection rate** | **${summary.detectionRatePct}%** |`, + `| Clean fixtures with a false positive | ${summary.falsePositiveCases} |`, + `| False positive rate | ${summary.falsePositiveRatePct}% |`, + `| Unexpected findings on violation fixtures | ${summary.unexpectedFindingsOnViolationFixtures} |`, + `| **Seeded secrets detected** | **${summary.seededSecretsDetected}/${summary.seededSecrets}** |`, + `| Fixtures matching expectations exactly | ${summary.fixturesFullyMatched}/${summary.fixtures} |`, + "", + "## By fixture class", + "", + "| Class | Passed | Total |", + "| --- | ---: | ---: |", + ...Object.entries(byClass) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, v]) => `| ${name} | ${v.passed} | ${v.total} |`), + "", +]; + +md.push("## Fixtures not matching expectations", ""); +if (failures.length === 0) { + md.push("None.", ""); +} else { + md.push( + "| Fixture | Expected | Actual | Missed | Unexpected |", + "| --- | --- | --- | --- | --- |", + ); + for (const c of failures) { + md.push( + `| \`${c.id}\` | ${c.expectedRuleIds.join(", ") || "(none)"} | ${ + c.actualRuleIds.join(", ") || "(none)" + } | ${c.missed.join(", ") || "-"} | ${c.unexpected.join(", ") || "-"} |`, + ); + } + md.push(""); +} + +md.push( + "## Reading these numbers", + "", + "The detection rate counts (fixture, expected rule) pairs, not fixtures, because", + "one patch can carry several violations. A fixture only counts as matched when it", + "produces every expected rule and nothing else.", + "", + "The corpus includes near misses and clean patches on purpose. A checker that", + "failed every input would score 100% detection and is caught instead by the false", + "positive columns, so the two have to be read together.", + "", + "Per-fixture results are in `benchmark/results/results.json`, and the expanded", + "corpus with every input diff is in `benchmark/results/corpus.json`.", + "", +); + +writeFileSync(join(resultsDir, "results.md"), md.join("\n")); + +// --- console -------------------------------------------------------------- + +console.log(`fixtures ${summary.fixtures}`); +console.log( + `detection rate ${summary.detectionRatePct}% (${summary.detectedSignals}/${summary.expectedSignals} signals)`, +); +console.log(`seeded secrets detected ${summary.seededSecretsDetected}/${summary.seededSecrets}`); +console.log( + `false positives ${summary.falsePositiveCases}/${summary.cleanFixtures} clean fixtures`, +); +console.log(`unexpected findings ${summary.unexpectedFindingsOnViolationFixtures}`); +console.log(`exact matches ${summary.fixturesFullyMatched}/${summary.fixtures}`); +if (failures.length > 0) { + console.log(`\n${failures.length} fixture(s) did not match expectations:`); + for (const c of failures) { + const pattern = c.patternMismatch ? ` pattern ${JSON.stringify(c.patternMismatch)}` : ""; + console.log(` ${c.id}: expected [${c.expectedRuleIds}] actual [${c.actualRuleIds}]${pattern}`); + } +} diff --git a/docs/benchmark-methodology.md b/docs/benchmark-methodology.md index 7dda719..e72dc5c 100644 --- a/docs/benchmark-methodology.md +++ b/docs/benchmark-methodology.md @@ -1,8 +1,35 @@ # Benchmark Methodology -Fencier has been tested with a fixture-based benchmark built from synthetic Git diff inputs. The benchmark exercised the deterministic policy engine without live agent calls, API keys, or hosted services. +Fencier is evaluated with a fixture-based benchmark built from synthetic Git diff +inputs. The benchmark exercises the deterministic policy engine without live agent +calls, API keys, or hosted services. -The evaluated fixtures covered the policy signals implemented by Fencier: +Results are published in [benchmark.md](./benchmark.md). The corpus, the runner, +and the policy snapshot are checked into `benchmark/`, so the numbers reproduce +from a clean clone: + +```bash +pnpm install +pnpm --filter @fencier/core build +node benchmark/run.mjs +``` + +## Method + +Each fixture defines an input diff and the policy outcome expected from it. The +runner evaluates every fixture through `evaluatePolicy` and compares the engine's +structured findings against that expectation, recording three things separately: + +- **detections**, expected rules the engine reported; +- **misses**, expected rules the engine did not report; +- **unexpected findings**, rules the engine reported that the fixture did not + expect. + +A fixture counts as matched only when it produces every expected rule and nothing +else. The detection rate is computed over (fixture, expected rule) pairs rather +than fixtures, because one patch can carry several violations. + +The evaluated fixtures cover the policy signals implemented by Fencier: - files and lines changed - files outside the configured scope @@ -13,8 +40,37 @@ The evaluated fixtures covered the policy signals implemented by Fencier: - file and line limits - resulting verification status and risk score -Each fixture defined an input diff and an expected policy outcome. The benchmark compared Fencier's structured findings with those expectations so detections, misses, and unexpected findings could be reviewed independently. +## Corpus design + +The corpus contains three kinds of case, and all three are needed for the numbers +to mean anything: + +- **Violations**, which must be detected. +- **Near misses**, which resemble violations and must not be. A change of exactly + 8 files against a limit of 8, a variable named `tokenizer`, a comment mentioning + an API key. +- **Clean patches**, ordinary work that must produce no finding at all. + +Without the second and third kinds a detection rate is unfalsifiable, because a +checker that failed every input would report 100%. False positives are therefore +reported next to the detection rate rather than separately. + +## Reproducibility + +- The policy is pinned in `benchmark/policy.snapshot.yaml`, not read from the + repository's own `fencier.yaml`, so changing the project policy does not move + published results. +- Runs are deterministic. No network, no model calls, no clock-dependent + behaviour in the evaluated path. +- Every run records the Fencier commit id in `benchmark/results/results.md` and + `results.json`, alongside fixture composition, per-fixture outcomes, and the + full input corpus in `corpus.json`. -Numeric results are intentionally omitted from this document until the fixture manifest, policy snapshot, and raw per-fixture results are published in the repository. This keeps the repository evidence separate from measurements that cannot yet be reproduced from the checked-in files. +## What the results do not establish -Future published results should include both Markdown and JSON artifacts, identify the exact Fencier commit and policy used, and report fixture composition, detections, misses, and false positives. +The published detection rate measures coverage of the rules the engine +implements, on inputs written to exercise those rules. It does not establish that +the policy model covers every way an AI agent can damage a repository, and it is +not an independent audit: the corpus and the detectors share an author. Known +false positives are listed in [benchmark.md](./benchmark.md) rather than removed +from the corpus. diff --git a/docs/benchmark.md b/docs/benchmark.md new file mode 100644 index 0000000..0e4a4d3 --- /dev/null +++ b/docs/benchmark.md @@ -0,0 +1,125 @@ +# Benchmark + +Fencier's policy engine is evaluated against a fixture corpus that is checked into +this repository and reproducible from a clean clone: + +```bash +pnpm install +pnpm --filter @fencier/core build +node benchmark/run.mjs +``` + +The run writes `benchmark/results/results.md`, `benchmark/results/results.json` +(per-fixture outcomes) and `benchmark/results/corpus.json` (every input diff). +Nothing in the benchmark reaches the network, calls an agent, or needs an API key, +so two runs of the same commit produce the same numbers. + +## What is measured + +Each fixture pairs an input diff with the policy outcome it should produce. The +runner compares the engine's structured findings against that expectation and +records detections, misses, and findings that were not expected. + +The policy is pinned in `benchmark/policy.snapshot.yaml` rather than read from the +repository's own `fencier.yaml`, so editing the project's policy does not silently +move the results. + +## Results + +Current run, commit `d9649dc`, corpus of 82 fixtures: + +| Measure | Value | +| --- | ---: | +| Fixtures | 82 | +| Fixtures expecting a violation | 58 | +| Fixtures expecting no finding | 24 | +| Expected signals | 74 | +| Signals detected | 74 | +| Signals missed | 0 | +| Detection rate | 100% | +| Clean fixtures with a false positive | 1 | +| False positive rate | 4.2% | +| Unexpected findings on violation fixtures | 0 | +| Seeded secrets detected | 25/25 | +| Fixtures matching expectations exactly | 81/82 | + +By fixture class: + +| Class | Passed | Total | +| --- | ---: | ---: | +| blocked_path | 6 | 6 | +| clean | 13 | 13 | +| combined | 3 | 3 | +| max_files_changed | 3 | 3 | +| max_lines_changed | 3 | 3 | +| missing_tests | 3 | 3 | +| near_miss | 13 | 14 | +| outside_allowed_paths | 6 | 6 | +| secret_pattern | 25 | 25 | +| sensitive_path | 6 | 6 | + +## Corpus composition + +- **58 fixtures carrying a violation**, covering all seven policy rules: + `blocked_path`, `outside_allowed_paths`, `sensitive_path`, `missing_tests`, + `max_files_changed`, `max_lines_changed`, `secret_pattern`. +- **25 seeded secrets**, at least two per detector pattern across all nine + patterns (private keys, bearer tokens, GitHub tokens, OpenAI keys, AWS access + key ids, database URLs with inline credentials, and the generic + `API_KEY` / `SECRET` / `TOKEN` assignment forms). Every value is synthetic and + structurally valid for its pattern. None is a real credential. +- **14 near misses**, inputs that resemble violations and must not be flagged: a + variable named `tokenizer`, a comment mentioning an API key, a change of exactly + 8 files against a limit of 8, a 499-line change against a limit of 500, a + protected path changed together with its test. +- **13 clean patches**, ordinary work that must produce no finding, including + files under `ignored_paths` that have to be dropped before evaluation. + +A detection rate on its own is not worth much: a checker that failed every input +would score 100%. The near misses and clean patches are what make the number +mean something, which is why the false positive columns are reported beside it +rather than in an appendix. + +## Known false positive + +One fixture does not match, and it is a real defect rather than a fixture error: + +``` +process.env.API_KEY = undefined; -> flagged as generic_api_key +``` + +The `generic_api_key` pattern matches `API_KEY` followed by `=` and eight or more +non-quote characters. `undefined;` satisfies that, so clearing a key is reported +as leaking one. Assigning `undefined` is the opposite of introducing a secret. + +This is left in the corpus, and in these results, deliberately. It is tracked +rather than removed so the number above stays honest. + +## Limitations + +These are the reasons not to read more into the results than they support. + +- **The corpus is synthetic.** Fixtures are hand-written diffs, not sampled from + real pull requests. They exercise the signals the policy engine implements. +- **100% detection measures coverage of implemented rules, not of real-world + risk.** It says the engine detects what it claims to detect on inputs built to + exercise those rules. It does not say the policy model covers every way an agent + can damage a repository. A secret shape with no pattern, an unsafe change inside + an allowed path, or a logic error in a permitted file are all invisible here and + would not lower this number. +- **The corpus was written against the current pattern set.** Seeded secrets use + forms the detectors are built to catch. A benchmark written by the same author + as the detectors is a check on implementation, not an independent audit. +- **The policy is one snapshot.** Results depend on + `benchmark/policy.snapshot.yaml`. A stricter or looser policy moves them. +- **Review-time effects are not measured here.** Any claim about how long a diff + takes a human to review is outside this benchmark, which is a deterministic + comparison of engine output against expected output. + +## Policy observation + +The snapshot policy blocks `.env.*`, which also matches `.env.example`. A +committed `.env.example` template is therefore reported as a blocked path. The +engine is behaving exactly as configured; whether the shipped default policy +should carve out `.env.example` is a policy question, recorded here so the +behaviour is not mistaken for a detector bug. diff --git a/package.json b/package.json index c15eb1c..d44f9ce 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "pack:cli": "pnpm --filter @fencier/cli pack", "release:check": "node scripts/release-check.mjs", "test": "vitest run", - "typecheck": "pnpm -r typecheck" + "typecheck": "pnpm -r typecheck", + "benchmark": "pnpm --filter @fencier/core build && node benchmark/run.mjs" }, "devDependencies": { "@biomejs/biome": "^1.9.4", From 7c1cd226e92d3d5bbe5e8918c70931570af9aad0 Mon Sep 17 00:00:00 2001 From: Lucas Zimmermann Date: Tue, 4 Aug 2026 18:54:33 -0300 Subject: [PATCH 2/2] bench: exclude generated results from lint, pin the run to a real commit CI failed on the previous commit: biome lints benchmark/results/, and the runner writes JSON with JSON.stringify(..., null, 2), which formats arrays across multiple lines where biome wants them inline. Those files are generated artifacts marked "do not edit by hand", so they are excluded from linting rather than formatted to match. I missed this locally because a Windows checkout gives every file CRLF, so `pnpm lint` already failed on all 45 files before my change and the real error was buried. Verified this time in a fresh LF clone, which is what CI sees: lint, typecheck, 54 tests, build and release:check all pass. The published results also recorded commit d9649dc, which predates the benchmark, so that provenance line could not be reproduced. Regenerated at c81d17d, which contains the benchmark code. Numbers are unchanged: 74/74 signals, 25/25 seeded secrets, 1 false positive across 24 clean fixtures. --- benchmark/results/results.json | 4 ++-- benchmark/results/results.md | 4 ++-- biome.json | 2 +- docs/benchmark.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmark/results/results.json b/benchmark/results/results.json index 40f08de..9006f40 100644 --- a/benchmark/results/results.json +++ b/benchmark/results/results.json @@ -1,7 +1,7 @@ { "summary": { - "generatedAt": "2026-08-04T21:03:53.900Z", - "commit": "d9649dc", + "generatedAt": "2026-08-04T21:53:58.133Z", + "commit": "c81d17d", "policySnapshot": "benchmark/policy.snapshot.yaml", "fixtures": 82, "violationFixtures": 58, diff --git a/benchmark/results/results.md b/benchmark/results/results.md index bd8bd14..0796833 100644 --- a/benchmark/results/results.md +++ b/benchmark/results/results.md @@ -2,9 +2,9 @@ Generated by `node benchmark/run.mjs`. Do not edit by hand. -- Fencier commit: `d9649dc` +- Fencier commit: `c81d17d` - Policy: `benchmark/policy.snapshot.yaml` -- Generated: 2026-08-04T21:03:53.900Z +- Generated: 2026-08-04T21:53:58.133Z ## Summary diff --git a/biome.json b/biome.json index 5921960..6e204c1 100644 --- a/biome.json +++ b/biome.json @@ -7,7 +7,7 @@ }, "files": { "ignoreUnknown": false, - "ignore": ["dist", "coverage", "node_modules"] + "ignore": ["dist", "coverage", "node_modules", "benchmark/results"] }, "formatter": { "enabled": true, diff --git a/docs/benchmark.md b/docs/benchmark.md index 0e4a4d3..0e45357 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -26,7 +26,7 @@ move the results. ## Results -Current run, commit `d9649dc`, corpus of 82 fixtures: +Current run, commit `c81d17d`, corpus of 82 fixtures: | Measure | Value | | --- | ---: |