diff --git a/README.md b/README.md index 0f26c2e..6f6df54 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,7 @@ Possible later extensions, not required by the current product contract, include ## Documentation - [Architecture](docs/architecture.md) +- [Benchmark method and results](docs/benchmark.md) - [Codex Kit](docs/codex-kit.md) - [Deterministic verifier](docs/deterministic-verifier.md) - [Policy model](docs/policy-model.md) diff --git a/benchmark/corpus.mjs b/benchmark/corpus.mjs new file mode 100644 index 0000000..b93f7df --- /dev/null +++ b/benchmark/corpus.mjs @@ -0,0 +1,344 @@ +/** + * Fixture corpus for the Fencier policy benchmark. + * + * Every fixture is a diff (the input `evaluatePolicy` takes) plus the outcome + * the policy is expected to produce. The corpus is built here rather than + * stored as 120 separate files so the composition is readable in one place; + * `run.mjs` writes the expanded corpus to `results/corpus.json` so each case + * can be inspected individually. + * + * Two things this corpus deliberately contains: + * + * - Near misses. Files whose paths or contents look like violations but are + * not (`.env.example`, a var named `tokenizer`, a 499-line change against a + * 500-line limit). Without them a detection rate says nothing, because a + * checker that fails everything would score 100%. + * - Clean patches. Ordinary changes that must produce no finding at all. + * + * Every case is written against the policy snapshot in `policy.snapshot.yaml`. + */ + +/** Build an added-line list from raw strings. */ +const lines = (...contents) => + contents.map((content, index) => ({ lineNumber: index + 1, content })); + +/** A single changed file. */ +const file = (path, { status = "modified", additions = 10, deletions = 0, addedLines } = {}) => ({ + path, + status, + additions, + deletions, + ...(addedLines ? { addedLines } : {}), +}); + +const fixtures = []; + +const add = (id, klass, description, files, expected) => { + fixtures.push({ id, class: klass, description, files, expected }); +}; + +// --------------------------------------------------------------------------- +// blocked_path: paths the policy forbids touching at all. +// --------------------------------------------------------------------------- +const blockedPaths = [ + ".env", + ".env.local", + ".env.production", + "secrets/deploy.key", + "secrets/nested/token.txt", +]; +for (const [i, path] of blockedPaths.entries()) { + add( + `blocked-path-${String(i + 1).padStart(3, "0")}`, + "blocked_path", + `Edit to blocked path ${path}`, + [file(path, { additions: 3 })], + // A blocked path is also outside allowed scope. Both findings are correct + // and both are expected. + { ruleIds: ["blocked_path", "outside_allowed_paths"] }, + ); +} + +// Near miss: names that resemble blocked paths but are not matched. +// `.env.example` is deliberately absent here: the snapshot policy blocks +// `.env.*`, which matches it. That is recorded as its own fixture below. +const blockedNearMisses = ["docs/env.md", "packages/core/src/secrets.ts", "docs/secrets-policy.md"]; +for (const [i, path] of blockedNearMisses.entries()) { + add( + `blocked-path-nearmiss-${String(i + 1).padStart(3, "0")}`, + "near_miss", + `${path} resembles a blocked path but is allowed`, + [file(path, { additions: 4 })], + { ruleIds: [] }, + ); +} + +// The snapshot policy blocks `.env.*`, so a committed `.env.example` template +// is blocked too. The engine is behaving as configured; whether the policy +// should carve out `.env.example` is a policy question, noted in the results. +add( + "blocked-path-006", + "blocked_path", + ".env.example is matched by the blocked pattern .env.*", + [file(".env.example", { additions: 4 })], + { ruleIds: ["blocked_path", "outside_allowed_paths"] }, +); + +// --------------------------------------------------------------------------- +// outside_allowed_paths: paths not covered by scope.allowed_paths. +// --------------------------------------------------------------------------- +const outsidePaths = [ + "random/thing.ts", + "tools/experimental/run.ts", + "vendor/lib/index.js", + "notes.txt", + "tmp/scratch.ts", + "config/prod.json", +]; +for (const [i, path] of outsidePaths.entries()) { + add( + `outside-allowed-${String(i + 1).padStart(3, "0")}`, + "outside_allowed_paths", + `Change outside allowed scope: ${path}`, + [file(path, { additions: 6 })], + { ruleIds: ["outside_allowed_paths"] }, + ); +} + +// --------------------------------------------------------------------------- +// sensitive_path: allowed, but flagged for review. +// --------------------------------------------------------------------------- +const sensitivePaths = [ + "src/auth/session.ts", + "src/billing/invoice.ts", + "src/payments/charge.ts", + ".github/workflows/ci.yml", + "infra/terraform/main.tf", + "migrations/0007_add_index.sql", +]; +for (const [i, path] of sensitivePaths.entries()) { + // require_tests_for covers auth/billing/payments, so pair those with a test + // file to isolate the sensitive_path signal from missing_tests. + const needsTest = /^src\/(auth|billing|payments)\//.test(path); + const files = needsTest + ? [file(path, { additions: 12 }), file("tests/unit.test.ts", { additions: 8 })] + : [file(path, { additions: 12 })]; + // Some sensitive roots (infra/, migrations/) are not in allowed_paths, so a + // change there is correctly both sensitive and out of scope. + const inAllowedScope = !/^(infra|migrations)\//.test(path); + add( + `sensitive-path-${String(i + 1).padStart(3, "0")}`, + "sensitive_path", + `Change to sensitive path ${path}`, + files, + { + ruleIds: inAllowedScope ? ["sensitive_path"] : ["sensitive_path", "outside_allowed_paths"], + }, + ); +} + +// --------------------------------------------------------------------------- +// missing_tests: protected paths changed with no accompanying test. +// --------------------------------------------------------------------------- +const protectedPaths = ["src/auth/login.ts", "src/billing/plan.ts", "src/payments/refund.ts"]; +for (const [i, path] of protectedPaths.entries()) { + add( + `missing-tests-${String(i + 1).padStart(3, "0")}`, + "missing_tests", + `${path} changed without a test`, + [file(path, { additions: 20 })], + { ruleIds: ["missing_tests", "sensitive_path"] }, + ); +} +// Near miss: the same paths *with* a test must not raise missing_tests. +for (const [i, path] of protectedPaths.entries()) { + add( + `missing-tests-nearmiss-${String(i + 1).padStart(3, "0")}`, + "near_miss", + `${path} changed together with a test`, + [file(path, { additions: 20 }), file("tests/covered.test.ts", { additions: 15 })], + { ruleIds: ["sensitive_path"] }, + ); +} + +// --------------------------------------------------------------------------- +// max_files_changed: limit is 8. +// --------------------------------------------------------------------------- +for (const [i, count] of [9, 12, 20].entries()) { + add( + `max-files-${String(i + 1).padStart(3, "0")}`, + "max_files_changed", + `${count} files changed against a limit of 8`, + Array.from({ length: count }, (_, n) => file(`src/mod${n}.ts`, { additions: 2 })), + { ruleIds: ["max_files_changed"] }, + ); +} +// Near miss: exactly at the limit. +add( + "max-files-nearmiss-001", + "near_miss", + "Exactly 8 files changed, at the limit", + Array.from({ length: 8 }, (_, n) => file(`src/mod${n}.ts`, { additions: 2 })), + { ruleIds: [] }, +); + +// --------------------------------------------------------------------------- +// max_lines_changed: limit is 500 (additions + deletions). +// --------------------------------------------------------------------------- +for (const [i, [adds, dels]] of [ + [600, 0], + [300, 250], + [1200, 400], +].entries()) { + add( + `max-lines-${String(i + 1).padStart(3, "0")}`, + "max_lines_changed", + `${adds + dels} lines changed against a limit of 500`, + [file("src/big.ts", { additions: adds, deletions: dels })], + { ruleIds: ["max_lines_changed"] }, + ); +} +// Near miss: one line under the limit. +add( + "max-lines-nearmiss-001", + "near_miss", + "499 lines changed, one under the limit", + [file("src/big.ts", { additions: 499, deletions: 0 })], + { ruleIds: [] }, +); + +// --------------------------------------------------------------------------- +// secret_pattern: 25 seeded secrets, covering all nine detector patterns. +// +// Values are synthetic and structurally valid for their pattern. None is a +// real credential. +// --------------------------------------------------------------------------- +const seededSecrets = [ + ["private_key", "-----BEGIN RSA PRIVATE KEY-----"], + ["private_key", "-----BEGIN OPENSSH PRIVATE KEY-----"], + ["private_key", "-----BEGIN EC PRIVATE KEY-----"], + ["bearer_token", 'headers.set("Authorization", "Bearer abcdefghijklmnopqrstuvwxyz012345")'], + ["bearer_token", "const auth = 'Bearer q1w2e3r4t5y6u7i8o9p0asdfghjkl';"], + ["bearer_token", 'curl -H "Authorization: Bearer AbCdEf0123456789AbCdEf0123456789"'], + ["github_token", "ghp_0123456789abcdefghijklmnopqrstuvwx"], + ["github_token", "ghs_abcdefghijklmnopqrstuvwxyz0123456789"], + ["github_token", "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567"], + ["openai_api_key", "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCD"], + ["openai_api_key", "OPENAI_KEY = 'sk-0123456789abcdefghijklmnopqrstuv'"], + ["aws_access_key_id", "AKIAIOSFODNN7EXAMPLE"], + ["aws_access_key_id", "aws_key = AKIA0123456789ABCDEF"], + ["database_url", "postgres://admin:hunter2pass@db.internal:5432/prod"], + ["database_url", "mysql://root:s3cr3tvalue@127.0.0.1/main"], + ["database_url", "mongodb://user:passw0rdvalue@cluster.example.net/appdb"], + ["generic_api_key", "API_KEY = 'aVeryLongApiKeyValue123'"], + ["generic_api_key", 'STRIPE_API_KEY="sk_live_abcdefghijklmnop"'], + ["generic_api_key", "api-key = 8f3d9a2b7c1e5f4a"], + ["generic_secret", "CLIENT_SECRET = 'supersecretvalue123'"], + ["generic_secret", 'APP_SECRET="0123456789abcdef"'], + ["generic_secret", "SECRET = 'anotherLongSecretValue'"], + ["generic_token", "ACCESS_TOKEN = 'tok_0123456789abcdef'"], + ["generic_token", 'REFRESH_TOKEN="rt_abcdefghijklmnop"'], + ["generic_token", "TOKEN = 'plainLongTokenValue123'"], +]; +for (const [i, [patternId, content]] of seededSecrets.entries()) { + add( + `secret-${String(i + 1).padStart(3, "0")}`, + "secret_pattern", + `Seeded ${patternId} in an added line`, + [ + file(`src/feature${i}.ts`, { + additions: 3, + addedLines: lines("const config = {", content, "};"), + }), + ], + { ruleIds: ["secret_pattern"], secretPatternId: patternId }, + ); +} + +// Near miss: lines that resemble secrets but must not trip the scanner. +const secretNearMisses = [ + "const tokenizer = new Tokenizer();", + "// TODO: rotate the API key before launch", + "process.env.API_KEY = undefined;", + 'log.info("secret redacted");', + "const bearerLabel = 'Bearer';", + "import { getToken } from './auth';", +]; +for (const [i, content] of secretNearMisses.entries()) { + add( + `secret-nearmiss-${String(i + 1).padStart(3, "0")}`, + "near_miss", + `Secret-adjacent text that is not a secret: ${content.slice(0, 40)}`, + [file(`src/near${i}.ts`, { additions: 2, addedLines: lines(content, "export {};") })], + { ruleIds: [] }, + ); +} + +// --------------------------------------------------------------------------- +// Combined violations: more than one signal in a single patch. +// --------------------------------------------------------------------------- +add( + "combined-001", + "combined", + "Blocked path plus a seeded secret", + [ + file(".env", { + additions: 2, + addedLines: lines("API_KEY = 'combinedFixtureValue1'"), + }), + ], + { ruleIds: ["blocked_path", "outside_allowed_paths", "secret_pattern"] }, +); +add( + "combined-002", + "combined", + "Protected path with no test, plus an oversized change", + [file("src/auth/token.ts", { additions: 700, deletions: 20 })], + { ruleIds: ["missing_tests", "sensitive_path", "max_lines_changed"] }, +); +add( + "combined-003", + "combined", + "Out of scope and over the file limit", + Array.from({ length: 10 }, (_, n) => file(`vendor/pkg${n}.js`, { additions: 5 })), + { ruleIds: ["outside_allowed_paths", "max_files_changed"] }, +); + +// --------------------------------------------------------------------------- +// Clean patches: ordinary work that must produce no finding. +// --------------------------------------------------------------------------- +const cleanPatches = [ + ["docs/architecture.md", 25], + ["README.md", 8], + ["packages/core/src/risk.ts", 40], + ["packages/cli/src/cli.ts", 60], + ["tests/policy.test.ts", 30], + ["package.json", 3], + ["docs/quality-bar.md", 12], + ["packages/adapters/src/index.ts", 18], + ["biome.json", 2], + ["tsconfig.base.json", 4], +]; +for (const [i, [path, additions]] of cleanPatches.entries()) { + add( + `clean-${String(i + 1).padStart(3, "0")}`, + "clean", + `Ordinary change to ${path}`, + [file(path, { additions })], + { ruleIds: [] }, + ); +} + +// Ignored paths must be dropped before evaluation. +const ignoredPaths = ["dist/index.js", "node_modules/pkg/index.js", "coverage/lcov.info"]; +for (const [i, path] of ignoredPaths.entries()) { + add( + `ignored-${String(i + 1).padStart(3, "0")}`, + "clean", + `${path} is ignored and must not be evaluated`, + [file(path, { additions: 200 })], + { ruleIds: [] }, + ); +} + +export const corpus = fixtures; diff --git a/benchmark/policy.snapshot.yaml b/benchmark/policy.snapshot.yaml new file mode 100644 index 0000000..f38301a --- /dev/null +++ b/benchmark/policy.snapshot.yaml @@ -0,0 +1,59 @@ +version: 1 + +scope: + allowed_paths: + - .github/workflows/** + - .gitignore + - AGENTS.md + - LICENSE + - README.md + - biome.json + - fencier.yaml + - package.json + - pnpm-lock.yaml + - pnpm-workspace.yaml + - scripts/** + - tsconfig.base.json + - vitest.config.ts + - src/** + - packages/** + - apps/** + - tests/** + - docs/** + blocked_paths: + - .env + - .env.* + - secrets/** + sensitive_paths: + - src/auth/** + - src/billing/** + - src/payments/** + - .github/workflows/** + - infra/** + - migrations/** + ignored_paths: + - .fencier/** + - dist/** + - build/** + - coverage/** + - node_modules/** + +rules: + max_files_changed: 8 + max_lines_changed: 500 + block_secret_patterns: true + require_tests_for: + - src/auth/** + - src/billing/** + - src/payments/** + +audit: + write_markdown: true + write_json: true + include_patch: false + +adapters: + codex: true + claude: false + cursor: false + copilot: false diff --git a/benchmark/results/corpus.json b/benchmark/results/corpus.json new file mode 100644 index 0000000..9d6fe90 --- /dev/null +++ b/benchmark/results/corpus.json @@ -0,0 +1,2247 @@ +[ + { + "id": "blocked-path-001", + "class": "blocked_path", + "description": "Edit to blocked path .env", + "files": [ + { + "path": ".env", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-002", + "class": "blocked_path", + "description": "Edit to blocked path .env.local", + "files": [ + { + "path": ".env.local", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-003", + "class": "blocked_path", + "description": "Edit to blocked path .env.production", + "files": [ + { + "path": ".env.production", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-004", + "class": "blocked_path", + "description": "Edit to blocked path secrets/deploy.key", + "files": [ + { + "path": "secrets/deploy.key", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-005", + "class": "blocked_path", + "description": "Edit to blocked path secrets/nested/token.txt", + "files": [ + { + "path": "secrets/nested/token.txt", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "blocked-path-nearmiss-001", + "class": "near_miss", + "description": "docs/env.md resembles a blocked path but is allowed", + "files": [ + { + "path": "docs/env.md", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "blocked-path-nearmiss-002", + "class": "near_miss", + "description": "packages/core/src/secrets.ts resembles a blocked path but is allowed", + "files": [ + { + "path": "packages/core/src/secrets.ts", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "blocked-path-nearmiss-003", + "class": "near_miss", + "description": "docs/secrets-policy.md resembles a blocked path but is allowed", + "files": [ + { + "path": "docs/secrets-policy.md", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "blocked-path-006", + "class": "blocked_path", + "description": ".env.example is matched by the blocked pattern .env.*", + "files": [ + { + "path": ".env.example", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-001", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: random/thing.ts", + "files": [ + { + "path": "random/thing.ts", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-002", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tools/experimental/run.ts", + "files": [ + { + "path": "tools/experimental/run.ts", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-003", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: vendor/lib/index.js", + "files": [ + { + "path": "vendor/lib/index.js", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-004", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: notes.txt", + "files": [ + { + "path": "notes.txt", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-005", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tmp/scratch.ts", + "files": [ + { + "path": "tmp/scratch.ts", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "outside-allowed-006", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: config/prod.json", + "files": [ + { + "path": "config/prod.json", + "status": "modified", + "additions": 6, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths" + ] + } + }, + { + "id": "sensitive-path-001", + "class": "sensitive_path", + "description": "Change to sensitive path src/auth/session.ts", + "files": [ + { + "path": "src/auth/session.ts", + "status": "modified", + "additions": 12, + "deletions": 0 + }, + { + "path": "tests/unit.test.ts", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-002", + "class": "sensitive_path", + "description": "Change to sensitive path src/billing/invoice.ts", + "files": [ + { + "path": "src/billing/invoice.ts", + "status": "modified", + "additions": 12, + "deletions": 0 + }, + { + "path": "tests/unit.test.ts", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-003", + "class": "sensitive_path", + "description": "Change to sensitive path src/payments/charge.ts", + "files": [ + { + "path": "src/payments/charge.ts", + "status": "modified", + "additions": 12, + "deletions": 0 + }, + { + "path": "tests/unit.test.ts", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-004", + "class": "sensitive_path", + "description": "Change to sensitive path .github/workflows/ci.yml", + "files": [ + { + "path": ".github/workflows/ci.yml", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "sensitive-path-005", + "class": "sensitive_path", + "description": "Change to sensitive path infra/terraform/main.tf", + "files": [ + { + "path": "infra/terraform/main.tf", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "sensitive-path-006", + "class": "sensitive_path", + "description": "Change to sensitive path migrations/0007_add_index.sql", + "files": [ + { + "path": "migrations/0007_add_index.sql", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path", + "outside_allowed_paths" + ] + } + }, + { + "id": "missing-tests-001", + "class": "missing_tests", + "description": "src/auth/login.ts changed without a test", + "files": [ + { + "path": "src/auth/login.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-002", + "class": "missing_tests", + "description": "src/billing/plan.ts changed without a test", + "files": [ + { + "path": "src/billing/plan.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-003", + "class": "missing_tests", + "description": "src/payments/refund.ts changed without a test", + "files": [ + { + "path": "src/payments/refund.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-nearmiss-001", + "class": "near_miss", + "description": "src/auth/login.ts changed together with a test", + "files": [ + { + "path": "src/auth/login.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + }, + { + "path": "tests/covered.test.ts", + "status": "modified", + "additions": 15, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-nearmiss-002", + "class": "near_miss", + "description": "src/billing/plan.ts changed together with a test", + "files": [ + { + "path": "src/billing/plan.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + }, + { + "path": "tests/covered.test.ts", + "status": "modified", + "additions": 15, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "missing-tests-nearmiss-003", + "class": "near_miss", + "description": "src/payments/refund.ts changed together with a test", + "files": [ + { + "path": "src/payments/refund.ts", + "status": "modified", + "additions": 20, + "deletions": 0 + }, + { + "path": "tests/covered.test.ts", + "status": "modified", + "additions": 15, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "sensitive_path" + ] + } + }, + { + "id": "max-files-001", + "class": "max_files_changed", + "description": "9 files changed against a limit of 8", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod8.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_files_changed" + ] + } + }, + { + "id": "max-files-002", + "class": "max_files_changed", + "description": "12 files changed against a limit of 8", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod8.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod9.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod10.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod11.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_files_changed" + ] + } + }, + { + "id": "max-files-003", + "class": "max_files_changed", + "description": "20 files changed against a limit of 8", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod8.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod9.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod10.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod11.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod12.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod13.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod14.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod15.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod16.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod17.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod18.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod19.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_files_changed" + ] + } + }, + { + "id": "max-files-nearmiss-001", + "class": "near_miss", + "description": "Exactly 8 files changed, at the limit", + "files": [ + { + "path": "src/mod0.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod1.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod2.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod3.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod4.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod5.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod6.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + }, + { + "path": "src/mod7.ts", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "max-lines-001", + "class": "max_lines_changed", + "description": "600 lines changed against a limit of 500", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 600, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "max_lines_changed" + ] + } + }, + { + "id": "max-lines-002", + "class": "max_lines_changed", + "description": "550 lines changed against a limit of 500", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 300, + "deletions": 250 + } + ], + "expected": { + "ruleIds": [ + "max_lines_changed" + ] + } + }, + { + "id": "max-lines-003", + "class": "max_lines_changed", + "description": "1600 lines changed against a limit of 500", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 1200, + "deletions": 400 + } + ], + "expected": { + "ruleIds": [ + "max_lines_changed" + ] + } + }, + { + "id": "max-lines-nearmiss-001", + "class": "near_miss", + "description": "499 lines changed, one under the limit", + "files": [ + { + "path": "src/big.ts", + "status": "modified", + "additions": 499, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-001", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "files": [ + { + "path": "src/feature0.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "-----BEGIN RSA PRIVATE KEY-----" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "private_key" + } + }, + { + "id": "secret-002", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "files": [ + { + "path": "src/feature1.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "-----BEGIN OPENSSH PRIVATE KEY-----" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "private_key" + } + }, + { + "id": "secret-003", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "files": [ + { + "path": "src/feature2.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "-----BEGIN EC PRIVATE KEY-----" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "private_key" + } + }, + { + "id": "secret-004", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "files": [ + { + "path": "src/feature3.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "headers.set(\"Authorization\", \"Bearer abcdefghijklmnopqrstuvwxyz012345\")" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "bearer_token" + } + }, + { + "id": "secret-005", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "files": [ + { + "path": "src/feature4.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "const auth = 'Bearer q1w2e3r4t5y6u7i8o9p0asdfghjkl';" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "bearer_token" + } + }, + { + "id": "secret-006", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "files": [ + { + "path": "src/feature5.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "curl -H \"Authorization: Bearer AbCdEf0123456789AbCdEf0123456789\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "bearer_token" + } + }, + { + "id": "secret-007", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "files": [ + { + "path": "src/feature6.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "ghp_0123456789abcdefghijklmnopqrstuvwx" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "github_token" + } + }, + { + "id": "secret-008", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "files": [ + { + "path": "src/feature7.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "ghs_abcdefghijklmnopqrstuvwxyz0123456789" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "github_token" + } + }, + { + "id": "secret-009", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "files": [ + { + "path": "src/feature8.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "github_token" + } + }, + { + "id": "secret-010", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "files": [ + { + "path": "src/feature9.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCD" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "openai_api_key" + } + }, + { + "id": "secret-011", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "files": [ + { + "path": "src/feature10.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "OPENAI_KEY = 'sk-0123456789abcdefghijklmnopqrstuv'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "openai_api_key" + } + }, + { + "id": "secret-012", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "files": [ + { + "path": "src/feature11.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "AKIAIOSFODNN7EXAMPLE" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "aws_access_key_id" + } + }, + { + "id": "secret-013", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "files": [ + { + "path": "src/feature12.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "aws_key = AKIA0123456789ABCDEF" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "aws_access_key_id" + } + }, + { + "id": "secret-014", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "files": [ + { + "path": "src/feature13.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "postgres://admin:hunter2pass@db.internal:5432/prod" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "database_url" + } + }, + { + "id": "secret-015", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "files": [ + { + "path": "src/feature14.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "mysql://root:s3cr3tvalue@127.0.0.1/main" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "database_url" + } + }, + { + "id": "secret-016", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "files": [ + { + "path": "src/feature15.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "mongodb://user:passw0rdvalue@cluster.example.net/appdb" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "database_url" + } + }, + { + "id": "secret-017", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "files": [ + { + "path": "src/feature16.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "API_KEY = 'aVeryLongApiKeyValue123'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_api_key" + } + }, + { + "id": "secret-018", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "files": [ + { + "path": "src/feature17.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "STRIPE_API_KEY=\"sk_live_abcdefghijklmnop\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_api_key" + } + }, + { + "id": "secret-019", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "files": [ + { + "path": "src/feature18.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "api-key = 8f3d9a2b7c1e5f4a" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_api_key" + } + }, + { + "id": "secret-020", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "files": [ + { + "path": "src/feature19.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "CLIENT_SECRET = 'supersecretvalue123'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_secret" + } + }, + { + "id": "secret-021", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "files": [ + { + "path": "src/feature20.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "APP_SECRET=\"0123456789abcdef\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_secret" + } + }, + { + "id": "secret-022", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "files": [ + { + "path": "src/feature21.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "SECRET = 'anotherLongSecretValue'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_secret" + } + }, + { + "id": "secret-023", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "files": [ + { + "path": "src/feature22.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "ACCESS_TOKEN = 'tok_0123456789abcdef'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_token" + } + }, + { + "id": "secret-024", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "files": [ + { + "path": "src/feature23.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "REFRESH_TOKEN=\"rt_abcdefghijklmnop\"" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_token" + } + }, + { + "id": "secret-025", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "files": [ + { + "path": "src/feature24.ts", + "status": "modified", + "additions": 3, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const config = {" + }, + { + "lineNumber": 2, + "content": "TOKEN = 'plainLongTokenValue123'" + }, + { + "lineNumber": 3, + "content": "};" + } + ] + } + ], + "expected": { + "ruleIds": [ + "secret_pattern" + ], + "secretPatternId": "generic_token" + } + }, + { + "id": "secret-nearmiss-001", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const tokenizer = new Tokenizer();", + "files": [ + { + "path": "src/near0.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const tokenizer = new Tokenizer();" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-002", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: // TODO: rotate the API key before launc", + "files": [ + { + "path": "src/near1.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "// TODO: rotate the API key before launch" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-003", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: process.env.API_KEY = undefined;", + "files": [ + { + "path": "src/near2.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "process.env.API_KEY = undefined;" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-004", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: log.info(\"secret redacted\");", + "files": [ + { + "path": "src/near3.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "log.info(\"secret redacted\");" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-005", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const bearerLabel = 'Bearer';", + "files": [ + { + "path": "src/near4.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "const bearerLabel = 'Bearer';" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "secret-nearmiss-006", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: import { getToken } from './auth';", + "files": [ + { + "path": "src/near5.ts", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "import { getToken } from './auth';" + }, + { + "lineNumber": 2, + "content": "export {};" + } + ] + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "combined-001", + "class": "combined", + "description": "Blocked path plus a seeded secret", + "files": [ + { + "path": ".env", + "status": "modified", + "additions": 2, + "deletions": 0, + "addedLines": [ + { + "lineNumber": 1, + "content": "API_KEY = 'combinedFixtureValue1'" + } + ] + } + ], + "expected": { + "ruleIds": [ + "blocked_path", + "outside_allowed_paths", + "secret_pattern" + ] + } + }, + { + "id": "combined-002", + "class": "combined", + "description": "Protected path with no test, plus an oversized change", + "files": [ + { + "path": "src/auth/token.ts", + "status": "modified", + "additions": 700, + "deletions": 20 + } + ], + "expected": { + "ruleIds": [ + "missing_tests", + "sensitive_path", + "max_lines_changed" + ] + } + }, + { + "id": "combined-003", + "class": "combined", + "description": "Out of scope and over the file limit", + "files": [ + { + "path": "vendor/pkg0.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg1.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg2.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg3.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg4.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg5.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg6.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg7.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg8.js", + "status": "modified", + "additions": 5, + "deletions": 0 + }, + { + "path": "vendor/pkg9.js", + "status": "modified", + "additions": 5, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [ + "outside_allowed_paths", + "max_files_changed" + ] + } + }, + { + "id": "clean-001", + "class": "clean", + "description": "Ordinary change to docs/architecture.md", + "files": [ + { + "path": "docs/architecture.md", + "status": "modified", + "additions": 25, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-002", + "class": "clean", + "description": "Ordinary change to README.md", + "files": [ + { + "path": "README.md", + "status": "modified", + "additions": 8, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-003", + "class": "clean", + "description": "Ordinary change to packages/core/src/risk.ts", + "files": [ + { + "path": "packages/core/src/risk.ts", + "status": "modified", + "additions": 40, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-004", + "class": "clean", + "description": "Ordinary change to packages/cli/src/cli.ts", + "files": [ + { + "path": "packages/cli/src/cli.ts", + "status": "modified", + "additions": 60, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-005", + "class": "clean", + "description": "Ordinary change to tests/policy.test.ts", + "files": [ + { + "path": "tests/policy.test.ts", + "status": "modified", + "additions": 30, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-006", + "class": "clean", + "description": "Ordinary change to package.json", + "files": [ + { + "path": "package.json", + "status": "modified", + "additions": 3, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-007", + "class": "clean", + "description": "Ordinary change to docs/quality-bar.md", + "files": [ + { + "path": "docs/quality-bar.md", + "status": "modified", + "additions": 12, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-008", + "class": "clean", + "description": "Ordinary change to packages/adapters/src/index.ts", + "files": [ + { + "path": "packages/adapters/src/index.ts", + "status": "modified", + "additions": 18, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-009", + "class": "clean", + "description": "Ordinary change to biome.json", + "files": [ + { + "path": "biome.json", + "status": "modified", + "additions": 2, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "clean-010", + "class": "clean", + "description": "Ordinary change to tsconfig.base.json", + "files": [ + { + "path": "tsconfig.base.json", + "status": "modified", + "additions": 4, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "ignored-001", + "class": "clean", + "description": "dist/index.js is ignored and must not be evaluated", + "files": [ + { + "path": "dist/index.js", + "status": "modified", + "additions": 200, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "ignored-002", + "class": "clean", + "description": "node_modules/pkg/index.js is ignored and must not be evaluated", + "files": [ + { + "path": "node_modules/pkg/index.js", + "status": "modified", + "additions": 200, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + }, + { + "id": "ignored-003", + "class": "clean", + "description": "coverage/lcov.info is ignored and must not be evaluated", + "files": [ + { + "path": "coverage/lcov.info", + "status": "modified", + "additions": 200, + "deletions": 0 + } + ], + "expected": { + "ruleIds": [] + } + } +] diff --git a/benchmark/results/results.json b/benchmark/results/results.json new file mode 100644 index 0000000..9006f40 --- /dev/null +++ b/benchmark/results/results.json @@ -0,0 +1,1562 @@ +{ + "summary": { + "generatedAt": "2026-08-04T21:53:58.133Z", + "commit": "c81d17d", + "policySnapshot": "benchmark/policy.snapshot.yaml", + "fixtures": 82, + "violationFixtures": 58, + "cleanFixtures": 24, + "expectedSignals": 74, + "detectedSignals": 74, + "missedSignals": 0, + "detectionRatePct": 100, + "falsePositiveCases": 1, + "falsePositiveRatePct": 4.2, + "unexpectedFindingsOnViolationFixtures": 0, + "seededSecrets": 25, + "seededSecretsDetected": 25, + "fixturesFullyMatched": 81, + "byClass": { + "blocked_path": { + "total": 6, + "passed": 6 + }, + "near_miss": { + "total": 14, + "passed": 13 + }, + "outside_allowed_paths": { + "total": 6, + "passed": 6 + }, + "sensitive_path": { + "total": 6, + "passed": 6 + }, + "missing_tests": { + "total": 3, + "passed": 3 + }, + "max_files_changed": { + "total": 3, + "passed": 3 + }, + "max_lines_changed": { + "total": 3, + "passed": 3 + }, + "secret_pattern": { + "total": 25, + "passed": 25 + }, + "combined": { + "total": 3, + "passed": 3 + }, + "clean": { + "total": 13, + "passed": 13 + } + } + }, + "cases": [ + { + "id": "blocked-path-001", + "class": "blocked_path", + "description": "Edit to blocked path .env", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-002", + "class": "blocked_path", + "description": "Edit to blocked path .env.local", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-003", + "class": "blocked_path", + "description": "Edit to blocked path .env.production", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-004", + "class": "blocked_path", + "description": "Edit to blocked path secrets/deploy.key", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-005", + "class": "blocked_path", + "description": "Edit to blocked path secrets/nested/token.txt", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "blocked-path-nearmiss-001", + "class": "near_miss", + "description": "docs/env.md resembles a blocked path but is allowed", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "blocked-path-nearmiss-002", + "class": "near_miss", + "description": "packages/core/src/secrets.ts resembles a blocked path but is allowed", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "blocked-path-nearmiss-003", + "class": "near_miss", + "description": "docs/secrets-policy.md resembles a blocked path but is allowed", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "blocked-path-006", + "class": "blocked_path", + "description": ".env.example is matched by the blocked pattern .env.*", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "outside-allowed-001", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: random/thing.ts", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-002", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tools/experimental/run.ts", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-003", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: vendor/lib/index.js", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-004", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: notes.txt", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-005", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: tmp/scratch.ts", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "outside-allowed-006", + "class": "outside_allowed_paths", + "description": "Change outside allowed scope: config/prod.json", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths" + ], + "actualRuleIds": [ + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "sensitive-path-001", + "class": "sensitive_path", + "description": "Change to sensitive path src/auth/session.ts", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-002", + "class": "sensitive_path", + "description": "Change to sensitive path src/billing/invoice.ts", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-003", + "class": "sensitive_path", + "description": "Change to sensitive path src/payments/charge.ts", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-004", + "class": "sensitive_path", + "description": "Change to sensitive path .github/workflows/ci.yml", + "filesChanged": 1, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "sensitive-path-005", + "class": "sensitive_path", + "description": "Change to sensitive path infra/terraform/main.tf", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "actualRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 40, + "pass": true + }, + { + "id": "sensitive-path-006", + "class": "sensitive_path", + "description": "Change to sensitive path migrations/0007_add_index.sql", + "filesChanged": 1, + "expectedRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "actualRuleIds": [ + "outside_allowed_paths", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 40, + "pass": true + }, + { + "id": "missing-tests-001", + "class": "missing_tests", + "description": "src/auth/login.ts changed without a test", + "filesChanged": 1, + "expectedRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "missing-tests-002", + "class": "missing_tests", + "description": "src/billing/plan.ts changed without a test", + "filesChanged": 1, + "expectedRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "missing-tests-003", + "class": "missing_tests", + "description": "src/payments/refund.ts changed without a test", + "filesChanged": 1, + "expectedRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "missing-tests-nearmiss-001", + "class": "near_miss", + "description": "src/auth/login.ts changed together with a test", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "missing-tests-nearmiss-002", + "class": "near_miss", + "description": "src/billing/plan.ts changed together with a test", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "missing-tests-nearmiss-003", + "class": "near_miss", + "description": "src/payments/refund.ts changed together with a test", + "filesChanged": 2, + "expectedRuleIds": [ + "sensitive_path" + ], + "actualRuleIds": [ + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "medium", + "score": 25, + "pass": true + }, + { + "id": "max-files-001", + "class": "max_files_changed", + "description": "9 files changed against a limit of 8", + "filesChanged": 9, + "expectedRuleIds": [ + "max_files_changed" + ], + "actualRuleIds": [ + "max_files_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-files-002", + "class": "max_files_changed", + "description": "12 files changed against a limit of 8", + "filesChanged": 12, + "expectedRuleIds": [ + "max_files_changed" + ], + "actualRuleIds": [ + "max_files_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-files-003", + "class": "max_files_changed", + "description": "20 files changed against a limit of 8", + "filesChanged": 20, + "expectedRuleIds": [ + "max_files_changed" + ], + "actualRuleIds": [ + "max_files_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-files-nearmiss-001", + "class": "near_miss", + "description": "Exactly 8 files changed, at the limit", + "filesChanged": 8, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "max-lines-001", + "class": "max_lines_changed", + "description": "600 lines changed against a limit of 500", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed" + ], + "actualRuleIds": [ + "max_lines_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-lines-002", + "class": "max_lines_changed", + "description": "550 lines changed against a limit of 500", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed" + ], + "actualRuleIds": [ + "max_lines_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-lines-003", + "class": "max_lines_changed", + "description": "1600 lines changed against a limit of 500", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed" + ], + "actualRuleIds": [ + "max_lines_changed" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "low", + "score": 15, + "pass": true + }, + { + "id": "max-lines-nearmiss-001", + "class": "near_miss", + "description": "499 lines changed, one under the limit", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-001", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-002", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-003", + "class": "secret_pattern", + "description": "Seeded private_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-004", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-005", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-006", + "class": "secret_pattern", + "description": "Seeded bearer_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-007", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-008", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-009", + "class": "secret_pattern", + "description": "Seeded github_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-010", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-011", + "class": "secret_pattern", + "description": "Seeded openai_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-012", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-013", + "class": "secret_pattern", + "description": "Seeded aws_access_key_id in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-014", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-015", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-016", + "class": "secret_pattern", + "description": "Seeded database_url in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-017", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-018", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-019", + "class": "secret_pattern", + "description": "Seeded generic_api_key in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-020", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-021", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-022", + "class": "secret_pattern", + "description": "Seeded generic_secret in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-023", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-024", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-025", + "class": "secret_pattern", + "description": "Seeded generic_token in an added line", + "filesChanged": 1, + "expectedRuleIds": [ + "secret_pattern" + ], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": true + }, + { + "id": "secret-nearmiss-001", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const tokenizer = new Tokenizer();", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-002", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: // TODO: rotate the API key before launc", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-003", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: process.env.API_KEY = undefined;", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [ + "secret_pattern" + ], + "missed": [], + "unexpected": [ + "secret_pattern" + ], + "patternMismatch": null, + "status": "fail", + "risk": "high", + "score": 50, + "pass": false + }, + { + "id": "secret-nearmiss-004", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: log.info(\"secret redacted\");", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-005", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: const bearerLabel = 'Bearer';", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "secret-nearmiss-006", + "class": "near_miss", + "description": "Secret-adjacent text that is not a secret: import { getToken } from './auth';", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "combined-001", + "class": "combined", + "description": "Blocked path plus a seeded secret", + "filesChanged": 1, + "expectedRuleIds": [ + "blocked_path", + "outside_allowed_paths", + "secret_pattern" + ], + "actualRuleIds": [ + "blocked_path", + "outside_allowed_paths", + "secret_pattern" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "fail", + "risk": "critical", + "score": 100, + "pass": true + }, + { + "id": "combined-002", + "class": "combined", + "description": "Protected path with no test, plus an oversized change", + "filesChanged": 1, + "expectedRuleIds": [ + "max_lines_changed", + "missing_tests", + "sensitive_path" + ], + "actualRuleIds": [ + "max_lines_changed", + "missing_tests", + "sensitive_path" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "high", + "score": 65, + "pass": true + }, + { + "id": "combined-003", + "class": "combined", + "description": "Out of scope and over the file limit", + "filesChanged": 10, + "expectedRuleIds": [ + "max_files_changed", + "outside_allowed_paths" + ], + "actualRuleIds": [ + "max_files_changed", + "outside_allowed_paths" + ], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "warn", + "risk": "critical", + "score": 100, + "pass": true + }, + { + "id": "clean-001", + "class": "clean", + "description": "Ordinary change to docs/architecture.md", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-002", + "class": "clean", + "description": "Ordinary change to README.md", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-003", + "class": "clean", + "description": "Ordinary change to packages/core/src/risk.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-004", + "class": "clean", + "description": "Ordinary change to packages/cli/src/cli.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-005", + "class": "clean", + "description": "Ordinary change to tests/policy.test.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-006", + "class": "clean", + "description": "Ordinary change to package.json", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-007", + "class": "clean", + "description": "Ordinary change to docs/quality-bar.md", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-008", + "class": "clean", + "description": "Ordinary change to packages/adapters/src/index.ts", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-009", + "class": "clean", + "description": "Ordinary change to biome.json", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "clean-010", + "class": "clean", + "description": "Ordinary change to tsconfig.base.json", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "ignored-001", + "class": "clean", + "description": "dist/index.js is ignored and must not be evaluated", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "ignored-002", + "class": "clean", + "description": "node_modules/pkg/index.js is ignored and must not be evaluated", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + }, + { + "id": "ignored-003", + "class": "clean", + "description": "coverage/lcov.info is ignored and must not be evaluated", + "filesChanged": 1, + "expectedRuleIds": [], + "actualRuleIds": [], + "missed": [], + "unexpected": [], + "patternMismatch": null, + "status": "pass", + "risk": "low", + "score": 0, + "pass": true + } + ] +} diff --git a/benchmark/results/results.md b/benchmark/results/results.md new file mode 100644 index 0000000..0796833 --- /dev/null +++ b/benchmark/results/results.md @@ -0,0 +1,58 @@ +# Benchmark results + +Generated by `node benchmark/run.mjs`. Do not edit by hand. + +- Fencier commit: `c81d17d` +- Policy: `benchmark/policy.snapshot.yaml` +- Generated: 2026-08-04T21:53:58.133Z + +## Summary + +| Measure | Value | +| --- | ---: | +| Fixtures | 82 | +| Fixtures expecting a violation | 58 | +| Fixtures expecting no finding | 24 | +| Expected signals | 74 | +| Signals detected | 74 | +| Signals missed | 0 | +| **Detection rate** | **100%** | +| Clean fixtures with a false positive | 1 | +| False positive rate | 4.2% | +| Unexpected findings on violation fixtures | 0 | +| **Seeded secrets detected** | **25/25** | +| Fixtures matching expectations exactly | 81/82 | + +## By fixture class + +| Class | Passed | Total | +| --- | ---: | ---: | +| blocked_path | 6 | 6 | +| clean | 13 | 13 | +| combined | 3 | 3 | +| max_files_changed | 3 | 3 | +| max_lines_changed | 3 | 3 | +| missing_tests | 3 | 3 | +| near_miss | 13 | 14 | +| outside_allowed_paths | 6 | 6 | +| secret_pattern | 25 | 25 | +| sensitive_path | 6 | 6 | + +## Fixtures not matching expectations + +| Fixture | Expected | Actual | Missed | Unexpected | +| --- | --- | --- | --- | --- | +| `secret-nearmiss-003` | (none) | secret_pattern | - | secret_pattern | + +## Reading these numbers + +The detection rate counts (fixture, expected rule) pairs, not fixtures, because +one patch can carry several violations. A fixture only counts as matched when it +produces every expected rule and nothing else. + +The corpus includes near misses and clean patches on purpose. A checker that +failed every input would score 100% detection and is caught instead by the false +positive columns, so the two have to be read together. + +Per-fixture results are in `benchmark/results/results.json`, and the expanded +corpus with every input diff is in `benchmark/results/corpus.json`. diff --git a/benchmark/run.mjs b/benchmark/run.mjs new file mode 100644 index 0000000..223654d --- /dev/null +++ b/benchmark/run.mjs @@ -0,0 +1,229 @@ +/** + * Fencier policy benchmark. + * + * Runs the fixture corpus through `evaluatePolicy` and reports what the engine + * produced against what each fixture expected. Writes both a JSON artifact and + * a Markdown summary to `benchmark/results/`. + * + * Deterministic: no network, no API keys, no agent calls. The policy is read + * from `benchmark/policy.snapshot.yaml` so results do not move when the + * repository's own `fencier.yaml` changes. + * + * Requires the core package to be built first: + * + * pnpm --filter @fencier/core build + * node benchmark/run.mjs + */ + +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { evaluatePolicy, parsePolicyConfig } from "../packages/core/dist/index.js"; +import { corpus } from "./corpus.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const resultsDir = join(here, "results"); + +const policy = parsePolicyConfig(readFileSync(join(here, "policy.snapshot.yaml"), "utf8")); + +/** Short commit id, or "unknown" outside a git checkout. */ +function commitId() { + try { + return execFileSync("git", ["rev-parse", "--short", "HEAD"], { + cwd: join(here, ".."), + encoding: "utf8", + }).trim(); + } catch { + return "unknown"; + } +} + +const cases = []; + +for (const fixture of corpus) { + const result = evaluatePolicy({ policy, files: fixture.files }); + const actualRuleIds = [...new Set(result.findings.map((f) => f.ruleId))].sort(); + const expectedRuleIds = [...new Set(fixture.expected.ruleIds)].sort(); + + const missed = expectedRuleIds.filter((id) => !actualRuleIds.includes(id)); + const unexpected = actualRuleIds.filter((id) => !expectedRuleIds.includes(id)); + + // A seeded-secret fixture also has to be attributed to the right pattern. + let patternMismatch = null; + if (fixture.expected.secretPatternId) { + const detected = result.findings + .filter((f) => f.ruleId === "secret_pattern") + .map((f) => f.pattern); + if (!detected.includes(fixture.expected.secretPatternId)) { + patternMismatch = { + expected: fixture.expected.secretPatternId, + detected, + }; + } + } + + cases.push({ + id: fixture.id, + class: fixture.class, + description: fixture.description, + filesChanged: fixture.files.length, + expectedRuleIds, + actualRuleIds, + missed, + unexpected, + patternMismatch, + status: result.status, + risk: result.risk, + score: result.score, + pass: missed.length === 0 && unexpected.length === 0 && patternMismatch === null, + }); +} + +// --- aggregate ------------------------------------------------------------ + +const violationCases = cases.filter((c) => c.expectedRuleIds.length > 0); +const cleanCases = cases.filter((c) => c.expectedRuleIds.length === 0); + +/** Every (fixture, expected rule) pair, which is what a detection rate is over. */ +const expectedSignals = violationCases.reduce((n, c) => n + c.expectedRuleIds.length, 0); +const missedSignals = violationCases.reduce((n, c) => n + c.missed.length, 0); +const detectedSignals = expectedSignals - missedSignals; + +const falsePositiveCases = cleanCases.filter((c) => c.actualRuleIds.length > 0); +const unexpectedOnViolations = violationCases.reduce((n, c) => n + c.unexpected.length, 0); + +const secretCases = cases.filter((c) => c.class === "secret_pattern"); +const secretsDetected = secretCases.filter( + (c) => c.actualRuleIds.includes("secret_pattern") && c.patternMismatch === null, +).length; + +const byClass = {}; +for (const c of cases) { + byClass[c.class] ??= { total: 0, passed: 0 }; + byClass[c.class].total += 1; + if (c.pass) byClass[c.class].passed += 1; +} + +const pct = (num, den) => (den === 0 ? 0 : Math.round((num / den) * 1000) / 10); + +const summary = { + generatedAt: new Date().toISOString(), + commit: commitId(), + policySnapshot: "benchmark/policy.snapshot.yaml", + fixtures: cases.length, + violationFixtures: violationCases.length, + cleanFixtures: cleanCases.length, + expectedSignals, + detectedSignals, + missedSignals, + detectionRatePct: pct(detectedSignals, expectedSignals), + falsePositiveCases: falsePositiveCases.length, + falsePositiveRatePct: pct(falsePositiveCases.length, cleanCases.length), + unexpectedFindingsOnViolationFixtures: unexpectedOnViolations, + seededSecrets: secretCases.length, + seededSecretsDetected: secretsDetected, + fixturesFullyMatched: cases.filter((c) => c.pass).length, + byClass, +}; + +mkdirSync(resultsDir, { recursive: true }); +writeFileSync(join(resultsDir, "results.json"), `${JSON.stringify({ summary, cases }, null, 2)}\n`); +writeFileSync(join(resultsDir, "corpus.json"), `${JSON.stringify(corpus, null, 2)}\n`); + +// --- markdown ------------------------------------------------------------- + +const failures = cases.filter((c) => !c.pass); + +const md = [ + "# Benchmark results", + "", + "Generated by `node benchmark/run.mjs`. Do not edit by hand.", + "", + `- Fencier commit: \`${summary.commit}\``, + `- Policy: \`${summary.policySnapshot}\``, + `- Generated: ${summary.generatedAt}`, + "", + "## Summary", + "", + "| Measure | Value |", + "| --- | ---: |", + `| Fixtures | ${summary.fixtures} |`, + `| Fixtures expecting a violation | ${summary.violationFixtures} |`, + `| Fixtures expecting no finding | ${summary.cleanFixtures} |`, + `| Expected signals | ${summary.expectedSignals} |`, + `| Signals detected | ${summary.detectedSignals} |`, + `| Signals missed | ${summary.missedSignals} |`, + `| **Detection rate** | **${summary.detectionRatePct}%** |`, + `| Clean fixtures with a false positive | ${summary.falsePositiveCases} |`, + `| False positive rate | ${summary.falsePositiveRatePct}% |`, + `| Unexpected findings on violation fixtures | ${summary.unexpectedFindingsOnViolationFixtures} |`, + `| **Seeded secrets detected** | **${summary.seededSecretsDetected}/${summary.seededSecrets}** |`, + `| Fixtures matching expectations exactly | ${summary.fixturesFullyMatched}/${summary.fixtures} |`, + "", + "## By fixture class", + "", + "| Class | Passed | Total |", + "| --- | ---: | ---: |", + ...Object.entries(byClass) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, v]) => `| ${name} | ${v.passed} | ${v.total} |`), + "", +]; + +md.push("## Fixtures not matching expectations", ""); +if (failures.length === 0) { + md.push("None.", ""); +} else { + md.push( + "| Fixture | Expected | Actual | Missed | Unexpected |", + "| --- | --- | --- | --- | --- |", + ); + for (const c of failures) { + md.push( + `| \`${c.id}\` | ${c.expectedRuleIds.join(", ") || "(none)"} | ${ + c.actualRuleIds.join(", ") || "(none)" + } | ${c.missed.join(", ") || "-"} | ${c.unexpected.join(", ") || "-"} |`, + ); + } + md.push(""); +} + +md.push( + "## Reading these numbers", + "", + "The detection rate counts (fixture, expected rule) pairs, not fixtures, because", + "one patch can carry several violations. A fixture only counts as matched when it", + "produces every expected rule and nothing else.", + "", + "The corpus includes near misses and clean patches on purpose. A checker that", + "failed every input would score 100% detection and is caught instead by the false", + "positive columns, so the two have to be read together.", + "", + "Per-fixture results are in `benchmark/results/results.json`, and the expanded", + "corpus with every input diff is in `benchmark/results/corpus.json`.", + "", +); + +writeFileSync(join(resultsDir, "results.md"), md.join("\n")); + +// --- console -------------------------------------------------------------- + +console.log(`fixtures ${summary.fixtures}`); +console.log( + `detection rate ${summary.detectionRatePct}% (${summary.detectedSignals}/${summary.expectedSignals} signals)`, +); +console.log(`seeded secrets detected ${summary.seededSecretsDetected}/${summary.seededSecrets}`); +console.log( + `false positives ${summary.falsePositiveCases}/${summary.cleanFixtures} clean fixtures`, +); +console.log(`unexpected findings ${summary.unexpectedFindingsOnViolationFixtures}`); +console.log(`exact matches ${summary.fixturesFullyMatched}/${summary.fixtures}`); +if (failures.length > 0) { + console.log(`\n${failures.length} fixture(s) did not match expectations:`); + for (const c of failures) { + const pattern = c.patternMismatch ? ` pattern ${JSON.stringify(c.patternMismatch)}` : ""; + console.log(` ${c.id}: expected [${c.expectedRuleIds}] actual [${c.actualRuleIds}]${pattern}`); + } +} diff --git a/biome.json b/biome.json index 5921960..6e204c1 100644 --- a/biome.json +++ b/biome.json @@ -7,7 +7,7 @@ }, "files": { "ignoreUnknown": false, - "ignore": ["dist", "coverage", "node_modules"] + "ignore": ["dist", "coverage", "node_modules", "benchmark/results"] }, "formatter": { "enabled": true, diff --git a/docs/benchmark-methodology.md b/docs/benchmark-methodology.md index 7dda719..e72dc5c 100644 --- a/docs/benchmark-methodology.md +++ b/docs/benchmark-methodology.md @@ -1,8 +1,35 @@ # Benchmark Methodology -Fencier has been tested with a fixture-based benchmark built from synthetic Git diff inputs. The benchmark exercised the deterministic policy engine without live agent calls, API keys, or hosted services. +Fencier is evaluated with a fixture-based benchmark built from synthetic Git diff +inputs. The benchmark exercises the deterministic policy engine without live agent +calls, API keys, or hosted services. -The evaluated fixtures covered the policy signals implemented by Fencier: +Results are published in [benchmark.md](./benchmark.md). The corpus, the runner, +and the policy snapshot are checked into `benchmark/`, so the numbers reproduce +from a clean clone: + +```bash +pnpm install +pnpm --filter @fencier/core build +node benchmark/run.mjs +``` + +## Method + +Each fixture defines an input diff and the policy outcome expected from it. The +runner evaluates every fixture through `evaluatePolicy` and compares the engine's +structured findings against that expectation, recording three things separately: + +- **detections**, expected rules the engine reported; +- **misses**, expected rules the engine did not report; +- **unexpected findings**, rules the engine reported that the fixture did not + expect. + +A fixture counts as matched only when it produces every expected rule and nothing +else. The detection rate is computed over (fixture, expected rule) pairs rather +than fixtures, because one patch can carry several violations. + +The evaluated fixtures cover the policy signals implemented by Fencier: - files and lines changed - files outside the configured scope @@ -13,8 +40,37 @@ The evaluated fixtures covered the policy signals implemented by Fencier: - file and line limits - resulting verification status and risk score -Each fixture defined an input diff and an expected policy outcome. The benchmark compared Fencier's structured findings with those expectations so detections, misses, and unexpected findings could be reviewed independently. +## Corpus design + +The corpus contains three kinds of case, and all three are needed for the numbers +to mean anything: + +- **Violations**, which must be detected. +- **Near misses**, which resemble violations and must not be. A change of exactly + 8 files against a limit of 8, a variable named `tokenizer`, a comment mentioning + an API key. +- **Clean patches**, ordinary work that must produce no finding at all. + +Without the second and third kinds a detection rate is unfalsifiable, because a +checker that failed every input would report 100%. False positives are therefore +reported next to the detection rate rather than separately. + +## Reproducibility + +- The policy is pinned in `benchmark/policy.snapshot.yaml`, not read from the + repository's own `fencier.yaml`, so changing the project policy does not move + published results. +- Runs are deterministic. No network, no model calls, no clock-dependent + behaviour in the evaluated path. +- Every run records the Fencier commit id in `benchmark/results/results.md` and + `results.json`, alongside fixture composition, per-fixture outcomes, and the + full input corpus in `corpus.json`. -Numeric results are intentionally omitted from this document until the fixture manifest, policy snapshot, and raw per-fixture results are published in the repository. This keeps the repository evidence separate from measurements that cannot yet be reproduced from the checked-in files. +## What the results do not establish -Future published results should include both Markdown and JSON artifacts, identify the exact Fencier commit and policy used, and report fixture composition, detections, misses, and false positives. +The published detection rate measures coverage of the rules the engine +implements, on inputs written to exercise those rules. It does not establish that +the policy model covers every way an AI agent can damage a repository, and it is +not an independent audit: the corpus and the detectors share an author. Known +false positives are listed in [benchmark.md](./benchmark.md) rather than removed +from the corpus. diff --git a/docs/benchmark.md b/docs/benchmark.md new file mode 100644 index 0000000..0e45357 --- /dev/null +++ b/docs/benchmark.md @@ -0,0 +1,125 @@ +# Benchmark + +Fencier's policy engine is evaluated against a fixture corpus that is checked into +this repository and reproducible from a clean clone: + +```bash +pnpm install +pnpm --filter @fencier/core build +node benchmark/run.mjs +``` + +The run writes `benchmark/results/results.md`, `benchmark/results/results.json` +(per-fixture outcomes) and `benchmark/results/corpus.json` (every input diff). +Nothing in the benchmark reaches the network, calls an agent, or needs an API key, +so two runs of the same commit produce the same numbers. + +## What is measured + +Each fixture pairs an input diff with the policy outcome it should produce. The +runner compares the engine's structured findings against that expectation and +records detections, misses, and findings that were not expected. + +The policy is pinned in `benchmark/policy.snapshot.yaml` rather than read from the +repository's own `fencier.yaml`, so editing the project's policy does not silently +move the results. + +## Results + +Current run, commit `c81d17d`, corpus of 82 fixtures: + +| Measure | Value | +| --- | ---: | +| Fixtures | 82 | +| Fixtures expecting a violation | 58 | +| Fixtures expecting no finding | 24 | +| Expected signals | 74 | +| Signals detected | 74 | +| Signals missed | 0 | +| Detection rate | 100% | +| Clean fixtures with a false positive | 1 | +| False positive rate | 4.2% | +| Unexpected findings on violation fixtures | 0 | +| Seeded secrets detected | 25/25 | +| Fixtures matching expectations exactly | 81/82 | + +By fixture class: + +| Class | Passed | Total | +| --- | ---: | ---: | +| blocked_path | 6 | 6 | +| clean | 13 | 13 | +| combined | 3 | 3 | +| max_files_changed | 3 | 3 | +| max_lines_changed | 3 | 3 | +| missing_tests | 3 | 3 | +| near_miss | 13 | 14 | +| outside_allowed_paths | 6 | 6 | +| secret_pattern | 25 | 25 | +| sensitive_path | 6 | 6 | + +## Corpus composition + +- **58 fixtures carrying a violation**, covering all seven policy rules: + `blocked_path`, `outside_allowed_paths`, `sensitive_path`, `missing_tests`, + `max_files_changed`, `max_lines_changed`, `secret_pattern`. +- **25 seeded secrets**, at least two per detector pattern across all nine + patterns (private keys, bearer tokens, GitHub tokens, OpenAI keys, AWS access + key ids, database URLs with inline credentials, and the generic + `API_KEY` / `SECRET` / `TOKEN` assignment forms). Every value is synthetic and + structurally valid for its pattern. None is a real credential. +- **14 near misses**, inputs that resemble violations and must not be flagged: a + variable named `tokenizer`, a comment mentioning an API key, a change of exactly + 8 files against a limit of 8, a 499-line change against a limit of 500, a + protected path changed together with its test. +- **13 clean patches**, ordinary work that must produce no finding, including + files under `ignored_paths` that have to be dropped before evaluation. + +A detection rate on its own is not worth much: a checker that failed every input +would score 100%. The near misses and clean patches are what make the number +mean something, which is why the false positive columns are reported beside it +rather than in an appendix. + +## Known false positive + +One fixture does not match, and it is a real defect rather than a fixture error: + +``` +process.env.API_KEY = undefined; -> flagged as generic_api_key +``` + +The `generic_api_key` pattern matches `API_KEY` followed by `=` and eight or more +non-quote characters. `undefined;` satisfies that, so clearing a key is reported +as leaking one. Assigning `undefined` is the opposite of introducing a secret. + +This is left in the corpus, and in these results, deliberately. It is tracked +rather than removed so the number above stays honest. + +## Limitations + +These are the reasons not to read more into the results than they support. + +- **The corpus is synthetic.** Fixtures are hand-written diffs, not sampled from + real pull requests. They exercise the signals the policy engine implements. +- **100% detection measures coverage of implemented rules, not of real-world + risk.** It says the engine detects what it claims to detect on inputs built to + exercise those rules. It does not say the policy model covers every way an agent + can damage a repository. A secret shape with no pattern, an unsafe change inside + an allowed path, or a logic error in a permitted file are all invisible here and + would not lower this number. +- **The corpus was written against the current pattern set.** Seeded secrets use + forms the detectors are built to catch. A benchmark written by the same author + as the detectors is a check on implementation, not an independent audit. +- **The policy is one snapshot.** Results depend on + `benchmark/policy.snapshot.yaml`. A stricter or looser policy moves them. +- **Review-time effects are not measured here.** Any claim about how long a diff + takes a human to review is outside this benchmark, which is a deterministic + comparison of engine output against expected output. + +## Policy observation + +The snapshot policy blocks `.env.*`, which also matches `.env.example`. A +committed `.env.example` template is therefore reported as a blocked path. The +engine is behaving exactly as configured; whether the shipped default policy +should carve out `.env.example` is a policy question, recorded here so the +behaviour is not mistaken for a detector bug. diff --git a/package.json b/package.json index c15eb1c..d44f9ce 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "pack:cli": "pnpm --filter @fencier/cli pack", "release:check": "node scripts/release-check.mjs", "test": "vitest run", - "typecheck": "pnpm -r typecheck" + "typecheck": "pnpm -r typecheck", + "benchmark": "pnpm --filter @fencier/core build && node benchmark/run.mjs" }, "devDependencies": { "@biomejs/biome": "^1.9.4",