bench: publish a reproducible fixture benchmark

README.md

@@ -361,6 +361,7 @@ Possible later extensions, not required by the current product contract, include
 ## Documentation
 
 - [Architecture](docs/architecture.md)
+- [Benchmark method and results](docs/benchmark.md)
 - [Codex Kit](docs/codex-kit.md)
 - [Deterministic verifier](docs/deterministic-verifier.md)
 - [Policy model](docs/policy-model.md)

benchmark/corpus.mjs

@@ -0,0 +1,344 @@
+/**
+ * Fixture corpus for the Fencier policy benchmark.
+ *
+ * Every fixture is a diff (the input `evaluatePolicy` takes) plus the outcome
+ * the policy is expected to produce. The corpus is built here rather than
+ * stored as 120 separate files so the composition is readable in one place;
+ * `run.mjs` writes the expanded corpus to `results/corpus.json` so each case
+ * can be inspected individually.
+ *
+ * Two things this corpus deliberately contains:
+ *
+ * - Near misses. Files whose paths or contents look like violations but are
+ *   not (`.env.example`, a var named `tokenizer`, a 499-line change against a
+ *   500-line limit). Without them a detection rate says nothing, because a
+ *   checker that fails everything would score 100%.
+ * - Clean patches. Ordinary changes that must produce no finding at all.
+ *
+ * Every case is written against the policy snapshot in `policy.snapshot.yaml`.
+ */
+
+/** Build an added-line list from raw strings. */
+const lines = (...contents) =>
+  contents.map((content, index) => ({ lineNumber: index + 1, content }));
+
+/** A single changed file. */
+const file = (path, { status = "modified", additions = 10, deletions = 0, addedLines } = {}) => ({
+  path,
+  status,
+  additions,
+  deletions,
+  ...(addedLines ? { addedLines } : {}),
+});
+
+const fixtures = [];
+
+const add = (id, klass, description, files, expected) => {
+  fixtures.push({ id, class: klass, description, files, expected });
+};
+
+// ---------------------------------------------------------------------------
+// blocked_path: paths the policy forbids touching at all.
+// ---------------------------------------------------------------------------
+const blockedPaths = [
+  ".env",
+  ".env.local",
+  ".env.production",
+  "secrets/deploy.key",
+  "secrets/nested/token.txt",
+];
+for (const [i, path] of blockedPaths.entries()) {
+  add(
+    `blocked-path-${String(i + 1).padStart(3, "0")}`,
+    "blocked_path",
+    `Edit to blocked path ${path}`,
+    [file(path, { additions: 3 })],
+    // A blocked path is also outside allowed scope. Both findings are correct
+    // and both are expected.
+    { ruleIds: ["blocked_path", "outside_allowed_paths"] },
+  );
+}
+
+// Near miss: names that resemble blocked paths but are not matched.
+// `.env.example` is deliberately absent here: the snapshot policy blocks
+// `.env.*`, which matches it. That is recorded as its own fixture below.
+const blockedNearMisses = ["docs/env.md", "packages/core/src/secrets.ts", "docs/secrets-policy.md"];
+for (const [i, path] of blockedNearMisses.entries()) {
+  add(
+    `blocked-path-nearmiss-${String(i + 1).padStart(3, "0")}`,
+    "near_miss",
+    `${path} resembles a blocked path but is allowed`,
+    [file(path, { additions: 4 })],
+    { ruleIds: [] },
+  );
+}
+
+// The snapshot policy blocks `.env.*`, so a committed `.env.example` template
+// is blocked too. The engine is behaving as configured; whether the policy
+// should carve out `.env.example` is a policy question, noted in the results.
+add(
+  "blocked-path-006",
+  "blocked_path",
+  ".env.example is matched by the blocked pattern .env.*",
+  [file(".env.example", { additions: 4 })],
+  { ruleIds: ["blocked_path", "outside_allowed_paths"] },
+);
+
+// ---------------------------------------------------------------------------
+// outside_allowed_paths: paths not covered by scope.allowed_paths.
+// ---------------------------------------------------------------------------
+const outsidePaths = [
+  "random/thing.ts",
+  "tools/experimental/run.ts",
+  "vendor/lib/index.js",
+  "notes.txt",
+  "tmp/scratch.ts",
+  "config/prod.json",
+];
+for (const [i, path] of outsidePaths.entries()) {
+  add(
+    `outside-allowed-${String(i + 1).padStart(3, "0")}`,
+    "outside_allowed_paths",
+    `Change outside allowed scope: ${path}`,
+    [file(path, { additions: 6 })],
+    { ruleIds: ["outside_allowed_paths"] },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// sensitive_path: allowed, but flagged for review.
+// ---------------------------------------------------------------------------
+const sensitivePaths = [
+  "src/auth/session.ts",
+  "src/billing/invoice.ts",
+  "src/payments/charge.ts",
+  ".github/workflows/ci.yml",
+  "infra/terraform/main.tf",
+  "migrations/0007_add_index.sql",
+];
+for (const [i, path] of sensitivePaths.entries()) {
+  // require_tests_for covers auth/billing/payments, so pair those with a test
+  // file to isolate the sensitive_path signal from missing_tests.
+  const needsTest = /^src\/(auth|billing|payments)\//.test(path);
+  const files = needsTest
+    ? [file(path, { additions: 12 }), file("tests/unit.test.ts", { additions: 8 })]
+    : [file(path, { additions: 12 })];
+  // Some sensitive roots (infra/, migrations/) are not in allowed_paths, so a
+  // change there is correctly both sensitive and out of scope.
+  const inAllowedScope = !/^(infra|migrations)\//.test(path);
+  add(
+    `sensitive-path-${String(i + 1).padStart(3, "0")}`,
+    "sensitive_path",
+    `Change to sensitive path ${path}`,
+    files,
+    {
+      ruleIds: inAllowedScope ? ["sensitive_path"] : ["sensitive_path", "outside_allowed_paths"],
+    },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// missing_tests: protected paths changed with no accompanying test.
+// ---------------------------------------------------------------------------
+const protectedPaths = ["src/auth/login.ts", "src/billing/plan.ts", "src/payments/refund.ts"];
+for (const [i, path] of protectedPaths.entries()) {
+  add(
+    `missing-tests-${String(i + 1).padStart(3, "0")}`,
+    "missing_tests",
+    `${path} changed without a test`,
+    [file(path, { additions: 20 })],
+    { ruleIds: ["missing_tests", "sensitive_path"] },
+  );
+}
+// Near miss: the same paths *with* a test must not raise missing_tests.
+for (const [i, path] of protectedPaths.entries()) {
+  add(
+    `missing-tests-nearmiss-${String(i + 1).padStart(3, "0")}`,
+    "near_miss",
+    `${path} changed together with a test`,
+    [file(path, { additions: 20 }), file("tests/covered.test.ts", { additions: 15 })],
+    { ruleIds: ["sensitive_path"] },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// max_files_changed: limit is 8.
+// ---------------------------------------------------------------------------
+for (const [i, count] of [9, 12, 20].entries()) {
+  add(
+    `max-files-${String(i + 1).padStart(3, "0")}`,
+    "max_files_changed",
+    `${count} files changed against a limit of 8`,
+    Array.from({ length: count }, (_, n) => file(`src/mod${n}.ts`, { additions: 2 })),
+    { ruleIds: ["max_files_changed"] },
+  );
+}
+// Near miss: exactly at the limit.
+add(
+  "max-files-nearmiss-001",
+  "near_miss",
+  "Exactly 8 files changed, at the limit",
+  Array.from({ length: 8 }, (_, n) => file(`src/mod${n}.ts`, { additions: 2 })),
+  { ruleIds: [] },
+);
+
+// ---------------------------------------------------------------------------
+// max_lines_changed: limit is 500 (additions + deletions).
+// ---------------------------------------------------------------------------
+for (const [i, [adds, dels]] of [
+  [600, 0],
+  [300, 250],
+  [1200, 400],
+].entries()) {
+  add(
+    `max-lines-${String(i + 1).padStart(3, "0")}`,
+    "max_lines_changed",
+    `${adds + dels} lines changed against a limit of 500`,
+    [file("src/big.ts", { additions: adds, deletions: dels })],
+    { ruleIds: ["max_lines_changed"] },
+  );
+}
+// Near miss: one line under the limit.
+add(
+  "max-lines-nearmiss-001",
+  "near_miss",
+  "499 lines changed, one under the limit",
+  [file("src/big.ts", { additions: 499, deletions: 0 })],
+  { ruleIds: [] },
+);
+
+// ---------------------------------------------------------------------------
+// secret_pattern: 25 seeded secrets, covering all nine detector patterns.
+//
+// Values are synthetic and structurally valid for their pattern. None is a
+// real credential.
+// ---------------------------------------------------------------------------
+const seededSecrets = [
+  ["private_key", "-----BEGIN RSA PRIVATE KEY-----"],
+  ["private_key", "-----BEGIN OPENSSH PRIVATE KEY-----"],
+  ["private_key", "-----BEGIN EC PRIVATE KEY-----"],
+  ["bearer_token", 'headers.set("Authorization", "Bearer abcdefghijklmnopqrstuvwxyz012345")'],
+  ["bearer_token", "const auth = 'Bearer q1w2e3r4t5y6u7i8o9p0asdfghjkl';"],
+  ["bearer_token", 'curl -H "Authorization: Bearer AbCdEf0123456789AbCdEf0123456789"'],
+  ["github_token", "ghp_0123456789abcdefghijklmnopqrstuvwx"],
+  ["github_token", "ghs_abcdefghijklmnopqrstuvwxyz0123456789"],
+  ["github_token", "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567"],
+  ["openai_api_key", "sk-abcdefghijklmnopqrstuvwxyz0123456789ABCD"],
+  ["openai_api_key", "OPENAI_KEY = 'sk-0123456789abcdefghijklmnopqrstuv'"],
+  ["aws_access_key_id", "AKIAIOSFODNN7EXAMPLE"],
+  ["aws_access_key_id", "aws_key = AKIA0123456789ABCDEF"],
+  ["database_url", "postgres://admin:hunter2pass@db.internal:5432/prod"],
+  ["database_url", "mysql://root:s3cr3tvalue@127.0.0.1/main"],
+  ["database_url", "mongodb://user:passw0rdvalue@cluster.example.net/appdb"],
+  ["generic_api_key", "API_KEY = 'aVeryLongApiKeyValue123'"],
+  ["generic_api_key", 'STRIPE_API_KEY="sk_live_abcdefghijklmnop"'],
+  ["generic_api_key", "api-key = 8f3d9a2b7c1e5f4a"],
+  ["generic_secret", "CLIENT_SECRET = 'supersecretvalue123'"],
+  ["generic_secret", 'APP_SECRET="0123456789abcdef"'],
+  ["generic_secret", "SECRET = 'anotherLongSecretValue'"],
+  ["generic_token", "ACCESS_TOKEN = 'tok_0123456789abcdef'"],
+  ["generic_token", 'REFRESH_TOKEN="rt_abcdefghijklmnop"'],
+  ["generic_token", "TOKEN = 'plainLongTokenValue123'"],
+];
+for (const [i, [patternId, content]] of seededSecrets.entries()) {
+  add(
+    `secret-${String(i + 1).padStart(3, "0")}`,
+    "secret_pattern",
+    `Seeded ${patternId} in an added line`,
+    [
+      file(`src/feature${i}.ts`, {
+        additions: 3,
+        addedLines: lines("const config = {", content, "};"),
+      }),
+    ],
+    { ruleIds: ["secret_pattern"], secretPatternId: patternId },
+  );
+}
+
+// Near miss: lines that resemble secrets but must not trip the scanner.
+const secretNearMisses = [
+  "const tokenizer = new Tokenizer();",
+  "// TODO: rotate the API key before launch",
+  "process.env.API_KEY = undefined;",
+  'log.info("secret redacted");',
+  "const bearerLabel = 'Bearer';",
+  "import { getToken } from './auth';",
+];
+for (const [i, content] of secretNearMisses.entries()) {
+  add(
+    `secret-nearmiss-${String(i + 1).padStart(3, "0")}`,
+    "near_miss",
+    `Secret-adjacent text that is not a secret: ${content.slice(0, 40)}`,
+    [file(`src/near${i}.ts`, { additions: 2, addedLines: lines(content, "export {};") })],
+    { ruleIds: [] },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Combined violations: more than one signal in a single patch.
+// ---------------------------------------------------------------------------
+add(
+  "combined-001",
+  "combined",
+  "Blocked path plus a seeded secret",
+  [
+    file(".env", {
+      additions: 2,
+      addedLines: lines("API_KEY = 'combinedFixtureValue1'"),
+    }),
+  ],
+  { ruleIds: ["blocked_path", "outside_allowed_paths", "secret_pattern"] },
+);
+add(
+  "combined-002",
+  "combined",
+  "Protected path with no test, plus an oversized change",
+  [file("src/auth/token.ts", { additions: 700, deletions: 20 })],
+  { ruleIds: ["missing_tests", "sensitive_path", "max_lines_changed"] },
+);
+add(
+  "combined-003",
+  "combined",
+  "Out of scope and over the file limit",
+  Array.from({ length: 10 }, (_, n) => file(`vendor/pkg${n}.js`, { additions: 5 })),
+  { ruleIds: ["outside_allowed_paths", "max_files_changed"] },
+);
+
+// ---------------------------------------------------------------------------
+// Clean patches: ordinary work that must produce no finding.
+// ---------------------------------------------------------------------------
+const cleanPatches = [
+  ["docs/architecture.md", 25],
+  ["README.md", 8],
+  ["packages/core/src/risk.ts", 40],
+  ["packages/cli/src/cli.ts", 60],
+  ["tests/policy.test.ts", 30],
+  ["package.json", 3],
+  ["docs/quality-bar.md", 12],
+  ["packages/adapters/src/index.ts", 18],
+  ["biome.json", 2],
+  ["tsconfig.base.json", 4],
+];
+for (const [i, [path, additions]] of cleanPatches.entries()) {
+  add(
+    `clean-${String(i + 1).padStart(3, "0")}`,
+    "clean",
+    `Ordinary change to ${path}`,
+    [file(path, { additions })],
+    { ruleIds: [] },
+  );
+}
+
+// Ignored paths must be dropped before evaluation.
+const ignoredPaths = ["dist/index.js", "node_modules/pkg/index.js", "coverage/lcov.info"];
+for (const [i, path] of ignoredPaths.entries()) {
+  add(
+    `ignored-${String(i + 1).padStart(3, "0")}`,
+    "clean",
+    `${path} is ignored and must not be evaluated`,
+    [file(path, { additions: 200 })],
+    { ruleIds: [] },
+  );
+}
+
+export const corpus = fixtures;