mvp benchmarking - WIP

.bumpy/bench-telemetry-host-override.md

@@ -0,0 +1,5 @@
+---
+varlock: patch
+---
+
+Allow overriding the telemetry endpoint with VARLOCK_POSTHOG_HOST, so tooling can point it at a local mock

.github/workflows/benchmarks.yaml

@@ -0,0 +1,166 @@
+name: Benchmarks
+
+# Runs against published npm packages (+ optional SEA binary). Triggered after a
+# varlock publish (via release.yaml) or manually for iteration.
+on:
+  workflow_dispatch:
+    inputs:
+      varlock_version:
+        description: 'Published varlock version (empty = latest on npm)'
+        required: false
+        type: string
+        default: ''
+      only:
+        description: 'Optional comma-separated scenario groups (empty = all)'
+        required: false
+        type: string
+        default: ''
+      iterations:
+        description: 'Measured iterations per scenario'
+        required: false
+        type: string
+        default: '5'
+      release_dispatch:
+        description: 'Set to true when invoked from release.yaml after publish'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: benchmarks-${{ github.event.inputs.varlock_version || 'latest' }}
+  cancel-in-progress: false
+
+jobs:
+  bench:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          token: ${{ secrets.BUMPY_GH_TOKEN }}
+          ref: main
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Use Node.js 24.x
+        uses: actions/setup-node@v6
+        with:
+          node-version: '24.x'
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+          cache: false
+
+      - name: Install benchmarks package deps
+        working-directory: benchmarks
+        # Frozen so the run cannot leave a modified bun.lock behind, which would
+        # block the rebase in the commit step at the very end of a long run.
+        run: bun install --frozen-lockfile
+
+      # Workflow inputs go through `env:` rather than being interpolated straight
+      # into the script, so a crafted input value cannot become shell syntax.
+      - name: Resolve varlock version
+        id: ver
+        env:
+          INPUT_VERSION: ${{ github.event.inputs.varlock_version }}
+        run: |
+          set -euo pipefail
+          if [[ -z "$INPUT_VERSION" || "$INPUT_VERSION" == "latest" ]]; then
+            V=$(npm view varlock version)
+          else
+            V="$INPUT_VERSION"
+          fi
+          echo "version=$V" >> "$GITHUB_OUTPUT"
+          echo "Resolved varlock@$V"
+
+      # Note: the suite itself waits for the version to appear on npm, so a
+      # release-triggered run can start before the registry has caught up.
+
+      - name: Download SEA binary (linux-x64)
+        id: sea
+        env:
+          GH_TOKEN: ${{ github.token }}
+          VERSION: ${{ steps.ver.outputs.version }}
+        run: |
+          set -euo pipefail
+          TAG="varlock@${VERSION}"
+          DEST="$RUNNER_TEMP/varlock-sea"
+          mkdir -p "$DEST"
+          if gh release download "$TAG" --pattern 'varlock-linux-x64.tar.gz' --dir "$DEST"; then
+            tar -xzf "$DEST/varlock-linux-x64.tar.gz" -C "$DEST"
+            BIN="$DEST/varlock"
+            chmod +x "$BIN"
+            echo "path=$BIN" >> "$GITHUB_OUTPUT"
+            echo "found=true" >> "$GITHUB_OUTPUT"
+            "$BIN" --version || true
+          else
+            echo "No SEA release asset for $TAG; continuing without SEA"
+            echo "found=false" >> "$GITHUB_OUTPUT"
+            echo "path=" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run benchmarks
+        working-directory: benchmarks
+        env:
+          VERSION: ${{ steps.ver.outputs.version }}
+          ONLY: ${{ github.event.inputs.only }}
+          ITERATIONS: ${{ github.event.inputs.iterations }}
+          RELEASE_DISPATCH: ${{ github.event.inputs.release_dispatch }}
+          SEA_FOUND: ${{ steps.sea.outputs.found }}
+          SEA_PATH: ${{ steps.sea.outputs.path }}
+        run: |
+          set -euo pipefail
+          if [[ "$RELEASE_DISPATCH" == "true" ]]; then
+            TRIGGER=release
+          else
+            TRIGGER=workflow_dispatch
+          fi
+          ARGS=(--version "$VERSION" --trigger "$TRIGGER")
+          if [[ -n "$ITERATIONS" ]]; then
+            ARGS+=(--iterations "$ITERATIONS")
+          fi
+          if [[ -n "$ONLY" ]]; then
+            ARGS+=(--only "$ONLY")
+          fi
+          if [[ "$SEA_FOUND" == "true" ]]; then
+            ARGS+=(--sea-path "$SEA_PATH")
+          fi
+          bun run src/run.ts "${ARGS[@]}"
+
+      - name: Commit results
+        env:
+          VERSION: ${{ steps.ver.outputs.version }}
+        run: |
+          set -euo pipefail
+          # Absolute path — `git add` resolves pathspecs against the cwd, so a
+          # repo-relative path only works from the repo root.
+          RESULT_PATH=$(cat benchmarks/.work/last-result-path.txt)
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -- "$RESULT_PATH"
+          if git diff --staged --quiet; then
+            echo "No results to commit"
+            exit 0
+          fi
+          git commit -m "chore(benchmarks): record varlock@${VERSION} [skip ci]"
+          # Commit first, then rebase: a rebase refuses to run with staged or
+          # unstaged changes present, and other commits may have landed on main
+          # during the (long) bench run.
+          for attempt in 1 2 3; do
+            git pull --rebase origin main
+            if git push origin HEAD:main; then
+              echo "Pushed results"
+              exit 0
+            fi
+            echo "Push rejected, retrying ($attempt/3)..."
+          done
+          echo "::error::Failed to push benchmark results after 3 attempts"
+          exit 1

.github/workflows/release.yaml

@@ -511,6 +511,15 @@ jobs:
         working-directory: packages/varlock/dist-sea
         run: gh release upload "${{ steps.ver.outputs.tag }}" *.{tar.gz,zip} checksums.txt checksums.txt.cosign.bundle ../SHA256SUMS.txt --clobber
 
+      - name: Dispatch benchmarks workflow
+        env:
+          GH_TOKEN: ${{ secrets.BUMPY_GH_TOKEN }}
+        run: |
+          gh workflow run benchmarks.yaml \
+            --ref main \
+            -f "varlock_version=${{ steps.ver.outputs.version }}" \
+            -f "release_dispatch=true"
+
       # Update the homebrew tap formula
       - name: Checkout homebrew tap repo
         uses: actions/checkout@v7

.github/workflows/test.yaml

@@ -50,6 +50,10 @@ jobs:
         run: bun run lint
       - name: TypeScript type check
         run: bun run typecheck:all
+      # benchmarks/ is not a workspace member, so turbo's typecheck does not reach it
+      - name: TypeScript type check (benchmarks)
+        working-directory: benchmarks
+        run: bun install --frozen-lockfile && bun run typecheck
       - name: Build libraries
         run: bun run build:libs
       - name: Run tests

benchmarks/.gitignore

@@ -0,0 +1,5 @@
+.work/
+node_modules/
+fixtures/**/env.d.ts
+fixtures/**/env.py
+fixtures/**/env/

benchmarks/README.md

@@ -0,0 +1,78 @@
+# Varlock benchmarks
+
+Release benchmarking suite for **memory footprint**, **execution time**, and **added latency** (redaction / leak prevention).
+
+Runs against **published** npm packages (and optionally the linux SEA binary), not workspace links. Results are committed under [`results/`](results/) so trends are visible in git history.
+
+## What it measures
+
+| Group | Scenarios |
+|-------|-----------|
+| `cli-load` | `load` cold (`--clear-cache`) and warm, for npm / bun / SEA, with **telemetry on/off** |
+| `cli-run` | Bare node baseline; `varlock run` wrap with **telemetry on/off**; stdout redaction on vs off (telemetry off) |
+| `cli-scan-audit` | Light `scan` and `audit` coverage (telemetry off) |
+| `integration-next` | Uses [`framework-tests/frameworks/nextjs`](../framework-tests/frameworks/nextjs): `next build` baseline vs varlock with **telemetry on/off**; request latency for `preventLeaks` and `redactLogs` |
+| `integration-vite` | Uses [`framework-tests/frameworks/vite`](../framework-tests/frameworks/vite): `vite build` baseline vs varlock with **telemetry on/off**; request latency for `preventLeaks` and `redactLogs` |
+| `lang-python` | `load`+codegen and `varlock run -- python3` |
+| `lang-go` | `load`+codegen and `varlock run` of a built Go binary |
+
+Install methods are three distinct runtimes, not three package managers: `npm` installs and runs under **node**, `bun` installs and runs under **bun**, `sea` is the compiled standalone binary.
+
+## Reading the results
+
+Each run prints a **Deltas** table before the raw numbers. The deltas are the point of the suite: absolute wall times on a shared CI runner are not comparable between runs, but the difference between two scenarios measured back to back within one run is.
+
+A delta smaller than the standard deviation of either side is tagged `(within noise)` and should not be read as a change.
+
+Every scenario records `wallMsMin`, `wallMsMedian`, `wallMsP95`, `wallMsStdDev` and `iterations`. Prefer **min** and **stddev**: min is the least noise-sensitive statistic for this kind of measurement, and p95 collapses onto the max at the iteration counts used here.
+
+`meta.notes` lists anything that was skipped or degraded (missing SEA binary, no Go toolchain, telemetry not measurable). Nothing is dropped silently.
+
+## Telemetry
+
+Telemetry-on scenarios exist to measure what the telemetry code path costs. **They never send real telemetry.** The suite starts a local mock collector and points varlock at it with `VARLOCK_POSTHOG_HOST`, which keeps the code path intact (payload building, the exit hook that waits on the in-flight request) without injecting synthetic events into product analytics, and without making the timings depend on network latency to the real collector.
+
+Before running any telemetry-on scenario the suite probes whether the version under test honours that override. If it does not (versions published before the override existed), those scenarios are skipped and a note is recorded.
+
+## Local usage
+
+```bash
+bun run bench
+```
+
+```bash
+bun run bench -- --version 1.13.0 --sea-path ./packages/varlock/dist-sea/varlock
+```
+
+```bash
+bun run bench -- --only cli-load,cli-run --iterations 3
+```
+
+```bash
+bun run bench -- --skip-install --only cli-load
+```
+
+The first form benchmarks the latest published varlock. The others pin a version and add a local SEA binary, restrict to a subset of scenario groups for faster iteration, and reuse the npm/bun installs left in `benchmarks/.work` by a previous run.
+
+From this directory, `bun install` first and then use `bun run bench` the same way.
+
+Integration benches drive [`FrameworkTestEnv`](../framework-tests/harness/fixture-env.ts) with `usePublished: true` so they install from npm (not packed workspace tarballs) while reusing the same Next/Vite templates as framework CI.
+
+Results are written to `results/<iso>-varlock@<ver>-<runid>.json`. CI commits those files; local runs leave them untracked unless you commit them yourself.
+
+## CI
+
+Workflow: [`.github/workflows/benchmarks.yaml`](../.github/workflows/benchmarks.yaml)
+
+- **Manual:** Actions → Benchmarks → Run workflow (optional version / scenario filter)
+- **After publish:** [`release.yaml`](../.github/workflows/release.yaml) dispatches this workflow once SEA binaries are uploaded for `varlock@<version>`
+
+The job installs from npm, downloads `varlock-linux-x64.tar.gz` when present, runs the suite, and commits the new JSON under `results/` with `[skip ci]` so the commit does not retrigger release/CI. The suite waits for the version to appear on npm itself, so a release-triggered run can start before the registry has caught up.
+
+v1 is informational only (no regression gate). Suite failures still fail the workflow.
+
+## Known gaps
+
+- **Linux/x64 only.** The SEA binary ships for macOS and Windows too, but nothing measures them. The non-Linux RSS sampling path (which shells out to `ps` once per sample, perturbing the timings it measures) is therefore only exercised by local runs.
+- **The `cli-load` fixture has nothing worth caching.** Cold vs warm is now a valid comparison in CI (`_VARLOCK_CACHE_KEY` forces the on-disk cache, which CI would otherwise skip in favour of a per-process memory cache), but the fixture is all static literals, so both arms measure roughly the same work. Exercising the cache meaningfully needs a fixture with expensive resolvers, e.g. a plugin-backed or `exec()` value.
+- **No regression gate and no cross-run comparison tooling.** Results accumulate in `results/` but nothing reads the history yet.

benchmarks/fixtures/cli-basic/.env.schema

@@ -0,0 +1,43 @@
+# @defaultSensitive=false
+# @redactLogs=true
+# @preventLeaks=true
+# ---
+
+PUBLIC_VAR=public-value
+
+# @sensitive
+SECRET_TOKEN=super-secret-token-12345
+# @sensitive
+SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa
+# @sensitive
+SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb
+# @sensitive
+SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig
+# @sensitive
+SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc
+# @sensitive
+SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD
+# @sensitive
+SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee
+# @sensitive
+SECRET_REDIS=redis-auth-bench-ffffffffffff
+# @sensitive
+SECRET_SMTP=smtp-pass-bench-gggggggggggg
+# @sensitive
+SECRET_OAUTH=oauth-client-secret-hhhhhhhh
+# @sensitive
+SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii
+# @sensitive
+SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj
+# @sensitive
+SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk
+# @sensitive
+SECRET_GITHUB=ghp_benchTokenLlllllllllllllll
+# @sensitive
+SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm
+# @sensitive
+SECRET_OPENAI=sk-proj-bench-openainnnnnnnn
+# @sensitive
+SECRET_SENTRY=sntrys_bench_oooooooooooooo
+# @sensitive
+SECRET_PRIVATE_KEY=-----BEGIN BENCH PRIVATE KEY-----MIIBenchKey-----END BENCH PRIVATE KEY-----

benchmarks/fixtures/cli-basic/app.js

@@ -0,0 +1,4 @@
+// Sample app source for audit (references env keys).
+const token = process.env.SECRET_TOKEN;
+const pub = process.env.PUBLIC_VAR;
+console.log(token, pub);

benchmarks/fixtures/cli-basic/child.js

@@ -0,0 +1,2 @@
+// Minimal child used for varlock run wrap overhead benchmarks.
+process.stdout.write('ok\n');

benchmarks/fixtures/cli-basic/emit-secret.js

@@ -0,0 +1,19 @@
+// Emits every SECRET_* env var many times so stdout redaction cost scales with secret count.
+// Line count comes from BENCH_EMIT_LINES: redaction cost is per byte of output, so the
+// volume has to be large enough to clear the fixed ~50ms of process-startup noise.
+const secrets = Object.entries(process.env)
+  .filter(([key]) => key.startsWith('SECRET_'))
+  .map(([, value]) => value)
+  .filter(Boolean);
+
+if (secrets.length === 0) {
+  process.stderr.write('emit-secret.js: no SECRET_* env vars found\n');
+  process.exit(1);
+}
+
+const parsedLines = Number(process.env.BENCH_EMIT_LINES);
+const chunks = Number.isInteger(parsedLines) && parsedLines > 0 ? parsedLines : 200;
+for (let i = 0; i < chunks; i++) {
+  const secret = secrets[i % secrets.length];
+  process.stdout.write(`line-${i}: prefix ${secret} suffix\n`);
+}

benchmarks/fixtures/lang-go/.env.schema

@@ -0,0 +1,9 @@
+# @defaultSensitive=false
+# @generateGoEnv(path=env/env.go)
+# ---
+# @type=port
+PORT=8080          # @required @public
+# @type=boolean
+DEBUG=true         # @required @public
+OPTIONAL_UNSET=    # @optional @public
+SECRET=shhh        # @required @sensitive

benchmarks/fixtures/lang-go/go.mod

@@ -0,0 +1,3 @@
+module benchlang
+
+go 1.21