From c51a00c4be2733d1dd9a47226b11b3c180f1704c Mon Sep 17 00:00:00 2001 From: philmillman Date: Wed, 22 Jul 2026 15:09:35 -0400 Subject: [PATCH 1/5] mvp benchmarking - WIP --- .github/workflows/benchmarks.yaml | 152 ++++++++ .github/workflows/release.yaml | 9 + benchmarks/.gitignore | 5 + benchmarks/README.md | 55 +++ benchmarks/bun.lock | 20 + benchmarks/fixtures/cli-basic/.env.schema | 43 +++ benchmarks/fixtures/cli-basic/app.js | 4 + benchmarks/fixtures/cli-basic/child.js | 2 + benchmarks/fixtures/cli-basic/emit-secret.js | 16 + benchmarks/fixtures/lang-go/.env.schema | 9 + benchmarks/fixtures/lang-go/go.mod | 3 + benchmarks/fixtures/lang-go/main.go | 29 ++ benchmarks/fixtures/lang-python/.env.schema | 9 + benchmarks/fixtures/lang-python/main.py | 8 + benchmarks/package.json | 15 + benchmarks/results/.gitkeep | 0 benchmarks/src/install.ts | 115 ++++++ benchmarks/src/many-secrets-schema.ts | 134 +++++++ benchmarks/src/measure.ts | 212 +++++++++++ benchmarks/src/report.ts | 42 +++ benchmarks/src/run.ts | 192 ++++++++++ benchmarks/src/scenarios/cli-load.ts | 45 +++ benchmarks/src/scenarios/cli-run.ts | 85 +++++ benchmarks/src/scenarios/cli-scan-audit.ts | 53 +++ benchmarks/src/scenarios/index.ts | 44 +++ benchmarks/src/scenarios/integration-next.ts | 336 +++++++++++++++++ benchmarks/src/scenarios/integration-vite.ts | 316 ++++++++++++++++ benchmarks/src/scenarios/lang-go.ts | 76 ++++ benchmarks/src/scenarios/lang-python.ts | 70 ++++ benchmarks/src/telemetry.ts | 21 ++ benchmarks/src/types.ts | 76 ++++ benchmarks/tsconfig.json | 15 + eslint.config.mjs | 3 + framework-tests/README.md | 4 +- framework-tests/harness/fixture-env.ts | 364 +++++++++++++++++++ framework-tests/harness/test-fixture.ts | 344 +----------------- framework-tests/harness/types.ts | 7 + package.json | 1 + 38 files changed, 2599 insertions(+), 335 deletions(-) create mode 100644 .github/workflows/benchmarks.yaml create mode 100644 benchmarks/.gitignore create mode 100644 benchmarks/README.md create mode 100644 benchmarks/bun.lock create mode 100644 benchmarks/fixtures/cli-basic/.env.schema create mode 100644 benchmarks/fixtures/cli-basic/app.js create mode 100644 benchmarks/fixtures/cli-basic/child.js create mode 100644 benchmarks/fixtures/cli-basic/emit-secret.js create mode 100644 benchmarks/fixtures/lang-go/.env.schema create mode 100644 benchmarks/fixtures/lang-go/go.mod create mode 100644 benchmarks/fixtures/lang-go/main.go create mode 100644 benchmarks/fixtures/lang-python/.env.schema create mode 100644 benchmarks/fixtures/lang-python/main.py create mode 100644 benchmarks/package.json create mode 100644 benchmarks/results/.gitkeep create mode 100644 benchmarks/src/install.ts create mode 100644 benchmarks/src/many-secrets-schema.ts create mode 100644 benchmarks/src/measure.ts create mode 100644 benchmarks/src/report.ts create mode 100644 benchmarks/src/run.ts create mode 100644 benchmarks/src/scenarios/cli-load.ts create mode 100644 benchmarks/src/scenarios/cli-run.ts create mode 100644 benchmarks/src/scenarios/cli-scan-audit.ts create mode 100644 benchmarks/src/scenarios/index.ts create mode 100644 benchmarks/src/scenarios/integration-next.ts create mode 100644 benchmarks/src/scenarios/integration-vite.ts create mode 100644 benchmarks/src/scenarios/lang-go.ts create mode 100644 benchmarks/src/scenarios/lang-python.ts create mode 100644 benchmarks/src/telemetry.ts create mode 100644 benchmarks/src/types.ts create mode 100644 benchmarks/tsconfig.json create mode 100644 framework-tests/harness/fixture-env.ts diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 000000000..1a3d0cf35 --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,152 @@ +name: Benchmarks + +# Runs against published npm packages (+ optional SEA binary). Triggered after a +# varlock publish (via release.yaml) or manually for iteration. +on: + workflow_dispatch: + inputs: + varlock_version: + description: 'Published varlock version (empty = latest on npm)' + required: false + type: string + default: '' + only: + description: 'Optional comma-separated scenario groups (empty = all)' + required: false + type: string + default: '' + iterations: + description: 'Measured iterations per scenario' + required: false + type: string + default: '5' + release_dispatch: + description: 'Set to true when invoked from release.yaml after publish' + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.event.inputs.varlock_version || 'latest' }} + cancel-in-progress: false + +jobs: + bench: + runs-on: ubuntu-latest + permissions: + contents: write + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + with: + token: ${{ secrets.BUMPY_GH_TOKEN }} + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Use Node.js 24.x + uses: actions/setup-node@v6 + with: + node-version: '24.x' + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + cache: false + + - name: Install benchmarks package deps + working-directory: benchmarks + run: bun install + + - name: Resolve varlock version + id: ver + run: | + INPUT="${{ github.event.inputs.varlock_version }}" + if [[ -z "$INPUT" || "$INPUT" == "latest" ]]; then + V=$(npm view varlock version) + else + V="$INPUT" + fi + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Resolved varlock@$V" + + - name: Wait for npm package + run: | + set -euo pipefail + V="${{ steps.ver.outputs.version }}" + for i in $(seq 1 30); do + if npm view "varlock@${V}" version >/dev/null 2>&1; then + echo "varlock@${V} is on npm" + exit 0 + fi + echo "waiting for varlock@${V} on npm ($i)..." + sleep 10 + done + echo "::error::Timed out waiting for varlock@${V} on npm" + exit 1 + + - name: Download SEA binary (linux-x64) + id: sea + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + V="${{ steps.ver.outputs.version }}" + TAG="varlock@${V}" + DEST="$RUNNER_TEMP/varlock-sea" + mkdir -p "$DEST" + if gh release download "$TAG" --pattern 'varlock-linux-x64.tar.gz' --dir "$DEST"; then + tar -xzf "$DEST/varlock-linux-x64.tar.gz" -C "$DEST" + BIN="$DEST/varlock" + chmod +x "$BIN" + echo "path=$BIN" >> "$GITHUB_OUTPUT" + echo "found=true" >> "$GITHUB_OUTPUT" + "$BIN" --version || true + else + echo "No SEA release asset for $TAG; continuing without SEA" + echo "found=false" >> "$GITHUB_OUTPUT" + echo "path=" >> "$GITHUB_OUTPUT" + fi + + - name: Run benchmarks + working-directory: benchmarks + run: | + set -euo pipefail + V="${{ steps.ver.outputs.version }}" + if [[ "${{ github.event.inputs.release_dispatch }}" == "true" ]]; then + TRIGGER=release + else + TRIGGER=workflow_dispatch + fi + ARGS=(--version "$V" --trigger "$TRIGGER" --iterations "${{ github.event.inputs.iterations }}") + ONLY="${{ github.event.inputs.only }}" + if [[ -n "$ONLY" ]]; then + ARGS+=(--only "$ONLY") + fi + if [[ "${{ steps.sea.outputs.found }}" == "true" ]]; then + ARGS+=(--sea-path "${{ steps.sea.outputs.path }}") + fi + bun run src/run.ts "${ARGS[@]}" + + - name: Commit results + working-directory: benchmarks + run: | + set -euo pipefail + RESULT_PATH=$(cat .work/last-result-path.txt) + REL="${RESULT_PATH#"$GITHUB_WORKSPACE"/}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Fetch latest main in case other commits landed during the long bench run + git pull --rebase origin main + git add -- "$REL" + if git diff --staged --quiet; then + echo "No results to commit" + exit 0 + fi + V="${{ steps.ver.outputs.version }}" + git commit -m "chore(benchmarks): record varlock@${V} [skip ci]" + git push origin HEAD:main diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e3d74dcf5..3d96300ff 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -511,6 +511,15 @@ jobs: working-directory: packages/varlock/dist-sea run: gh release upload "${{ steps.ver.outputs.tag }}" *.{tar.gz,zip} checksums.txt checksums.txt.cosign.bundle ../SHA256SUMS.txt --clobber + - name: Dispatch benchmarks workflow + env: + GH_TOKEN: ${{ secrets.BUMPY_GH_TOKEN }} + run: | + gh workflow run benchmarks.yaml \ + --ref main \ + -f "varlock_version=${{ steps.ver.outputs.version }}" \ + -f "release_dispatch=true" + # Update the homebrew tap formula - name: Checkout homebrew tap repo uses: actions/checkout@v7 diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 000000000..a20843f47 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,5 @@ +.work/ +node_modules/ +fixtures/**/env.d.ts +fixtures/**/env.py +fixtures/**/env/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..4033aa522 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,55 @@ +# Varlock benchmarks + +Release benchmarking suite for **memory footprint**, **execution time**, and **added latency** (redaction / leak prevention). + +Runs against **published** npm packages (and optionally the linux SEA binary), not workspace links. Results are committed under [`results/`](results/) so trends are visible in git history. + +## What it measures + +| Group | Scenarios | +|-------|-----------| +| `cli-load` | `load` cold (`--clear-cache`) and warm, for npm / bun / SEA, with **telemetry on/off** | +| `cli-run` | Bare node baseline; `varlock run` wrap with **telemetry on/off**; stdout redaction on vs off (telemetry off) | +| `cli-scan-audit` | Light `scan` and `audit` coverage (telemetry off) | +| `integration-next` | Uses [`framework-tests/frameworks/nextjs`](../framework-tests/frameworks/nextjs): `next build` baseline vs varlock with **telemetry on/off**; request latency for `preventLeaks` and `redactLogs` | +| `integration-vite` | Uses [`framework-tests/frameworks/vite`](../framework-tests/frameworks/vite): `vite build` baseline vs varlock with **telemetry on/off**; request latency for `preventLeaks` and `redactLogs` | +| `lang-python` | `load`+codegen and `varlock run -- python3` | +| `lang-go` | `load`+codegen and `varlock run` of a built Go binary | + +## Local usage + +```bash +# From repo root (latest published varlock) +bun run bench + +# Specific version + local SEA binary +bun run bench -- --version 1.13.0 --sea-path ./packages/varlock/dist-sea/varlock + +# Subset of scenario groups (faster iteration) +bun run bench -- --only cli-load,cli-run --iterations 3 + +# Reuse prior npm/bun installs under benchmarks/.work +bun run bench -- --skip-install --only cli-load +``` + +Or from this directory: + +```bash +bun install +bun run bench -- --version latest --only cli-load +``` + +Integration benches drive [`FrameworkTestEnv`](../framework-tests/harness/fixture-env.ts) with `usePublished: true` so they install from npm (not packed workspace tarballs) while reusing the same Next/Vite templates as framework CI. + +Results are written to `results/-varlock@-.json`. CI commits those files; local runs leave them untracked unless you commit them yourself. + +## CI + +Workflow: [`.github/workflows/benchmarks.yaml`](../.github/workflows/benchmarks.yaml) + +- **Manual:** Actions → Benchmarks → Run workflow (optional version / scenario filter) +- **After publish:** [`release.yaml`](../.github/workflows/release.yaml) dispatches this workflow once SEA binaries are uploaded for `varlock@` + +The job installs from npm, downloads `varlock-linux-x64.tar.gz` when present, runs the suite, and commits the new JSON under `results/` with `[skip ci]` so the commit does not retrigger release/CI. + +v1 is informational only (no regression gate). Suite failures still fail the workflow. diff --git a/benchmarks/bun.lock b/benchmarks/bun.lock new file mode 100644 index 000000000..df2fd36f3 --- /dev/null +++ b/benchmarks/bun.lock @@ -0,0 +1,20 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "varlock-benchmarks", + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.9.3", + }, + }, + }, + "packages": { + "@types/node": ["@types/node@22.20.1", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + } +} diff --git a/benchmarks/fixtures/cli-basic/.env.schema b/benchmarks/fixtures/cli-basic/.env.schema new file mode 100644 index 000000000..8280ce402 --- /dev/null +++ b/benchmarks/fixtures/cli-basic/.env.schema @@ -0,0 +1,43 @@ +# @defaultSensitive=false +# @redactLogs=true +# @preventLeaks=true +# --- + +PUBLIC_VAR=public-value + +# @sensitive +SECRET_TOKEN=super-secret-token-12345 +# @sensitive +SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa +# @sensitive +SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb +# @sensitive +SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig +# @sensitive +SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc +# @sensitive +SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD +# @sensitive +SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee +# @sensitive +SECRET_REDIS=redis-auth-bench-ffffffffffff +# @sensitive +SECRET_SMTP=smtp-pass-bench-gggggggggggg +# @sensitive +SECRET_OAUTH=oauth-client-secret-hhhhhhhh +# @sensitive +SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii +# @sensitive +SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj +# @sensitive +SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk +# @sensitive +SECRET_GITHUB=ghp_benchTokenLlllllllllllllll +# @sensitive +SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm +# @sensitive +SECRET_OPENAI=sk-proj-bench-openainnnnnnnn +# @sensitive +SECRET_SENTRY=sntrys_bench_oooooooooooooo +# @sensitive +SECRET_PRIVATE_KEY=-----BEGIN BENCH PRIVATE KEY-----MIIBenchKey-----END BENCH PRIVATE KEY----- diff --git a/benchmarks/fixtures/cli-basic/app.js b/benchmarks/fixtures/cli-basic/app.js new file mode 100644 index 000000000..897635a67 --- /dev/null +++ b/benchmarks/fixtures/cli-basic/app.js @@ -0,0 +1,4 @@ +// Sample app source for audit (references env keys). +const token = process.env.SECRET_TOKEN; +const pub = process.env.PUBLIC_VAR; +console.log(token, pub); diff --git a/benchmarks/fixtures/cli-basic/child.js b/benchmarks/fixtures/cli-basic/child.js new file mode 100644 index 000000000..bbfdbb22a --- /dev/null +++ b/benchmarks/fixtures/cli-basic/child.js @@ -0,0 +1,2 @@ +// Minimal child used for varlock run wrap overhead benchmarks. +process.stdout.write('ok\n'); diff --git a/benchmarks/fixtures/cli-basic/emit-secret.js b/benchmarks/fixtures/cli-basic/emit-secret.js new file mode 100644 index 000000000..272ea13f1 --- /dev/null +++ b/benchmarks/fixtures/cli-basic/emit-secret.js @@ -0,0 +1,16 @@ +// Emits every SECRET_* env var many times so stdout redaction cost scales with secret count. +const secrets = Object.entries(process.env) + .filter(([key]) => key.startsWith('SECRET_')) + .map(([, value]) => value) + .filter(Boolean); + +if (secrets.length === 0) { + process.stderr.write('emit-secret.js: no SECRET_* env vars found\n'); + process.exit(1); +} + +const chunks = 200; +for (let i = 0; i < chunks; i++) { + const secret = secrets[i % secrets.length]; + process.stdout.write(`line-${i}: prefix ${secret} suffix\n`); +} diff --git a/benchmarks/fixtures/lang-go/.env.schema b/benchmarks/fixtures/lang-go/.env.schema new file mode 100644 index 000000000..6247e56c7 --- /dev/null +++ b/benchmarks/fixtures/lang-go/.env.schema @@ -0,0 +1,9 @@ +# @defaultSensitive=false +# @generateGoEnv(path=env/env.go) +# --- +# @type=port +PORT=8080 # @required @public +# @type=boolean +DEBUG=true # @required @public +OPTIONAL_UNSET= # @optional @public +SECRET=shhh # @required @sensitive diff --git a/benchmarks/fixtures/lang-go/go.mod b/benchmarks/fixtures/lang-go/go.mod new file mode 100644 index 000000000..773fe1adf --- /dev/null +++ b/benchmarks/fixtures/lang-go/go.mod @@ -0,0 +1,3 @@ +module benchlang + +go 1.21 diff --git a/benchmarks/fixtures/lang-go/main.go b/benchmarks/fixtures/lang-go/main.go new file mode 100644 index 000000000..333383137 --- /dev/null +++ b/benchmarks/fixtures/lang-go/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "os" + + "benchlang/env" +) + +func main() { + e, err := env.Load() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if e.Port != 8080 || !e.Debug { + fmt.Fprintf(os.Stderr, "unexpected: port=%d debug=%t\n", e.Port, e.Debug) + os.Exit(1) + } + if e.OptionalUnset != nil { + fmt.Fprintf(os.Stderr, "expected OptionalUnset to be nil, got %v\n", *e.OptionalUnset) + os.Exit(1) + } + if !env.SensitiveKeys["SECRET"] { + fmt.Fprintln(os.Stderr, "SECRET not marked sensitive") + os.Exit(1) + } + fmt.Println("OK") +} diff --git a/benchmarks/fixtures/lang-python/.env.schema b/benchmarks/fixtures/lang-python/.env.schema new file mode 100644 index 000000000..b9ecbede1 --- /dev/null +++ b/benchmarks/fixtures/lang-python/.env.schema @@ -0,0 +1,9 @@ +# @defaultSensitive=false +# @generatePythonEnv(path=env.py) +# --- +# @type=port +PORT=8080 # @required @public +# @type=boolean +DEBUG=true # @required @public +OPTIONAL_UNSET= # @optional @public +SECRET=shhh # @required @sensitive diff --git a/benchmarks/fixtures/lang-python/main.py b/benchmarks/fixtures/lang-python/main.py new file mode 100644 index 000000000..eaf2adeb0 --- /dev/null +++ b/benchmarks/fixtures/lang-python/main.py @@ -0,0 +1,8 @@ +from env import load_env, SENSITIVE_KEYS + +ENV = load_env() +assert ENV["PORT"] == 8080, ENV["PORT"] +assert ENV["DEBUG"] is True, ENV["DEBUG"] +assert "OPTIONAL_UNSET" not in ENV, ENV +assert "SECRET" in SENSITIVE_KEYS +print("OK") diff --git a/benchmarks/package.json b/benchmarks/package.json new file mode 100644 index 000000000..d5f47a3e1 --- /dev/null +++ b/benchmarks/package.json @@ -0,0 +1,15 @@ +{ + "name": "varlock-benchmarks", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Release benchmarking suite for varlock (CLI, integrations, codegen)", + "scripts": { + "bench": "bun run src/run.ts", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.9.3" + } +} diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/src/install.ts b/benchmarks/src/install.ts new file mode 100644 index 000000000..382d1e4f9 --- /dev/null +++ b/benchmarks/src/install.ts @@ -0,0 +1,115 @@ +import { + mkdirSync, existsSync, chmodSync, rmSync, readFileSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { CliInvocation } from './types.ts'; + +function runOrThrow( + command: string, + args: Array, + opts: { cwd?: string; env?: NodeJS.ProcessEnv }, +): void { + const result = spawnSync(command, args, { + cwd: opts.cwd, + env: opts.env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(' ')} failed (exit ${result.status}):\n${result.stderr}\n${result.stdout}`, + ); + } +} + +/** + * Install published `varlock@version` with npm into workDir/installs/npm + * and return a CliInvocation that runs it via node. + */ +export function installVarlockNpm( + workDir: string, + version: string, +): CliInvocation { + const dir = join(workDir, 'installs', 'npm'); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + runOrThrow('npm', ['init', '-y'], { cwd: dir }); + runOrThrow('npm', ['install', `varlock@${version}`, '--no-fund', '--no-audit'], { cwd: dir }); + const cliJs = join(dir, 'node_modules', 'varlock', 'bin', 'cli.js'); + if (!existsSync(cliJs)) { + throw new Error(`npm install did not produce CLI at ${cliJs}`); + } + return { + command: [process.execPath, cliJs], + label: 'npm', + packageManager: 'npm', + }; +} + +/** + * Install published `varlock@version` with bun into workDir/installs/bun. + */ +export function installVarlockBun( + workDir: string, + version: string, +): CliInvocation { + const dir = join(workDir, 'installs', 'bun'); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + runOrThrow('bun', ['init', '-y'], { cwd: dir }); + runOrThrow('bun', ['add', `varlock@${version}`], { cwd: dir }); + const cliJs = join(dir, 'node_modules', 'varlock', 'bin', 'cli.js'); + if (!existsSync(cliJs)) { + throw new Error(`bun install did not produce CLI at ${cliJs}`); + } + return { + command: [process.execPath, cliJs], + label: 'bun', + packageManager: 'bun', + }; +} + +export function seaInvocation(seaPath: string): CliInvocation { + if (!existsSync(seaPath)) { + throw new Error(`SEA binary not found at ${seaPath}`); + } + chmodSync(seaPath, 0o755); + return { + command: [seaPath], + label: 'sea', + }; +} + +/** Resolve package version of an installed package under an install root. */ +export function readInstalledVersion(installRoot: string, pkgName: string): string | null { + try { + const pkgPath = join(installRoot, 'node_modules', ...pkgName.split('/'), 'package.json'); + const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { version?: string }; + return pkg.version ?? null; + } catch { + return null; + } +} + +export function npmViewVersion(pkgSpec: string): string { + const result = spawnSync('npm', ['view', pkgSpec, 'version'], { encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error(`npm view ${pkgSpec} failed: ${result.stderr}`); + } + return result.stdout.trim(); +} + +export async function waitForNpmPackage(pkgSpec: string, attempts = 30, delayMs = 10_000): Promise { + for (let i = 1; i <= attempts; i++) { + const result = spawnSync('npm', ['view', pkgSpec, 'version'], { encoding: 'utf8' }); + if (result.status === 0 && result.stdout.trim()) { + return result.stdout.trim(); + } + console.log(`waiting for ${pkgSpec} on npm (${i}/${attempts})...`); + await new Promise((r) => { + setTimeout(r, delayMs); + }); + } + throw new Error(`Timed out waiting for ${pkgSpec} on npm`); +} diff --git a/benchmarks/src/many-secrets-schema.ts b/benchmarks/src/many-secrets-schema.ts new file mode 100644 index 000000000..4c5d551d3 --- /dev/null +++ b/benchmarks/src/many-secrets-schema.ts @@ -0,0 +1,134 @@ +/** + * Shared multi-secret schema bodies for integration latency/build benches. + * Keep framework-required public keys, then add ~18 distinct sensitive values. + */ + +export const NEXT_MANY_SECRETS_SCHEMA = `# @defaultSensitive=false @defaultRequired=infer +# @generateTypes(lang="ts", path="env.d.ts") +# @currentEnv=$APP_ENV +# --- + +# @type=enum(dev, preview, prod, test) +APP_ENV=dev + +NEXT_PUBLIC_VAR=next-prefixed-public-var +PUBLIC_VAR=unprefixed-public-var +ENV_SPECIFIC_VAR=env-specific-var--default + +# Kept for framework-test page templates that reference ENV.SENSITIVE_VAR +# @sensitive +SENSITIVE_VAR=super-secret-var + +# @sensitive +SECRET_TOKEN=super-secret-token-12345 +# @sensitive +SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa +# @sensitive +SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb +# @sensitive +SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig +# @sensitive +SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc +# @sensitive +SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD +# @sensitive +SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee +# @sensitive +SECRET_REDIS=redis-auth-bench-ffffffffffff +# @sensitive +SECRET_SMTP=smtp-pass-bench-gggggggggggg +# @sensitive +SECRET_OAUTH=oauth-client-secret-hhhhhhhh +# @sensitive +SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii +# @sensitive +SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj +# @sensitive +SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk +# @sensitive +SECRET_GITHUB=ghp_benchTokenLlllllllllllllll +# @sensitive +SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm +# @sensitive +SECRET_OPENAI=sk-proj-bench-openainnnnnnnn +# @sensitive +SECRET_SENTRY=sntrys_bench_oooooooooooooo +`; + +export const VITE_MANY_SECRETS_SCHEMA = `# @defaultSensitive=false @defaultRequired=infer +# @currentEnv=$APP_ENV +# --- + +# @type=enum(dev, prod) +APP_ENV=dev + +PUBLIC_VAR=public-test-value +API_URL=https://api.example.com +ENV_SPECIFIC_VAR=env-specific-default + +# Kept for framework-test page templates that reference ENV.SECRET_KEY +# @sensitive +SECRET_KEY=super-secret-value + +# @sensitive +SECRET_TOKEN=super-secret-token-12345 +# @sensitive +SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa +# @sensitive +SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb +# @sensitive +SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig +# @sensitive +SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc +# @sensitive +SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD +# @sensitive +SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee +# @sensitive +SECRET_REDIS=redis-auth-bench-ffffffffffff +# @sensitive +SECRET_SMTP=smtp-pass-bench-gggggggggggg +# @sensitive +SECRET_OAUTH=oauth-client-secret-hhhhhhhh +# @sensitive +SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii +# @sensitive +SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj +# @sensitive +SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk +# @sensitive +SECRET_GITHUB=ghp_benchTokenLlllllllllllllll +# @sensitive +SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm +# @sensitive +SECRET_OPENAI=sk-proj-bench-openainnnnnnnn +# @sensitive +SECRET_SENTRY=sntrys_bench_oooooooooooooo +`; + +/** Apply preventLeaks / redactLogs root flags onto a schema body. */ +export function withSchemaFlags( + schema: string, + preventLeaks: boolean, + redactLogs: boolean, +): string { + const flags = `# @preventLeaks=${preventLeaks}\n# @redactLogs=${redactLogs}`; + if (!schema.includes('@preventLeaks=') && !schema.includes('@redactLogs=')) { + return schema.replace( + '# @defaultSensitive=false @defaultRequired=infer', + `# @defaultSensitive=false @defaultRequired=infer\n${flags}`, + ); + } + let out = schema; + if (out.includes('@preventLeaks=')) { + out = out.replace(/@preventLeaks=\w+/, `@preventLeaks=${preventLeaks}`); + } else { + out = `# @preventLeaks=${preventLeaks}\n${out}`; + } + if (out.includes('@redactLogs=')) { + out = out.replace(/@redactLogs=\w+/, `@redactLogs=${redactLogs}`); + } else { + out = `# @redactLogs=${redactLogs}\n${out}`; + } + return out; +} diff --git a/benchmarks/src/measure.ts b/benchmarks/src/measure.ts new file mode 100644 index 000000000..e6fd72670 --- /dev/null +++ b/benchmarks/src/measure.ts @@ -0,0 +1,212 @@ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import type { Sample, ScenarioMetrics } from './types.ts'; + +export function rssKiB(pid: number): number | null { + if (process.platform === 'linux') { + try { + const status = readFileSync(`/proc/${pid}/status`, 'utf8'); + const match = status.match(/^VmRSS:\s+(\d+)/m); + return match ? Number(match[1]) : null; + } catch { + return null; + } + } + + const result = spawnSync('ps', ['-o', 'rss=', '-p', String(pid)], { encoding: 'utf8' }); + if (result.status !== 0) return null; + const n = Number(result.stdout.trim()); + return Number.isFinite(n) ? n : null; +} + +function percentile(sorted: Array, p: number): number { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.max(0, idx)]!; +} + +function median(values: Array): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return (sorted[mid - 1]! + sorted[mid]!) / 2; + } + return sorted[mid]!; +} + +export function summarizeSamples(samples: Array): ScenarioMetrics { + const walls = samples.map((s) => s.wallMs).sort((a, b) => a - b); + const rssValues = samples + .map((s) => s.rssPeakBytes) + .filter((v): v is number => v !== null); + + return { + wallMsMedian: median(walls), + wallMsP95: percentile(walls, 95), + rssPeakBytesMedian: rssValues.length > 0 ? median(rssValues) : null, + samples, + }; +} + +export type MeasureCommandOptions = { + cwd?: string; + env?: Record; + input?: string; + /** Sample RSS of the spawned process while it runs. Default true. */ + sampleRss?: boolean; + sampleIntervalMs?: number; + timeoutMs?: number; +}; + +/** + * Spawn a command, measure wall time and optional peak RSS of the child. + */ +export function measureCommand( + command: Array, + options: MeasureCommandOptions = {}, +): Promise { + const [bin, ...args] = command; + if (!bin) { + return Promise.reject(new Error('measureCommand: empty command')); + } + + const sampleRss = options.sampleRss !== false; + const sampleIntervalMs = options.sampleIntervalMs ?? 25; + const timeoutMs = options.timeoutMs ?? 120_000; + + return new Promise((resolve, reject) => { + const start = performance.now(); + let peakRssKiB: number | null = null; + let stdout = ''; + let stderr = ''; + let settled = false; + let sampling: ReturnType | undefined; + const timers: { timeout?: ReturnType } = {}; + + const finish = (err: Error) => { + if (settled) return; + settled = true; + if (sampling) clearInterval(sampling); + if (timers.timeout) clearTimeout(timers.timeout); + reject(err); + }; + + const childEnv: NodeJS.ProcessEnv = { ...process.env }; + if (options.env) { + for (const [key, value] of Object.entries(options.env)) { + if (value === undefined) { + delete childEnv[key]; + } else { + childEnv[key] = value; + } + } + } + + const child: ChildProcess = spawn(bin, args, { + cwd: options.cwd, + env: childEnv, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + if (sampleRss) { + sampling = setInterval(() => { + if (child.pid) { + const rss = rssKiB(child.pid); + if (rss !== null) { + peakRssKiB = peakRssKiB === null ? rss : Math.max(peakRssKiB, rss); + } + } + }, sampleIntervalMs); + } + + timers.timeout = setTimeout(() => { + child.kill('SIGKILL'); + finish(new Error(`Command timed out after ${timeoutMs}ms: ${command.join(' ')}`)); + }, timeoutMs); + + if (options.input) { + child.stdin?.write(options.input); + } + child.stdin?.end(); + + child.stdout?.on('data', (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr?.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + child.on('error', (err) => { + finish(err); + }); + + child.on('close', (code) => { + const wallMs = performance.now() - start; + if (sampling) clearInterval(sampling); + if (timers.timeout) clearTimeout(timers.timeout); + if (settled) return; + settled = true; + resolve({ + wallMs, + rssPeakBytes: peakRssKiB !== null ? peakRssKiB * 1024 : null, + exitCode: code ?? 1, + stdout, + stderr, + }); + }); + }); +} + +export type RepeatOptions = { + iterations: number; + warmup: number; + /** Throw if any measured iteration exits non-zero. Default true. */ + expectSuccess?: boolean; +}; + +/** + * Run warmup + measured iterations of an async sample factory. + */ +export async function repeatMeasure( + factory: () => Promise, + options: RepeatOptions, +): Promise { + const expectSuccess = options.expectSuccess !== false; + + for (let i = 0; i < options.warmup; i++) { + const warm = await factory(); + if (expectSuccess && warm.exitCode !== 0) { + throw new Error(`Warmup failed with exit ${warm.exitCode}`); + } + } + + const samples: Array = []; + for (let i = 0; i < options.iterations; i++) { + const sample = await factory(); + if (expectSuccess && sample.exitCode !== 0) { + const extra = 'stderr' in sample || 'stdout' in sample + ? `\nstdout:\n${(sample as { stdout?: string }).stdout ?? ''}\nstderr:\n${(sample as { stderr?: string }).stderr ?? ''}` + : ''; + throw new Error(`Iteration ${i} failed with exit ${sample.exitCode}${extra}`); + } + samples.push({ + wallMs: sample.wallMs, + rssPeakBytes: sample.rssPeakBytes, + exitCode: sample.exitCode, + }); + } + + return summarizeSamples(samples); +} + +/** Copy a fixture directory into a unique work subdirectory. */ +export async function copyFixture( + sourceDir: string, + destDir: string, +): Promise { + const { cpSync, mkdirSync, rmSync } = await import('node:fs'); + rmSync(destDir, { recursive: true, force: true }); + mkdirSync(destDir, { recursive: true }); + cpSync(sourceDir, destDir, { recursive: true }); +} diff --git a/benchmarks/src/report.ts b/benchmarks/src/report.ts new file mode 100644 index 000000000..10a11b23c --- /dev/null +++ b/benchmarks/src/report.ts @@ -0,0 +1,42 @@ +import type { BenchRunResult, ScenarioResult } from './types.ts'; + +function fmtMs(n: number): string { + return `${n.toFixed(1)}ms`; +} + +function fmtRss(bytes: number | null): string { + if (bytes === null) return '-'; + return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`; +} + +export function formatSummaryMarkdown(result: BenchRunResult): string { + const lines: Array = []; + lines.push('## Varlock benchmarks'); + lines.push(''); + lines.push(`- **varlock:** ${result.meta.versions.varlock}`); + lines.push(`- **trigger:** ${result.meta.trigger}`); + lines.push(`- **runner:** ${result.meta.runnerOs}/${result.meta.runnerArch}`); + lines.push(`- **timestamp:** ${result.meta.timestamp}`); + if (result.meta.gitSha) lines.push(`- **git:** ${result.meta.gitSha.slice(0, 12)}`); + lines.push(''); + lines.push('| Scenario | Install | Telemetry | Median | p95 | Peak RSS |'); + lines.push('|----------|---------|-----------|--------|-----|----------|'); + + const sorted = [...result.scenarios].sort((a, b) => a.id.localeCompare(b.id)); + for (const s of sorted) { + lines.push( + `| ${s.id} | ${s.installMethod} | ${s.telemetry} | ${fmtMs(s.metrics.wallMsMedian)} | ${fmtMs(s.metrics.wallMsP95)} | ${fmtRss(s.metrics.rssPeakBytesMedian)} |`, + ); + } + lines.push(''); + return lines.join('\n'); +} + +export function printScenarioLine(s: ScenarioResult): void { + const rss = s.metrics.rssPeakBytesMedian !== null + ? ` rss=${fmtRss(s.metrics.rssPeakBytesMedian)}` + : ''; + console.log( + ` ${s.id} [${s.installMethod} telemetry=${s.telemetry}] median=${fmtMs(s.metrics.wallMsMedian)} p95=${fmtMs(s.metrics.wallMsP95)}${rss}`, + ); +} diff --git a/benchmarks/src/run.ts b/benchmarks/src/run.ts new file mode 100644 index 000000000..940d7dd8c --- /dev/null +++ b/benchmarks/src/run.ts @@ -0,0 +1,192 @@ +import { mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + installVarlockBun, + installVarlockNpm, + npmViewVersion, + readInstalledVersion, + seaInvocation, +} from './install.ts'; +import { runAllScenarios, SCENARIO_GROUPS } from './scenarios/index.ts'; +import { formatSummaryMarkdown } from './report.ts'; +import type { BenchContext, BenchRunResult, TriggerKind } from './types.ts'; + +const ROOT_DIR = resolve(import.meta.dirname, '..'); +const FIXTURES_DIR = join(ROOT_DIR, 'fixtures'); +const RESULTS_DIR = join(ROOT_DIR, 'results'); +const WORK_DIR = join(ROOT_DIR, '.work'); + +type Args = { + version: string; + seaPath: string | null; + out: string | null; + iterations: number; + warmup: number; + only: Array; + skipInstall: boolean; + trigger: TriggerKind; + help: boolean; +}; + +function parseArgs(argv: Array): Args { + const args: Args = { + version: 'latest', + seaPath: null, + out: null, + iterations: 5, + warmup: 1, + only: [], + skipInstall: false, + trigger: (process.env.BENCH_TRIGGER as TriggerKind | undefined) ?? 'local', + help: false, + }; + + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + if (a === '--help' || a === '-h') args.help = true; + else if (a === '--version') args.version = argv[++i] ?? args.version; + else if (a === '--sea-path') args.seaPath = argv[++i] ?? null; + else if (a === '--out') args.out = argv[++i] ?? null; + else if (a === '--iterations') args.iterations = Number(argv[++i]); + else if (a === '--warmup') args.warmup = Number(argv[++i]); + else if (a === '--only') args.only = (argv[++i] ?? '').split(',').filter(Boolean); + else if (a === '--skip-install') args.skipInstall = true; + else if (a === '--trigger') args.trigger = (argv[++i] as TriggerKind) ?? args.trigger; + else throw new Error(`Unknown argument: ${a}`); + } + return args; +} + +function usage(): string { + const groups = SCENARIO_GROUPS.map((g) => g.name).join(', '); + return `Usage: bun run src/run.ts [options] + +Options: + --version Published varlock version (default: latest) + --sea-path Path to SEA binary (enables sea install method) + --out Output JSON path (default: results/-varlock@-.json) + --iterations Measured iterations (default: 5) + --warmup Warmup iterations (default: 1) + --only Comma-separated scenario groups: ${groups} + --skip-install Reuse .work/installs from a previous run + --trigger release | workflow_dispatch | local + --help +`; +} + +function gitSha(): string | null { + const r = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', cwd: ROOT_DIR }); + return r.status === 0 ? r.stdout.trim() : null; +} + +function defaultOutPath(version: string): string { + const iso = new Date().toISOString().replace(/[:.]/g, '-'); + const runId = process.env.GITHUB_RUN_ID ?? 'local'; + mkdirSync(RESULTS_DIR, { recursive: true }); + return join(RESULTS_DIR, `${iso}-varlock@${version}-${runId}.json`); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const resolvedVersion = args.version === 'latest' + ? npmViewVersion('varlock') + : args.version; + + console.log(`Benchmarking varlock@${resolvedVersion}`); + mkdirSync(WORK_DIR, { recursive: true }); + + const clis = []; + if (args.skipInstall) { + const npmCli = join(WORK_DIR, 'installs', 'npm', 'node_modules', 'varlock', 'bin', 'cli.js'); + const bunCli = join(WORK_DIR, 'installs', 'bun', 'node_modules', 'varlock', 'bin', 'cli.js'); + if (!existsSync(npmCli) || !existsSync(bunCli)) { + throw new Error('--skip-install requires existing .work/installs/{npm,bun}'); + } + clis.push( + { command: [process.execPath, npmCli], label: 'npm' as const, packageManager: 'npm' as const }, + { command: [process.execPath, bunCli], label: 'bun' as const, packageManager: 'bun' as const }, + ); + } else { + console.log('Installing varlock via npm...'); + clis.push(installVarlockNpm(WORK_DIR, resolvedVersion)); + console.log('Installing varlock via bun...'); + clis.push(installVarlockBun(WORK_DIR, resolvedVersion)); + } + + if (args.seaPath) { + console.log(`Using SEA binary at ${args.seaPath}`); + clis.push(seaInvocation(resolve(args.seaPath))); + } else { + console.log('No --sea-path; skipping SEA scenarios'); + } + + const ctx: BenchContext = { + version: resolvedVersion, + rootDir: ROOT_DIR, + fixturesDir: FIXTURES_DIR, + workDir: WORK_DIR, + iterations: args.iterations, + warmup: args.warmup, + clis, + seaPath: args.seaPath, + }; + + const scenarios = await runAllScenarios(ctx, args.only.length ? args.only : undefined); + + const npmRoot = join(WORK_DIR, 'installs', 'npm'); + const result: BenchRunResult = { + meta: { + timestamp: new Date().toISOString(), + gitSha: gitSha(), + githubRunId: process.env.GITHUB_RUN_ID ?? null, + runnerOs: process.platform, + runnerArch: process.arch, + versions: { + varlock: resolvedVersion, + nextjsIntegration: (() => { + try { + return npmViewVersion('@varlock/nextjs-integration'); + } catch { + return undefined; + } + })(), + viteIntegration: (() => { + try { + return npmViewVersion('@varlock/vite-integration'); + } catch { + return undefined; + } + })(), + '@env-spec/parser': readInstalledVersion(npmRoot, '@env-spec/parser') ?? undefined, + }, + trigger: args.trigger, + }, + scenarios, + }; + + const outPath = args.out ? resolve(args.out) : defaultOutPath(resolvedVersion); + mkdirSync(join(outPath, '..'), { recursive: true }); + writeFileSync(outPath, `${JSON.stringify(result, null, 2)}\n`); + console.log(`\nWrote ${outPath}`); + + const summary = formatSummaryMarkdown(result); + console.log(`\n${summary}`); + + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' }); + } + + // Also write a pointer file used by CI commit step + writeFileSync(join(WORK_DIR, 'last-result-path.txt'), `${outPath}\n`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/benchmarks/src/scenarios/cli-load.ts b/benchmarks/src/scenarios/cli-load.ts new file mode 100644 index 000000000..4b4a1eefb --- /dev/null +++ b/benchmarks/src/scenarios/cli-load.ts @@ -0,0 +1,45 @@ +import { join } from 'node:path'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { TELEMETRY_MODES, telemetryEnv } from '../telemetry.ts'; + +export async function runCliLoadScenarios(ctx: BenchContext): Promise> { + const cwd = join(ctx.fixturesDir, 'cli-basic'); + const results: Array = []; + + for (const cli of ctx.clis) { + for (const telemetry of TELEMETRY_MODES) { + const env = telemetryEnv(telemetry); + + const cold = await repeatMeasure( + async () => measureCommand([...cli.command, 'load', '--clear-cache'], { cwd, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: `cli.load.cold.telemetry.${telemetry}`, + facet: 'cli-load', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry, + metrics: cold, + }); + + // Warm: one clear then repeated loads without clear + await measureCommand([...cli.command, 'load', '--clear-cache'], { cwd, env }); + const warm = await repeatMeasure( + async () => measureCommand([...cli.command, 'load'], { cwd, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: `cli.load.warm.telemetry.${telemetry}`, + facet: 'cli-load', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry, + metrics: warm, + }); + } + } + + return results; +} diff --git a/benchmarks/src/scenarios/cli-run.ts b/benchmarks/src/scenarios/cli-run.ts new file mode 100644 index 000000000..da677d130 --- /dev/null +++ b/benchmarks/src/scenarios/cli-run.ts @@ -0,0 +1,85 @@ +import { join } from 'node:path'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { TELEMETRY_MODES, telemetryEnv } from '../telemetry.ts'; + +export async function runCliRunScenarios(ctx: BenchContext): Promise> { + const cwd = join(ctx.fixturesDir, 'cli-basic'); + const childJs = join(cwd, 'child.js'); + const emitJs = join(cwd, 'emit-secret.js'); + const results: Array = []; + + // Bare node baseline (not tied to an install method; recorded once under npm label) + const bare = await repeatMeasure( + async () => measureCommand([process.execPath, childJs], { cwd }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: 'cli.run.bare-node', + facet: 'cli-run', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: bare, + notes: 'Baseline without varlock wrap', + }); + + for (const cli of ctx.clis) { + // Wrap overhead measured with telemetry on/off (exit-hook wait hits here) + for (const telemetry of TELEMETRY_MODES) { + const env = telemetryEnv(telemetry); + const wrapped = await repeatMeasure( + async () => measureCommand( + [...cli.command, 'run', '--', process.execPath, childJs], + { cwd, env }, + ), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: `cli.run.wrap.telemetry.${telemetry}`, + facet: 'cli-run', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry, + metrics: wrapped, + notes: 'varlock run wrap overhead vs bare-node', + }); + } + + // Redaction comparison: telemetry off so we isolate redact-stdout cost + const envOff = telemetryEnv('off'); + const redactOn = await repeatMeasure( + async () => measureCommand( + [...cli.command, 'run', '--redact-stdout', '--', process.execPath, emitJs], + { cwd, env: envOff }, + ), + { iterations: ctx.iterations, warmup: Math.max(1, ctx.warmup) }, + ); + results.push({ + id: 'cli.run.redact-stdout.on', + facet: 'cli-run', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: redactOn, + }); + + const redactOff = await repeatMeasure( + async () => measureCommand( + [...cli.command, 'run', '--no-redact-stdout', '--', process.execPath, emitJs], + { cwd, env: envOff }, + ), + { iterations: ctx.iterations, warmup: Math.max(1, ctx.warmup) }, + ); + results.push({ + id: 'cli.run.redact-stdout.off', + facet: 'cli-run', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: redactOff, + }); + } + + return results; +} diff --git a/benchmarks/src/scenarios/cli-scan-audit.ts b/benchmarks/src/scenarios/cli-scan-audit.ts new file mode 100644 index 000000000..8012de821 --- /dev/null +++ b/benchmarks/src/scenarios/cli-scan-audit.ts @@ -0,0 +1,53 @@ +import { join } from 'node:path'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; + +export async function runCliScanAuditScenarios(ctx: BenchContext): Promise> { + const cwd = join(ctx.fixturesDir, 'cli-basic'); + const results: Array = []; + const env = telemetryEnv('off'); + + // Lighter coverage: fewer iterations than load/run + const lightIterations = Math.max(2, Math.min(3, ctx.iterations)); + const lightWarmup = Math.min(1, ctx.warmup); + + for (const cli of ctx.clis) { + const scan = await repeatMeasure( + async () => measureCommand([...cli.command, 'scan', './child.js'], { + cwd, + env, + timeoutMs: 180_000, + }), + { iterations: lightIterations, warmup: lightWarmup }, + ); + results.push({ + id: 'cli.scan', + facet: 'cli-scan', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: scan, + notes: 'scan of a clean file (load + scan cost)', + }); + + const audit = await repeatMeasure( + async () => measureCommand([...cli.command, 'audit', '.'], { + cwd, + env, + timeoutMs: 180_000, + }), + { iterations: lightIterations, warmup: lightWarmup }, + ); + results.push({ + id: 'cli.audit', + facet: 'cli-audit', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: audit, + }); + } + + return results; +} diff --git a/benchmarks/src/scenarios/index.ts b/benchmarks/src/scenarios/index.ts new file mode 100644 index 000000000..44901f8b4 --- /dev/null +++ b/benchmarks/src/scenarios/index.ts @@ -0,0 +1,44 @@ +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { runCliLoadScenarios } from './cli-load.ts'; +import { runCliRunScenarios } from './cli-run.ts'; +import { runCliScanAuditScenarios } from './cli-scan-audit.ts'; +import { runNextScenarios } from './integration-next.ts'; +import { runViteScenarios } from './integration-vite.ts'; +import { runPythonScenarios } from './lang-python.ts'; +import { runGoScenarios } from './lang-go.ts'; +import { printScenarioLine } from '../report.ts'; + +export type ScenarioGroup = { + name: string; + run: (ctx: BenchContext) => Promise>; +}; + +export const SCENARIO_GROUPS: Array = [ + { name: 'cli-load', run: runCliLoadScenarios }, + { name: 'cli-run', run: runCliRunScenarios }, + { name: 'cli-scan-audit', run: runCliScanAuditScenarios }, + { name: 'integration-next', run: runNextScenarios }, + { name: 'integration-vite', run: runViteScenarios }, + { name: 'lang-python', run: runPythonScenarios }, + { name: 'lang-go', run: runGoScenarios }, +]; + +export async function runAllScenarios( + ctx: BenchContext, + only?: Array, +): Promise> { + const groups = only?.length + ? SCENARIO_GROUPS.filter((g) => only.includes(g.name)) + : SCENARIO_GROUPS; + + const all: Array = []; + for (const group of groups) { + console.log(`\n== ${group.name} ==`); + const results = await group.run(ctx); + for (const r of results) { + printScenarioLine(r); + all.push(r); + } + } + return all; +} diff --git a/benchmarks/src/scenarios/integration-next.ts b/benchmarks/src/scenarios/integration-next.ts new file mode 100644 index 000000000..b7983fd2f --- /dev/null +++ b/benchmarks/src/scenarios/integration-next.ts @@ -0,0 +1,336 @@ +import { rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { spawn } from 'node:child_process'; +import { FrameworkTestEnv } from '../../../framework-tests/harness/fixture-env.ts'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { TELEMETRY_MODES, telemetryEnv } from '../telemetry.ts'; +import { NEXT_MANY_SECRETS_SCHEMA, withSchemaFlags } from '../many-secrets-schema.ts'; + +const NEXT_TEST_DIR = resolve(import.meta.dirname, '../../../framework-tests/frameworks/nextjs'); + +const BASELINE_NEXT_CONFIG = `/** @type {import('next').NextConfig} */ +const nextConfig = { + productionBrowserSourceMaps: true, + typescript: { ignoreBuildErrors: true }, + eslint: { ignoreDuringBuilds: true }, +}; + +export default nextConfig; +`; + +const BASELINE_PAGE = `export default function Page() { + return ( +
+

bench baseline

+

{process.env.NEXT_PUBLIC_VAR || process.env.PUBLIC_VAR || 'none'}

+
+ ); +} +`; + +const ECHO_ROUTE = `import { NextResponse } from 'next/server'; + +export async function GET() { + // Large body without the sensitive value so preventLeaks scanning still runs + // but the request succeeds (no leak throw). + const body = \`ok padding=\${'x'.repeat(16_384)}\`; + return new NextResponse(body, { + headers: { 'content-type': 'text/plain' }, + }); +} +`; + +const LOG_ROUTE = `import { NextResponse } from 'next/server'; +import { ENV } from 'varlock/env'; + +const SECRET_KEYS = [ + 'SENSITIVE_VAR', + 'SECRET_TOKEN', + 'SECRET_API_KEY', + 'SECRET_DB_PASSWORD', + 'SECRET_JWT', + 'SECRET_STRIPE', + 'SECRET_AWS_ACCESS', + 'SECRET_AWS_SECRET', + 'SECRET_REDIS', + 'SECRET_SMTP', + 'SECRET_OAUTH', + 'SECRET_WEBHOOK', + 'SECRET_ENCRYPTION', + 'SECRET_SESSION', + 'SECRET_GITHUB', + 'SECRET_SLACK', + 'SECRET_OPENAI', + 'SECRET_SENTRY', +]; + +export async function GET() { + for (let i = 0; i < 200; i++) { + const key = SECRET_KEYS[i % SECRET_KEYS.length]; + console.log(\`bench-log-\${i}:\`, ENV[key]); + } + return new NextResponse('ok', { + headers: { 'content-type': 'text/plain' }, + }); +} +`; + +function createNextEnv( + ctx: BenchContext, + mode: 'baseline' | 'varlock', + labelSuffix = '', +): FrameworkTestEnv { + const withVarlock = mode === 'varlock'; + return new FrameworkTestEnv({ + testDir: NEXT_TEST_DIR, + framework: `bench-next-${mode}${labelSuffix}`, + packageManager: 'npm', + usePublished: true, + installTimeout: 180_000, + dependencies: { + next: '^15', + react: '^19', + 'react-dom': '^19', + '@types/react': '^19', + typescript: '^5.9.3', + ...(withVarlock + ? { + varlock: ctx.version, + '@varlock/nextjs-integration': 'latest', + } + : {}), + }, + ...(withVarlock + ? { + overrides: { + '@next/env': '', + }, + } + : {}), + templateFiles: { + '.env.schema': 'schemas/.env.schema', + '.env.dev': 'schemas/.env.dev', + '.env.prod': 'schemas/.env.prod', + }, + }); +} + +async function waitForUrl(url: string, timeoutMs: number): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + if (res.status > 0) return; + } catch { + // retry + } + await new Promise((r) => { + setTimeout(r, 250); + }); + } + throw new Error(`Timed out waiting for ${url}`); +} + +async function measurePathLatency( + baseUrl: string, + path: string, + iterations: number, + warmup: number, +): Promise { + return repeatMeasure( + async () => { + const start = performance.now(); + const res = await fetch(`${baseUrl}${path}`); + const wallMs = performance.now() - start; + if (!res.ok) { + throw new Error(`Request failed: ${res.status} ${await res.text()}`); + } + await res.text(); + return { wallMs, rssPeakBytes: null, exitCode: 0 }; + }, + { iterations, warmup }, + ); +} + +async function withNextServer( + projectDir: string, + port: number, + readyPath: string, + fn: () => Promise, +): Promise { + const server = spawn('npx', ['next', 'start', '-H', '127.0.0.1', '-p', String(port)], { + cwd: projectDir, + env: { + ...process.env, + ...Object.fromEntries( + Object.entries(telemetryEnv('off')).filter(([, v]) => v !== undefined), + ), + APP_ENV: 'dev', + PORT: String(port), + HOSTNAME: '127.0.0.1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stderr = ''; + server.stderr?.on('data', (c: Buffer) => { + stderr += c.toString(); + }); + + try { + await waitForUrl(`http://127.0.0.1:${port}${readyPath}`, 90_000); + return await fn(); + } catch (err) { + throw new Error(`${String(err)}\nnext start stderr:\n${stderr}`); + } finally { + server.kill('SIGTERM'); + await new Promise((r) => { + setTimeout(r, 500); + }); + if (!server.killed) server.kill('SIGKILL'); + } +} + +function prepareNextFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { + if (mode === 'baseline') { + env.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + }, + files: [ + { path: '.env.schema', content: NEXT_MANY_SECRETS_SCHEMA }, + { path: 'next.config.mjs', content: BASELINE_NEXT_CONFIG }, + { path: 'app/page.tsx', content: BASELINE_PAGE }, + ], + }); + return; + } + + env.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'app/page.tsx': 'pages/basic-page.tsx', + }, + files: [{ path: '.env.schema', content: NEXT_MANY_SECRETS_SCHEMA }], + }); +} + +export async function runNextScenarios(ctx: BenchContext): Promise> { + const results: Array = []; + const buildIterations = Math.max(2, Math.min(3, ctx.iterations)); + + console.log(' preparing next baseline (framework-tests)...'); + { + const fixture = createNextEnv(ctx, 'baseline'); + await fixture.setup(); + prepareNextFiles(fixture, 'baseline'); + const build = await repeatMeasure( + async () => { + rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); + return measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + }); + }, + { iterations: buildIterations, warmup: 0 }, + ); + results.push({ + id: 'integration.next.build.baseline', + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: build, + }); + await fixture.teardown(); + } + + // Varlock build: telemetry on/off (next shells out to varlock load via @next/env override) + for (const telemetry of TELEMETRY_MODES) { + console.log(` preparing next varlock telemetry.${telemetry} (framework-tests)...`); + const fixture = createNextEnv(ctx, 'varlock', `-telemetry-${telemetry}`); + await fixture.setup(); + prepareNextFiles(fixture, 'varlock'); + const build = await repeatMeasure( + async () => { + rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); + return measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv(telemetry), APP_ENV: 'dev', CI: '1' }, + }); + }, + { iterations: buildIterations, warmup: 0 }, + ); + results.push({ + id: `integration.next.build.varlock.telemetry.${telemetry}`, + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry, + metrics: build, + notes: 'Cold next build; telemetry affects sync varlock load spawn', + }); + await fixture.teardown(); + } + + console.log(' measuring next request latency (preventLeaks / redactLogs)...'); + const requestIterations = Math.max(10, ctx.iterations * 2); + for (const [label, preventLeaks, redactLogs, port, path] of [ + ['preventLeaks.on', true, true, 3451, '/api/echo'], + ['preventLeaks.off', false, true, 3452, '/api/echo'], + ['redactLogs.on', true, true, 3453, '/api/log'], + ['redactLogs.off', true, false, 3454, '/api/log'], + ] as const) { + const fixture = createNextEnv(ctx, 'varlock', `-${label}`); + await fixture.setup(); + + fixture.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'app/page.tsx': 'pages/basic-page.tsx', + }, + files: [ + { path: '.env.schema', content: withSchemaFlags(NEXT_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, + { path: 'app/api/echo/route.js', content: ECHO_ROUTE }, + { path: 'app/api/log/route.js', content: LOG_ROUTE }, + ], + }); + + const buildResult = await measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + }); + if (buildResult.exitCode !== 0) { + await fixture.teardown(); + throw new Error(`next build failed for ${label}:\n${buildResult.stderr}\n${buildResult.stdout}`); + } + + try { + const latency = await withNextServer(fixture.dir, port, path, () => measurePathLatency( + `http://127.0.0.1:${port}`, + path, + requestIterations, + 3, + )); + results.push({ + id: `integration.next.request.${label}`, + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: latency, + notes: path === '/api/echo' + ? 'Large safe body; preventLeaks scan cost' + : '200 console.log lines with secret; redactLogs cost', + }); + } finally { + await fixture.teardown(); + } + } + + return results; +} diff --git a/benchmarks/src/scenarios/integration-vite.ts b/benchmarks/src/scenarios/integration-vite.ts new file mode 100644 index 000000000..a57f1c75f --- /dev/null +++ b/benchmarks/src/scenarios/integration-vite.ts @@ -0,0 +1,316 @@ +import { rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { spawn } from 'node:child_process'; +import { FrameworkTestEnv } from '../../../framework-tests/harness/fixture-env.ts'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { TELEMETRY_MODES, telemetryEnv } from '../telemetry.ts'; +import { VITE_MANY_SECRETS_SCHEMA, withSchemaFlags } from '../many-secrets-schema.ts'; + +const VITE_TEST_DIR = resolve(import.meta.dirname, '../../../framework-tests/frameworks/vite'); + +const BASELINE_VITE_CONFIG = `import { defineConfig } from 'vite'; + +export default defineConfig({}); +`; + +const BASELINE_MAIN = `document.querySelector('#app')!.textContent = 'bench'; +`; + +/** + * Dev-server middleware used for latency benches: + * - /api/echo: large body without secrets (preventLeaks still scans) + * - /api/log: many console.log lines containing the secret (redactLogs cost) + */ +const LATENCY_VITE_CONFIG = `import { defineConfig } from 'vite'; +import { varlockVitePlugin } from '@varlock/vite-integration'; +import { ENV } from 'varlock/env'; + +const SECRET_KEYS = [ + 'SECRET_KEY', + 'SECRET_TOKEN', + 'SECRET_API_KEY', + 'SECRET_DB_PASSWORD', + 'SECRET_JWT', + 'SECRET_STRIPE', + 'SECRET_AWS_ACCESS', + 'SECRET_AWS_SECRET', + 'SECRET_REDIS', + 'SECRET_SMTP', + 'SECRET_OAUTH', + 'SECRET_WEBHOOK', + 'SECRET_ENCRYPTION', + 'SECRET_SESSION', + 'SECRET_GITHUB', + 'SECRET_SLACK', + 'SECRET_OPENAI', + 'SECRET_SENTRY', +]; + +export default defineConfig({ + plugins: [ + varlockVitePlugin(), + { + name: 'bench-latency-middleware', + configureServer(server) { + server.middlewares.use('/api/echo', (_req, res) => { + res.setHeader('content-type', 'text/plain'); + res.end(\`ok padding=\${'x'.repeat(16_384)}\`); + }); + server.middlewares.use('/api/log', (_req, res) => { + for (let i = 0; i < 200; i++) { + const key = SECRET_KEYS[i % SECRET_KEYS.length]; + console.log(\`bench-log-\${i}:\`, ENV[key]); + } + res.setHeader('content-type', 'text/plain'); + res.end('ok'); + }); + }, + }, + ], +}); +`; + +function createViteEnv( + ctx: BenchContext, + mode: 'baseline' | 'varlock', + labelSuffix = '', +): FrameworkTestEnv { + const withVarlock = mode === 'varlock'; + return new FrameworkTestEnv({ + testDir: VITE_TEST_DIR, + framework: `bench-vite-${mode}${labelSuffix}`, + packageManager: 'npm', + usePublished: true, + installTimeout: 180_000, + dependencies: { + vite: '^6', + ...(withVarlock + ? { + varlock: ctx.version, + '@varlock/vite-integration': 'latest', + } + : {}), + }, + templateFiles: { + '.env.schema': 'schemas/.env.schema', + '.env.dev': 'schemas/.env.dev', + '.env.prod': 'schemas/.env.prod', + }, + }); +} + +function prepareViteFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { + if (mode === 'baseline') { + env.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'index.html': 'html/basic.html', + }, + files: [ + { path: '.env.schema', content: VITE_MANY_SECRETS_SCHEMA }, + { path: 'vite.config.ts', content: BASELINE_VITE_CONFIG }, + { path: 'src/main.ts', content: BASELINE_MAIN }, + ], + }); + return; + } + + env.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'vite.config.ts': 'vite-configs/vite.config.ts', + 'index.html': 'html/basic.html', + 'src/main.ts': 'pages/basic-page.ts', + }, + files: [{ path: '.env.schema', content: VITE_MANY_SECRETS_SCHEMA }], + }); +} + +async function waitForUrl(url: string, timeoutMs: number): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + if (res.status > 0) return; + } catch { + // retry + } + await new Promise((r) => { + setTimeout(r, 250); + }); + } + throw new Error(`Timed out waiting for ${url}`); +} + +async function measurePathLatency( + baseUrl: string, + path: string, + iterations: number, + warmup: number, +): Promise { + return repeatMeasure( + async () => { + const start = performance.now(); + const res = await fetch(`${baseUrl}${path}`); + const wallMs = performance.now() - start; + if (!res.ok) { + throw new Error(`Request failed: ${res.status} ${await res.text()}`); + } + await res.text(); + return { wallMs, rssPeakBytes: null, exitCode: 0 }; + }, + { iterations, warmup }, + ); +} + +async function withViteDevServer( + projectDir: string, + port: number, + fn: () => Promise, +): Promise { + const server = spawn('npx', ['vite', 'dev', '--host', '127.0.0.1', '--port', String(port)], { + cwd: projectDir, + env: { + ...process.env, + ...Object.fromEntries( + Object.entries(telemetryEnv('off')).filter(([, v]) => v !== undefined), + ), + APP_ENV: 'dev', + CI: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stderr = ''; + let stdout = ''; + server.stderr?.on('data', (c: Buffer) => { + stderr += c.toString(); + }); + server.stdout?.on('data', (c: Buffer) => { + stdout += c.toString(); + }); + + try { + await waitForUrl(`http://127.0.0.1:${port}/api/echo`, 90_000); + return await fn(); + } catch (err) { + throw new Error(`${String(err)}\nvite dev stdout:\n${stdout}\nstderr:\n${stderr}`); + } finally { + server.kill('SIGTERM'); + await new Promise((r) => { + setTimeout(r, 500); + }); + if (!server.killed) server.kill('SIGKILL'); + } +} + +export async function runViteScenarios(ctx: BenchContext): Promise> { + const results: Array = []; + const buildIterations = Math.max(2, Math.min(4, ctx.iterations)); + const requestIterations = Math.max(10, ctx.iterations * 2); + + // Baseline: telemetry N/A (no varlock CLI). Tag as off for schema consistency. + console.log(' preparing vite baseline (framework-tests)...'); + { + const fixture = createViteEnv(ctx, 'baseline'); + await fixture.setup(); + prepareViteFiles(fixture, 'baseline'); + const build = await repeatMeasure( + async () => { + rmSync(join(fixture.dir, 'dist'), { recursive: true, force: true }); + return measureCommand(['npx', 'vite', 'build'], { + cwd: fixture.dir, + timeoutMs: 180_000, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + }); + }, + { iterations: buildIterations, warmup: 0 }, + ); + results.push({ + id: 'integration.vite.build.baseline', + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: build, + }); + await fixture.teardown(); + } + + // Varlock build: telemetry on/off (plugin shells out to `varlock load`) + for (const telemetry of TELEMETRY_MODES) { + console.log(` preparing vite varlock telemetry.${telemetry} (framework-tests)...`); + const fixture = createViteEnv(ctx, 'varlock', `-telemetry-${telemetry}`); + await fixture.setup(); + prepareViteFiles(fixture, 'varlock'); + const build = await repeatMeasure( + async () => { + rmSync(join(fixture.dir, 'dist'), { recursive: true, force: true }); + return measureCommand(['npx', 'vite', 'build'], { + cwd: fixture.dir, + timeoutMs: 180_000, + env: { ...telemetryEnv(telemetry), APP_ENV: 'dev', CI: '1' }, + }); + }, + { iterations: buildIterations, warmup: 0 }, + ); + results.push({ + id: `integration.vite.build.varlock.telemetry.${telemetry}`, + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry, + metrics: build, + notes: 'Cold vite build; telemetry affects sync varlock load spawn', + }); + await fixture.teardown(); + } + + console.log(' measuring vite request latency (preventLeaks / redactLogs)...'); + for (const [label, preventLeaks, redactLogs, port, path] of [ + ['preventLeaks.on', true, true, 3461, '/api/echo'], + ['preventLeaks.off', false, true, 3462, '/api/echo'], + ['redactLogs.on', true, true, 3463, '/api/log'], + ['redactLogs.off', true, false, 3464, '/api/log'], + ] as const) { + const fixture = createViteEnv(ctx, 'varlock', `-${label}`); + await fixture.setup(); + + fixture.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'index.html': 'html/basic.html', + 'src/main.ts': 'pages/minimal-page.ts', + }, + files: [ + { path: '.env.schema', content: withSchemaFlags(VITE_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, + { path: 'vite.config.ts', content: LATENCY_VITE_CONFIG }, + ], + }); + + try { + const latency = await withViteDevServer(fixture.dir, port, () => measurePathLatency( + `http://127.0.0.1:${port}`, + path, + requestIterations, + 3, + )); + results.push({ + id: `integration.vite.request.${label}`, + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: latency, + notes: path === '/api/echo' + ? 'Large safe body; preventLeaks scan cost' + : '200 console.log lines with secret; redactLogs cost', + }); + } finally { + await fixture.teardown(); + } + } + + return results; +} diff --git a/benchmarks/src/scenarios/lang-go.ts b/benchmarks/src/scenarios/lang-go.ts new file mode 100644 index 000000000..3c16c028b --- /dev/null +++ b/benchmarks/src/scenarios/lang-go.ts @@ -0,0 +1,76 @@ +import { + cpSync, mkdirSync, rmSync, existsSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; + +function hasBinary(name: string): boolean { + const result = spawnSync(name, ['version'], { encoding: 'utf8' }); + return result.status === 0; +} + +export async function runGoScenarios(ctx: BenchContext): Promise> { + if (!hasBinary('go')) { + console.log(' skipping go: go not found'); + return []; + } + + const results: Array = []; + const cli = ctx.clis.find((c) => c.label === 'npm') ?? ctx.clis[0]; + if (!cli) return []; + const env = telemetryEnv('off'); + + const dest = join(ctx.workDir, 'lang-go'); + rmSync(dest, { recursive: true, force: true }); + mkdirSync(dest, { recursive: true }); + cpSync(join(ctx.fixturesDir, 'lang-go'), dest, { recursive: true }); + + const codegen = await repeatMeasure( + async () => { + const genDir = join(dest, 'env'); + if (existsSync(genDir)) { + rmSync(genDir, { recursive: true, force: true }); + } + return measureCommand([...cli.command, 'load', '--clear-cache'], { cwd: dest, env }); + }, + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: 'lang.go.load-codegen', + facet: 'lang-go', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: codegen, + notes: 'load triggers @generateGoEnv', + }); + + await measureCommand([...cli.command, 'load'], { cwd: dest, env }); + + // Build once so `go run` is not dominated by compile in every sample + const build = spawnSync('go', ['build', '-o', 'main.bin', '.'], { + cwd: dest, + encoding: 'utf8', + }); + if (build.status !== 0) { + throw new Error(`go build failed:\n${build.stderr}\n${build.stdout}`); + } + + const wrapped = await repeatMeasure( + async () => measureCommand([...cli.command, 'run', '--', join(dest, 'main.bin')], { cwd: dest, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: 'lang.go.run', + facet: 'lang-go', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: wrapped, + }); + + return results; +} diff --git a/benchmarks/src/scenarios/lang-python.ts b/benchmarks/src/scenarios/lang-python.ts new file mode 100644 index 000000000..68dea591a --- /dev/null +++ b/benchmarks/src/scenarios/lang-python.ts @@ -0,0 +1,70 @@ +import { + cpSync, mkdirSync, rmSync, existsSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; + +function hasBinary(name: string): boolean { + const result = spawnSync(name, ['--version'], { encoding: 'utf8' }); + return result.status === 0; +} + +export async function runPythonScenarios(ctx: BenchContext): Promise> { + if (!hasBinary('python3')) { + console.log(' skipping python: python3 not found'); + return []; + } + + const results: Array = []; + // Use npm-installed CLI for lang scenarios (one representative install method) + const cli = ctx.clis.find((c) => c.label === 'npm') ?? ctx.clis[0]; + if (!cli) return []; + const env = telemetryEnv('off'); + + const dest = join(ctx.workDir, 'lang-python'); + rmSync(dest, { recursive: true, force: true }); + mkdirSync(dest, { recursive: true }); + cpSync(join(ctx.fixturesDir, 'lang-python'), dest, { recursive: true }); + + const codegen = await repeatMeasure( + async () => { + // Remove generated file so codegen has work each time + const gen = join(dest, 'env.py'); + if (existsSync(gen)) { + rmSync(gen); + } + return measureCommand([...cli.command, 'load', '--clear-cache'], { cwd: dest, env }); + }, + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: 'lang.python.load-codegen', + facet: 'lang-python', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: codegen, + notes: 'load triggers @generatePythonEnv', + }); + + // Ensure generated file exists for run + await measureCommand([...cli.command, 'load'], { cwd: dest, env }); + + const wrapped = await repeatMeasure( + async () => measureCommand([...cli.command, 'run', '--', 'python3', 'main.py'], { cwd: dest, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: 'lang.python.run', + facet: 'lang-python', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: wrapped, + }); + + return results; +} diff --git a/benchmarks/src/telemetry.ts b/benchmarks/src/telemetry.ts new file mode 100644 index 000000000..1ba41fac5 --- /dev/null +++ b/benchmarks/src/telemetry.ts @@ -0,0 +1,21 @@ +import type { TelemetryMode } from './types.ts'; + +export type { TelemetryMode }; + +/** Env overlay for telemetry on/off. Pass through measureCommand. */ +export function telemetryEnv(mode: TelemetryMode): Record { + if (mode === 'off') { + return { + VARLOCK_TELEMETRY_DISABLED: '1', + // Clear legacy opt-out so "off" is unambiguous + PH_OPT_OUT: undefined, + }; + } + // Explicitly clear disable flags so a parent-shell opt-out does not leak in + return { + VARLOCK_TELEMETRY_DISABLED: undefined, + PH_OPT_OUT: undefined, + }; +} + +export const TELEMETRY_MODES: Array = ['off', 'on']; diff --git a/benchmarks/src/types.ts b/benchmarks/src/types.ts new file mode 100644 index 000000000..4d442e65f --- /dev/null +++ b/benchmarks/src/types.ts @@ -0,0 +1,76 @@ +export type InstallMethod = 'npm' | 'bun' | 'sea'; + +export type TelemetryMode = 'on' | 'off'; + +export type ScenarioFacet = | 'cli-load' + | 'cli-run' + | 'cli-scan' + | 'cli-audit' + | 'integration-next' + | 'integration-vite' + | 'lang-python' + | 'lang-go'; + +export type TriggerKind = 'release' | 'workflow_dispatch' | 'local'; + +export type Sample = { + wallMs: number; + rssPeakBytes: number | null; + exitCode: number; +}; + +export type ScenarioMetrics = { + wallMsMedian: number; + wallMsP95: number; + rssPeakBytesMedian: number | null; + samples: Array; +}; + +export type ScenarioResult = { + id: string; + facet: ScenarioFacet; + installMethod: InstallMethod; + packageManager?: 'npm' | 'bun'; + /** Whether VARLOCK_TELEMETRY_DISABLED was cleared (on) or set (off). */ + telemetry: TelemetryMode; + metrics: ScenarioMetrics; + notes?: string; +}; + +export type BenchRunMeta = { + timestamp: string; + gitSha: string | null; + githubRunId: string | null; + runnerOs: string; + runnerArch: string; + versions: { + varlock: string; + nextjsIntegration?: string; + viteIntegration?: string; + '@env-spec/parser'?: string; + }; + trigger: TriggerKind; +}; + +export type BenchRunResult = { + meta: BenchRunMeta; + scenarios: Array; +}; + +export type CliInvocation = { + /** Executable + args that invoke varlock (without the subcommand). */ + command: Array; + label: InstallMethod; + packageManager?: 'npm' | 'bun'; +}; + +export type BenchContext = { + version: string; + rootDir: string; + fixturesDir: string; + workDir: string; + iterations: number; + warmup: number; + clis: Array; + seaPath: string | null; +}; diff --git a/benchmarks/tsconfig.json b/benchmarks/tsconfig.json new file mode 100644 index 000000000..6359a2536 --- /dev/null +++ b/benchmarks/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2023"], + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/eslint.config.mjs b/eslint.config.mjs index f0bb8f985..498323524 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -58,6 +58,8 @@ export default tseslint.config( '.claude', 'framework-tests/.test-projects', 'framework-tests/.packed', + 'benchmarks/.work', + 'benchmarks/results/*.json', ], }, @@ -171,6 +173,7 @@ export default tseslint.config( 'packages/varlock/scripts/**', 'smoke-tests/**', 'framework-tests/**', + 'benchmarks/**', 'packages/encryption-binary-swift/scripts/**', 'packages/encryption-binary-rust/scripts/**', 'packages/vscode-plugin/scripts/**', diff --git a/framework-tests/README.md b/framework-tests/README.md index 6a39bf1fd..43252e1f5 100644 --- a/framework-tests/README.md +++ b/framework-tests/README.md @@ -35,11 +35,13 @@ bun run --filter varlock-framework-tests test The shared harness provides `FrameworkTestEnv`, which manages the full lifecycle: -1. **Pack** — varlock packages are built and packed into `.tgz` tarballs (cached in `.packed/`; run `bun run repack` to refresh after source changes) +1. **Pack** — varlock packages are built and packed into `.tgz` tarballs (cached in `.packed/`; run `bun run repack` to refresh after source changes). Set `usePublished: true` on the fixture config to skip packing and install declared versions from npm instead (used by the release benchmarking suite). 2. **Setup** — a temp project is created in `.test-projects/`, deps are installed via pnpm 3. **Scenario** — template files are copied, a build command runs, and output is asserted 4. **Teardown** — temp project is removed (set `KEEP_TEST_DIRS=1` to preserve for debugging) +Imperative APIs (`setup`, `prepareFiles`, `runScenario`, `teardown`) live in `harness/fixture-env.ts` and can be imported without Vitest. Vitest helpers (`describeScenario`, `describeDevScenario`, `runTest`) are layered in `harness/test-fixture.ts`. + ### Adding a new framework Create a directory under `frameworks//` with: diff --git a/framework-tests/harness/fixture-env.ts b/framework-tests/harness/fixture-env.ts new file mode 100644 index 000000000..dd12ba032 --- /dev/null +++ b/framework-tests/harness/fixture-env.ts @@ -0,0 +1,364 @@ +import { + cpSync, writeFileSync, readFileSync, readdirSync, + rmSync, existsSync, mkdirSync, +} from 'node:fs'; +import { + join, dirname, basename, resolve, +} from 'node:path'; +import { runCommand } from './command-runner.js'; +import { runDevServer as runDevServerProcess } from './dev-server.js'; +import { packPackages, getPackedDeps } from './pack.js'; +import type { + TestFixtureConfig, TestScenario, DevServerScenario, DevServerResult, + BuildResult, TemplateFileSource, +} from './types.js'; + +const FRAMEWORK_TESTS_DIR = resolve(import.meta.dirname, '..'); + +/** Insert text after JS directives ('use client', 'use server') at the top of a file */ +export function insertAfterDirectives(content: string, text: string): string { + const lines = content.split('\n'); + let insertIdx = 0; + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].trim(); + if (trimmed === '') { + continue; + } + if (/^['"]use (client|server)['"];?\s*$/.test(trimmed)) { + insertIdx = i + 1; + continue; + } + break; + } + lines.splice(insertIdx, 0, text); + return lines.join('\n'); +} + +/** Get the exec prefix for running binaries from node_modules/.bin */ +export function execPrefix(pm: string): string { + if (pm === 'bun') return 'bunx'; + if (pm === 'yarn') return 'yarn exec'; + if (pm === 'npm') return 'npx'; + return `${pm} exec`; +} + +/** + * Imperative fixture environment (no Vitest dependency). + * Used by framework tests and by the release benchmarking suite. + */ +export class FrameworkTestEnv { + dir!: string; + protected filesDir: string; + protected label: string; + /** Files written by the previous scenario, restored before the next one runs */ + private prevScenarioFiles = new Set(); + + constructor(public config: TestFixtureConfig) { + this.filesDir = join(config.testDir, 'files'); + this.label = config.framework ?? basename(config.testDir); + } + + /** + * Create temp dir, copy base template, build package.json, install deps. + */ + async setup(): Promise { + this.dir = join(FRAMEWORK_TESTS_DIR, '.test-projects', this.label); + if (existsSync(this.dir)) { + rmSync(this.dir, { recursive: true, force: true }); + } + mkdirSync(this.dir, { recursive: true }); + console.log(`[${this.label}] Setting up fixture in ${this.dir}`); + + // Isolate from parent workspaces so the test project + // is treated as its own independent root + + // Prevent bun from inheriting the repo root's bunfig.toml which has + // minimumReleaseAge and a security scanner that can block/hang installs + writeFileSync(join(this.dir, 'bunfig.toml'), [ + '[install]', + 'minimumReleaseAge = 0', + '', + ].join('\n')); + + // pnpm v11 defaults: block all build scripts, and error on packages missing time metadata. + // Allow known native packages that need postinstall scripts. + // minimumReleaseAge is 0 here (matching bunfig.toml) so framework tests can use + // recently published versions (e.g. new Astro majors) without waiting 72h. + writeFileSync(join(this.dir, 'pnpm-workspace.yaml'), [ + 'minimumReleaseAge: 0', + 'onlyBuiltDependencies:', + ' - esbuild', + ' - sharp', + ' - lightningcss', + ' - workerd', + '', + ].join('\n')); + writeFileSync(join(this.dir, '.npmrc'), 'ignore-workspace-root-check=true\n'); + + // Copy _base/ skeleton into the project + const baseDir = join(this.filesDir, '_base'); + if (existsSync(baseDir)) { + cpSync(baseDir, this.dir, { recursive: true }); + } + + // Pack required varlock packages (unless measuring published registry versions) + const varlockPackageNames = Object.keys(this.config.dependencies) + .filter((dep) => dep === 'varlock' || dep.startsWith('@varlock/')); + const usePublished = this.config.usePublished === true; + let packedDeps: Record = {}; + if (!usePublished) { + packPackages(varlockPackageNames); + packedDeps = getPackedDeps(varlockPackageNames); + } + + // Build package.json + const templatePkgPath = join(this.dir, 'package.json'); + const templatePkg = existsSync(templatePkgPath) + ? JSON.parse(readFileSync(templatePkgPath, 'utf-8')) + : {}; + + const pm = this.config.packageManager ?? 'pnpm'; + + const pkg = { + name: 'framework-test-project', + version: '0.0.0', + private: true, + type: 'module', + ...templatePkg, + dependencies: { + ...templatePkg.dependencies, + ...this.config.dependencies, + ...packedDeps, // override varlock deps with packed file: paths (skipped when usePublished) + }, + ...(this.config.devDependencies ? { + devDependencies: { + ...templatePkg.devDependencies, + ...this.config.devDependencies, + }, + } : {}), + ...(this.config.scripts ? { + scripts: { ...templatePkg.scripts, ...this.config.scripts }, + } : {}), + }; + + // Apply overrides — pnpm nests them under `pnpm.overrides` + if (this.config.overrides) { + if (pm === 'pnpm') { + pkg.pnpm = { ...templatePkg.pnpm, overrides: { ...templatePkg.pnpm?.overrides, ...this.config.overrides } }; + } else { + pkg.overrides = { ...templatePkg.overrides, ...this.config.overrides }; + } + } + + // Apply packageJsonMerge (deep merge one level) + if (this.config.packageJsonMerge) { + for (const [key, value] of Object.entries(this.config.packageJsonMerge)) { + if (typeof value === 'object' && !Array.isArray(value) && value !== null) { + pkg[key] = { ...pkg[key], ...value }; + } else { + pkg[key] = value; + } + } + } + + // Replace placeholders in any overrides sections + const overrideSections = [ + pkg.overrides, // npm/bun top-level overrides + pkg.pnpm?.overrides, // pnpm overrides + ]; + for (const overrides of overrideSections) { + if (!overrides) continue; + for (const [key, val] of Object.entries(overrides)) { + if (typeof val === 'string' && val.startsWith('): void { + for (const dest of this.prevScenarioFiles) { + const basePath = join(this.filesDir, '_base', dest); + const destPath = join(this.dir, dest); + if (existsSync(basePath)) { + cpSync(basePath, destPath); + } else { + rmSync(destPath, { force: true }); + } + } + this.prevScenarioFiles = new Set(); + + // Copy template files: fixture defaults merged with scenario overrides + const templateFiles = { + ...this.config.templateFiles, + ...scenario.templateFiles, + }; + for (const [dest, source] of Object.entries(templateFiles)) { + this.prevScenarioFiles.add(dest); + const srcRef: Exclude = typeof source === 'string' + ? { path: source } + : source; + const srcPath = join(this.filesDir, srcRef.path); + const destPath = join(this.dir, dest); + mkdirSync(dirname(destPath), { recursive: true }); + cpSync(srcPath, destPath); + + // Apply transformations (replacements, prepend, append) + if (typeof source !== 'string') { + let content = readFileSync(destPath, 'utf-8'); + if (srcRef.replacements) { + for (const [find, replace] of Object.entries(srcRef.replacements)) { + content = content.replaceAll(find, replace); + } + } + if (srcRef.prepend) { + content = `${srcRef.prepend}\n${content}`; + } + if (srcRef.insertAfterDirectives) { + content = insertAfterDirectives(content, srcRef.insertAfterDirectives); + } + if (srcRef.append) { + content = `${content}\n${srcRef.append}`; + } + writeFileSync(destPath, content); + } + } + + // Write inline files + if (scenario.files) { + for (const file of scenario.files) { + this.prevScenarioFiles.add(file.path); + const destPath = join(this.dir, file.path); + mkdirSync(dirname(destPath), { recursive: true }); + writeFileSync(destPath, file.content); + } + } + } + + /** + * Apply template / inline files without running a command. + * Useful for benchmarks that measure with their own timers. + */ + prepareFiles(scenario: Pick): void { + this.applyFiles(scenario); + } + + /** + * Run a test scenario: write files, build, return result. + */ + async runScenario(scenario: TestScenario): Promise { + // Clean previous build artifacts + this.cleanBuildArtifacts(); + + this.applyFiles(scenario); + + // Run command, auto-prefixed with package manager exec + const pm = this.config.packageManager ?? 'pnpm'; + const buildCmd = `${execPrefix(pm)} ${scenario.command}`; + + return runCommand(this.dir, buildCmd, { + env: scenario.env, + timeout: scenario.timeout ?? 120_000, + killAfterPattern: scenario.killAfterPattern, + }); + } + + /** + * Run a dev server scenario: write files, start server, make requests, return result. + */ + async runDevServer(scenario: DevServerScenario): Promise { + this.cleanBuildArtifacts(); + this.applyFiles(scenario); + + const pm = this.config.packageManager ?? 'pnpm'; + const command = `${execPrefix(pm)} ${scenario.command}`; + + return runDevServerProcess(this.dir, command, scenario); + } + + /** + * Clean build artifacts between scenarios. + */ + cleanBuildArtifacts(): void { + // Clean .next but preserve the cache directory so turbopack/webpack + // compilation cache speeds up subsequent builds within the same fixture + const nextDir = join(this.dir, '.next'); + if (existsSync(nextDir)) { + for (const entry of readdirSync(nextDir)) { + if (entry === 'cache') continue; + rmSync(join(nextDir, entry), { recursive: true, force: true }); + } + } + + const artifactDirs = ['out', 'dist', '.turbo', '.wrangler']; + for (const dir of artifactDirs) { + const fullPath = join(this.dir, dir); + if (existsSync(fullPath)) { + rmSync(fullPath, { recursive: true, force: true }); + } + } + } + + /** + * Remove the entire temp directory. + */ + async teardown(): Promise { + if (this.dir && existsSync(this.dir)) { + if (process.env.KEEP_TEST_DIRS) { + console.log(`[${this.label}] Preserving test dir: ${this.dir}`); + } else { + // retry removal — on CI, child processes (e.g. wrangler) may still be + // releasing file handles when teardown runs, causing ENOTEMPTY + for (let attempt = 0; attempt < 3; attempt++) { + try { + rmSync(this.dir, { recursive: true, force: true }); + break; + } catch (err) { + if (attempt < 2) { + await new Promise((r) => { + setTimeout(r, 500); + }); + } else { + console.warn(`[${this.label}] Failed to clean up ${this.dir}: ${(err as Error).message}`); + } + } + } + console.log(`[${this.label}] Cleaned up ${this.dir}`); + } + } + } +} diff --git a/framework-tests/harness/test-fixture.ts b/framework-tests/harness/test-fixture.ts index 4c799be69..94c50cee3 100644 --- a/framework-tests/harness/test-fixture.ts +++ b/framework-tests/harness/test-fixture.ts @@ -1,23 +1,14 @@ -import { - cpSync, writeFileSync, readFileSync, readdirSync, - rmSync, existsSync, mkdirSync, -} from 'node:fs'; -import { - join, dirname, basename, resolve, -} from 'node:path'; - -const FRAMEWORK_TESTS_DIR = resolve(import.meta.dirname, '..'); import { describe, test, beforeAll, expect, } from 'vitest'; -import { runCommand } from './command-runner.js'; -import { runDevServer as runDevServerProcess } from './dev-server.js'; -import { packPackages, getPackedDeps } from './pack.js'; import { assertBuildResult, assertOutput, assertFiles, } from './assertions.js'; +import { + FrameworkTestEnv as BaseFrameworkTestEnv, +} from './fixture-env.js'; import type { - TestFixtureConfig, TestScenario, DevServerScenario, DevServerResult, + TestScenario, DevServerScenario, DevServerResult, BuildResult, TemplateFileMap, TemplateFileSource, } from './types.js'; @@ -40,25 +31,6 @@ function addEdgeRuntimeToTemplateFiles(templateFiles?: TemplateFileMap): Templat return result; } -/** Insert text after JS directives ('use client', 'use server') at the top of a file */ -function insertAfterDirectives(content: string, text: string): string { - const lines = content.split('\n'); - let insertIdx = 0; - for (let i = 0; i < lines.length; i++) { - const trimmed = lines[i].trim(); - if (trimmed === '') { - continue; - } - if (/^['"]use (client|server)['"];?\s*$/.test(trimmed)) { - insertIdx = i + 1; - continue; - } - break; - } - lines.splice(insertIdx, 0, text); - return lines.join('\n'); -} - /** Apply .only or .skip modifier to a vitest describe/test function based on flags */ function withSkipOrOnly( fn: T, @@ -69,243 +41,12 @@ function withSkipOrOnly( return fn; } -/** Get the exec prefix for running binaries from node_modules/.bin */ -function execPrefix(pm: string): string { - if (pm === 'bun') return 'bunx'; - if (pm === 'yarn') return 'yarn exec'; - if (pm === 'npm') return 'npx'; - return `${pm} exec`; -} - -export class FrameworkTestEnv { - dir!: string; - private filesDir: string; - private label: string; - /** Files written by the previous scenario, restored before the next one runs */ - private prevScenarioFiles = new Set(); - - constructor(public config: TestFixtureConfig) { - this.filesDir = join(config.testDir, 'files'); - this.label = config.framework ?? basename(config.testDir); - } - - /** - * Create temp dir, copy base template, build package.json, install deps. - */ - async setup(): Promise { - this.dir = join(FRAMEWORK_TESTS_DIR, '.test-projects', this.label); - if (existsSync(this.dir)) { - rmSync(this.dir, { recursive: true, force: true }); - } - mkdirSync(this.dir, { recursive: true }); - console.log(`[${this.label}] Setting up fixture in ${this.dir}`); - - // Isolate from parent workspaces so the test project - // is treated as its own independent root - - // Prevent bun from inheriting the repo root's bunfig.toml which has - // minimumReleaseAge and a security scanner that can block/hang installs - writeFileSync(join(this.dir, 'bunfig.toml'), [ - '[install]', - 'minimumReleaseAge = 0', - '', - ].join('\n')); - - // pnpm v11 defaults: block all build scripts, and error on packages missing time metadata. - // Allow known native packages that need postinstall scripts. - // minimumReleaseAge is 0 here (matching bunfig.toml) so framework tests can use - // recently published versions (e.g. new Astro majors) without waiting 72h. - writeFileSync(join(this.dir, 'pnpm-workspace.yaml'), [ - 'minimumReleaseAge: 0', - 'onlyBuiltDependencies:', - ' - esbuild', - ' - sharp', - ' - lightningcss', - ' - workerd', - '', - ].join('\n')); - writeFileSync(join(this.dir, '.npmrc'), 'ignore-workspace-root-check=true\n'); - - // Copy _base/ skeleton into the project - const baseDir = join(this.filesDir, '_base'); - if (existsSync(baseDir)) { - cpSync(baseDir, this.dir, { recursive: true }); - } - - // Pack required varlock packages - const varlockPackageNames = Object.keys(this.config.dependencies) - .filter((dep) => dep === 'varlock' || dep.startsWith('@varlock/')); - packPackages(varlockPackageNames); - const packedDeps = getPackedDeps(varlockPackageNames); - - // Build package.json - const templatePkgPath = join(this.dir, 'package.json'); - const templatePkg = existsSync(templatePkgPath) - ? JSON.parse(readFileSync(templatePkgPath, 'utf-8')) - : {}; - - const pm = this.config.packageManager ?? 'pnpm'; - - const pkg = { - name: 'framework-test-project', - version: '0.0.0', - private: true, - type: 'module', - ...templatePkg, - dependencies: { - ...templatePkg.dependencies, - ...this.config.dependencies, - ...packedDeps, // override varlock deps with packed file: paths - }, - ...(this.config.devDependencies ? { - devDependencies: { - ...templatePkg.devDependencies, - ...this.config.devDependencies, - }, - } : {}), - ...(this.config.scripts ? { - scripts: { ...templatePkg.scripts, ...this.config.scripts }, - } : {}), - }; - - // Apply overrides — pnpm nests them under `pnpm.overrides` - if (this.config.overrides) { - if (pm === 'pnpm') { - pkg.pnpm = { ...templatePkg.pnpm, overrides: { ...templatePkg.pnpm?.overrides, ...this.config.overrides } }; - } else { - pkg.overrides = { ...templatePkg.overrides, ...this.config.overrides }; - } - } - - // Apply packageJsonMerge (deep merge one level) - if (this.config.packageJsonMerge) { - for (const [key, value] of Object.entries(this.config.packageJsonMerge)) { - if (typeof value === 'object' && !Array.isArray(value) && value !== null) { - pkg[key] = { ...pkg[key], ...value }; - } else { - pkg[key] = value; - } - } - } - - // Replace placeholders in any overrides sections - const overrideSections = [ - pkg.overrides, // npm/bun top-level overrides - pkg.pnpm?.overrides, // pnpm overrides - ]; - for (const overrides of overrideSections) { - if (!overrides) continue; - for (const [key, val] of Object.entries(overrides)) { - if (typeof val === 'string' && val.startsWith('): void { - for (const dest of this.prevScenarioFiles) { - const basePath = join(this.filesDir, '_base', dest); - const destPath = join(this.dir, dest); - if (existsSync(basePath)) { - cpSync(basePath, destPath); - } else { - rmSync(destPath, { force: true }); - } - } - this.prevScenarioFiles = new Set(); - - // Copy template files: fixture defaults merged with scenario overrides - const templateFiles = { - ...this.config.templateFiles, - ...scenario.templateFiles, - }; - for (const [dest, source] of Object.entries(templateFiles)) { - this.prevScenarioFiles.add(dest); - const srcRef = typeof source === 'string' ? { path: source } : source; - const srcPath = join(this.filesDir, srcRef.path); - const destPath = join(this.dir, dest); - mkdirSync(dirname(destPath), { recursive: true }); - cpSync(srcPath, destPath); - - // Apply transformations (replacements, prepend, append) - if (typeof source !== 'string') { - let content = readFileSync(destPath, 'utf-8'); - if (srcRef.replacements) { - for (const [find, replace] of Object.entries(srcRef.replacements)) { - content = content.replaceAll(find, replace); - } - } - if (srcRef.prepend) { - content = `${srcRef.prepend}\n${content}`; - } - if (srcRef.insertAfterDirectives) { - content = insertAfterDirectives(content, srcRef.insertAfterDirectives); - } - if (srcRef.append) { - content = `${content}\n${srcRef.append}`; - } - writeFileSync(destPath, content); - } - } - - // Write inline files - if (scenario.files) { - for (const file of scenario.files) { - this.prevScenarioFiles.add(file.path); - const destPath = join(this.dir, file.path); - mkdirSync(dirname(destPath), { recursive: true }); - writeFileSync(destPath, file.content); - } - } - } - - /** - * Run a test scenario: write files, build, return result. - */ - async runScenario(scenario: TestScenario): Promise { - // Clean previous build artifacts - this.cleanBuildArtifacts(); - - this.applyFiles(scenario); - - // Run command, auto-prefixed with package manager exec - const pm = this.config.packageManager ?? 'pnpm'; - const buildCmd = `${execPrefix(pm)} ${scenario.command}`; - - return runCommand(this.dir, buildCmd, { - env: scenario.env, - timeout: scenario.timeout ?? 120_000, - killAfterPattern: scenario.killAfterPattern, - }); - } - +/** + * Vitest-aware fixture env used by framework test suites. + * Benchmarks should import the base class from `./fixture-env.js` instead, + * so Vitest is not loaded outside a test runner. + */ +export class FrameworkTestEnv extends BaseFrameworkTestEnv { /** * Run a scenario and assert results in a single test. */ @@ -371,19 +112,6 @@ export class FrameworkTestEnv { }); } - /** - * Run a dev server scenario: write files, start server, make requests, return result. - */ - async runDevServer(scenario: DevServerScenario): Promise { - this.cleanBuildArtifacts(); - this.applyFiles(scenario); - - const pm = this.config.packageManager ?? 'pnpm'; - const command = `${execPrefix(pm)} ${scenario.command}`; - - return runDevServerProcess(this.dir, command, scenario); - } - /** * Create a describe block that starts a dev server once and runs each assertion as a separate test. */ @@ -442,56 +170,4 @@ export class FrameworkTestEnv { } }); } - - /** - * Clean build artifacts between scenarios. - */ - cleanBuildArtifacts(): void { - // Clean .next but preserve the cache directory so turbopack/webpack - // compilation cache speeds up subsequent builds within the same fixture - const nextDir = join(this.dir, '.next'); - if (existsSync(nextDir)) { - for (const entry of readdirSync(nextDir)) { - if (entry === 'cache') continue; - rmSync(join(nextDir, entry), { recursive: true, force: true }); - } - } - - const artifactDirs = ['out', 'dist', '.turbo', '.wrangler']; - for (const dir of artifactDirs) { - const fullPath = join(this.dir, dir); - if (existsSync(fullPath)) { - rmSync(fullPath, { recursive: true, force: true }); - } - } - } - - /** - * Remove the entire temp directory. - */ - async teardown(): Promise { - if (this.dir && existsSync(this.dir)) { - if (process.env.KEEP_TEST_DIRS) { - console.log(`[${this.label}] Preserving test dir: ${this.dir}`); - } else { - // retry removal — on CI, child processes (e.g. wrangler) may still be - // releasing file handles when teardown runs, causing ENOTEMPTY - for (let attempt = 0; attempt < 3; attempt++) { - try { - rmSync(this.dir, { recursive: true, force: true }); - break; - } catch (err) { - if (attempt < 2) { - await new Promise((r) => { - setTimeout(r, 500); - }); - } else { - console.warn(`[${this.label}] Failed to clean up ${this.dir}: ${(err as Error).message}`); - } - } - } - console.log(`[${this.label}] Cleaned up ${this.dir}`); - } - } - } } diff --git a/framework-tests/harness/types.ts b/framework-tests/harness/types.ts index 0237da874..995604f03 100644 --- a/framework-tests/harness/types.ts +++ b/framework-tests/harness/types.ts @@ -74,6 +74,13 @@ export interface TestFixtureConfig { templateFiles?: TemplateFileMap; /** Timeout for dependency installation in ms (default: 120_000) */ installTimeout?: number; + /** + * Install varlock / `@varlock/*` from the npm registry using the versions in + * `dependencies`, instead of packing workspace tarballs. Also resolves + * `` override placeholders to `npm:pkg`. + * Used by release benchmarks that measure published packages. + */ + usePublished?: boolean; } /** diff --git a/package.json b/package.json index 49c3aee1e..dddd47dbd 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "test:ci": "turbo test:ci --filter=\"!smoke-test-*\"", "smoke-test": "cd smoke-tests && bun run test", "test:frameworks": "cd framework-tests && bun run test", + "bench": "bun run --cwd benchmarks src/run.ts", "typecheck": "tsc --noEmit", "typecheck:all": "turbo typecheck --filter=\"!@varlock/website\" --filter=\"!smoke-test-*\" --filter=\"!varlock-docs-mcp\"", "check": "bun run lint && bun run typecheck:all && bun run build:libs && bun run test:ci", From 28f8a3c9dd109466ef3eaf410b346c902eb11b13 Mon Sep 17 00:00:00 2001 From: philmillman Date: Fri, 31 Jul 2026 18:11:57 -0400 Subject: [PATCH 2/5] varlock: allow overriding the telemetry endpoint via VARLOCK_POSTHOG_HOST Lets tooling exercise the telemetry code path against a local mock instead of the production collector. Not an opt-out knob - VARLOCK_TELEMETRY_DISABLED still disables telemetry entirely. --- .bumpy/bench-telemetry-host-override.md | 5 +++++ packages/varlock/src/config.ts | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .bumpy/bench-telemetry-host-override.md diff --git a/.bumpy/bench-telemetry-host-override.md b/.bumpy/bench-telemetry-host-override.md new file mode 100644 index 000000000..b4e51a9b0 --- /dev/null +++ b/.bumpy/bench-telemetry-host-override.md @@ -0,0 +1,5 @@ +--- +varlock: patch +--- + +Allow overriding the telemetry endpoint with VARLOCK_POSTHOG_HOST, so tooling can point it at a local mock diff --git a/packages/varlock/src/config.ts b/packages/varlock/src/config.ts index 0d5bf6f03..42a89aa09 100644 --- a/packages/varlock/src/config.ts +++ b/packages/varlock/src/config.ts @@ -7,5 +7,11 @@ export const CONFIG = { VARLOCK_API_URL: 'https://api.varlock.dev', GITHUB_APP_CLIENT_ID: 'Iv23li50gB8bMxLauiJQ', // varlock.dev app POSTHOG_API_KEY: 'phc_bfzH97VIta8yQa8HrsgmitqS6rTydjMISs0m8aqJTnq', - POSTHOG_HOST: 'https://ph.varlock.dev', + /** + * Telemetry collector endpoint. Overridable so tooling that needs the telemetry + * code path to actually run (the benchmark suite measures its cost) can point it + * at a local mock instead of the real collector. This is not an opt-out knob: + * use VARLOCK_TELEMETRY_DISABLED to disable telemetry entirely. + */ + POSTHOG_HOST: process.env.VARLOCK_POSTHOG_HOST || 'https://ph.varlock.dev', }; From ffa73f0a42f6c61b8295b4e63a54b568fe42405e Mon Sep 17 00:00:00 2001 From: philmillman Date: Fri, 31 Jul 2026 18:12:23 -0400 Subject: [PATCH 3/5] benchmarks: fix result publishing, measurement validity, and telemetry mocking CI: - git add used a repo-relative path from the benchmarks/ working dir, so the commit step failed on every run; use the absolute path - commit before rebasing (a rebase refuses to run with changes staged) and retry the push - install with --frozen-lockfile so a rewritten bun.lock cannot block the rebase - pass workflow inputs through env: instead of interpolating into shell - typecheck benchmarks/ in CI; turbo does not reach it Telemetry: - telemetry-on scenarios now run against a local mock collector instead of production PostHog, and refuse to run at all if the version under test does not honour VARLOCK_POSTHOG_HOST Measurement: - drain server stdout; the redactLogs benches filled the 64KB pipe and turned a latency measurement into a measurement of pipe backpressure - kill servers by process group; npx execs the real server as a grandchild - sample RSS across the whole process tree, and sample immediately so short-lived commands are not missed entirely - report min/stddev and flag deltas that fall within noise - run cli scenarios against a copy of the fixture, not the fixture itself - force the on-disk cache so warm-vs-cold load is meaningful in CI - run the bun install under bun; it was running the same node code twice - scale the redaction/leak workloads above the noise floor (they previously could not resolve the thing they exist to measure) - align baseline and varlock arms so the delta is not also a source diff - reuse one fixture per framework: 14 npm installs down to 4 - unique scenario ids, validated CLI args, recorded skips instead of silent ones --- .github/workflows/benchmarks.yaml | 83 +++-- .github/workflows/test.yaml | 4 + benchmarks/README.md | 43 ++- benchmarks/fixtures/cli-basic/emit-secret.js | 5 +- benchmarks/src/install.ts | 44 ++- benchmarks/src/many-secrets-schema.ts | 32 +- benchmarks/src/measure.ts | 177 ++++++++-- benchmarks/src/report.ts | 101 +++++- benchmarks/src/run.ts | 278 +++++++++++---- benchmarks/src/scenarios/cli-load.ts | 23 +- benchmarks/src/scenarios/cli-run.ts | 53 ++- benchmarks/src/scenarios/cli-scan-audit.ts | 8 +- benchmarks/src/scenarios/integration-next.ts | 339 ++++++++----------- benchmarks/src/scenarios/integration-vite.ts | 289 ++++++---------- benchmarks/src/scenarios/lang-go.ts | 20 +- benchmarks/src/scenarios/lang-python.ts | 20 +- benchmarks/src/scenarios/util.ts | 33 ++ benchmarks/src/server.ts | 136 ++++++++ benchmarks/src/telemetry-mock.ts | 51 +++ benchmarks/src/telemetry.ts | 20 +- benchmarks/src/types.ts | 38 +++ 21 files changed, 1201 insertions(+), 596 deletions(-) create mode 100644 benchmarks/src/scenarios/util.ts create mode 100644 benchmarks/src/server.ts create mode 100644 benchmarks/src/telemetry-mock.ts diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 1a3d0cf35..46f95c9cc 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -60,43 +60,37 @@ jobs: - name: Install benchmarks package deps working-directory: benchmarks - run: bun install + # Frozen so the run cannot leave a modified bun.lock behind, which would + # block the rebase in the commit step at the very end of a long run. + run: bun install --frozen-lockfile + # Workflow inputs go through `env:` rather than being interpolated straight + # into the script, so a crafted input value cannot become shell syntax. - name: Resolve varlock version id: ver + env: + INPUT_VERSION: ${{ github.event.inputs.varlock_version }} run: | - INPUT="${{ github.event.inputs.varlock_version }}" - if [[ -z "$INPUT" || "$INPUT" == "latest" ]]; then + set -euo pipefail + if [[ -z "$INPUT_VERSION" || "$INPUT_VERSION" == "latest" ]]; then V=$(npm view varlock version) else - V="$INPUT" + V="$INPUT_VERSION" fi echo "version=$V" >> "$GITHUB_OUTPUT" echo "Resolved varlock@$V" - - name: Wait for npm package - run: | - set -euo pipefail - V="${{ steps.ver.outputs.version }}" - for i in $(seq 1 30); do - if npm view "varlock@${V}" version >/dev/null 2>&1; then - echo "varlock@${V} is on npm" - exit 0 - fi - echo "waiting for varlock@${V} on npm ($i)..." - sleep 10 - done - echo "::error::Timed out waiting for varlock@${V} on npm" - exit 1 + # Note: the suite itself waits for the version to appear on npm, so a + # release-triggered run can start before the registry has caught up. - name: Download SEA binary (linux-x64) id: sea env: GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.ver.outputs.version }} run: | set -euo pipefail - V="${{ steps.ver.outputs.version }}" - TAG="varlock@${V}" + TAG="varlock@${VERSION}" DEST="$RUNNER_TEMP/varlock-sea" mkdir -p "$DEST" if gh release download "$TAG" --pattern 'varlock-linux-x64.tar.gz' --dir "$DEST"; then @@ -114,39 +108,58 @@ jobs: - name: Run benchmarks working-directory: benchmarks + env: + VERSION: ${{ steps.ver.outputs.version }} + ONLY: ${{ github.event.inputs.only }} + ITERATIONS: ${{ github.event.inputs.iterations }} + RELEASE_DISPATCH: ${{ github.event.inputs.release_dispatch }} + SEA_FOUND: ${{ steps.sea.outputs.found }} + SEA_PATH: ${{ steps.sea.outputs.path }} run: | set -euo pipefail - V="${{ steps.ver.outputs.version }}" - if [[ "${{ github.event.inputs.release_dispatch }}" == "true" ]]; then + if [[ "$RELEASE_DISPATCH" == "true" ]]; then TRIGGER=release else TRIGGER=workflow_dispatch fi - ARGS=(--version "$V" --trigger "$TRIGGER" --iterations "${{ github.event.inputs.iterations }}") - ONLY="${{ github.event.inputs.only }}" + ARGS=(--version "$VERSION" --trigger "$TRIGGER") + if [[ -n "$ITERATIONS" ]]; then + ARGS+=(--iterations "$ITERATIONS") + fi if [[ -n "$ONLY" ]]; then ARGS+=(--only "$ONLY") fi - if [[ "${{ steps.sea.outputs.found }}" == "true" ]]; then - ARGS+=(--sea-path "${{ steps.sea.outputs.path }}") + if [[ "$SEA_FOUND" == "true" ]]; then + ARGS+=(--sea-path "$SEA_PATH") fi bun run src/run.ts "${ARGS[@]}" - name: Commit results - working-directory: benchmarks + env: + VERSION: ${{ steps.ver.outputs.version }} run: | set -euo pipefail - RESULT_PATH=$(cat .work/last-result-path.txt) - REL="${RESULT_PATH#"$GITHUB_WORKSPACE"/}" + # Absolute path — `git add` resolves pathspecs against the cwd, so a + # repo-relative path only works from the repo root. + RESULT_PATH=$(cat benchmarks/.work/last-result-path.txt) git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - # Fetch latest main in case other commits landed during the long bench run - git pull --rebase origin main - git add -- "$REL" + git add -- "$RESULT_PATH" if git diff --staged --quiet; then echo "No results to commit" exit 0 fi - V="${{ steps.ver.outputs.version }}" - git commit -m "chore(benchmarks): record varlock@${V} [skip ci]" - git push origin HEAD:main + git commit -m "chore(benchmarks): record varlock@${VERSION} [skip ci]" + # Commit first, then rebase: a rebase refuses to run with staged or + # unstaged changes present, and other commits may have landed on main + # during the (long) bench run. + for attempt in 1 2 3; do + git pull --rebase origin main + if git push origin HEAD:main; then + echo "Pushed results" + exit 0 + fi + echo "Push rejected, retrying ($attempt/3)..." + done + echo "::error::Failed to push benchmark results after 3 attempts" + exit 1 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fc17ad34d..4f371f058 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,6 +50,10 @@ jobs: run: bun run lint - name: TypeScript type check run: bun run typecheck:all + # benchmarks/ is not a workspace member, so turbo's typecheck does not reach it + - name: TypeScript type check (benchmarks) + working-directory: benchmarks + run: bun install --frozen-lockfile && bun run typecheck - name: Build libraries run: bun run build:libs - name: Run tests diff --git a/benchmarks/README.md b/benchmarks/README.md index 4033aa522..c604ccf85 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -16,28 +16,45 @@ Runs against **published** npm packages (and optionally the linux SEA binary), n | `lang-python` | `load`+codegen and `varlock run -- python3` | | `lang-go` | `load`+codegen and `varlock run` of a built Go binary | +Install methods are three distinct runtimes, not three package managers: `npm` installs and runs under **node**, `bun` installs and runs under **bun**, `sea` is the compiled standalone binary. + +## Reading the results + +Each run prints a **Deltas** table before the raw numbers. The deltas are the point of the suite: absolute wall times on a shared CI runner are not comparable between runs, but the difference between two scenarios measured back to back within one run is. + +A delta smaller than the standard deviation of either side is tagged `(within noise)` and should not be read as a change. + +Every scenario records `wallMsMin`, `wallMsMedian`, `wallMsP95`, `wallMsStdDev` and `iterations`. Prefer **min** and **stddev**: min is the least noise-sensitive statistic for this kind of measurement, and p95 collapses onto the max at the iteration counts used here. + +`meta.notes` lists anything that was skipped or degraded (missing SEA binary, no Go toolchain, telemetry not measurable). Nothing is dropped silently. + +## Telemetry + +Telemetry-on scenarios exist to measure what the telemetry code path costs. **They never send real telemetry.** The suite starts a local mock collector and points varlock at it with `VARLOCK_POSTHOG_HOST`, which keeps the code path intact (payload building, the exit hook that waits on the in-flight request) without injecting synthetic events into product analytics, and without making the timings depend on network latency to the real collector. + +Before running any telemetry-on scenario the suite probes whether the version under test honours that override. If it does not (versions published before the override existed), those scenarios are skipped and a note is recorded. + ## Local usage ```bash -# From repo root (latest published varlock) bun run bench +``` -# Specific version + local SEA binary +```bash bun run bench -- --version 1.13.0 --sea-path ./packages/varlock/dist-sea/varlock +``` -# Subset of scenario groups (faster iteration) +```bash bun run bench -- --only cli-load,cli-run --iterations 3 +``` -# Reuse prior npm/bun installs under benchmarks/.work +```bash bun run bench -- --skip-install --only cli-load ``` -Or from this directory: +The first form benchmarks the latest published varlock. The others pin a version and add a local SEA binary, restrict to a subset of scenario groups for faster iteration, and reuse the npm/bun installs left in `benchmarks/.work` by a previous run. -```bash -bun install -bun run bench -- --version latest --only cli-load -``` +From this directory, `bun install` first and then use `bun run bench` the same way. Integration benches drive [`FrameworkTestEnv`](../framework-tests/harness/fixture-env.ts) with `usePublished: true` so they install from npm (not packed workspace tarballs) while reusing the same Next/Vite templates as framework CI. @@ -50,6 +67,12 @@ Workflow: [`.github/workflows/benchmarks.yaml`](../.github/workflows/benchmarks. - **Manual:** Actions → Benchmarks → Run workflow (optional version / scenario filter) - **After publish:** [`release.yaml`](../.github/workflows/release.yaml) dispatches this workflow once SEA binaries are uploaded for `varlock@` -The job installs from npm, downloads `varlock-linux-x64.tar.gz` when present, runs the suite, and commits the new JSON under `results/` with `[skip ci]` so the commit does not retrigger release/CI. +The job installs from npm, downloads `varlock-linux-x64.tar.gz` when present, runs the suite, and commits the new JSON under `results/` with `[skip ci]` so the commit does not retrigger release/CI. The suite waits for the version to appear on npm itself, so a release-triggered run can start before the registry has caught up. v1 is informational only (no regression gate). Suite failures still fail the workflow. + +## Known gaps + +- **Linux/x64 only.** The SEA binary ships for macOS and Windows too, but nothing measures them. The non-Linux RSS sampling path (which shells out to `ps` once per sample, perturbing the timings it measures) is therefore only exercised by local runs. +- **The `cli-load` fixture has nothing worth caching.** Cold vs warm is now a valid comparison in CI (`_VARLOCK_CACHE_KEY` forces the on-disk cache, which CI would otherwise skip in favour of a per-process memory cache), but the fixture is all static literals, so both arms measure roughly the same work. Exercising the cache meaningfully needs a fixture with expensive resolvers, e.g. a plugin-backed or `exec()` value. +- **No regression gate and no cross-run comparison tooling.** Results accumulate in `results/` but nothing reads the history yet. diff --git a/benchmarks/fixtures/cli-basic/emit-secret.js b/benchmarks/fixtures/cli-basic/emit-secret.js index 272ea13f1..4442b66b8 100644 --- a/benchmarks/fixtures/cli-basic/emit-secret.js +++ b/benchmarks/fixtures/cli-basic/emit-secret.js @@ -1,4 +1,6 @@ // Emits every SECRET_* env var many times so stdout redaction cost scales with secret count. +// Line count comes from BENCH_EMIT_LINES: redaction cost is per byte of output, so the +// volume has to be large enough to clear the fixed ~50ms of process-startup noise. const secrets = Object.entries(process.env) .filter(([key]) => key.startsWith('SECRET_')) .map(([, value]) => value) @@ -9,7 +11,8 @@ if (secrets.length === 0) { process.exit(1); } -const chunks = 200; +const parsedLines = Number(process.env.BENCH_EMIT_LINES); +const chunks = Number.isInteger(parsedLines) && parsedLines > 0 ? parsedLines : 200; for (let i = 0; i < chunks; i++) { const secret = secrets[i % secrets.length]; process.stdout.write(`line-${i}: prefix ${secret} suffix\n`); diff --git a/benchmarks/src/install.ts b/benchmarks/src/install.ts index 382d1e4f9..ec5e321db 100644 --- a/benchmarks/src/install.ts +++ b/benchmarks/src/install.ts @@ -23,6 +23,14 @@ function runOrThrow( } } +export function npmInstallDir(workDir: string): string { + return join(workDir, 'installs', 'npm'); +} + +export function bunInstallDir(workDir: string): string { + return join(workDir, 'installs', 'bun'); +} + /** * Install published `varlock@version` with npm into workDir/installs/npm * and return a CliInvocation that runs it via node. @@ -31,7 +39,7 @@ export function installVarlockNpm( workDir: string, version: string, ): CliInvocation { - const dir = join(workDir, 'installs', 'npm'); + const dir = npmInstallDir(workDir); rmSync(dir, { recursive: true, force: true }); mkdirSync(dir, { recursive: true }); runOrThrow('npm', ['init', '-y'], { cwd: dir }); @@ -48,13 +56,18 @@ export function installVarlockNpm( } /** - * Install published `varlock@version` with bun into workDir/installs/bun. + * Install published `varlock@version` with bun into workDir/installs/bun and run + * it with the bun runtime. + * + * Running it under node would execute byte-identical code to the npm invocation + * above, so the two would measure the same thing twice. Bun as the runtime is the + * axis worth measuring. */ export function installVarlockBun( workDir: string, version: string, ): CliInvocation { - const dir = join(workDir, 'installs', 'bun'); + const dir = bunInstallDir(workDir); rmSync(dir, { recursive: true, force: true }); mkdirSync(dir, { recursive: true }); runOrThrow('bun', ['init', '-y'], { cwd: dir }); @@ -64,7 +77,7 @@ export function installVarlockBun( throw new Error(`bun install did not produce CLI at ${cliJs}`); } return { - command: [process.execPath, cliJs], + command: ['bun', cliJs], label: 'bun', packageManager: 'bun', }; @@ -81,6 +94,17 @@ export function seaInvocation(seaPath: string): CliInvocation { }; } +/** Check that an invocation actually runs, so a broken one skips instead of failing the suite. */ +export function probeCli(cli: CliInvocation): { ok: true } | { ok: false; error: string } { + const [bin, ...args] = cli.command; + const result = spawnSync(bin!, [...args, '--version'], { encoding: 'utf8' }); + if (result.error) return { ok: false, error: result.error.message }; + if (result.status !== 0) { + return { ok: false, error: `exit ${result.status}: ${(result.stderr || result.stdout || '').trim().slice(0, 500)}` }; + } + return { ok: true }; +} + /** Resolve package version of an installed package under an install root. */ export function readInstalledVersion(installRoot: string, pkgName: string): string | null { try { @@ -100,6 +124,18 @@ export function npmViewVersion(pkgSpec: string): string { return result.stdout.trim(); } +export function tryNpmViewVersion(pkgSpec: string): string | undefined { + try { + return npmViewVersion(pkgSpec) || undefined; + } catch { + return undefined; + } +} + +/** + * Wait for a version to be resolvable on npm. Returns immediately when it already + * is — release-triggered runs can start before the registry has caught up. + */ export async function waitForNpmPackage(pkgSpec: string, attempts = 30, delayMs = 10_000): Promise { for (let i = 1; i <= attempts; i++) { const result = spawnSync('npm', ['view', pkgSpec, 'version'], { encoding: 'utf8' }); diff --git a/benchmarks/src/many-secrets-schema.ts b/benchmarks/src/many-secrets-schema.ts index 4c5d551d3..02b3b8458 100644 --- a/benchmarks/src/many-secrets-schema.ts +++ b/benchmarks/src/many-secrets-schema.ts @@ -106,29 +106,27 @@ SECRET_OPENAI=sk-proj-bench-openainnnnnnnn SECRET_SENTRY=sntrys_bench_oooooooooooooo `; -/** Apply preventLeaks / redactLogs root flags onto a schema body. */ +const HEADER_DIVIDER = '# ---'; + +/** + * Apply preventLeaks / redactLogs root flags onto a schema body — replacing an + * existing flag, or adding it to the header section when absent. + */ export function withSchemaFlags( schema: string, preventLeaks: boolean, redactLogs: boolean, ): string { - const flags = `# @preventLeaks=${preventLeaks}\n# @redactLogs=${redactLogs}`; - if (!schema.includes('@preventLeaks=') && !schema.includes('@redactLogs=')) { - return schema.replace( - '# @defaultSensitive=false @defaultRequired=infer', - `# @defaultSensitive=false @defaultRequired=infer\n${flags}`, - ); - } let out = schema; - if (out.includes('@preventLeaks=')) { - out = out.replace(/@preventLeaks=\w+/, `@preventLeaks=${preventLeaks}`); - } else { - out = `# @preventLeaks=${preventLeaks}\n${out}`; - } - if (out.includes('@redactLogs=')) { - out = out.replace(/@redactLogs=\w+/, `@redactLogs=${redactLogs}`); - } else { - out = `# @redactLogs=${redactLogs}\n${out}`; + for (const [flag, value] of [['preventLeaks', preventLeaks], ['redactLogs', redactLogs]] as const) { + const existing = new RegExp(`@${flag}=\\w+`); + if (existing.test(out)) { + out = out.replace(existing, `@${flag}=${value}`); + } else if (out.includes(HEADER_DIVIDER)) { + out = out.replace(HEADER_DIVIDER, `# @${flag}=${value}\n${HEADER_DIVIDER}`); + } else { + throw new Error(`withSchemaFlags: schema has no "${HEADER_DIVIDER}" header divider to add @${flag} to`); + } } return out; } diff --git a/benchmarks/src/measure.ts b/benchmarks/src/measure.ts index e6fd72670..5cdf1732a 100644 --- a/benchmarks/src/measure.ts +++ b/benchmarks/src/measure.ts @@ -1,22 +1,101 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; -import { readFileSync } from 'node:fs'; +import { + cpSync, mkdirSync, readFileSync, readdirSync, rmSync, +} from 'node:fs'; import type { Sample, ScenarioMetrics } from './types.ts'; -export function rssKiB(pid: number): number | null { - if (process.platform === 'linux') { +/** RSS (KiB) of a single pid, or null if it is gone / unreadable. */ +function rssKiBForPid(pid: number): number | null { + try { + const status = readFileSync(`/proc/${pid}/status`, 'utf8'); + const match = status.match(/^VmRSS:\s+(\d+)/m); + return match ? Number(match[1]) : null; + } catch { + return null; + } +} + +/** Direct children of a pid, via /proc//task//children. */ +function childPidsLinux(pid: number): Array { + const out: Array = []; + let tids: Array; + try { + tids = readdirSync(`/proc/${pid}/task`); + } catch { + return out; + } + for (const tid of tids) { try { - const status = readFileSync(`/proc/${pid}/status`, 'utf8'); - const match = status.match(/^VmRSS:\s+(\d+)/m); - return match ? Number(match[1]) : null; + const raw = readFileSync(`/proc/${pid}/task/${tid}/children`, 'utf8').trim(); + if (!raw) continue; + for (const part of raw.split(/\s+/)) { + const n = Number(part); + if (Number.isInteger(n)) out.push(n); + } } catch { - return null; + // thread exited between readdir and read + } + } + return out; +} + +/** + * Summed RSS (KiB) of a process and all of its descendants. + * + * The tree matters: `varlock run -- node app.js` and `npx next build` both do the + * real work in a grandchild, so sampling only the direct child would report the + * footprint of a wrapper process. + * + * On Linux this walks /proc with no subprocess spawns. Elsewhere it shells out to + * `ps` once per sample, which perturbs the very timings we are measuring — so + * non-Linux runs sample at a coarser interval and are best treated as indicative. + */ +export function rssTreeKiB(rootPid: number): number | null { + if (process.platform === 'linux') { + let total = 0; + let found = false; + const seen = new Set(); + const queue = [rootPid]; + while (queue.length) { + const pid = queue.pop()!; + if (seen.has(pid)) continue; + seen.add(pid); + const rss = rssKiBForPid(pid); + if (rss !== null) { + total += rss; + found = true; + } + queue.push(...childPidsLinux(pid)); } + return found ? total : null; } - const result = spawnSync('ps', ['-o', 'rss=', '-p', String(pid)], { encoding: 'utf8' }); + const result = spawnSync('ps', ['-eo', 'pid=,ppid=,rss='], { encoding: 'utf8' }); if (result.status !== 0) return null; - const n = Number(result.stdout.trim()); - return Number.isFinite(n) ? n : null; + const rssByPid = new Map(); + const childrenByPid = new Map>(); + for (const line of result.stdout.split('\n')) { + const parts = line.trim().split(/\s+/); + if (parts.length < 3) continue; + const [pid, ppid, rss] = parts.map(Number); + if (!Number.isInteger(pid) || !Number.isInteger(ppid) || !Number.isFinite(rss)) continue; + rssByPid.set(pid, rss!); + const siblings = childrenByPid.get(ppid!) ?? []; + siblings.push(pid!); + childrenByPid.set(ppid!, siblings); + } + if (!rssByPid.has(rootPid)) return null; + let total = 0; + const seen = new Set(); + const queue = [rootPid]; + while (queue.length) { + const pid = queue.pop()!; + if (seen.has(pid)) continue; + seen.add(pid); + total += rssByPid.get(pid) ?? 0; + queue.push(...(childrenByPid.get(pid) ?? [])); + } + return total; } function percentile(sorted: Array, p: number): number { @@ -35,16 +114,32 @@ function median(values: Array): number { return sorted[mid]!; } +function stdDev(values: Array): number { + if (values.length < 2) return 0; + const mean = values.reduce((sum, v) => sum + v, 0) / values.length; + const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (values.length - 1); + return Math.sqrt(variance); +} + export function summarizeSamples(samples: Array): ScenarioMetrics { + if (samples.length === 0) { + throw new Error('summarizeSamples: no samples — refusing to report zeroed metrics'); + } const walls = samples.map((s) => s.wallMs).sort((a, b) => a - b); const rssValues = samples .map((s) => s.rssPeakBytes) .filter((v): v is number => v !== null); return { + iterations: samples.length, + wallMsMin: walls[0]!, wallMsMedian: median(walls), + // With few iterations p95 collapses onto the max — it is kept for continuity + // but wallMsMin / wallMsStdDev are the numbers to reason about. wallMsP95: percentile(walls, 95), + wallMsStdDev: stdDev(walls), rssPeakBytesMedian: rssValues.length > 0 ? median(rssValues) : null, + rssSampleCount: rssValues.length, samples, }; } @@ -53,14 +148,17 @@ export type MeasureCommandOptions = { cwd?: string; env?: Record; input?: string; - /** Sample RSS of the spawned process while it runs. Default true. */ + /** Sample RSS of the spawned process tree while it runs. Default true. */ sampleRss?: boolean; sampleIntervalMs?: number; timeoutMs?: number; }; +/** `ps` costs a spawn per sample, so back off when we cannot read /proc. */ +const DEFAULT_RSS_INTERVAL_MS = process.platform === 'linux' ? 10 : 50; + /** - * Spawn a command, measure wall time and optional peak RSS of the child. + * Spawn a command, measure wall time and peak RSS of the whole process tree. */ export function measureCommand( command: Array, @@ -72,7 +170,7 @@ export function measureCommand( } const sampleRss = options.sampleRss !== false; - const sampleIntervalMs = options.sampleIntervalMs ?? 25; + const sampleIntervalMs = options.sampleIntervalMs ?? DEFAULT_RSS_INTERVAL_MS; const timeoutMs = options.timeoutMs ?? 120_000; return new Promise((resolve, reject) => { @@ -110,14 +208,17 @@ export function measureCommand( }); if (sampleRss) { - sampling = setInterval(() => { - if (child.pid) { - const rss = rssKiB(child.pid); - if (rss !== null) { - peakRssKiB = peakRssKiB === null ? rss : Math.max(peakRssKiB, rss); - } + const takeSample = () => { + if (!child.pid) return; + const rss = rssTreeKiB(child.pid); + if (rss !== null) { + peakRssKiB = peakRssKiB === null ? rss : Math.max(peakRssKiB, rss); } - }, sampleIntervalMs); + }; + // Sample immediately — short-lived commands can finish inside one interval, + // which previously reported no RSS at all. + takeSample(); + sampling = setInterval(takeSample, sampleIntervalMs); } timers.timeout = setTimeout(() => { @@ -163,8 +264,19 @@ export type RepeatOptions = { warmup: number; /** Throw if any measured iteration exits non-zero. Default true. */ expectSuccess?: boolean; + /** + * Truncate captured output in failure messages. Scenarios that deliberately + * print secret values to stdout (redaction benches) set this low so fixture + * secrets do not end up in CI logs. + */ + maxFailureOutputChars?: number; }; +function truncate(value: string, max: number): string { + if (value.length <= max) return value; + return `${value.slice(0, max)}… [${value.length - max} more chars truncated]`; +} + /** * Run warmup + measured iterations of an async sample factory. */ @@ -173,6 +285,14 @@ export async function repeatMeasure( options: RepeatOptions, ): Promise { const expectSuccess = options.expectSuccess !== false; + const maxOutput = options.maxFailureOutputChars ?? 4_000; + + if (!Number.isInteger(options.iterations) || options.iterations < 1) { + throw new Error(`repeatMeasure: iterations must be a positive integer, got ${options.iterations}`); + } + if (!Number.isInteger(options.warmup) || options.warmup < 0) { + throw new Error(`repeatMeasure: warmup must be a non-negative integer, got ${options.warmup}`); + } for (let i = 0; i < options.warmup; i++) { const warm = await factory(); @@ -185,8 +305,9 @@ export async function repeatMeasure( for (let i = 0; i < options.iterations; i++) { const sample = await factory(); if (expectSuccess && sample.exitCode !== 0) { - const extra = 'stderr' in sample || 'stdout' in sample - ? `\nstdout:\n${(sample as { stdout?: string }).stdout ?? ''}\nstderr:\n${(sample as { stderr?: string }).stderr ?? ''}` + const withOutput = sample as { stdout?: string; stderr?: string }; + const extra = withOutput.stdout !== undefined || withOutput.stderr !== undefined + ? `\nstdout:\n${truncate(withOutput.stdout ?? '', maxOutput)}\nstderr:\n${truncate(withOutput.stderr ?? '', maxOutput)}` : ''; throw new Error(`Iteration ${i} failed with exit ${sample.exitCode}${extra}`); } @@ -200,13 +321,13 @@ export async function repeatMeasure( return summarizeSamples(samples); } -/** Copy a fixture directory into a unique work subdirectory. */ -export async function copyFixture( - sourceDir: string, - destDir: string, -): Promise { - const { cpSync, mkdirSync, rmSync } = await import('node:fs'); +/** + * Copy a fixture directory into a work subdirectory, so scenarios never mutate + * the checked-in fixtures (codegen output, caches, build artifacts). + */ +export function copyFixture(sourceDir: string, destDir: string): string { rmSync(destDir, { recursive: true, force: true }); mkdirSync(destDir, { recursive: true }); cpSync(sourceDir, destDir, { recursive: true }); + return destDir; } diff --git a/benchmarks/src/report.ts b/benchmarks/src/report.ts index 10a11b23c..515415563 100644 --- a/benchmarks/src/report.ts +++ b/benchmarks/src/report.ts @@ -9,23 +9,114 @@ function fmtRss(bytes: number | null): string { return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`; } +function fmtDelta(deltaMs: number, basisMs: number): string { + const sign = deltaMs >= 0 ? '+' : ''; + const pct = basisMs > 0 ? ` (${sign}${((deltaMs / basisMs) * 100).toFixed(1)}%)` : ''; + return `${sign}${deltaMs.toFixed(1)}ms${pct}`; +} + +type Comparison = { + label: string; + baseId: string; + againstId: string; +}; + +/** + * The interesting numbers are all differences: what varlock adds over a plain + * build, what telemetry adds, what redaction adds. Computing them here means the + * committed results are readable without doing arithmetic by hand. + * + * Ids are matched by suffix so one entry covers every install method. + */ +const COMPARISONS: Array = [ + { label: 'varlock run wrap overhead', baseId: 'cli.run.bare-node', againstId: 'cli.run.wrap.telemetry.off' }, + { label: 'telemetry cost (cli run)', baseId: 'cli.run.wrap.telemetry.off', againstId: 'cli.run.wrap.telemetry.on' }, + { label: 'telemetry cost (cli load, cold)', baseId: 'cli.load.cold.telemetry.off', againstId: 'cli.load.cold.telemetry.on' }, + { label: 'cache benefit (cold - warm)', baseId: 'cli.load.warm.telemetry.off', againstId: 'cli.load.cold.telemetry.off' }, + { label: 'stdout redaction cost', baseId: 'cli.run.redact-stdout.off', againstId: 'cli.run.redact-stdout.on' }, + { label: 'next build: varlock overhead', baseId: 'integration.next.build.baseline', againstId: 'integration.next.build.varlock.telemetry.off' }, + { label: 'next build: telemetry cost', baseId: 'integration.next.build.varlock.telemetry.off', againstId: 'integration.next.build.varlock.telemetry.on' }, + { label: 'next request: preventLeaks cost', baseId: 'integration.next.request.preventLeaks.off', againstId: 'integration.next.request.preventLeaks.on' }, + { label: 'next request: redactLogs cost', baseId: 'integration.next.request.redactLogs.off', againstId: 'integration.next.request.redactLogs.on' }, + { label: 'vite build: varlock overhead', baseId: 'integration.vite.build.baseline', againstId: 'integration.vite.build.varlock.telemetry.off' }, + { label: 'vite build: telemetry cost', baseId: 'integration.vite.build.varlock.telemetry.off', againstId: 'integration.vite.build.varlock.telemetry.on' }, + { label: 'vite request: preventLeaks cost', baseId: 'integration.vite.request.preventLeaks.off', againstId: 'integration.vite.request.preventLeaks.on' }, + { label: 'vite request: redactLogs cost', baseId: 'integration.vite.request.redactLogs.off', againstId: 'integration.vite.request.redactLogs.on' }, +]; + +/** Match `cli.load.cold.telemetry.off` against `cli.load.cold.telemetry.off.install.npm`. */ +function findByBaseId(scenarios: Array, baseId: string): Array { + return scenarios.filter((s) => s.id === baseId || s.id.startsWith(`${baseId}.install.`)); +} + +function installOf(s: ScenarioResult): string { + return s.id.includes('.install.') ? s.installMethod : 'n/a'; +} + +function comparisonRows(scenarios: Array): Array { + const rows: Array = []; + for (const cmp of COMPARISONS) { + const bases = findByBaseId(scenarios, cmp.baseId); + const againsts = findByBaseId(scenarios, cmp.againstId); + if (bases.length === 0 || againsts.length === 0) continue; + + for (const against of againsts) { + // Prefer the same install method, fall back to the single shared baseline + const base = bases.find((b) => b.installMethod === against.installMethod) + ?? (bases.length === 1 ? bases[0] : undefined); + if (!base) continue; + const deltaMedian = against.metrics.wallMsMedian - base.metrics.wallMsMedian; + const deltaMin = against.metrics.wallMsMin - base.metrics.wallMsMin; + // Noise guard: a delta smaller than the spread of either side is not a signal. + const noise = Math.max(base.metrics.wallMsStdDev, against.metrics.wallMsStdDev); + const verdict = Math.abs(deltaMedian) < noise ? ' _(within noise)_' : ''; + rows.push( + `| ${cmp.label} | ${installOf(against)} | ${fmtDelta(deltaMedian, base.metrics.wallMsMedian)} | ${fmtDelta(deltaMin, base.metrics.wallMsMin)}${verdict} |`, + ); + } + } + return rows; +} + export function formatSummaryMarkdown(result: BenchRunResult): string { const lines: Array = []; lines.push('## Varlock benchmarks'); lines.push(''); lines.push(`- **varlock:** ${result.meta.versions.varlock}`); lines.push(`- **trigger:** ${result.meta.trigger}`); - lines.push(`- **runner:** ${result.meta.runnerOs}/${result.meta.runnerArch}`); + lines.push(`- **runner:** ${result.meta.runnerOs}/${result.meta.runnerArch} (node ${result.meta.nodeVersion})`); lines.push(`- **timestamp:** ${result.meta.timestamp}`); if (result.meta.gitSha) lines.push(`- **git:** ${result.meta.gitSha.slice(0, 12)}`); + lines.push(`- **telemetry:** ${result.meta.telemetryMocked ? 'mocked locally' : 'not measured'}`); + lines.push(''); + + if (result.meta.notes.length > 0) { + lines.push('### Notes'); + lines.push(''); + for (const note of result.meta.notes) lines.push(`- ${note}`); + lines.push(''); + } + + const deltas = comparisonRows(result.scenarios); + if (deltas.length > 0) { + lines.push('### Deltas'); + lines.push(''); + lines.push('| Comparison | Install | Δ median | Δ min |'); + lines.push('|------------|---------|----------|-------|'); + lines.push(...deltas); + lines.push(''); + } + + lines.push('### Raw'); lines.push(''); - lines.push('| Scenario | Install | Telemetry | Median | p95 | Peak RSS |'); - lines.push('|----------|---------|-----------|--------|-----|----------|'); + lines.push('| Scenario | Install | Telemetry | Min | Median | p95 | StdDev | Peak RSS |'); + lines.push('|----------|---------|-----------|-----|--------|-----|--------|----------|'); const sorted = [...result.scenarios].sort((a, b) => a.id.localeCompare(b.id)); for (const s of sorted) { + const m = s.metrics; lines.push( - `| ${s.id} | ${s.installMethod} | ${s.telemetry} | ${fmtMs(s.metrics.wallMsMedian)} | ${fmtMs(s.metrics.wallMsP95)} | ${fmtRss(s.metrics.rssPeakBytesMedian)} |`, + `| ${s.id} | ${s.installMethod} | ${s.telemetry} | ${fmtMs(m.wallMsMin)} | ${fmtMs(m.wallMsMedian)} | ${fmtMs(m.wallMsP95)} | ${fmtMs(m.wallMsStdDev)} | ${fmtRss(m.rssPeakBytesMedian)} |`, ); } lines.push(''); @@ -37,6 +128,6 @@ export function printScenarioLine(s: ScenarioResult): void { ? ` rss=${fmtRss(s.metrics.rssPeakBytesMedian)}` : ''; console.log( - ` ${s.id} [${s.installMethod} telemetry=${s.telemetry}] median=${fmtMs(s.metrics.wallMsMedian)} p95=${fmtMs(s.metrics.wallMsP95)}${rss}`, + ` ${s.id} [${s.installMethod} telemetry=${s.telemetry}] min=${fmtMs(s.metrics.wallMsMin)} median=${fmtMs(s.metrics.wallMsMedian)} sd=${fmtMs(s.metrics.wallMsStdDev)}${rss}`, ); } diff --git a/benchmarks/src/run.ts b/benchmarks/src/run.ts index 940d7dd8c..530e994f3 100644 --- a/benchmarks/src/run.ts +++ b/benchmarks/src/run.ts @@ -1,22 +1,34 @@ import { mkdirSync, writeFileSync, existsSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { join, resolve, dirname } from 'node:path'; import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; import { installVarlockBun, installVarlockNpm, - npmViewVersion, + npmInstallDir, + bunInstallDir, + probeCli, readInstalledVersion, seaInvocation, + tryNpmViewVersion, + waitForNpmPackage, } from './install.ts'; import { runAllScenarios, SCENARIO_GROUPS } from './scenarios/index.ts'; import { formatSummaryMarkdown } from './report.ts'; -import type { BenchContext, BenchRunResult, TriggerKind } from './types.ts'; +import { measureCommand } from './measure.ts'; +import { startTelemetryMock } from './telemetry-mock.ts'; +import { ALL_TELEMETRY_MODES, telemetryEnv } from './telemetry.ts'; +import type { + BenchContext, BenchRunResult, CliInvocation, TelemetryMode, TriggerKind, +} from './types.ts'; const ROOT_DIR = resolve(import.meta.dirname, '..'); const FIXTURES_DIR = join(ROOT_DIR, 'fixtures'); const RESULTS_DIR = join(ROOT_DIR, 'results'); const WORK_DIR = join(ROOT_DIR, '.work'); +const TRIGGER_KINDS: Array = ['release', 'workflow_dispatch', 'local']; + type Args = { version: string; seaPath: string | null; @@ -29,6 +41,15 @@ type Args = { help: boolean; }; +function parsePositiveInt(raw: string | undefined, flag: string, { allowZero = false } = {}): number { + const n = Number(raw); + const min = allowZero ? 0 : 1; + if (raw === undefined || raw === '' || !Number.isInteger(n) || n < min) { + throw new Error(`${flag} expects an integer >= ${min}, got ${JSON.stringify(raw)}`); + } + return n; +} + function parseArgs(argv: Array): Args { const args: Args = { version: 'latest', @@ -38,23 +59,44 @@ function parseArgs(argv: Array): Args { warmup: 1, only: [], skipInstall: false, - trigger: (process.env.BENCH_TRIGGER as TriggerKind | undefined) ?? 'local', + trigger: 'local', help: false, }; + const envTrigger = process.env.BENCH_TRIGGER; + if (envTrigger) { + if (!TRIGGER_KINDS.includes(envTrigger as TriggerKind)) { + throw new Error(`BENCH_TRIGGER must be one of ${TRIGGER_KINDS.join(', ')}, got ${JSON.stringify(envTrigger)}`); + } + args.trigger = envTrigger as TriggerKind; + } + for (let i = 0; i < argv.length; i++) { const a = argv[i]!; if (a === '--help' || a === '-h') args.help = true; else if (a === '--version') args.version = argv[++i] ?? args.version; else if (a === '--sea-path') args.seaPath = argv[++i] ?? null; else if (a === '--out') args.out = argv[++i] ?? null; - else if (a === '--iterations') args.iterations = Number(argv[++i]); - else if (a === '--warmup') args.warmup = Number(argv[++i]); - else if (a === '--only') args.only = (argv[++i] ?? '').split(',').filter(Boolean); + else if (a === '--iterations') args.iterations = parsePositiveInt(argv[++i], '--iterations'); + else if (a === '--warmup') args.warmup = parsePositiveInt(argv[++i], '--warmup', { allowZero: true }); + else if (a === '--only') args.only = (argv[++i] ?? '').split(',').map((s) => s.trim()).filter(Boolean); else if (a === '--skip-install') args.skipInstall = true; - else if (a === '--trigger') args.trigger = (argv[++i] as TriggerKind) ?? args.trigger; - else throw new Error(`Unknown argument: ${a}`); + else if (a === '--trigger') { + const value = argv[++i]; + if (!value || !TRIGGER_KINDS.includes(value as TriggerKind)) { + throw new Error(`--trigger must be one of ${TRIGGER_KINDS.join(', ')}, got ${JSON.stringify(value)}`); + } + args.trigger = value as TriggerKind; + } else throw new Error(`Unknown argument: ${a}`); + } + + // Unknown group names used to produce an empty run that still looked successful. + const knownGroups = SCENARIO_GROUPS.map((g) => g.name); + const unknown = args.only.filter((name) => !knownGroups.includes(name)); + if (unknown.length > 0) { + throw new Error(`--only got unknown scenario group(s): ${unknown.join(', ')}\nKnown groups: ${knownGroups.join(', ')}`); } + return args; } @@ -70,11 +112,22 @@ Options: --warmup Warmup iterations (default: 1) --only Comma-separated scenario groups: ${groups} --skip-install Reuse .work/installs from a previous run - --trigger release | workflow_dispatch | local + --trigger ${TRIGGER_KINDS.join(' | ')} --help `; } +/** Bad flags are user error, not a crash — print the problem and the usage, no stack. */ +function parseArgsOrExit(argv: Array): Args { + try { + return parseArgs(argv); + } catch (err) { + console.error(`${(err as Error).message}\n`); + console.error(usage()); + process.exit(1); + } +} + function gitSha(): string | null { const r = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', cwd: ROOT_DIR }); return r.status === 0 ? r.stdout.trim() : null; @@ -83,39 +136,75 @@ function gitSha(): string | null { function defaultOutPath(version: string): string { const iso = new Date().toISOString().replace(/[:.]/g, '-'); const runId = process.env.GITHUB_RUN_ID ?? 'local'; - mkdirSync(RESULTS_DIR, { recursive: true }); return join(RESULTS_DIR, `${iso}-varlock@${version}-${runId}.json`); } +/** + * Confirm the tested varlock build honours VARLOCK_POSTHOG_HOST before running any + * telemetry-on scenario. A version published before that override existed would + * send real events to the production collector instead, so those scenarios get + * dropped rather than measured. + */ +async function checkTelemetryMockable( + cli: CliInvocation, + cwd: string, + mockEnv: Record, + received: () => number, +): Promise { + const before = received(); + const result = await measureCommand([...cli.command, 'load'], { + cwd, + env: telemetryEnv('on', mockEnv), + timeoutMs: 60_000, + }); + if (result.exitCode !== 0) return false; + // The CLI's exit hook only waits ~500ms on the in-flight request; give the + // loopback round-trip a moment longer before concluding nothing arrived. + for (let i = 0; i < 20 && received() === before; i++) { + await new Promise((r) => { + setTimeout(r, 100); + }); + } + return received() > before; +} + async function main(): Promise { - const args = parseArgs(process.argv.slice(2)); + const args = parseArgsOrExit(process.argv.slice(2)); if (args.help) { console.log(usage()); return; } - const resolvedVersion = args.version === 'latest' - ? npmViewVersion('varlock') - : args.version; + // Waits when the version is not on npm yet — a release-triggered run can start + // before the registry has caught up. + const resolvedVersion = await waitForNpmPackage( + args.version === 'latest' ? 'varlock@latest' : `varlock@${args.version}`, + ); console.log(`Benchmarking varlock@${resolvedVersion}`); mkdirSync(WORK_DIR, { recursive: true }); - const clis = []; + const notes: Array = []; + const note = (message: string) => { + notes.push(message); + console.log(` note: ${message}`); + }; + + const clis: Array = []; if (args.skipInstall) { - const npmCli = join(WORK_DIR, 'installs', 'npm', 'node_modules', 'varlock', 'bin', 'cli.js'); - const bunCli = join(WORK_DIR, 'installs', 'bun', 'node_modules', 'varlock', 'bin', 'cli.js'); + const npmCli = join(npmInstallDir(WORK_DIR), 'node_modules', 'varlock', 'bin', 'cli.js'); + const bunCli = join(bunInstallDir(WORK_DIR), 'node_modules', 'varlock', 'bin', 'cli.js'); if (!existsSync(npmCli) || !existsSync(bunCli)) { throw new Error('--skip-install requires existing .work/installs/{npm,bun}'); } clis.push( - { command: [process.execPath, npmCli], label: 'npm' as const, packageManager: 'npm' as const }, - { command: [process.execPath, bunCli], label: 'bun' as const, packageManager: 'bun' as const }, + { command: [process.execPath, npmCli], label: 'npm', packageManager: 'npm' }, + { command: ['bun', bunCli], label: 'bun', packageManager: 'bun' }, ); } else { - console.log('Installing varlock via npm...'); + console.log('Installing varlock via npm (run with node)...'); clis.push(installVarlockNpm(WORK_DIR, resolvedVersion)); - console.log('Installing varlock via bun...'); + console.log('Installing varlock via bun (run with bun)...'); clis.push(installVarlockBun(WORK_DIR, resolvedVersion)); } @@ -123,67 +212,106 @@ async function main(): Promise { console.log(`Using SEA binary at ${args.seaPath}`); clis.push(seaInvocation(resolve(args.seaPath))); } else { - console.log('No --sea-path; skipping SEA scenarios'); + note('SEA scenarios skipped: no --sea-path given'); } - const ctx: BenchContext = { - version: resolvedVersion, - rootDir: ROOT_DIR, - fixturesDir: FIXTURES_DIR, - workDir: WORK_DIR, - iterations: args.iterations, - warmup: args.warmup, - clis, - seaPath: args.seaPath, - }; + // A CLI that cannot even print its version would fail every scenario it appears + // in. Drop it with a recorded note rather than taking the whole suite down. + const usableClis = clis.filter((cli) => { + const probe = probeCli(cli); + if (!probe.ok) { + note(`${cli.label} install skipped: \`varlock --version\` failed (${probe.error})`); + return false; + } + return true; + }); + if (usableClis.length === 0) { + throw new Error('No usable varlock CLI invocations — nothing to benchmark'); + } + + const telemetryMock = await startTelemetryMock(); + const telemetryMockEnv = { VARLOCK_POSTHOG_HOST: telemetryMock.url }; + let telemetryModes: Array = ['off']; + + try { + const mockable = await checkTelemetryMockable( + usableClis[0]!, + join(FIXTURES_DIR, 'cli-basic'), + telemetryMockEnv, + telemetryMock.requestCount, + ); + if (mockable) { + telemetryModes = ALL_TELEMETRY_MODES; + console.log(`Telemetry mock reachable at ${telemetryMock.url} — telemetry-on scenarios enabled`); + } else { + note( + `telemetry-on scenarios skipped: varlock@${resolvedVersion} does not honour VARLOCK_POSTHOG_HOST, ` + + 'and benchmarks never send real telemetry', + ); + } - const scenarios = await runAllScenarios(ctx, args.only.length ? args.only : undefined); - - const npmRoot = join(WORK_DIR, 'installs', 'npm'); - const result: BenchRunResult = { - meta: { - timestamp: new Date().toISOString(), - gitSha: gitSha(), - githubRunId: process.env.GITHUB_RUN_ID ?? null, - runnerOs: process.platform, - runnerArch: process.arch, - versions: { - varlock: resolvedVersion, - nextjsIntegration: (() => { - try { - return npmViewVersion('@varlock/nextjs-integration'); - } catch { - return undefined; - } - })(), - viteIntegration: (() => { - try { - return npmViewVersion('@varlock/vite-integration'); - } catch { - return undefined; - } - })(), - '@env-spec/parser': readInstalledVersion(npmRoot, '@env-spec/parser') ?? undefined, + const ctx: BenchContext = { + version: resolvedVersion, + integrationVersions: { + nextjs: tryNpmViewVersion('@varlock/nextjs-integration'), + vite: tryNpmViewVersion('@varlock/vite-integration'), }, - trigger: args.trigger, - }, - scenarios, - }; + rootDir: ROOT_DIR, + fixturesDir: FIXTURES_DIR, + workDir: WORK_DIR, + iterations: args.iterations, + warmup: args.warmup, + clis: usableClis, + seaPath: args.seaPath, + telemetryModes, + telemetryMockEnv: telemetryModes.includes('on') ? telemetryMockEnv : {}, + // Enables the on-disk resolver cache even in CI, where varlock otherwise + // falls back to a per-process memory cache that cannot survive between + // invocations — which would make the warm-load scenario measure nothing. + cacheKey: randomBytes(32).toString('hex'), + note, + }; - const outPath = args.out ? resolve(args.out) : defaultOutPath(resolvedVersion); - mkdirSync(join(outPath, '..'), { recursive: true }); - writeFileSync(outPath, `${JSON.stringify(result, null, 2)}\n`); - console.log(`\nWrote ${outPath}`); + const scenarios = await runAllScenarios(ctx, args.only.length ? args.only : undefined); - const summary = formatSummaryMarkdown(result); - console.log(`\n${summary}`); + const result: BenchRunResult = { + meta: { + timestamp: new Date().toISOString(), + gitSha: gitSha(), + githubRunId: process.env.GITHUB_RUN_ID ?? null, + runnerOs: process.platform, + runnerArch: process.arch, + nodeVersion: process.version, + versions: { + varlock: resolvedVersion, + nextjsIntegration: ctx.integrationVersions.nextjs, + viteIntegration: ctx.integrationVersions.vite, + '@env-spec/parser': readInstalledVersion(npmInstallDir(WORK_DIR), '@env-spec/parser') ?? undefined, + }, + trigger: args.trigger, + telemetryMocked: telemetryModes.includes('on'), + notes, + }, + scenarios, + }; - if (process.env.GITHUB_STEP_SUMMARY) { - writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' }); - } + const outPath = args.out ? resolve(args.out) : defaultOutPath(resolvedVersion); + mkdirSync(dirname(outPath), { recursive: true }); + writeFileSync(outPath, `${JSON.stringify(result, null, 2)}\n`); + console.log(`\nWrote ${outPath}`); - // Also write a pointer file used by CI commit step - writeFileSync(join(WORK_DIR, 'last-result-path.txt'), `${outPath}\n`); + const summary = formatSummaryMarkdown(result); + console.log(`\n${summary}`); + + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' }); + } + + // Also write a pointer file used by CI commit step + writeFileSync(join(WORK_DIR, 'last-result-path.txt'), `${outPath}\n`); + } finally { + await telemetryMock.close(); + } } main().catch((err) => { diff --git a/benchmarks/src/scenarios/cli-load.ts b/benchmarks/src/scenarios/cli-load.ts index 4b4a1eefb..2828217fe 100644 --- a/benchmarks/src/scenarios/cli-load.ts +++ b/benchmarks/src/scenarios/cli-load.ts @@ -1,27 +1,35 @@ -import { join } from 'node:path'; import type { BenchContext, ScenarioResult } from '../types.ts'; import { measureCommand, repeatMeasure } from '../measure.ts'; -import { TELEMETRY_MODES, telemetryEnv } from '../telemetry.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; export async function runCliLoadScenarios(ctx: BenchContext): Promise> { - const cwd = join(ctx.fixturesDir, 'cli-basic'); + const cwd = fixtureWorkDir(ctx, 'cli-basic', '-load'); const results: Array = []; for (const cli of ctx.clis) { - for (const telemetry of TELEMETRY_MODES) { - const env = telemetryEnv(telemetry); + for (const telemetry of ctx.telemetryModes) { + // _VARLOCK_CACHE_KEY forces the on-disk resolver cache. Without it, CI falls + // back to an in-process memory cache (see loader.ts cache policy), which + // cannot survive between CLI invocations — so "warm" would silently measure + // exactly the same work as "cold". + const env = { + ...telemetryEnv(telemetry, ctx.telemetryMockEnv), + _VARLOCK_CACHE_KEY: ctx.cacheKey, + }; const cold = await repeatMeasure( async () => measureCommand([...cli.command, 'load', '--clear-cache'], { cwd, env }), { iterations: ctx.iterations, warmup: ctx.warmup }, ); results.push({ - id: `cli.load.cold.telemetry.${telemetry}`, + id: cliScenarioId(`cli.load.cold.telemetry.${telemetry}`, cli), facet: 'cli-load', installMethod: cli.label, packageManager: cli.packageManager, telemetry, metrics: cold, + notes: 'Disk cache cleared before every iteration', }); // Warm: one clear then repeated loads without clear @@ -31,12 +39,13 @@ export async function runCliLoadScenarios(ctx: BenchContext): Promise> { - const cwd = join(ctx.fixturesDir, 'cli-basic'); + const cwd = fixtureWorkDir(ctx, 'cli-basic', '-run'); const childJs = join(cwd, 'child.js'); const emitJs = join(cwd, 'emit-secret.js'); const results: Array = []; - // Bare node baseline (not tied to an install method; recorded once under npm label) + // Bare node baseline (no varlock in the picture; installMethod is not meaningful here) const bare = await repeatMeasure( async () => measureCommand([process.execPath, childJs], { cwd }), { iterations: ctx.iterations, warmup: ctx.warmup }, @@ -21,13 +33,13 @@ export async function runCliRunScenarios(ctx: BenchContext): Promise measureCommand( [...cli.command, 'run', '--', process.execPath, childJs], @@ -36,7 +48,7 @@ export async function runCliRunScenarios(ctx: BenchContext): Promise measureCommand( [...cli.command, 'run', '--redact-stdout', '--', process.execPath, emitJs], - { cwd, env: envOff }, + { cwd, env: envOff, timeoutMs: 300_000 }, ), - { iterations: ctx.iterations, warmup: Math.max(1, ctx.warmup) }, + redactRepeatOpts, ); results.push({ - id: 'cli.run.redact-stdout.on', + id: cliScenarioId('cli.run.redact-stdout.on', cli), facet: 'cli-run', installMethod: cli.label, packageManager: cli.packageManager, telemetry: 'off', metrics: redactOn, + notes: `${REDACT_BENCH_LINES} stdout lines containing secrets`, }); const redactOff = await repeatMeasure( async () => measureCommand( [...cli.command, 'run', '--no-redact-stdout', '--', process.execPath, emitJs], - { cwd, env: envOff }, + { cwd, env: envOff, timeoutMs: 300_000 }, ), - { iterations: ctx.iterations, warmup: Math.max(1, ctx.warmup) }, + redactRepeatOpts, ); results.push({ - id: 'cli.run.redact-stdout.off', + id: cliScenarioId('cli.run.redact-stdout.off', cli), facet: 'cli-run', installMethod: cli.label, packageManager: cli.packageManager, telemetry: 'off', metrics: redactOff, + notes: `${REDACT_BENCH_LINES} stdout lines containing secrets`, }); } diff --git a/benchmarks/src/scenarios/cli-scan-audit.ts b/benchmarks/src/scenarios/cli-scan-audit.ts index 8012de821..68eafccc0 100644 --- a/benchmarks/src/scenarios/cli-scan-audit.ts +++ b/benchmarks/src/scenarios/cli-scan-audit.ts @@ -1,10 +1,10 @@ -import { join } from 'node:path'; import type { BenchContext, ScenarioResult } from '../types.ts'; import { measureCommand, repeatMeasure } from '../measure.ts'; import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; export async function runCliScanAuditScenarios(ctx: BenchContext): Promise> { - const cwd = join(ctx.fixturesDir, 'cli-basic'); + const cwd = fixtureWorkDir(ctx, 'cli-basic', '-scan'); const results: Array = []; const env = telemetryEnv('off'); @@ -22,7 +22,7 @@ export async function runCliScanAuditScenarios(ctx: BenchContext): Promise string): string { + return `export default function Page() { + const hasSensitive = !!${read('SENSITIVE_VAR')}; + + console.log('secret-log-test:', ${read('SENSITIVE_VAR')}); + return (
-

bench baseline

-

{process.env.NEXT_PUBLIC_VAR || process.env.PUBLIC_VAR || 'none'}

+

bench

+

Next prefixed var: {${read('NEXT_PUBLIC_VAR')}}

+

Unprefixed var: {${read('PUBLIC_VAR')}}

+

Env specific var: {${read('ENV_SPECIFIC_VAR')}}

+

Has sensitive: {hasSensitive ? 'yes' : 'no'}

); } `; +} + +const BASELINE_PAGE = pageSource((key) => `process.env.${key}`); +const VARLOCK_PAGE = `import { ENV } from 'varlock/env'; +${pageSource((key) => `ENV.${key}`)}`; + +// force-dynamic keeps these handlers running per request. Next 15 already treats +// GET handlers as dynamic by default, but the bench should not silently start +// measuring a prerendered response if that default ever changes. const ECHO_ROUTE = `import { NextResponse } from 'next/server'; +export const dynamic = 'force-dynamic'; + export async function GET() { // Large body without the sensitive value so preventLeaks scanning still runs // but the request succeeds (no leak throw). - const body = \`ok padding=\${'x'.repeat(16_384)}\`; + const body = \`ok padding=\${'x'.repeat(${LEAK_SCAN_BODY_BYTES})}\`; return new NextResponse(body, { headers: { 'content-type': 'text/plain' }, }); @@ -44,6 +78,8 @@ export async function GET() { const LOG_ROUTE = `import { NextResponse } from 'next/server'; import { ENV } from 'varlock/env'; +export const dynamic = 'force-dynamic'; + const SECRET_KEYS = [ 'SENSITIVE_VAR', 'SECRET_TOKEN', @@ -66,7 +102,7 @@ const SECRET_KEYS = [ ]; export async function GET() { - for (let i = 0; i < 200; i++) { + for (let i = 0; i < ${REDACT_LOG_LINES}; i++) { const key = SECRET_KEYS[i % SECRET_KEYS.length]; console.log(\`bench-log-\${i}:\`, ENV[key]); } @@ -76,15 +112,12 @@ export async function GET() { } `; -function createNextEnv( - ctx: BenchContext, - mode: 'baseline' | 'varlock', - labelSuffix = '', -): FrameworkTestEnv { +function createNextEnv(ctx: BenchContext, mode: 'baseline' | 'varlock'): FrameworkTestEnv { const withVarlock = mode === 'varlock'; + const integrationVersion = ctx.integrationVersions.nextjs ?? 'latest'; return new FrameworkTestEnv({ testDir: NEXT_TEST_DIR, - framework: `bench-next-${mode}${labelSuffix}`, + framework: `bench-next-${mode}`, packageManager: 'npm', usePublished: true, installTimeout: 180_000, @@ -97,7 +130,7 @@ function createNextEnv( ...(withVarlock ? { varlock: ctx.version, - '@varlock/nextjs-integration': 'latest', + '@varlock/nextjs-integration': integrationVersion, } : {}), }, @@ -116,206 +149,126 @@ function createNextEnv( }); } -async function waitForUrl(url: string, timeoutMs: number): Promise { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - try { - const res = await fetch(url); - if (res.status > 0) return; - } catch { - // retry - } - await new Promise((r) => { - setTimeout(r, 250); - }); - } - throw new Error(`Timed out waiting for ${url}`); +function prepareBuildFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { + env.prepareFiles({ + templateFiles: { '.env.dev': 'schemas/.env.dev' }, + files: [ + { path: '.env.schema', content: NEXT_MANY_SECRETS_SCHEMA }, + { + path: 'next.config.mjs', + content: mode === 'baseline' ? BASELINE_NEXT_CONFIG : VARLOCK_NEXT_CONFIG, + }, + { path: 'app/page.tsx', content: mode === 'baseline' ? BASELINE_PAGE : VARLOCK_PAGE }, + ], + }); } -async function measurePathLatency( - baseUrl: string, - path: string, +async function measureBuild( + fixture: FrameworkTestEnv, iterations: number, - warmup: number, -): Promise { + telemetry: TelemetryMode, + mockEnv: Record, +) { return repeatMeasure( async () => { - const start = performance.now(); - const res = await fetch(`${baseUrl}${path}`); - const wallMs = performance.now() - start; - if (!res.ok) { - throw new Error(`Request failed: ${res.status} ${await res.text()}`); - } - await res.text(); - return { wallMs, rssPeakBytes: null, exitCode: 0 }; + rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); + return measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv(telemetry, mockEnv), APP_ENV: 'dev', CI: '1' }, + }); }, - { iterations, warmup }, + { iterations, warmup: 0 }, ); } -async function withNextServer( - projectDir: string, - port: number, - readyPath: string, - fn: () => Promise, -): Promise { - const server = spawn('npx', ['next', 'start', '-H', '127.0.0.1', '-p', String(port)], { - cwd: projectDir, - env: { - ...process.env, - ...Object.fromEntries( - Object.entries(telemetryEnv('off')).filter(([, v]) => v !== undefined), - ), - APP_ENV: 'dev', - PORT: String(port), - HOSTNAME: '127.0.0.1', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - let stderr = ''; - server.stderr?.on('data', (c: Buffer) => { - stderr += c.toString(); - }); - - try { - await waitForUrl(`http://127.0.0.1:${port}${readyPath}`, 90_000); - return await fn(); - } catch (err) { - throw new Error(`${String(err)}\nnext start stderr:\n${stderr}`); - } finally { - server.kill('SIGTERM'); - await new Promise((r) => { - setTimeout(r, 500); - }); - if (!server.killed) server.kill('SIGKILL'); - } -} - -function prepareNextFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { - if (mode === 'baseline') { - env.prepareFiles({ - templateFiles: { - '.env.dev': 'schemas/.env.dev', - }, - files: [ - { path: '.env.schema', content: NEXT_MANY_SECRETS_SCHEMA }, - { path: 'next.config.mjs', content: BASELINE_NEXT_CONFIG }, - { path: 'app/page.tsx', content: BASELINE_PAGE }, - ], - }); - return; - } - - env.prepareFiles({ - templateFiles: { - '.env.dev': 'schemas/.env.dev', - 'app/page.tsx': 'pages/basic-page.tsx', - }, - files: [{ path: '.env.schema', content: NEXT_MANY_SECRETS_SCHEMA }], - }); -} - export async function runNextScenarios(ctx: BenchContext): Promise> { const results: Array = []; const buildIterations = Math.max(2, Math.min(3, ctx.iterations)); + const requestIterations = Math.max(10, ctx.iterations * 2); console.log(' preparing next baseline (framework-tests)...'); - { - const fixture = createNextEnv(ctx, 'baseline'); - await fixture.setup(); - prepareNextFiles(fixture, 'baseline'); - const build = await repeatMeasure( - async () => { - rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); - return measureCommand(['npx', 'next', 'build'], { - cwd: fixture.dir, - timeoutMs: 300_000, - env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, - }); - }, - { iterations: buildIterations, warmup: 0 }, - ); + const baseline = createNextEnv(ctx, 'baseline'); + await baseline.setup(); + try { + prepareBuildFiles(baseline, 'baseline'); results.push({ id: 'integration.next.build.baseline', facet: 'integration-next', installMethod: 'npm', packageManager: 'npm', telemetry: 'off', - metrics: build, + metrics: await measureBuild(baseline, buildIterations, 'off', ctx.telemetryMockEnv), + notes: 'Cold next build, no varlock installed', }); - await fixture.teardown(); + } finally { + await baseline.teardown(); } - // Varlock build: telemetry on/off (next shells out to varlock load via @next/env override) - for (const telemetry of TELEMETRY_MODES) { - console.log(` preparing next varlock telemetry.${telemetry} (framework-tests)...`); - const fixture = createNextEnv(ctx, 'varlock', `-telemetry-${telemetry}`); - await fixture.setup(); - prepareNextFiles(fixture, 'varlock'); - const build = await repeatMeasure( - async () => { - rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); - return measureCommand(['npx', 'next', 'build'], { - cwd: fixture.dir, - timeoutMs: 300_000, - env: { ...telemetryEnv(telemetry), APP_ENV: 'dev', CI: '1' }, - }); - }, - { iterations: buildIterations, warmup: 0 }, - ); - results.push({ - id: `integration.next.build.varlock.telemetry.${telemetry}`, - facet: 'integration-next', - installMethod: 'npm', - packageManager: 'npm', - telemetry, - metrics: build, - notes: 'Cold next build; telemetry affects sync varlock load spawn', - }); - await fixture.teardown(); - } + // One varlock fixture serves every varlock arm. Installing a fresh project per + // telemetry mode and per latency config meant 7 npm installs where 2 will do, + // which dominated the wall-clock of this group. + console.log(' preparing next varlock (framework-tests)...'); + const fixture = createNextEnv(ctx, 'varlock'); + await fixture.setup(); + try { + prepareBuildFiles(fixture, 'varlock'); + for (const telemetry of ctx.telemetryModes) { + console.log(` next varlock build, telemetry=${telemetry}...`); + results.push({ + id: `integration.next.build.varlock.telemetry.${telemetry}`, + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry, + metrics: await measureBuild(fixture, buildIterations, telemetry, ctx.telemetryMockEnv), + notes: 'Cold next build; telemetry affects sync varlock load spawn', + }); + } - console.log(' measuring next request latency (preventLeaks / redactLogs)...'); - const requestIterations = Math.max(10, ctx.iterations * 2); - for (const [label, preventLeaks, redactLogs, port, path] of [ - ['preventLeaks.on', true, true, 3451, '/api/echo'], - ['preventLeaks.off', false, true, 3452, '/api/echo'], - ['redactLogs.on', true, true, 3453, '/api/log'], - ['redactLogs.off', true, false, 3454, '/api/log'], - ] as const) { - const fixture = createNextEnv(ctx, 'varlock', `-${label}`); - await fixture.setup(); + console.log(' measuring next request latency (preventLeaks / redactLogs)...'); + for (const [label, preventLeaks, redactLogs, port, path] of [ + ['preventLeaks.on', true, true, 3451, '/api/echo'], + ['preventLeaks.off', false, true, 3452, '/api/echo'], + ['redactLogs.on', true, true, 3453, '/api/log'], + ['redactLogs.off', true, false, 3454, '/api/log'], + ] as const) { + fixture.prepareFiles({ + templateFiles: { '.env.dev': 'schemas/.env.dev' }, + files: [ + { path: '.env.schema', content: withSchemaFlags(NEXT_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, + { path: 'next.config.mjs', content: VARLOCK_NEXT_CONFIG }, + { path: 'app/page.tsx', content: VARLOCK_PAGE }, + { path: 'app/api/echo/route.js', content: ECHO_ROUTE }, + { path: 'app/api/log/route.js', content: LOG_ROUTE }, + ], + }); - fixture.prepareFiles({ - templateFiles: { - '.env.dev': 'schemas/.env.dev', - 'app/page.tsx': 'pages/basic-page.tsx', - }, - files: [ - { path: '.env.schema', content: withSchemaFlags(NEXT_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, - { path: 'app/api/echo/route.js', content: ECHO_ROUTE }, - { path: 'app/api/log/route.js', content: LOG_ROUTE }, - ], - }); + rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); + const buildResult = await measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + }); + if (buildResult.exitCode !== 0) { + throw new Error(`next build failed for ${label}:\n${buildResult.stderr}\n${buildResult.stdout}`); + } - const buildResult = await measureCommand(['npx', 'next', 'build'], { - cwd: fixture.dir, - timeoutMs: 300_000, - env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, - }); - if (buildResult.exitCode !== 0) { - await fixture.teardown(); - throw new Error(`next build failed for ${label}:\n${buildResult.stderr}\n${buildResult.stdout}`); - } + const latency = await withServer( + ['npx', 'next', 'start', '-H', '127.0.0.1', '-p', String(port)], + { + cwd: fixture.dir, + env: { + ...telemetryEnv('off'), + APP_ENV: 'dev', + PORT: String(port), + HOSTNAME: '127.0.0.1', + }, + readyUrl: `http://127.0.0.1:${port}${path}`, + }, + () => measurePathLatency(`http://127.0.0.1:${port}`, path, requestIterations, 3), + ); - try { - const latency = await withNextServer(fixture.dir, port, path, () => measurePathLatency( - `http://127.0.0.1:${port}`, - path, - requestIterations, - 3, - )); results.push({ id: `integration.next.request.${label}`, facet: 'integration-next', @@ -324,12 +277,12 @@ export async function runNextScenarios(ctx: BenchContext): Promise string): string { + return `document.getElementById('app')!.innerHTML = \` +

bench

+

\${${read('PUBLIC_VAR')}}

+

\${${read('API_URL')}}

+

\${${read('ENV_SPECIFIC_VAR')}}

+\`; +`; +} + +const BASELINE_MAIN = mainSource((key) => `import.meta.env.${key}`); +const VARLOCK_MAIN = `import { ENV } from 'varlock/env'; + +${mainSource((key) => `ENV.${key}`)}`; + /** * Dev-server middleware used for latency benches: * - /api/echo: large body without secrets (preventLeaks still scans) @@ -55,10 +77,10 @@ export default defineConfig({ configureServer(server) { server.middlewares.use('/api/echo', (_req, res) => { res.setHeader('content-type', 'text/plain'); - res.end(\`ok padding=\${'x'.repeat(16_384)}\`); + res.end(\`ok padding=\${'x'.repeat(${LEAK_SCAN_BODY_BYTES})}\`); }); server.middlewares.use('/api/log', (_req, res) => { - for (let i = 0; i < 200; i++) { + for (let i = 0; i < ${REDACT_LOG_LINES}; i++) { const key = SECRET_KEYS[i % SECRET_KEYS.length]; console.log(\`bench-log-\${i}:\`, ENV[key]); } @@ -71,15 +93,12 @@ export default defineConfig({ }); `; -function createViteEnv( - ctx: BenchContext, - mode: 'baseline' | 'varlock', - labelSuffix = '', -): FrameworkTestEnv { +function createViteEnv(ctx: BenchContext, mode: 'baseline' | 'varlock'): FrameworkTestEnv { const withVarlock = mode === 'varlock'; + const integrationVersion = ctx.integrationVersions.vite ?? 'latest'; return new FrameworkTestEnv({ testDir: VITE_TEST_DIR, - framework: `bench-vite-${mode}${labelSuffix}`, + framework: `bench-vite-${mode}`, packageManager: 'npm', usePublished: true, installTimeout: 180_000, @@ -88,7 +107,7 @@ function createViteEnv( ...(withVarlock ? { varlock: ctx.version, - '@varlock/vite-integration': 'latest', + '@varlock/vite-integration': integrationVersion, } : {}), }, @@ -100,202 +119,112 @@ function createViteEnv( }); } -function prepareViteFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { - if (mode === 'baseline') { - env.prepareFiles({ - templateFiles: { - '.env.dev': 'schemas/.env.dev', - 'index.html': 'html/basic.html', - }, - files: [ - { path: '.env.schema', content: VITE_MANY_SECRETS_SCHEMA }, - { path: 'vite.config.ts', content: BASELINE_VITE_CONFIG }, - { path: 'src/main.ts', content: BASELINE_MAIN }, - ], - }); - return; - } - +function prepareBuildFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { env.prepareFiles({ templateFiles: { '.env.dev': 'schemas/.env.dev', - 'vite.config.ts': 'vite-configs/vite.config.ts', 'index.html': 'html/basic.html', - 'src/main.ts': 'pages/basic-page.ts', }, - files: [{ path: '.env.schema', content: VITE_MANY_SECRETS_SCHEMA }], + files: [ + { path: '.env.schema', content: VITE_MANY_SECRETS_SCHEMA }, + { + path: 'vite.config.ts', + content: mode === 'baseline' ? BASELINE_VITE_CONFIG : VARLOCK_VITE_CONFIG, + }, + { path: 'src/main.ts', content: mode === 'baseline' ? BASELINE_MAIN : VARLOCK_MAIN }, + ], }); } -async function waitForUrl(url: string, timeoutMs: number): Promise { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - try { - const res = await fetch(url); - if (res.status > 0) return; - } catch { - // retry - } - await new Promise((r) => { - setTimeout(r, 250); - }); - } - throw new Error(`Timed out waiting for ${url}`); -} - -async function measurePathLatency( - baseUrl: string, - path: string, +async function measureBuild( + fixture: FrameworkTestEnv, iterations: number, - warmup: number, -): Promise { + telemetry: TelemetryMode, + mockEnv: Record, +) { return repeatMeasure( async () => { - const start = performance.now(); - const res = await fetch(`${baseUrl}${path}`); - const wallMs = performance.now() - start; - if (!res.ok) { - throw new Error(`Request failed: ${res.status} ${await res.text()}`); - } - await res.text(); - return { wallMs, rssPeakBytes: null, exitCode: 0 }; + rmSync(join(fixture.dir, 'dist'), { recursive: true, force: true }); + return measureCommand(['npx', 'vite', 'build'], { + cwd: fixture.dir, + timeoutMs: 180_000, + env: { ...telemetryEnv(telemetry, mockEnv), APP_ENV: 'dev', CI: '1' }, + }); }, - { iterations, warmup }, + { iterations, warmup: 0 }, ); } -async function withViteDevServer( - projectDir: string, - port: number, - fn: () => Promise, -): Promise { - const server = spawn('npx', ['vite', 'dev', '--host', '127.0.0.1', '--port', String(port)], { - cwd: projectDir, - env: { - ...process.env, - ...Object.fromEntries( - Object.entries(telemetryEnv('off')).filter(([, v]) => v !== undefined), - ), - APP_ENV: 'dev', - CI: '1', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - let stderr = ''; - let stdout = ''; - server.stderr?.on('data', (c: Buffer) => { - stderr += c.toString(); - }); - server.stdout?.on('data', (c: Buffer) => { - stdout += c.toString(); - }); - - try { - await waitForUrl(`http://127.0.0.1:${port}/api/echo`, 90_000); - return await fn(); - } catch (err) { - throw new Error(`${String(err)}\nvite dev stdout:\n${stdout}\nstderr:\n${stderr}`); - } finally { - server.kill('SIGTERM'); - await new Promise((r) => { - setTimeout(r, 500); - }); - if (!server.killed) server.kill('SIGKILL'); - } -} - export async function runViteScenarios(ctx: BenchContext): Promise> { const results: Array = []; const buildIterations = Math.max(2, Math.min(4, ctx.iterations)); const requestIterations = Math.max(10, ctx.iterations * 2); - // Baseline: telemetry N/A (no varlock CLI). Tag as off for schema consistency. console.log(' preparing vite baseline (framework-tests)...'); - { - const fixture = createViteEnv(ctx, 'baseline'); - await fixture.setup(); - prepareViteFiles(fixture, 'baseline'); - const build = await repeatMeasure( - async () => { - rmSync(join(fixture.dir, 'dist'), { recursive: true, force: true }); - return measureCommand(['npx', 'vite', 'build'], { - cwd: fixture.dir, - timeoutMs: 180_000, - env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, - }); - }, - { iterations: buildIterations, warmup: 0 }, - ); + const baseline = createViteEnv(ctx, 'baseline'); + await baseline.setup(); + try { + prepareBuildFiles(baseline, 'baseline'); results.push({ id: 'integration.vite.build.baseline', facet: 'integration-vite', installMethod: 'npm', packageManager: 'npm', telemetry: 'off', - metrics: build, + metrics: await measureBuild(baseline, buildIterations, 'off', ctx.telemetryMockEnv), + notes: 'Cold vite build, no varlock installed', }); - await fixture.teardown(); + } finally { + await baseline.teardown(); } - // Varlock build: telemetry on/off (plugin shells out to `varlock load`) - for (const telemetry of TELEMETRY_MODES) { - console.log(` preparing vite varlock telemetry.${telemetry} (framework-tests)...`); - const fixture = createViteEnv(ctx, 'varlock', `-telemetry-${telemetry}`); - await fixture.setup(); - prepareViteFiles(fixture, 'varlock'); - const build = await repeatMeasure( - async () => { - rmSync(join(fixture.dir, 'dist'), { recursive: true, force: true }); - return measureCommand(['npx', 'vite', 'build'], { - cwd: fixture.dir, - timeoutMs: 180_000, - env: { ...telemetryEnv(telemetry), APP_ENV: 'dev', CI: '1' }, - }); - }, - { iterations: buildIterations, warmup: 0 }, - ); - results.push({ - id: `integration.vite.build.varlock.telemetry.${telemetry}`, - facet: 'integration-vite', - installMethod: 'npm', - packageManager: 'npm', - telemetry, - metrics: build, - notes: 'Cold vite build; telemetry affects sync varlock load spawn', - }); - await fixture.teardown(); - } + console.log(' preparing vite varlock (framework-tests)...'); + const fixture = createViteEnv(ctx, 'varlock'); + await fixture.setup(); + try { + prepareBuildFiles(fixture, 'varlock'); + for (const telemetry of ctx.telemetryModes) { + console.log(` vite varlock build, telemetry=${telemetry}...`); + results.push({ + id: `integration.vite.build.varlock.telemetry.${telemetry}`, + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry, + metrics: await measureBuild(fixture, buildIterations, telemetry, ctx.telemetryMockEnv), + notes: 'Cold vite build; telemetry affects sync varlock load spawn', + }); + } - console.log(' measuring vite request latency (preventLeaks / redactLogs)...'); - for (const [label, preventLeaks, redactLogs, port, path] of [ - ['preventLeaks.on', true, true, 3461, '/api/echo'], - ['preventLeaks.off', false, true, 3462, '/api/echo'], - ['redactLogs.on', true, true, 3463, '/api/log'], - ['redactLogs.off', true, false, 3464, '/api/log'], - ] as const) { - const fixture = createViteEnv(ctx, 'varlock', `-${label}`); - await fixture.setup(); + console.log(' measuring vite request latency (preventLeaks / redactLogs)...'); + for (const [label, preventLeaks, redactLogs, port, path] of [ + ['preventLeaks.on', true, true, 3461, '/api/echo'], + ['preventLeaks.off', false, true, 3462, '/api/echo'], + ['redactLogs.on', true, true, 3463, '/api/log'], + ['redactLogs.off', true, false, 3464, '/api/log'], + ] as const) { + fixture.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'index.html': 'html/basic.html', + 'src/main.ts': 'pages/minimal-page.ts', + }, + files: [ + { path: '.env.schema', content: withSchemaFlags(VITE_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, + { path: 'vite.config.ts', content: LATENCY_VITE_CONFIG }, + ], + }); - fixture.prepareFiles({ - templateFiles: { - '.env.dev': 'schemas/.env.dev', - 'index.html': 'html/basic.html', - 'src/main.ts': 'pages/minimal-page.ts', - }, - files: [ - { path: '.env.schema', content: withSchemaFlags(VITE_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, - { path: 'vite.config.ts', content: LATENCY_VITE_CONFIG }, - ], - }); + const latency = await withServer( + ['npx', 'vite', 'dev', '--host', '127.0.0.1', '--port', String(port)], + { + cwd: fixture.dir, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + readyUrl: `http://127.0.0.1:${port}${path}`, + }, + () => measurePathLatency(`http://127.0.0.1:${port}`, path, requestIterations, 3), + ); - try { - const latency = await withViteDevServer(fixture.dir, port, () => measurePathLatency( - `http://127.0.0.1:${port}`, - path, - requestIterations, - 3, - )); results.push({ id: `integration.vite.request.${label}`, facet: 'integration-vite', @@ -304,12 +233,12 @@ export async function runViteScenarios(ctx: BenchContext): Promise> { if (!hasBinary('go')) { + ctx.note('lang-go skipped: go not found on the runner'); console.log(' skipping go: go not found'); return []; } const results: Array = []; const cli = ctx.clis.find((c) => c.label === 'npm') ?? ctx.clis[0]; - if (!cli) return []; + if (!cli) { + ctx.note('lang-go skipped: no usable varlock CLI'); + return []; + } const env = telemetryEnv('off'); - const dest = join(ctx.workDir, 'lang-go'); - rmSync(dest, { recursive: true, force: true }); - mkdirSync(dest, { recursive: true }); - cpSync(join(ctx.fixturesDir, 'lang-go'), dest, { recursive: true }); + const dest = fixtureWorkDir(ctx, 'lang-go'); const codegen = await repeatMeasure( async () => { @@ -39,7 +39,7 @@ export async function runGoScenarios(ctx: BenchContext): Promise> { if (!hasBinary('python3')) { + ctx.note('lang-python skipped: python3 not found on the runner'); console.log(' skipping python: python3 not found'); return []; } @@ -21,13 +21,13 @@ export async function runPythonScenarios(ctx: BenchContext): Promise = []; // Use npm-installed CLI for lang scenarios (one representative install method) const cli = ctx.clis.find((c) => c.label === 'npm') ?? ctx.clis[0]; - if (!cli) return []; + if (!cli) { + ctx.note('lang-python skipped: no usable varlock CLI'); + return []; + } const env = telemetryEnv('off'); - const dest = join(ctx.workDir, 'lang-python'); - rmSync(dest, { recursive: true, force: true }); - mkdirSync(dest, { recursive: true }); - cpSync(join(ctx.fixturesDir, 'lang-python'), dest, { recursive: true }); + const dest = fixtureWorkDir(ctx, 'lang-python'); const codegen = await repeatMeasure( async () => { @@ -41,7 +41,7 @@ export async function runPythonScenarios(ctx: BenchContext): Promise { + const start = Date.now(); + let lastStatus: number | string = 'no response'; + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + // A 5xx means the server is up but the route is broken — keep polling so a + // slow boot still succeeds, but report the status if we time out. + if (res.ok) return; + lastStatus = res.status; + } catch (err) { + lastStatus = (err as Error).message; + } + await new Promise((r) => { + setTimeout(r, 250); + }); + } + throw new Error(`Timed out waiting for ${url} (last: ${lastStatus})`); +} + +/** Measure request latency against an already-running server. */ +export async function measurePathLatency( + baseUrl: string, + path: string, + iterations: number, + warmup: number, +): Promise { + return repeatMeasure( + async () => { + const start = performance.now(); + const res = await fetch(`${baseUrl}${path}`); + const body = await res.text(); + const wallMs = performance.now() - start; + if (!res.ok) { + throw new Error(`Request failed: ${res.status} ${body.slice(0, 500)}`); + } + return { wallMs, rssPeakBytes: null, exitCode: 0 }; + }, + { iterations, warmup }, + ); +} + +export type ServerOptions = { + cwd: string; + env: Record; + /** URL polled until it answers before `fn` runs. */ + readyUrl: string; + readyTimeoutMs?: number; +}; + +/** + * Run a dev/prod server for the duration of `fn`, then tear down the whole + * process group. + * + * Two things here are load-bearing: + * - stdout AND stderr are drained. The redactLogs benches make the server write + * hundreds of log lines per request; an unread pipe fills at ~64KB and the + * server then blocks on write(), which would turn a latency measurement into a + * measurement of pipe backpressure. + * - the child is detached and killed by process group. `npx next start` and + * `npx vite dev` both exec the real server as a grandchild, so signalling only + * the direct child leaves an orphan holding the port and burning CPU during + * later scenarios. + */ +export async function withServer( + command: Array, + options: ServerOptions, + fn: () => Promise, +): Promise { + const [bin, ...args] = command; + const env: NodeJS.ProcessEnv = { ...process.env }; + for (const [key, value] of Object.entries(options.env)) { + if (value === undefined) delete env[key]; + else env[key] = value; + } + + const server: ChildProcess = spawn(bin!, args, { + cwd: options.cwd, + env, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + + let output = ''; + const capture = (chunk: Buffer) => { + output += chunk.toString(); + // Bound the buffer — these servers can be very chatty under the log benches. + if (output.length > 200_000) output = output.slice(-100_000); + }; + server.stdout?.on('data', capture); + server.stderr?.on('data', capture); + + const exited = once(server, 'exit'); + let exitedEarly = false; + exited.then( + () => { + exitedEarly = true; + }, + () => { + exitedEarly = true; + }, + ); + + try { + await waitForUrl(options.readyUrl, options.readyTimeoutMs ?? 90_000); + return await fn(); + } catch (err) { + throw new Error(`${String(err)}\nserver output:\n${output.slice(-8_000)}`); + } finally { + if (!exitedEarly && server.pid) { + try { + process.kill(-server.pid, 'SIGTERM'); + } catch { + server.kill('SIGTERM'); + } + const timer = new Promise<'timeout'>((r) => { + setTimeout(() => r('timeout'), 5_000); + }); + const result = await Promise.race([exited.then(() => 'exited' as const), timer]); + if (result === 'timeout') { + try { + process.kill(-server.pid, 'SIGKILL'); + } catch { + server.kill('SIGKILL'); + } + await Promise.race([exited, timer]).catch(() => undefined); + } + } + } +} diff --git a/benchmarks/src/telemetry-mock.ts b/benchmarks/src/telemetry-mock.ts new file mode 100644 index 000000000..c143cbe5c --- /dev/null +++ b/benchmarks/src/telemetry-mock.ts @@ -0,0 +1,51 @@ +import { createServer, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; + +export type TelemetryMock = { + /** Base URL to hand to varlock via VARLOCK_POSTHOG_HOST. */ + url: string; + /** Number of capture requests received so far. */ + requestCount: () => number; + close: () => Promise; +}; + +/** + * Local stand-in for the telemetry collector. + * + * Telemetry-on scenarios exist to measure what the telemetry code path costs + * (payload building plus the exit hook that waits on the in-flight request). + * Pointing that at the real collector would inject hundreds of synthetic events + * into product analytics on every run, and would make the committed timings a + * function of runner-to-collector network latency. A local mock keeps the code + * path intact and the numbers comparable between runs. + */ +export async function startTelemetryMock(): Promise { + let count = 0; + + const server: Server = createServer((req, res) => { + // Drain the body so the client sees a complete request/response cycle. + req.resume(); + req.on('end', () => { + count += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + // Shape of a real PostHog capture response. + res.end('{"status":1}'); + }); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }); + + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}`, + requestCount: () => count, + close: () => new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + }; +} diff --git a/benchmarks/src/telemetry.ts b/benchmarks/src/telemetry.ts index 1ba41fac5..ccf60442f 100644 --- a/benchmarks/src/telemetry.ts +++ b/benchmarks/src/telemetry.ts @@ -2,8 +2,18 @@ import type { TelemetryMode } from './types.ts'; export type { TelemetryMode }; -/** Env overlay for telemetry on/off. Pass through measureCommand. */ -export function telemetryEnv(mode: TelemetryMode): Record { +/** + * Env overlay for telemetry on/off. Pass through measureCommand. + * + * `mockEnv` points the collector at the local mock (see telemetry-mock.ts) and is + * required for 'on' — benchmarks never send real telemetry. Callers get it from + * `ctx.telemetryMockEnv`, which is empty when the mock is unavailable, in which + * case 'on' has already been dropped from `ctx.telemetryModes`. + */ +export function telemetryEnv( + mode: TelemetryMode, + mockEnv: Record = {}, +): Record { if (mode === 'off') { return { VARLOCK_TELEMETRY_DISABLED: '1', @@ -11,11 +21,15 @@ export function telemetryEnv(mode: TelemetryMode): Record = ['off', 'on']; +export const ALL_TELEMETRY_MODES: Array = ['off', 'on']; diff --git a/benchmarks/src/types.ts b/benchmarks/src/types.ts index 4d442e65f..c55b16470 100644 --- a/benchmarks/src/types.ts +++ b/benchmarks/src/types.ts @@ -1,3 +1,9 @@ +/** + * How the measured CLI was installed and invoked: + * - `npm` — installed with npm, run with node + * - `bun` — installed with bun, run with the bun runtime + * - `sea` — the standalone compiled binary + */ export type InstallMethod = 'npm' | 'bun' | 'sea'; export type TelemetryMode = 'on' | 'off'; @@ -20,13 +26,21 @@ export type Sample = { }; export type ScenarioMetrics = { + /** Number of measured (non-warmup) iterations behind these numbers. */ + iterations: number; + wallMsMin: number; wallMsMedian: number; + /** Collapses onto the max at low iteration counts — read with wallMsStdDev. */ wallMsP95: number; + wallMsStdDev: number; rssPeakBytesMedian: number | null; + /** How many iterations produced an RSS reading (0 means RSS was never sampled). */ + rssSampleCount: number; samples: Array; }; export type ScenarioResult = { + /** Unique within a run — includes the install method for CLI scenarios. */ id: string; facet: ScenarioFacet; installMethod: InstallMethod; @@ -43,6 +57,7 @@ export type BenchRunMeta = { githubRunId: string | null; runnerOs: string; runnerArch: string; + nodeVersion: string; versions: { varlock: string; nextjsIntegration?: string; @@ -50,6 +65,13 @@ export type BenchRunMeta = { '@env-spec/parser'?: string; }; trigger: TriggerKind; + /** + * True when telemetry-on scenarios were pointed at a local mock collector. + * When false those scenarios are skipped — benchmarks never emit real telemetry. + */ + telemetryMocked: boolean; + /** Anything skipped, degraded, or otherwise worth knowing when reading the numbers. */ + notes: Array; }; export type BenchRunResult = { @@ -66,6 +88,14 @@ export type CliInvocation = { export type BenchContext = { version: string; + /** + * Resolved (not floating) integration versions, so a run records the exact pair + * of packages it measured. + */ + integrationVersions: { + nextjs?: string; + vite?: string; + }; rootDir: string; fixturesDir: string; workDir: string; @@ -73,4 +103,12 @@ export type BenchContext = { warmup: number; clis: Array; seaPath: string | null; + /** Telemetry modes to measure — 'on' is dropped when the mock is unavailable. */ + telemetryModes: Array; + /** Env overlay that points telemetry at the local mock (empty when unavailable). */ + telemetryMockEnv: Record; + /** 64-char hex key that enables the on-disk resolver cache, including in CI. */ + cacheKey: string; + /** Record a skip / degradation so it shows up in the committed results. */ + note: (message: string) => void; }; From 323f219ef4a3a65c5202ff4cc020619831238e5e Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:50:31 +0000 Subject: [PATCH 4/5] benchmarks: make telemetry capability probe safe --- benchmarks/src/run.ts | 61 ++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/benchmarks/src/run.ts b/benchmarks/src/run.ts index 530e994f3..193e70220 100644 --- a/benchmarks/src/run.ts +++ b/benchmarks/src/run.ts @@ -1,4 +1,6 @@ -import { mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { + mkdirSync, writeFileSync, existsSync, readFileSync, readdirSync, +} from 'node:fs'; import { join, resolve, dirname } from 'node:path'; import { spawnSync } from 'node:child_process'; import { randomBytes } from 'node:crypto'; @@ -15,9 +17,8 @@ import { } from './install.ts'; import { runAllScenarios, SCENARIO_GROUPS } from './scenarios/index.ts'; import { formatSummaryMarkdown } from './report.ts'; -import { measureCommand } from './measure.ts'; import { startTelemetryMock } from './telemetry-mock.ts'; -import { ALL_TELEMETRY_MODES, telemetryEnv } from './telemetry.ts'; +import { ALL_TELEMETRY_MODES } from './telemetry.ts'; import type { BenchContext, BenchRunResult, CliInvocation, TelemetryMode, TriggerKind, } from './types.ts'; @@ -139,33 +140,26 @@ function defaultOutPath(version: string): string { return join(RESULTS_DIR, `${iso}-varlock@${version}-${runId}.json`); } -/** - * Confirm the tested varlock build honours VARLOCK_POSTHOG_HOST before running any - * telemetry-on scenario. A version published before that override existed would - * send real events to the production collector instead, so those scenarios get - * dropped rather than measured. - */ -async function checkTelemetryMockable( - cli: CliInvocation, - cwd: string, - mockEnv: Record, - received: () => number, -): Promise { - const before = received(); - const result = await measureCommand([...cli.command, 'load'], { - cwd, - env: telemetryEnv('on', mockEnv), - timeoutMs: 60_000, - }); - if (result.exitCode !== 0) return false; - // The CLI's exit hook only waits ~500ms on the in-flight request; give the - // loopback round-trip a moment longer before concluding nothing arrived. - for (let i = 0; i < 20 && received() === before; i++) { - await new Promise((r) => { - setTimeout(r, 100); - }); +/** Inspect the installed package without executing telemetry-enabled code. */ +function checkTelemetryMockable(cli: CliInvocation): boolean { + const cliScript = cli.command.find((part) => part.endsWith('/bin/cli.js')); + if (!cliScript) return false; + + const pending = [join(dirname(dirname(cliScript)), 'dist')]; + try { + while (pending.length > 0) { + const dir = pending.pop()!; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) pending.push(path); + else if (/\.[cm]?js$/.test(entry.name) + && readFileSync(path, 'utf8').includes('VARLOCK_POSTHOG_HOST')) return true; + } + } + } catch { + return false; } - return received() > before; + return false; } async function main(): Promise { @@ -234,15 +228,10 @@ async function main(): Promise { let telemetryModes: Array = ['off']; try { - const mockable = await checkTelemetryMockable( - usableClis[0]!, - join(FIXTURES_DIR, 'cli-basic'), - telemetryMockEnv, - telemetryMock.requestCount, - ); + const mockable = checkTelemetryMockable(usableClis[0]!); if (mockable) { telemetryModes = ALL_TELEMETRY_MODES; - console.log(`Telemetry mock reachable at ${telemetryMock.url} — telemetry-on scenarios enabled`); + console.log(`Telemetry endpoint override supported: telemetry-on scenarios enabled with ${telemetryMock.url}`); } else { note( `telemetry-on scenarios skipped: varlock@${resolvedVersion} does not honour VARLOCK_POSTHOG_HOST, ` From a7a86f6316121607ae62fb445fc117b4e7d461bd Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:23:13 +0000 Subject: [PATCH 5/5] benchmarks: isolate framework build overhead --- .github/workflows/benchmarks.yaml | 1 + benchmarks/src/scenarios/integration-next.ts | 10 +++++++++- benchmarks/src/scenarios/integration-vite.ts | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 46f95c9cc..1ca497d9d 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -43,6 +43,7 @@ jobs: - uses: actions/checkout@v7 with: token: ${{ secrets.BUMPY_GH_TOKEN }} + ref: main - name: Setup Bun uses: oven-sh/setup-bun@v2 diff --git a/benchmarks/src/scenarios/integration-next.ts b/benchmarks/src/scenarios/integration-next.ts index cb45d672f..1183cd39e 100644 --- a/benchmarks/src/scenarios/integration-next.ts +++ b/benchmarks/src/scenarios/integration-next.ts @@ -10,6 +10,14 @@ import { LEAK_SCAN_BODY_BYTES, REDACT_LOG_LINES } from './util.ts'; const NEXT_TEST_DIR = resolve(import.meta.dirname, '../../../framework-tests/frameworks/nextjs'); +const BUILD_ENV = { + APP_ENV: 'dev', + NEXT_PUBLIC_VAR: 'next-prefixed-public-var', + PUBLIC_VAR: 'unprefixed-public-var', + ENV_SPECIFIC_VAR: 'env-specific-var--dev', + SENSITIVE_VAR: 'super-secret-var', +}; + /** * Baseline and varlock arms must compile the same app with the same next config, * or the "varlock overhead" number also contains a page-content and config diff. @@ -175,7 +183,7 @@ async function measureBuild( return measureCommand(['npx', 'next', 'build'], { cwd: fixture.dir, timeoutMs: 300_000, - env: { ...telemetryEnv(telemetry, mockEnv), APP_ENV: 'dev', CI: '1' }, + env: { ...telemetryEnv(telemetry, mockEnv), ...BUILD_ENV, CI: '1' }, }); }, { iterations, warmup: 0 }, diff --git a/benchmarks/src/scenarios/integration-vite.ts b/benchmarks/src/scenarios/integration-vite.ts index 6efa21039..1bbe3fcc3 100644 --- a/benchmarks/src/scenarios/integration-vite.ts +++ b/benchmarks/src/scenarios/integration-vite.ts @@ -10,6 +10,16 @@ import { LEAK_SCAN_BODY_BYTES, REDACT_LOG_LINES } from './util.ts'; const VITE_TEST_DIR = resolve(import.meta.dirname, '../../../framework-tests/frameworks/vite'); +const BUILD_ENV = { + APP_ENV: 'dev', + PUBLIC_VAR: 'public-test-value', + API_URL: 'https://api.example.com', + ENV_SPECIFIC_VAR: 'env-specific-dev', + VITE_PUBLIC_VAR: 'public-test-value', + VITE_API_URL: 'https://api.example.com', + VITE_ENV_SPECIFIC_VAR: 'env-specific-dev', +}; + /** Identical apart from the plugin — see the note in integration-next.ts. */ const BASELINE_VITE_CONFIG = `import { defineConfig } from 'vite'; @@ -34,7 +44,7 @@ function mainSource(read: (key: string) => string): string { `; } -const BASELINE_MAIN = mainSource((key) => `import.meta.env.${key}`); +const BASELINE_MAIN = mainSource((key) => `import.meta.env.VITE_${key}`); const VARLOCK_MAIN = `import { ENV } from 'varlock/env'; ${mainSource((key) => `ENV.${key}`)}`; @@ -148,7 +158,7 @@ async function measureBuild( return measureCommand(['npx', 'vite', 'build'], { cwd: fixture.dir, timeoutMs: 180_000, - env: { ...telemetryEnv(telemetry, mockEnv), APP_ENV: 'dev', CI: '1' }, + env: { ...telemetryEnv(telemetry, mockEnv), ...BUILD_ENV, CI: '1' }, }); }, { iterations, warmup: 0 },