diff --git a/.bumpy/bench-telemetry-host-override.md b/.bumpy/bench-telemetry-host-override.md new file mode 100644 index 000000000..b4e51a9b0 --- /dev/null +++ b/.bumpy/bench-telemetry-host-override.md @@ -0,0 +1,5 @@ +--- +varlock: patch +--- + +Allow overriding the telemetry endpoint with VARLOCK_POSTHOG_HOST, so tooling can point it at a local mock diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 000000000..1ca497d9d --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,166 @@ +name: Benchmarks + +# Runs against published npm packages (+ optional SEA binary). Triggered after a +# varlock publish (via release.yaml) or manually for iteration. +on: + workflow_dispatch: + inputs: + varlock_version: + description: 'Published varlock version (empty = latest on npm)' + required: false + type: string + default: '' + only: + description: 'Optional comma-separated scenario groups (empty = all)' + required: false + type: string + default: '' + iterations: + description: 'Measured iterations per scenario' + required: false + type: string + default: '5' + release_dispatch: + description: 'Set to true when invoked from release.yaml after publish' + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.event.inputs.varlock_version || 'latest' }} + cancel-in-progress: false + +jobs: + bench: + runs-on: ubuntu-latest + permissions: + contents: write + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + with: + token: ${{ secrets.BUMPY_GH_TOKEN }} + ref: main + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Use Node.js 24.x + uses: actions/setup-node@v6 + with: + node-version: '24.x' + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + cache: false + + - name: Install benchmarks package deps + working-directory: benchmarks + # Frozen so the run cannot leave a modified bun.lock behind, which would + # block the rebase in the commit step at the very end of a long run. + run: bun install --frozen-lockfile + + # Workflow inputs go through `env:` rather than being interpolated straight + # into the script, so a crafted input value cannot become shell syntax. + - name: Resolve varlock version + id: ver + env: + INPUT_VERSION: ${{ github.event.inputs.varlock_version }} + run: | + set -euo pipefail + if [[ -z "$INPUT_VERSION" || "$INPUT_VERSION" == "latest" ]]; then + V=$(npm view varlock version) + else + V="$INPUT_VERSION" + fi + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Resolved varlock@$V" + + # Note: the suite itself waits for the version to appear on npm, so a + # release-triggered run can start before the registry has caught up. + + - name: Download SEA binary (linux-x64) + id: sea + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.ver.outputs.version }} + run: | + set -euo pipefail + TAG="varlock@${VERSION}" + DEST="$RUNNER_TEMP/varlock-sea" + mkdir -p "$DEST" + if gh release download "$TAG" --pattern 'varlock-linux-x64.tar.gz' --dir "$DEST"; then + tar -xzf "$DEST/varlock-linux-x64.tar.gz" -C "$DEST" + BIN="$DEST/varlock" + chmod +x "$BIN" + echo "path=$BIN" >> "$GITHUB_OUTPUT" + echo "found=true" >> "$GITHUB_OUTPUT" + "$BIN" --version || true + else + echo "No SEA release asset for $TAG; continuing without SEA" + echo "found=false" >> "$GITHUB_OUTPUT" + echo "path=" >> "$GITHUB_OUTPUT" + fi + + - name: Run benchmarks + working-directory: benchmarks + env: + VERSION: ${{ steps.ver.outputs.version }} + ONLY: ${{ github.event.inputs.only }} + ITERATIONS: ${{ github.event.inputs.iterations }} + RELEASE_DISPATCH: ${{ github.event.inputs.release_dispatch }} + SEA_FOUND: ${{ steps.sea.outputs.found }} + SEA_PATH: ${{ steps.sea.outputs.path }} + run: | + set -euo pipefail + if [[ "$RELEASE_DISPATCH" == "true" ]]; then + TRIGGER=release + else + TRIGGER=workflow_dispatch + fi + ARGS=(--version "$VERSION" --trigger "$TRIGGER") + if [[ -n "$ITERATIONS" ]]; then + ARGS+=(--iterations "$ITERATIONS") + fi + if [[ -n "$ONLY" ]]; then + ARGS+=(--only "$ONLY") + fi + if [[ "$SEA_FOUND" == "true" ]]; then + ARGS+=(--sea-path "$SEA_PATH") + fi + bun run src/run.ts "${ARGS[@]}" + + - name: Commit results + env: + VERSION: ${{ steps.ver.outputs.version }} + run: | + set -euo pipefail + # Absolute path — `git add` resolves pathspecs against the cwd, so a + # repo-relative path only works from the repo root. + RESULT_PATH=$(cat benchmarks/.work/last-result-path.txt) + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -- "$RESULT_PATH" + if git diff --staged --quiet; then + echo "No results to commit" + exit 0 + fi + git commit -m "chore(benchmarks): record varlock@${VERSION} [skip ci]" + # Commit first, then rebase: a rebase refuses to run with staged or + # unstaged changes present, and other commits may have landed on main + # during the (long) bench run. + for attempt in 1 2 3; do + git pull --rebase origin main + if git push origin HEAD:main; then + echo "Pushed results" + exit 0 + fi + echo "Push rejected, retrying ($attempt/3)..." + done + echo "::error::Failed to push benchmark results after 3 attempts" + exit 1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e3d74dcf5..3d96300ff 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -511,6 +511,15 @@ jobs: working-directory: packages/varlock/dist-sea run: gh release upload "${{ steps.ver.outputs.tag }}" *.{tar.gz,zip} checksums.txt checksums.txt.cosign.bundle ../SHA256SUMS.txt --clobber + - name: Dispatch benchmarks workflow + env: + GH_TOKEN: ${{ secrets.BUMPY_GH_TOKEN }} + run: | + gh workflow run benchmarks.yaml \ + --ref main \ + -f "varlock_version=${{ steps.ver.outputs.version }}" \ + -f "release_dispatch=true" + # Update the homebrew tap formula - name: Checkout homebrew tap repo uses: actions/checkout@v7 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fc17ad34d..4f371f058 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,6 +50,10 @@ jobs: run: bun run lint - name: TypeScript type check run: bun run typecheck:all + # benchmarks/ is not a workspace member, so turbo's typecheck does not reach it + - name: TypeScript type check (benchmarks) + working-directory: benchmarks + run: bun install --frozen-lockfile && bun run typecheck - name: Build libraries run: bun run build:libs - name: Run tests diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 000000000..a20843f47 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,5 @@ +.work/ +node_modules/ +fixtures/**/env.d.ts +fixtures/**/env.py +fixtures/**/env/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..c604ccf85 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,78 @@ +# Varlock benchmarks + +Release benchmarking suite for **memory footprint**, **execution time**, and **added latency** (redaction / leak prevention). + +Runs against **published** npm packages (and optionally the linux SEA binary), not workspace links. Results are committed under [`results/`](results/) so trends are visible in git history. + +## What it measures + +| Group | Scenarios | +|-------|-----------| +| `cli-load` | `load` cold (`--clear-cache`) and warm, for npm / bun / SEA, with **telemetry on/off** | +| `cli-run` | Bare node baseline; `varlock run` wrap with **telemetry on/off**; stdout redaction on vs off (telemetry off) | +| `cli-scan-audit` | Light `scan` and `audit` coverage (telemetry off) | +| `integration-next` | Uses [`framework-tests/frameworks/nextjs`](../framework-tests/frameworks/nextjs): `next build` baseline vs varlock with **telemetry on/off**; request latency for `preventLeaks` and `redactLogs` | +| `integration-vite` | Uses [`framework-tests/frameworks/vite`](../framework-tests/frameworks/vite): `vite build` baseline vs varlock with **telemetry on/off**; request latency for `preventLeaks` and `redactLogs` | +| `lang-python` | `load`+codegen and `varlock run -- python3` | +| `lang-go` | `load`+codegen and `varlock run` of a built Go binary | + +Install methods are three distinct runtimes, not three package managers: `npm` installs and runs under **node**, `bun` installs and runs under **bun**, `sea` is the compiled standalone binary. + +## Reading the results + +Each run prints a **Deltas** table before the raw numbers. The deltas are the point of the suite: absolute wall times on a shared CI runner are not comparable between runs, but the difference between two scenarios measured back to back within one run is. + +A delta smaller than the standard deviation of either side is tagged `(within noise)` and should not be read as a change. + +Every scenario records `wallMsMin`, `wallMsMedian`, `wallMsP95`, `wallMsStdDev` and `iterations`. Prefer **min** and **stddev**: min is the least noise-sensitive statistic for this kind of measurement, and p95 collapses onto the max at the iteration counts used here. + +`meta.notes` lists anything that was skipped or degraded (missing SEA binary, no Go toolchain, telemetry not measurable). Nothing is dropped silently. + +## Telemetry + +Telemetry-on scenarios exist to measure what the telemetry code path costs. **They never send real telemetry.** The suite starts a local mock collector and points varlock at it with `VARLOCK_POSTHOG_HOST`, which keeps the code path intact (payload building, the exit hook that waits on the in-flight request) without injecting synthetic events into product analytics, and without making the timings depend on network latency to the real collector. + +Before running any telemetry-on scenario the suite probes whether the version under test honours that override. If it does not (versions published before the override existed), those scenarios are skipped and a note is recorded. + +## Local usage + +```bash +bun run bench +``` + +```bash +bun run bench -- --version 1.13.0 --sea-path ./packages/varlock/dist-sea/varlock +``` + +```bash +bun run bench -- --only cli-load,cli-run --iterations 3 +``` + +```bash +bun run bench -- --skip-install --only cli-load +``` + +The first form benchmarks the latest published varlock. The others pin a version and add a local SEA binary, restrict to a subset of scenario groups for faster iteration, and reuse the npm/bun installs left in `benchmarks/.work` by a previous run. + +From this directory, `bun install` first and then use `bun run bench` the same way. + +Integration benches drive [`FrameworkTestEnv`](../framework-tests/harness/fixture-env.ts) with `usePublished: true` so they install from npm (not packed workspace tarballs) while reusing the same Next/Vite templates as framework CI. + +Results are written to `results/-varlock@-.json`. CI commits those files; local runs leave them untracked unless you commit them yourself. + +## CI + +Workflow: [`.github/workflows/benchmarks.yaml`](../.github/workflows/benchmarks.yaml) + +- **Manual:** Actions → Benchmarks → Run workflow (optional version / scenario filter) +- **After publish:** [`release.yaml`](../.github/workflows/release.yaml) dispatches this workflow once SEA binaries are uploaded for `varlock@` + +The job installs from npm, downloads `varlock-linux-x64.tar.gz` when present, runs the suite, and commits the new JSON under `results/` with `[skip ci]` so the commit does not retrigger release/CI. The suite waits for the version to appear on npm itself, so a release-triggered run can start before the registry has caught up. + +v1 is informational only (no regression gate). Suite failures still fail the workflow. + +## Known gaps + +- **Linux/x64 only.** The SEA binary ships for macOS and Windows too, but nothing measures them. The non-Linux RSS sampling path (which shells out to `ps` once per sample, perturbing the timings it measures) is therefore only exercised by local runs. +- **The `cli-load` fixture has nothing worth caching.** Cold vs warm is now a valid comparison in CI (`_VARLOCK_CACHE_KEY` forces the on-disk cache, which CI would otherwise skip in favour of a per-process memory cache), but the fixture is all static literals, so both arms measure roughly the same work. Exercising the cache meaningfully needs a fixture with expensive resolvers, e.g. a plugin-backed or `exec()` value. +- **No regression gate and no cross-run comparison tooling.** Results accumulate in `results/` but nothing reads the history yet. diff --git a/benchmarks/bun.lock b/benchmarks/bun.lock new file mode 100644 index 000000000..df2fd36f3 --- /dev/null +++ b/benchmarks/bun.lock @@ -0,0 +1,20 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "varlock-benchmarks", + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.9.3", + }, + }, + }, + "packages": { + "@types/node": ["@types/node@22.20.1", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + } +} diff --git a/benchmarks/fixtures/cli-basic/.env.schema b/benchmarks/fixtures/cli-basic/.env.schema new file mode 100644 index 000000000..8280ce402 --- /dev/null +++ b/benchmarks/fixtures/cli-basic/.env.schema @@ -0,0 +1,43 @@ +# @defaultSensitive=false +# @redactLogs=true +# @preventLeaks=true +# --- + +PUBLIC_VAR=public-value + +# @sensitive +SECRET_TOKEN=super-secret-token-12345 +# @sensitive +SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa +# @sensitive +SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb +# @sensitive +SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig +# @sensitive +SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc +# @sensitive +SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD +# @sensitive +SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee +# @sensitive +SECRET_REDIS=redis-auth-bench-ffffffffffff +# @sensitive +SECRET_SMTP=smtp-pass-bench-gggggggggggg +# @sensitive +SECRET_OAUTH=oauth-client-secret-hhhhhhhh +# @sensitive +SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii +# @sensitive +SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj +# @sensitive +SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk +# @sensitive +SECRET_GITHUB=ghp_benchTokenLlllllllllllllll +# @sensitive +SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm +# @sensitive +SECRET_OPENAI=sk-proj-bench-openainnnnnnnn +# @sensitive +SECRET_SENTRY=sntrys_bench_oooooooooooooo +# @sensitive +SECRET_PRIVATE_KEY=-----BEGIN BENCH PRIVATE KEY-----MIIBenchKey-----END BENCH PRIVATE KEY----- diff --git a/benchmarks/fixtures/cli-basic/app.js b/benchmarks/fixtures/cli-basic/app.js new file mode 100644 index 000000000..897635a67 --- /dev/null +++ b/benchmarks/fixtures/cli-basic/app.js @@ -0,0 +1,4 @@ +// Sample app source for audit (references env keys). +const token = process.env.SECRET_TOKEN; +const pub = process.env.PUBLIC_VAR; +console.log(token, pub); diff --git a/benchmarks/fixtures/cli-basic/child.js b/benchmarks/fixtures/cli-basic/child.js new file mode 100644 index 000000000..bbfdbb22a --- /dev/null +++ b/benchmarks/fixtures/cli-basic/child.js @@ -0,0 +1,2 @@ +// Minimal child used for varlock run wrap overhead benchmarks. +process.stdout.write('ok\n'); diff --git a/benchmarks/fixtures/cli-basic/emit-secret.js b/benchmarks/fixtures/cli-basic/emit-secret.js new file mode 100644 index 000000000..4442b66b8 --- /dev/null +++ b/benchmarks/fixtures/cli-basic/emit-secret.js @@ -0,0 +1,19 @@ +// Emits every SECRET_* env var many times so stdout redaction cost scales with secret count. +// Line count comes from BENCH_EMIT_LINES: redaction cost is per byte of output, so the +// volume has to be large enough to clear the fixed ~50ms of process-startup noise. +const secrets = Object.entries(process.env) + .filter(([key]) => key.startsWith('SECRET_')) + .map(([, value]) => value) + .filter(Boolean); + +if (secrets.length === 0) { + process.stderr.write('emit-secret.js: no SECRET_* env vars found\n'); + process.exit(1); +} + +const parsedLines = Number(process.env.BENCH_EMIT_LINES); +const chunks = Number.isInteger(parsedLines) && parsedLines > 0 ? parsedLines : 200; +for (let i = 0; i < chunks; i++) { + const secret = secrets[i % secrets.length]; + process.stdout.write(`line-${i}: prefix ${secret} suffix\n`); +} diff --git a/benchmarks/fixtures/lang-go/.env.schema b/benchmarks/fixtures/lang-go/.env.schema new file mode 100644 index 000000000..6247e56c7 --- /dev/null +++ b/benchmarks/fixtures/lang-go/.env.schema @@ -0,0 +1,9 @@ +# @defaultSensitive=false +# @generateGoEnv(path=env/env.go) +# --- +# @type=port +PORT=8080 # @required @public +# @type=boolean +DEBUG=true # @required @public +OPTIONAL_UNSET= # @optional @public +SECRET=shhh # @required @sensitive diff --git a/benchmarks/fixtures/lang-go/go.mod b/benchmarks/fixtures/lang-go/go.mod new file mode 100644 index 000000000..773fe1adf --- /dev/null +++ b/benchmarks/fixtures/lang-go/go.mod @@ -0,0 +1,3 @@ +module benchlang + +go 1.21 diff --git a/benchmarks/fixtures/lang-go/main.go b/benchmarks/fixtures/lang-go/main.go new file mode 100644 index 000000000..333383137 --- /dev/null +++ b/benchmarks/fixtures/lang-go/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "os" + + "benchlang/env" +) + +func main() { + e, err := env.Load() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if e.Port != 8080 || !e.Debug { + fmt.Fprintf(os.Stderr, "unexpected: port=%d debug=%t\n", e.Port, e.Debug) + os.Exit(1) + } + if e.OptionalUnset != nil { + fmt.Fprintf(os.Stderr, "expected OptionalUnset to be nil, got %v\n", *e.OptionalUnset) + os.Exit(1) + } + if !env.SensitiveKeys["SECRET"] { + fmt.Fprintln(os.Stderr, "SECRET not marked sensitive") + os.Exit(1) + } + fmt.Println("OK") +} diff --git a/benchmarks/fixtures/lang-python/.env.schema b/benchmarks/fixtures/lang-python/.env.schema new file mode 100644 index 000000000..b9ecbede1 --- /dev/null +++ b/benchmarks/fixtures/lang-python/.env.schema @@ -0,0 +1,9 @@ +# @defaultSensitive=false +# @generatePythonEnv(path=env.py) +# --- +# @type=port +PORT=8080 # @required @public +# @type=boolean +DEBUG=true # @required @public +OPTIONAL_UNSET= # @optional @public +SECRET=shhh # @required @sensitive diff --git a/benchmarks/fixtures/lang-python/main.py b/benchmarks/fixtures/lang-python/main.py new file mode 100644 index 000000000..eaf2adeb0 --- /dev/null +++ b/benchmarks/fixtures/lang-python/main.py @@ -0,0 +1,8 @@ +from env import load_env, SENSITIVE_KEYS + +ENV = load_env() +assert ENV["PORT"] == 8080, ENV["PORT"] +assert ENV["DEBUG"] is True, ENV["DEBUG"] +assert "OPTIONAL_UNSET" not in ENV, ENV +assert "SECRET" in SENSITIVE_KEYS +print("OK") diff --git a/benchmarks/package.json b/benchmarks/package.json new file mode 100644 index 000000000..d5f47a3e1 --- /dev/null +++ b/benchmarks/package.json @@ -0,0 +1,15 @@ +{ + "name": "varlock-benchmarks", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Release benchmarking suite for varlock (CLI, integrations, codegen)", + "scripts": { + "bench": "bun run src/run.ts", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.9.3" + } +} diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/src/install.ts b/benchmarks/src/install.ts new file mode 100644 index 000000000..ec5e321db --- /dev/null +++ b/benchmarks/src/install.ts @@ -0,0 +1,151 @@ +import { + mkdirSync, existsSync, chmodSync, rmSync, readFileSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { CliInvocation } from './types.ts'; + +function runOrThrow( + command: string, + args: Array, + opts: { cwd?: string; env?: NodeJS.ProcessEnv }, +): void { + const result = spawnSync(command, args, { + cwd: opts.cwd, + env: opts.env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(' ')} failed (exit ${result.status}):\n${result.stderr}\n${result.stdout}`, + ); + } +} + +export function npmInstallDir(workDir: string): string { + return join(workDir, 'installs', 'npm'); +} + +export function bunInstallDir(workDir: string): string { + return join(workDir, 'installs', 'bun'); +} + +/** + * Install published `varlock@version` with npm into workDir/installs/npm + * and return a CliInvocation that runs it via node. + */ +export function installVarlockNpm( + workDir: string, + version: string, +): CliInvocation { + const dir = npmInstallDir(workDir); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + runOrThrow('npm', ['init', '-y'], { cwd: dir }); + runOrThrow('npm', ['install', `varlock@${version}`, '--no-fund', '--no-audit'], { cwd: dir }); + const cliJs = join(dir, 'node_modules', 'varlock', 'bin', 'cli.js'); + if (!existsSync(cliJs)) { + throw new Error(`npm install did not produce CLI at ${cliJs}`); + } + return { + command: [process.execPath, cliJs], + label: 'npm', + packageManager: 'npm', + }; +} + +/** + * Install published `varlock@version` with bun into workDir/installs/bun and run + * it with the bun runtime. + * + * Running it under node would execute byte-identical code to the npm invocation + * above, so the two would measure the same thing twice. Bun as the runtime is the + * axis worth measuring. + */ +export function installVarlockBun( + workDir: string, + version: string, +): CliInvocation { + const dir = bunInstallDir(workDir); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + runOrThrow('bun', ['init', '-y'], { cwd: dir }); + runOrThrow('bun', ['add', `varlock@${version}`], { cwd: dir }); + const cliJs = join(dir, 'node_modules', 'varlock', 'bin', 'cli.js'); + if (!existsSync(cliJs)) { + throw new Error(`bun install did not produce CLI at ${cliJs}`); + } + return { + command: ['bun', cliJs], + label: 'bun', + packageManager: 'bun', + }; +} + +export function seaInvocation(seaPath: string): CliInvocation { + if (!existsSync(seaPath)) { + throw new Error(`SEA binary not found at ${seaPath}`); + } + chmodSync(seaPath, 0o755); + return { + command: [seaPath], + label: 'sea', + }; +} + +/** Check that an invocation actually runs, so a broken one skips instead of failing the suite. */ +export function probeCli(cli: CliInvocation): { ok: true } | { ok: false; error: string } { + const [bin, ...args] = cli.command; + const result = spawnSync(bin!, [...args, '--version'], { encoding: 'utf8' }); + if (result.error) return { ok: false, error: result.error.message }; + if (result.status !== 0) { + return { ok: false, error: `exit ${result.status}: ${(result.stderr || result.stdout || '').trim().slice(0, 500)}` }; + } + return { ok: true }; +} + +/** Resolve package version of an installed package under an install root. */ +export function readInstalledVersion(installRoot: string, pkgName: string): string | null { + try { + const pkgPath = join(installRoot, 'node_modules', ...pkgName.split('/'), 'package.json'); + const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { version?: string }; + return pkg.version ?? null; + } catch { + return null; + } +} + +export function npmViewVersion(pkgSpec: string): string { + const result = spawnSync('npm', ['view', pkgSpec, 'version'], { encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error(`npm view ${pkgSpec} failed: ${result.stderr}`); + } + return result.stdout.trim(); +} + +export function tryNpmViewVersion(pkgSpec: string): string | undefined { + try { + return npmViewVersion(pkgSpec) || undefined; + } catch { + return undefined; + } +} + +/** + * Wait for a version to be resolvable on npm. Returns immediately when it already + * is — release-triggered runs can start before the registry has caught up. + */ +export async function waitForNpmPackage(pkgSpec: string, attempts = 30, delayMs = 10_000): Promise { + for (let i = 1; i <= attempts; i++) { + const result = spawnSync('npm', ['view', pkgSpec, 'version'], { encoding: 'utf8' }); + if (result.status === 0 && result.stdout.trim()) { + return result.stdout.trim(); + } + console.log(`waiting for ${pkgSpec} on npm (${i}/${attempts})...`); + await new Promise((r) => { + setTimeout(r, delayMs); + }); + } + throw new Error(`Timed out waiting for ${pkgSpec} on npm`); +} diff --git a/benchmarks/src/many-secrets-schema.ts b/benchmarks/src/many-secrets-schema.ts new file mode 100644 index 000000000..02b3b8458 --- /dev/null +++ b/benchmarks/src/many-secrets-schema.ts @@ -0,0 +1,132 @@ +/** + * Shared multi-secret schema bodies for integration latency/build benches. + * Keep framework-required public keys, then add ~18 distinct sensitive values. + */ + +export const NEXT_MANY_SECRETS_SCHEMA = `# @defaultSensitive=false @defaultRequired=infer +# @generateTypes(lang="ts", path="env.d.ts") +# @currentEnv=$APP_ENV +# --- + +# @type=enum(dev, preview, prod, test) +APP_ENV=dev + +NEXT_PUBLIC_VAR=next-prefixed-public-var +PUBLIC_VAR=unprefixed-public-var +ENV_SPECIFIC_VAR=env-specific-var--default + +# Kept for framework-test page templates that reference ENV.SENSITIVE_VAR +# @sensitive +SENSITIVE_VAR=super-secret-var + +# @sensitive +SECRET_TOKEN=super-secret-token-12345 +# @sensitive +SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa +# @sensitive +SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb +# @sensitive +SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig +# @sensitive +SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc +# @sensitive +SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD +# @sensitive +SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee +# @sensitive +SECRET_REDIS=redis-auth-bench-ffffffffffff +# @sensitive +SECRET_SMTP=smtp-pass-bench-gggggggggggg +# @sensitive +SECRET_OAUTH=oauth-client-secret-hhhhhhhh +# @sensitive +SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii +# @sensitive +SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj +# @sensitive +SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk +# @sensitive +SECRET_GITHUB=ghp_benchTokenLlllllllllllllll +# @sensitive +SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm +# @sensitive +SECRET_OPENAI=sk-proj-bench-openainnnnnnnn +# @sensitive +SECRET_SENTRY=sntrys_bench_oooooooooooooo +`; + +export const VITE_MANY_SECRETS_SCHEMA = `# @defaultSensitive=false @defaultRequired=infer +# @currentEnv=$APP_ENV +# --- + +# @type=enum(dev, prod) +APP_ENV=dev + +PUBLIC_VAR=public-test-value +API_URL=https://api.example.com +ENV_SPECIFIC_VAR=env-specific-default + +# Kept for framework-test page templates that reference ENV.SECRET_KEY +# @sensitive +SECRET_KEY=super-secret-value + +# @sensitive +SECRET_TOKEN=super-secret-token-12345 +# @sensitive +SECRET_API_KEY=sk-live-bench-api-key-aaaaaaaa +# @sensitive +SECRET_DB_PASSWORD=db-pass-bench-bbbbbbbbbbbb +# @sensitive +SECRET_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.benchpayload.sig +# @sensitive +SECRET_STRIPE=sk_test_bench_stripe_cccccccccccc +# @sensitive +SECRET_AWS_ACCESS=AKIA_BENCH_ACCESS_KEY_DDDD +# @sensitive +SECRET_AWS_SECRET=awsSecretBenchKeyEeeeeeeeeeee +# @sensitive +SECRET_REDIS=redis-auth-bench-ffffffffffff +# @sensitive +SECRET_SMTP=smtp-pass-bench-gggggggggggg +# @sensitive +SECRET_OAUTH=oauth-client-secret-hhhhhhhh +# @sensitive +SECRET_WEBHOOK=whsec_bench_iiiiiiiiiiiiiiii +# @sensitive +SECRET_ENCRYPTION=enc-key-bench-jjjjjjjjjjjjjj +# @sensitive +SECRET_SESSION=sess-bench-kkkkkkkkkkkkkkkk +# @sensitive +SECRET_GITHUB=ghp_benchTokenLlllllllllllllll +# @sensitive +SECRET_SLACK=xoxb-bench-slack-mmmmmmmmmmmm +# @sensitive +SECRET_OPENAI=sk-proj-bench-openainnnnnnnn +# @sensitive +SECRET_SENTRY=sntrys_bench_oooooooooooooo +`; + +const HEADER_DIVIDER = '# ---'; + +/** + * Apply preventLeaks / redactLogs root flags onto a schema body — replacing an + * existing flag, or adding it to the header section when absent. + */ +export function withSchemaFlags( + schema: string, + preventLeaks: boolean, + redactLogs: boolean, +): string { + let out = schema; + for (const [flag, value] of [['preventLeaks', preventLeaks], ['redactLogs', redactLogs]] as const) { + const existing = new RegExp(`@${flag}=\\w+`); + if (existing.test(out)) { + out = out.replace(existing, `@${flag}=${value}`); + } else if (out.includes(HEADER_DIVIDER)) { + out = out.replace(HEADER_DIVIDER, `# @${flag}=${value}\n${HEADER_DIVIDER}`); + } else { + throw new Error(`withSchemaFlags: schema has no "${HEADER_DIVIDER}" header divider to add @${flag} to`); + } + } + return out; +} diff --git a/benchmarks/src/measure.ts b/benchmarks/src/measure.ts new file mode 100644 index 000000000..5cdf1732a --- /dev/null +++ b/benchmarks/src/measure.ts @@ -0,0 +1,333 @@ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import { + cpSync, mkdirSync, readFileSync, readdirSync, rmSync, +} from 'node:fs'; +import type { Sample, ScenarioMetrics } from './types.ts'; + +/** RSS (KiB) of a single pid, or null if it is gone / unreadable. */ +function rssKiBForPid(pid: number): number | null { + try { + const status = readFileSync(`/proc/${pid}/status`, 'utf8'); + const match = status.match(/^VmRSS:\s+(\d+)/m); + return match ? Number(match[1]) : null; + } catch { + return null; + } +} + +/** Direct children of a pid, via /proc//task//children. */ +function childPidsLinux(pid: number): Array { + const out: Array = []; + let tids: Array; + try { + tids = readdirSync(`/proc/${pid}/task`); + } catch { + return out; + } + for (const tid of tids) { + try { + const raw = readFileSync(`/proc/${pid}/task/${tid}/children`, 'utf8').trim(); + if (!raw) continue; + for (const part of raw.split(/\s+/)) { + const n = Number(part); + if (Number.isInteger(n)) out.push(n); + } + } catch { + // thread exited between readdir and read + } + } + return out; +} + +/** + * Summed RSS (KiB) of a process and all of its descendants. + * + * The tree matters: `varlock run -- node app.js` and `npx next build` both do the + * real work in a grandchild, so sampling only the direct child would report the + * footprint of a wrapper process. + * + * On Linux this walks /proc with no subprocess spawns. Elsewhere it shells out to + * `ps` once per sample, which perturbs the very timings we are measuring — so + * non-Linux runs sample at a coarser interval and are best treated as indicative. + */ +export function rssTreeKiB(rootPid: number): number | null { + if (process.platform === 'linux') { + let total = 0; + let found = false; + const seen = new Set(); + const queue = [rootPid]; + while (queue.length) { + const pid = queue.pop()!; + if (seen.has(pid)) continue; + seen.add(pid); + const rss = rssKiBForPid(pid); + if (rss !== null) { + total += rss; + found = true; + } + queue.push(...childPidsLinux(pid)); + } + return found ? total : null; + } + + const result = spawnSync('ps', ['-eo', 'pid=,ppid=,rss='], { encoding: 'utf8' }); + if (result.status !== 0) return null; + const rssByPid = new Map(); + const childrenByPid = new Map>(); + for (const line of result.stdout.split('\n')) { + const parts = line.trim().split(/\s+/); + if (parts.length < 3) continue; + const [pid, ppid, rss] = parts.map(Number); + if (!Number.isInteger(pid) || !Number.isInteger(ppid) || !Number.isFinite(rss)) continue; + rssByPid.set(pid, rss!); + const siblings = childrenByPid.get(ppid!) ?? []; + siblings.push(pid!); + childrenByPid.set(ppid!, siblings); + } + if (!rssByPid.has(rootPid)) return null; + let total = 0; + const seen = new Set(); + const queue = [rootPid]; + while (queue.length) { + const pid = queue.pop()!; + if (seen.has(pid)) continue; + seen.add(pid); + total += rssByPid.get(pid) ?? 0; + queue.push(...(childrenByPid.get(pid) ?? [])); + } + return total; +} + +function percentile(sorted: Array, p: number): number { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.max(0, idx)]!; +} + +function median(values: Array): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return (sorted[mid - 1]! + sorted[mid]!) / 2; + } + return sorted[mid]!; +} + +function stdDev(values: Array): number { + if (values.length < 2) return 0; + const mean = values.reduce((sum, v) => sum + v, 0) / values.length; + const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (values.length - 1); + return Math.sqrt(variance); +} + +export function summarizeSamples(samples: Array): ScenarioMetrics { + if (samples.length === 0) { + throw new Error('summarizeSamples: no samples — refusing to report zeroed metrics'); + } + const walls = samples.map((s) => s.wallMs).sort((a, b) => a - b); + const rssValues = samples + .map((s) => s.rssPeakBytes) + .filter((v): v is number => v !== null); + + return { + iterations: samples.length, + wallMsMin: walls[0]!, + wallMsMedian: median(walls), + // With few iterations p95 collapses onto the max — it is kept for continuity + // but wallMsMin / wallMsStdDev are the numbers to reason about. + wallMsP95: percentile(walls, 95), + wallMsStdDev: stdDev(walls), + rssPeakBytesMedian: rssValues.length > 0 ? median(rssValues) : null, + rssSampleCount: rssValues.length, + samples, + }; +} + +export type MeasureCommandOptions = { + cwd?: string; + env?: Record; + input?: string; + /** Sample RSS of the spawned process tree while it runs. Default true. */ + sampleRss?: boolean; + sampleIntervalMs?: number; + timeoutMs?: number; +}; + +/** `ps` costs a spawn per sample, so back off when we cannot read /proc. */ +const DEFAULT_RSS_INTERVAL_MS = process.platform === 'linux' ? 10 : 50; + +/** + * Spawn a command, measure wall time and peak RSS of the whole process tree. + */ +export function measureCommand( + command: Array, + options: MeasureCommandOptions = {}, +): Promise { + const [bin, ...args] = command; + if (!bin) { + return Promise.reject(new Error('measureCommand: empty command')); + } + + const sampleRss = options.sampleRss !== false; + const sampleIntervalMs = options.sampleIntervalMs ?? DEFAULT_RSS_INTERVAL_MS; + const timeoutMs = options.timeoutMs ?? 120_000; + + return new Promise((resolve, reject) => { + const start = performance.now(); + let peakRssKiB: number | null = null; + let stdout = ''; + let stderr = ''; + let settled = false; + let sampling: ReturnType | undefined; + const timers: { timeout?: ReturnType } = {}; + + const finish = (err: Error) => { + if (settled) return; + settled = true; + if (sampling) clearInterval(sampling); + if (timers.timeout) clearTimeout(timers.timeout); + reject(err); + }; + + const childEnv: NodeJS.ProcessEnv = { ...process.env }; + if (options.env) { + for (const [key, value] of Object.entries(options.env)) { + if (value === undefined) { + delete childEnv[key]; + } else { + childEnv[key] = value; + } + } + } + + const child: ChildProcess = spawn(bin, args, { + cwd: options.cwd, + env: childEnv, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + if (sampleRss) { + const takeSample = () => { + if (!child.pid) return; + const rss = rssTreeKiB(child.pid); + if (rss !== null) { + peakRssKiB = peakRssKiB === null ? rss : Math.max(peakRssKiB, rss); + } + }; + // Sample immediately — short-lived commands can finish inside one interval, + // which previously reported no RSS at all. + takeSample(); + sampling = setInterval(takeSample, sampleIntervalMs); + } + + timers.timeout = setTimeout(() => { + child.kill('SIGKILL'); + finish(new Error(`Command timed out after ${timeoutMs}ms: ${command.join(' ')}`)); + }, timeoutMs); + + if (options.input) { + child.stdin?.write(options.input); + } + child.stdin?.end(); + + child.stdout?.on('data', (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr?.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + child.on('error', (err) => { + finish(err); + }); + + child.on('close', (code) => { + const wallMs = performance.now() - start; + if (sampling) clearInterval(sampling); + if (timers.timeout) clearTimeout(timers.timeout); + if (settled) return; + settled = true; + resolve({ + wallMs, + rssPeakBytes: peakRssKiB !== null ? peakRssKiB * 1024 : null, + exitCode: code ?? 1, + stdout, + stderr, + }); + }); + }); +} + +export type RepeatOptions = { + iterations: number; + warmup: number; + /** Throw if any measured iteration exits non-zero. Default true. */ + expectSuccess?: boolean; + /** + * Truncate captured output in failure messages. Scenarios that deliberately + * print secret values to stdout (redaction benches) set this low so fixture + * secrets do not end up in CI logs. + */ + maxFailureOutputChars?: number; +}; + +function truncate(value: string, max: number): string { + if (value.length <= max) return value; + return `${value.slice(0, max)}… [${value.length - max} more chars truncated]`; +} + +/** + * Run warmup + measured iterations of an async sample factory. + */ +export async function repeatMeasure( + factory: () => Promise, + options: RepeatOptions, +): Promise { + const expectSuccess = options.expectSuccess !== false; + const maxOutput = options.maxFailureOutputChars ?? 4_000; + + if (!Number.isInteger(options.iterations) || options.iterations < 1) { + throw new Error(`repeatMeasure: iterations must be a positive integer, got ${options.iterations}`); + } + if (!Number.isInteger(options.warmup) || options.warmup < 0) { + throw new Error(`repeatMeasure: warmup must be a non-negative integer, got ${options.warmup}`); + } + + for (let i = 0; i < options.warmup; i++) { + const warm = await factory(); + if (expectSuccess && warm.exitCode !== 0) { + throw new Error(`Warmup failed with exit ${warm.exitCode}`); + } + } + + const samples: Array = []; + for (let i = 0; i < options.iterations; i++) { + const sample = await factory(); + if (expectSuccess && sample.exitCode !== 0) { + const withOutput = sample as { stdout?: string; stderr?: string }; + const extra = withOutput.stdout !== undefined || withOutput.stderr !== undefined + ? `\nstdout:\n${truncate(withOutput.stdout ?? '', maxOutput)}\nstderr:\n${truncate(withOutput.stderr ?? '', maxOutput)}` + : ''; + throw new Error(`Iteration ${i} failed with exit ${sample.exitCode}${extra}`); + } + samples.push({ + wallMs: sample.wallMs, + rssPeakBytes: sample.rssPeakBytes, + exitCode: sample.exitCode, + }); + } + + return summarizeSamples(samples); +} + +/** + * Copy a fixture directory into a work subdirectory, so scenarios never mutate + * the checked-in fixtures (codegen output, caches, build artifacts). + */ +export function copyFixture(sourceDir: string, destDir: string): string { + rmSync(destDir, { recursive: true, force: true }); + mkdirSync(destDir, { recursive: true }); + cpSync(sourceDir, destDir, { recursive: true }); + return destDir; +} diff --git a/benchmarks/src/report.ts b/benchmarks/src/report.ts new file mode 100644 index 000000000..515415563 --- /dev/null +++ b/benchmarks/src/report.ts @@ -0,0 +1,133 @@ +import type { BenchRunResult, ScenarioResult } from './types.ts'; + +function fmtMs(n: number): string { + return `${n.toFixed(1)}ms`; +} + +function fmtRss(bytes: number | null): string { + if (bytes === null) return '-'; + return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`; +} + +function fmtDelta(deltaMs: number, basisMs: number): string { + const sign = deltaMs >= 0 ? '+' : ''; + const pct = basisMs > 0 ? ` (${sign}${((deltaMs / basisMs) * 100).toFixed(1)}%)` : ''; + return `${sign}${deltaMs.toFixed(1)}ms${pct}`; +} + +type Comparison = { + label: string; + baseId: string; + againstId: string; +}; + +/** + * The interesting numbers are all differences: what varlock adds over a plain + * build, what telemetry adds, what redaction adds. Computing them here means the + * committed results are readable without doing arithmetic by hand. + * + * Ids are matched by suffix so one entry covers every install method. + */ +const COMPARISONS: Array = [ + { label: 'varlock run wrap overhead', baseId: 'cli.run.bare-node', againstId: 'cli.run.wrap.telemetry.off' }, + { label: 'telemetry cost (cli run)', baseId: 'cli.run.wrap.telemetry.off', againstId: 'cli.run.wrap.telemetry.on' }, + { label: 'telemetry cost (cli load, cold)', baseId: 'cli.load.cold.telemetry.off', againstId: 'cli.load.cold.telemetry.on' }, + { label: 'cache benefit (cold - warm)', baseId: 'cli.load.warm.telemetry.off', againstId: 'cli.load.cold.telemetry.off' }, + { label: 'stdout redaction cost', baseId: 'cli.run.redact-stdout.off', againstId: 'cli.run.redact-stdout.on' }, + { label: 'next build: varlock overhead', baseId: 'integration.next.build.baseline', againstId: 'integration.next.build.varlock.telemetry.off' }, + { label: 'next build: telemetry cost', baseId: 'integration.next.build.varlock.telemetry.off', againstId: 'integration.next.build.varlock.telemetry.on' }, + { label: 'next request: preventLeaks cost', baseId: 'integration.next.request.preventLeaks.off', againstId: 'integration.next.request.preventLeaks.on' }, + { label: 'next request: redactLogs cost', baseId: 'integration.next.request.redactLogs.off', againstId: 'integration.next.request.redactLogs.on' }, + { label: 'vite build: varlock overhead', baseId: 'integration.vite.build.baseline', againstId: 'integration.vite.build.varlock.telemetry.off' }, + { label: 'vite build: telemetry cost', baseId: 'integration.vite.build.varlock.telemetry.off', againstId: 'integration.vite.build.varlock.telemetry.on' }, + { label: 'vite request: preventLeaks cost', baseId: 'integration.vite.request.preventLeaks.off', againstId: 'integration.vite.request.preventLeaks.on' }, + { label: 'vite request: redactLogs cost', baseId: 'integration.vite.request.redactLogs.off', againstId: 'integration.vite.request.redactLogs.on' }, +]; + +/** Match `cli.load.cold.telemetry.off` against `cli.load.cold.telemetry.off.install.npm`. */ +function findByBaseId(scenarios: Array, baseId: string): Array { + return scenarios.filter((s) => s.id === baseId || s.id.startsWith(`${baseId}.install.`)); +} + +function installOf(s: ScenarioResult): string { + return s.id.includes('.install.') ? s.installMethod : 'n/a'; +} + +function comparisonRows(scenarios: Array): Array { + const rows: Array = []; + for (const cmp of COMPARISONS) { + const bases = findByBaseId(scenarios, cmp.baseId); + const againsts = findByBaseId(scenarios, cmp.againstId); + if (bases.length === 0 || againsts.length === 0) continue; + + for (const against of againsts) { + // Prefer the same install method, fall back to the single shared baseline + const base = bases.find((b) => b.installMethod === against.installMethod) + ?? (bases.length === 1 ? bases[0] : undefined); + if (!base) continue; + const deltaMedian = against.metrics.wallMsMedian - base.metrics.wallMsMedian; + const deltaMin = against.metrics.wallMsMin - base.metrics.wallMsMin; + // Noise guard: a delta smaller than the spread of either side is not a signal. + const noise = Math.max(base.metrics.wallMsStdDev, against.metrics.wallMsStdDev); + const verdict = Math.abs(deltaMedian) < noise ? ' _(within noise)_' : ''; + rows.push( + `| ${cmp.label} | ${installOf(against)} | ${fmtDelta(deltaMedian, base.metrics.wallMsMedian)} | ${fmtDelta(deltaMin, base.metrics.wallMsMin)}${verdict} |`, + ); + } + } + return rows; +} + +export function formatSummaryMarkdown(result: BenchRunResult): string { + const lines: Array = []; + lines.push('## Varlock benchmarks'); + lines.push(''); + lines.push(`- **varlock:** ${result.meta.versions.varlock}`); + lines.push(`- **trigger:** ${result.meta.trigger}`); + lines.push(`- **runner:** ${result.meta.runnerOs}/${result.meta.runnerArch} (node ${result.meta.nodeVersion})`); + lines.push(`- **timestamp:** ${result.meta.timestamp}`); + if (result.meta.gitSha) lines.push(`- **git:** ${result.meta.gitSha.slice(0, 12)}`); + lines.push(`- **telemetry:** ${result.meta.telemetryMocked ? 'mocked locally' : 'not measured'}`); + lines.push(''); + + if (result.meta.notes.length > 0) { + lines.push('### Notes'); + lines.push(''); + for (const note of result.meta.notes) lines.push(`- ${note}`); + lines.push(''); + } + + const deltas = comparisonRows(result.scenarios); + if (deltas.length > 0) { + lines.push('### Deltas'); + lines.push(''); + lines.push('| Comparison | Install | Δ median | Δ min |'); + lines.push('|------------|---------|----------|-------|'); + lines.push(...deltas); + lines.push(''); + } + + lines.push('### Raw'); + lines.push(''); + lines.push('| Scenario | Install | Telemetry | Min | Median | p95 | StdDev | Peak RSS |'); + lines.push('|----------|---------|-----------|-----|--------|-----|--------|----------|'); + + const sorted = [...result.scenarios].sort((a, b) => a.id.localeCompare(b.id)); + for (const s of sorted) { + const m = s.metrics; + lines.push( + `| ${s.id} | ${s.installMethod} | ${s.telemetry} | ${fmtMs(m.wallMsMin)} | ${fmtMs(m.wallMsMedian)} | ${fmtMs(m.wallMsP95)} | ${fmtMs(m.wallMsStdDev)} | ${fmtRss(m.rssPeakBytesMedian)} |`, + ); + } + lines.push(''); + return lines.join('\n'); +} + +export function printScenarioLine(s: ScenarioResult): void { + const rss = s.metrics.rssPeakBytesMedian !== null + ? ` rss=${fmtRss(s.metrics.rssPeakBytesMedian)}` + : ''; + console.log( + ` ${s.id} [${s.installMethod} telemetry=${s.telemetry}] min=${fmtMs(s.metrics.wallMsMin)} median=${fmtMs(s.metrics.wallMsMedian)} sd=${fmtMs(s.metrics.wallMsStdDev)}${rss}`, + ); +} diff --git a/benchmarks/src/run.ts b/benchmarks/src/run.ts new file mode 100644 index 000000000..193e70220 --- /dev/null +++ b/benchmarks/src/run.ts @@ -0,0 +1,309 @@ +import { + mkdirSync, writeFileSync, existsSync, readFileSync, readdirSync, +} from 'node:fs'; +import { join, resolve, dirname } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { + installVarlockBun, + installVarlockNpm, + npmInstallDir, + bunInstallDir, + probeCli, + readInstalledVersion, + seaInvocation, + tryNpmViewVersion, + waitForNpmPackage, +} from './install.ts'; +import { runAllScenarios, SCENARIO_GROUPS } from './scenarios/index.ts'; +import { formatSummaryMarkdown } from './report.ts'; +import { startTelemetryMock } from './telemetry-mock.ts'; +import { ALL_TELEMETRY_MODES } from './telemetry.ts'; +import type { + BenchContext, BenchRunResult, CliInvocation, TelemetryMode, TriggerKind, +} from './types.ts'; + +const ROOT_DIR = resolve(import.meta.dirname, '..'); +const FIXTURES_DIR = join(ROOT_DIR, 'fixtures'); +const RESULTS_DIR = join(ROOT_DIR, 'results'); +const WORK_DIR = join(ROOT_DIR, '.work'); + +const TRIGGER_KINDS: Array = ['release', 'workflow_dispatch', 'local']; + +type Args = { + version: string; + seaPath: string | null; + out: string | null; + iterations: number; + warmup: number; + only: Array; + skipInstall: boolean; + trigger: TriggerKind; + help: boolean; +}; + +function parsePositiveInt(raw: string | undefined, flag: string, { allowZero = false } = {}): number { + const n = Number(raw); + const min = allowZero ? 0 : 1; + if (raw === undefined || raw === '' || !Number.isInteger(n) || n < min) { + throw new Error(`${flag} expects an integer >= ${min}, got ${JSON.stringify(raw)}`); + } + return n; +} + +function parseArgs(argv: Array): Args { + const args: Args = { + version: 'latest', + seaPath: null, + out: null, + iterations: 5, + warmup: 1, + only: [], + skipInstall: false, + trigger: 'local', + help: false, + }; + + const envTrigger = process.env.BENCH_TRIGGER; + if (envTrigger) { + if (!TRIGGER_KINDS.includes(envTrigger as TriggerKind)) { + throw new Error(`BENCH_TRIGGER must be one of ${TRIGGER_KINDS.join(', ')}, got ${JSON.stringify(envTrigger)}`); + } + args.trigger = envTrigger as TriggerKind; + } + + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + if (a === '--help' || a === '-h') args.help = true; + else if (a === '--version') args.version = argv[++i] ?? args.version; + else if (a === '--sea-path') args.seaPath = argv[++i] ?? null; + else if (a === '--out') args.out = argv[++i] ?? null; + else if (a === '--iterations') args.iterations = parsePositiveInt(argv[++i], '--iterations'); + else if (a === '--warmup') args.warmup = parsePositiveInt(argv[++i], '--warmup', { allowZero: true }); + else if (a === '--only') args.only = (argv[++i] ?? '').split(',').map((s) => s.trim()).filter(Boolean); + else if (a === '--skip-install') args.skipInstall = true; + else if (a === '--trigger') { + const value = argv[++i]; + if (!value || !TRIGGER_KINDS.includes(value as TriggerKind)) { + throw new Error(`--trigger must be one of ${TRIGGER_KINDS.join(', ')}, got ${JSON.stringify(value)}`); + } + args.trigger = value as TriggerKind; + } else throw new Error(`Unknown argument: ${a}`); + } + + // Unknown group names used to produce an empty run that still looked successful. + const knownGroups = SCENARIO_GROUPS.map((g) => g.name); + const unknown = args.only.filter((name) => !knownGroups.includes(name)); + if (unknown.length > 0) { + throw new Error(`--only got unknown scenario group(s): ${unknown.join(', ')}\nKnown groups: ${knownGroups.join(', ')}`); + } + + return args; +} + +function usage(): string { + const groups = SCENARIO_GROUPS.map((g) => g.name).join(', '); + return `Usage: bun run src/run.ts [options] + +Options: + --version Published varlock version (default: latest) + --sea-path Path to SEA binary (enables sea install method) + --out Output JSON path (default: results/-varlock@-.json) + --iterations Measured iterations (default: 5) + --warmup Warmup iterations (default: 1) + --only Comma-separated scenario groups: ${groups} + --skip-install Reuse .work/installs from a previous run + --trigger ${TRIGGER_KINDS.join(' | ')} + --help +`; +} + +/** Bad flags are user error, not a crash — print the problem and the usage, no stack. */ +function parseArgsOrExit(argv: Array): Args { + try { + return parseArgs(argv); + } catch (err) { + console.error(`${(err as Error).message}\n`); + console.error(usage()); + process.exit(1); + } +} + +function gitSha(): string | null { + const r = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', cwd: ROOT_DIR }); + return r.status === 0 ? r.stdout.trim() : null; +} + +function defaultOutPath(version: string): string { + const iso = new Date().toISOString().replace(/[:.]/g, '-'); + const runId = process.env.GITHUB_RUN_ID ?? 'local'; + return join(RESULTS_DIR, `${iso}-varlock@${version}-${runId}.json`); +} + +/** Inspect the installed package without executing telemetry-enabled code. */ +function checkTelemetryMockable(cli: CliInvocation): boolean { + const cliScript = cli.command.find((part) => part.endsWith('/bin/cli.js')); + if (!cliScript) return false; + + const pending = [join(dirname(dirname(cliScript)), 'dist')]; + try { + while (pending.length > 0) { + const dir = pending.pop()!; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) pending.push(path); + else if (/\.[cm]?js$/.test(entry.name) + && readFileSync(path, 'utf8').includes('VARLOCK_POSTHOG_HOST')) return true; + } + } + } catch { + return false; + } + return false; +} + +async function main(): Promise { + const args = parseArgsOrExit(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + // Waits when the version is not on npm yet — a release-triggered run can start + // before the registry has caught up. + const resolvedVersion = await waitForNpmPackage( + args.version === 'latest' ? 'varlock@latest' : `varlock@${args.version}`, + ); + + console.log(`Benchmarking varlock@${resolvedVersion}`); + mkdirSync(WORK_DIR, { recursive: true }); + + const notes: Array = []; + const note = (message: string) => { + notes.push(message); + console.log(` note: ${message}`); + }; + + const clis: Array = []; + if (args.skipInstall) { + const npmCli = join(npmInstallDir(WORK_DIR), 'node_modules', 'varlock', 'bin', 'cli.js'); + const bunCli = join(bunInstallDir(WORK_DIR), 'node_modules', 'varlock', 'bin', 'cli.js'); + if (!existsSync(npmCli) || !existsSync(bunCli)) { + throw new Error('--skip-install requires existing .work/installs/{npm,bun}'); + } + clis.push( + { command: [process.execPath, npmCli], label: 'npm', packageManager: 'npm' }, + { command: ['bun', bunCli], label: 'bun', packageManager: 'bun' }, + ); + } else { + console.log('Installing varlock via npm (run with node)...'); + clis.push(installVarlockNpm(WORK_DIR, resolvedVersion)); + console.log('Installing varlock via bun (run with bun)...'); + clis.push(installVarlockBun(WORK_DIR, resolvedVersion)); + } + + if (args.seaPath) { + console.log(`Using SEA binary at ${args.seaPath}`); + clis.push(seaInvocation(resolve(args.seaPath))); + } else { + note('SEA scenarios skipped: no --sea-path given'); + } + + // A CLI that cannot even print its version would fail every scenario it appears + // in. Drop it with a recorded note rather than taking the whole suite down. + const usableClis = clis.filter((cli) => { + const probe = probeCli(cli); + if (!probe.ok) { + note(`${cli.label} install skipped: \`varlock --version\` failed (${probe.error})`); + return false; + } + return true; + }); + if (usableClis.length === 0) { + throw new Error('No usable varlock CLI invocations — nothing to benchmark'); + } + + const telemetryMock = await startTelemetryMock(); + const telemetryMockEnv = { VARLOCK_POSTHOG_HOST: telemetryMock.url }; + let telemetryModes: Array = ['off']; + + try { + const mockable = checkTelemetryMockable(usableClis[0]!); + if (mockable) { + telemetryModes = ALL_TELEMETRY_MODES; + console.log(`Telemetry endpoint override supported: telemetry-on scenarios enabled with ${telemetryMock.url}`); + } else { + note( + `telemetry-on scenarios skipped: varlock@${resolvedVersion} does not honour VARLOCK_POSTHOG_HOST, ` + + 'and benchmarks never send real telemetry', + ); + } + + const ctx: BenchContext = { + version: resolvedVersion, + integrationVersions: { + nextjs: tryNpmViewVersion('@varlock/nextjs-integration'), + vite: tryNpmViewVersion('@varlock/vite-integration'), + }, + rootDir: ROOT_DIR, + fixturesDir: FIXTURES_DIR, + workDir: WORK_DIR, + iterations: args.iterations, + warmup: args.warmup, + clis: usableClis, + seaPath: args.seaPath, + telemetryModes, + telemetryMockEnv: telemetryModes.includes('on') ? telemetryMockEnv : {}, + // Enables the on-disk resolver cache even in CI, where varlock otherwise + // falls back to a per-process memory cache that cannot survive between + // invocations — which would make the warm-load scenario measure nothing. + cacheKey: randomBytes(32).toString('hex'), + note, + }; + + const scenarios = await runAllScenarios(ctx, args.only.length ? args.only : undefined); + + const result: BenchRunResult = { + meta: { + timestamp: new Date().toISOString(), + gitSha: gitSha(), + githubRunId: process.env.GITHUB_RUN_ID ?? null, + runnerOs: process.platform, + runnerArch: process.arch, + nodeVersion: process.version, + versions: { + varlock: resolvedVersion, + nextjsIntegration: ctx.integrationVersions.nextjs, + viteIntegration: ctx.integrationVersions.vite, + '@env-spec/parser': readInstalledVersion(npmInstallDir(WORK_DIR), '@env-spec/parser') ?? undefined, + }, + trigger: args.trigger, + telemetryMocked: telemetryModes.includes('on'), + notes, + }, + scenarios, + }; + + const outPath = args.out ? resolve(args.out) : defaultOutPath(resolvedVersion); + mkdirSync(dirname(outPath), { recursive: true }); + writeFileSync(outPath, `${JSON.stringify(result, null, 2)}\n`); + console.log(`\nWrote ${outPath}`); + + const summary = formatSummaryMarkdown(result); + console.log(`\n${summary}`); + + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' }); + } + + // Also write a pointer file used by CI commit step + writeFileSync(join(WORK_DIR, 'last-result-path.txt'), `${outPath}\n`); + } finally { + await telemetryMock.close(); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/benchmarks/src/scenarios/cli-load.ts b/benchmarks/src/scenarios/cli-load.ts new file mode 100644 index 000000000..2828217fe --- /dev/null +++ b/benchmarks/src/scenarios/cli-load.ts @@ -0,0 +1,54 @@ +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; + +export async function runCliLoadScenarios(ctx: BenchContext): Promise> { + const cwd = fixtureWorkDir(ctx, 'cli-basic', '-load'); + const results: Array = []; + + for (const cli of ctx.clis) { + for (const telemetry of ctx.telemetryModes) { + // _VARLOCK_CACHE_KEY forces the on-disk resolver cache. Without it, CI falls + // back to an in-process memory cache (see loader.ts cache policy), which + // cannot survive between CLI invocations — so "warm" would silently measure + // exactly the same work as "cold". + const env = { + ...telemetryEnv(telemetry, ctx.telemetryMockEnv), + _VARLOCK_CACHE_KEY: ctx.cacheKey, + }; + + const cold = await repeatMeasure( + async () => measureCommand([...cli.command, 'load', '--clear-cache'], { cwd, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId(`cli.load.cold.telemetry.${telemetry}`, cli), + facet: 'cli-load', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry, + metrics: cold, + notes: 'Disk cache cleared before every iteration', + }); + + // Warm: one clear then repeated loads without clear + await measureCommand([...cli.command, 'load', '--clear-cache'], { cwd, env }); + const warm = await repeatMeasure( + async () => measureCommand([...cli.command, 'load'], { cwd, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId(`cli.load.warm.telemetry.${telemetry}`, cli), + facet: 'cli-load', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry, + metrics: warm, + notes: 'Disk cache populated (_VARLOCK_CACHE_KEY set)', + }); + } + } + + return results; +} diff --git a/benchmarks/src/scenarios/cli-run.ts b/benchmarks/src/scenarios/cli-run.ts new file mode 100644 index 000000000..6464b6e9d --- /dev/null +++ b/benchmarks/src/scenarios/cli-run.ts @@ -0,0 +1,110 @@ +import { join } from 'node:path'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; + +/** + * Output volume for the redaction benches. + * + * Redaction cost is per byte of stdout, while everything around it (process + * spawn, varlock load, node startup) is a fixed ~50ms. At a few hundred lines the + * on/off delta sits well inside run-to-run noise on a shared runner, so the + * scenario cannot answer the question it exists to answer. ~3MB of output puts + * the redaction work above the noise floor. + */ +const REDACT_BENCH_LINES = 50_000; + +export async function runCliRunScenarios(ctx: BenchContext): Promise> { + const cwd = fixtureWorkDir(ctx, 'cli-basic', '-run'); + const childJs = join(cwd, 'child.js'); + const emitJs = join(cwd, 'emit-secret.js'); + const results: Array = []; + + // Bare node baseline (no varlock in the picture; installMethod is not meaningful here) + const bare = await repeatMeasure( + async () => measureCommand([process.execPath, childJs], { cwd }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: 'cli.run.bare-node', + facet: 'cli-run', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: bare, + notes: 'Baseline without varlock wrap — subtract from cli.run.wrap.* for wrap overhead', + }); + + for (const cli of ctx.clis) { + // Wrap overhead measured with telemetry on/off (exit-hook wait hits here) + for (const telemetry of ctx.telemetryModes) { + const env = telemetryEnv(telemetry, ctx.telemetryMockEnv); + const wrapped = await repeatMeasure( + async () => measureCommand( + [...cli.command, 'run', '--', process.execPath, childJs], + { cwd, env }, + ), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId(`cli.run.wrap.telemetry.${telemetry}`, cli), + facet: 'cli-run', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry, + metrics: wrapped, + notes: 'varlock run wrap overhead vs bare-node', + }); + } + + // Redaction comparison: telemetry off so we isolate redact-stdout cost + const envOff = { + ...telemetryEnv('off'), + BENCH_EMIT_LINES: String(REDACT_BENCH_LINES), + }; + // These scenarios deliberately print (fixture) secret values, so failure output + // is truncated hard rather than dumped into CI logs. + const redactRepeatOpts = { + iterations: ctx.iterations, + warmup: Math.max(1, ctx.warmup), + maxFailureOutputChars: 500, + }; + + const redactOn = await repeatMeasure( + async () => measureCommand( + [...cli.command, 'run', '--redact-stdout', '--', process.execPath, emitJs], + { cwd, env: envOff, timeoutMs: 300_000 }, + ), + redactRepeatOpts, + ); + results.push({ + id: cliScenarioId('cli.run.redact-stdout.on', cli), + facet: 'cli-run', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: redactOn, + notes: `${REDACT_BENCH_LINES} stdout lines containing secrets`, + }); + + const redactOff = await repeatMeasure( + async () => measureCommand( + [...cli.command, 'run', '--no-redact-stdout', '--', process.execPath, emitJs], + { cwd, env: envOff, timeoutMs: 300_000 }, + ), + redactRepeatOpts, + ); + results.push({ + id: cliScenarioId('cli.run.redact-stdout.off', cli), + facet: 'cli-run', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: redactOff, + notes: `${REDACT_BENCH_LINES} stdout lines containing secrets`, + }); + } + + return results; +} diff --git a/benchmarks/src/scenarios/cli-scan-audit.ts b/benchmarks/src/scenarios/cli-scan-audit.ts new file mode 100644 index 000000000..68eafccc0 --- /dev/null +++ b/benchmarks/src/scenarios/cli-scan-audit.ts @@ -0,0 +1,53 @@ +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; + +export async function runCliScanAuditScenarios(ctx: BenchContext): Promise> { + const cwd = fixtureWorkDir(ctx, 'cli-basic', '-scan'); + const results: Array = []; + const env = telemetryEnv('off'); + + // Lighter coverage: fewer iterations than load/run + const lightIterations = Math.max(2, Math.min(3, ctx.iterations)); + const lightWarmup = Math.min(1, ctx.warmup); + + for (const cli of ctx.clis) { + const scan = await repeatMeasure( + async () => measureCommand([...cli.command, 'scan', './child.js'], { + cwd, + env, + timeoutMs: 180_000, + }), + { iterations: lightIterations, warmup: lightWarmup }, + ); + results.push({ + id: cliScenarioId('cli.scan', cli), + facet: 'cli-scan', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: scan, + notes: 'scan of a clean file (load + scan cost)', + }); + + const audit = await repeatMeasure( + async () => measureCommand([...cli.command, 'audit', '.'], { + cwd, + env, + timeoutMs: 180_000, + }), + { iterations: lightIterations, warmup: lightWarmup }, + ); + results.push({ + id: cliScenarioId('cli.audit', cli), + facet: 'cli-audit', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: audit, + }); + } + + return results; +} diff --git a/benchmarks/src/scenarios/index.ts b/benchmarks/src/scenarios/index.ts new file mode 100644 index 000000000..44901f8b4 --- /dev/null +++ b/benchmarks/src/scenarios/index.ts @@ -0,0 +1,44 @@ +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { runCliLoadScenarios } from './cli-load.ts'; +import { runCliRunScenarios } from './cli-run.ts'; +import { runCliScanAuditScenarios } from './cli-scan-audit.ts'; +import { runNextScenarios } from './integration-next.ts'; +import { runViteScenarios } from './integration-vite.ts'; +import { runPythonScenarios } from './lang-python.ts'; +import { runGoScenarios } from './lang-go.ts'; +import { printScenarioLine } from '../report.ts'; + +export type ScenarioGroup = { + name: string; + run: (ctx: BenchContext) => Promise>; +}; + +export const SCENARIO_GROUPS: Array = [ + { name: 'cli-load', run: runCliLoadScenarios }, + { name: 'cli-run', run: runCliRunScenarios }, + { name: 'cli-scan-audit', run: runCliScanAuditScenarios }, + { name: 'integration-next', run: runNextScenarios }, + { name: 'integration-vite', run: runViteScenarios }, + { name: 'lang-python', run: runPythonScenarios }, + { name: 'lang-go', run: runGoScenarios }, +]; + +export async function runAllScenarios( + ctx: BenchContext, + only?: Array, +): Promise> { + const groups = only?.length + ? SCENARIO_GROUPS.filter((g) => only.includes(g.name)) + : SCENARIO_GROUPS; + + const all: Array = []; + for (const group of groups) { + console.log(`\n== ${group.name} ==`); + const results = await group.run(ctx); + for (const r of results) { + printScenarioLine(r); + all.push(r); + } + } + return all; +} diff --git a/benchmarks/src/scenarios/integration-next.ts b/benchmarks/src/scenarios/integration-next.ts new file mode 100644 index 000000000..1183cd39e --- /dev/null +++ b/benchmarks/src/scenarios/integration-next.ts @@ -0,0 +1,297 @@ +import { rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { FrameworkTestEnv } from '../../../framework-tests/harness/fixture-env.ts'; +import type { BenchContext, ScenarioResult, TelemetryMode } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { measurePathLatency, withServer } from '../server.ts'; +import { NEXT_MANY_SECRETS_SCHEMA, withSchemaFlags } from '../many-secrets-schema.ts'; +import { LEAK_SCAN_BODY_BYTES, REDACT_LOG_LINES } from './util.ts'; + +const NEXT_TEST_DIR = resolve(import.meta.dirname, '../../../framework-tests/frameworks/nextjs'); + +const BUILD_ENV = { + APP_ENV: 'dev', + NEXT_PUBLIC_VAR: 'next-prefixed-public-var', + PUBLIC_VAR: 'unprefixed-public-var', + ENV_SPECIFIC_VAR: 'env-specific-var--dev', + SENSITIVE_VAR: 'super-secret-var', +}; + +/** + * Baseline and varlock arms must compile the same app with the same next config, + * or the "varlock overhead" number also contains a page-content and config diff. + * Both configs below are identical apart from the plugin wrapper, and both pages + * render identical markup — one reads process.env, the other reads ENV. + */ +const NEXT_CONFIG_BODY = `/** @type {import('next').NextConfig} */ +const nextConfig = { + productionBrowserSourceMaps: true, + typescript: { ignoreBuildErrors: true }, + eslint: { ignoreDuringBuilds: true }, +}; +`; + +const BASELINE_NEXT_CONFIG = `${NEXT_CONFIG_BODY} +export default nextConfig; +`; + +const VARLOCK_NEXT_CONFIG = `import { varlockNextConfigPlugin } from '@varlock/nextjs-integration/plugin'; + +${NEXT_CONFIG_BODY} +export default varlockNextConfigPlugin()(nextConfig); +`; + +function pageSource(read: (key: string) => string): string { + return `export default function Page() { + const hasSensitive = !!${read('SENSITIVE_VAR')}; + + console.log('secret-log-test:', ${read('SENSITIVE_VAR')}); + + return ( +
+

bench

+

Next prefixed var: {${read('NEXT_PUBLIC_VAR')}}

+

Unprefixed var: {${read('PUBLIC_VAR')}}

+

Env specific var: {${read('ENV_SPECIFIC_VAR')}}

+

Has sensitive: {hasSensitive ? 'yes' : 'no'}

+
+ ); +} +`; +} + +const BASELINE_PAGE = pageSource((key) => `process.env.${key}`); +const VARLOCK_PAGE = `import { ENV } from 'varlock/env'; + +${pageSource((key) => `ENV.${key}`)}`; + +// force-dynamic keeps these handlers running per request. Next 15 already treats +// GET handlers as dynamic by default, but the bench should not silently start +// measuring a prerendered response if that default ever changes. +const ECHO_ROUTE = `import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +export async function GET() { + // Large body without the sensitive value so preventLeaks scanning still runs + // but the request succeeds (no leak throw). + const body = \`ok padding=\${'x'.repeat(${LEAK_SCAN_BODY_BYTES})}\`; + return new NextResponse(body, { + headers: { 'content-type': 'text/plain' }, + }); +} +`; + +const LOG_ROUTE = `import { NextResponse } from 'next/server'; +import { ENV } from 'varlock/env'; + +export const dynamic = 'force-dynamic'; + +const SECRET_KEYS = [ + 'SENSITIVE_VAR', + 'SECRET_TOKEN', + 'SECRET_API_KEY', + 'SECRET_DB_PASSWORD', + 'SECRET_JWT', + 'SECRET_STRIPE', + 'SECRET_AWS_ACCESS', + 'SECRET_AWS_SECRET', + 'SECRET_REDIS', + 'SECRET_SMTP', + 'SECRET_OAUTH', + 'SECRET_WEBHOOK', + 'SECRET_ENCRYPTION', + 'SECRET_SESSION', + 'SECRET_GITHUB', + 'SECRET_SLACK', + 'SECRET_OPENAI', + 'SECRET_SENTRY', +]; + +export async function GET() { + for (let i = 0; i < ${REDACT_LOG_LINES}; i++) { + const key = SECRET_KEYS[i % SECRET_KEYS.length]; + console.log(\`bench-log-\${i}:\`, ENV[key]); + } + return new NextResponse('ok', { + headers: { 'content-type': 'text/plain' }, + }); +} +`; + +function createNextEnv(ctx: BenchContext, mode: 'baseline' | 'varlock'): FrameworkTestEnv { + const withVarlock = mode === 'varlock'; + const integrationVersion = ctx.integrationVersions.nextjs ?? 'latest'; + return new FrameworkTestEnv({ + testDir: NEXT_TEST_DIR, + framework: `bench-next-${mode}`, + packageManager: 'npm', + usePublished: true, + installTimeout: 180_000, + dependencies: { + next: '^15', + react: '^19', + 'react-dom': '^19', + '@types/react': '^19', + typescript: '^5.9.3', + ...(withVarlock + ? { + varlock: ctx.version, + '@varlock/nextjs-integration': integrationVersion, + } + : {}), + }, + ...(withVarlock + ? { + overrides: { + '@next/env': '', + }, + } + : {}), + templateFiles: { + '.env.schema': 'schemas/.env.schema', + '.env.dev': 'schemas/.env.dev', + '.env.prod': 'schemas/.env.prod', + }, + }); +} + +function prepareBuildFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { + env.prepareFiles({ + templateFiles: { '.env.dev': 'schemas/.env.dev' }, + files: [ + { path: '.env.schema', content: NEXT_MANY_SECRETS_SCHEMA }, + { + path: 'next.config.mjs', + content: mode === 'baseline' ? BASELINE_NEXT_CONFIG : VARLOCK_NEXT_CONFIG, + }, + { path: 'app/page.tsx', content: mode === 'baseline' ? BASELINE_PAGE : VARLOCK_PAGE }, + ], + }); +} + +async function measureBuild( + fixture: FrameworkTestEnv, + iterations: number, + telemetry: TelemetryMode, + mockEnv: Record, +) { + return repeatMeasure( + async () => { + rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); + return measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv(telemetry, mockEnv), ...BUILD_ENV, CI: '1' }, + }); + }, + { iterations, warmup: 0 }, + ); +} + +export async function runNextScenarios(ctx: BenchContext): Promise> { + const results: Array = []; + const buildIterations = Math.max(2, Math.min(3, ctx.iterations)); + const requestIterations = Math.max(10, ctx.iterations * 2); + + console.log(' preparing next baseline (framework-tests)...'); + const baseline = createNextEnv(ctx, 'baseline'); + await baseline.setup(); + try { + prepareBuildFiles(baseline, 'baseline'); + results.push({ + id: 'integration.next.build.baseline', + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: await measureBuild(baseline, buildIterations, 'off', ctx.telemetryMockEnv), + notes: 'Cold next build, no varlock installed', + }); + } finally { + await baseline.teardown(); + } + + // One varlock fixture serves every varlock arm. Installing a fresh project per + // telemetry mode and per latency config meant 7 npm installs where 2 will do, + // which dominated the wall-clock of this group. + console.log(' preparing next varlock (framework-tests)...'); + const fixture = createNextEnv(ctx, 'varlock'); + await fixture.setup(); + try { + prepareBuildFiles(fixture, 'varlock'); + for (const telemetry of ctx.telemetryModes) { + console.log(` next varlock build, telemetry=${telemetry}...`); + results.push({ + id: `integration.next.build.varlock.telemetry.${telemetry}`, + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry, + metrics: await measureBuild(fixture, buildIterations, telemetry, ctx.telemetryMockEnv), + notes: 'Cold next build; telemetry affects sync varlock load spawn', + }); + } + + console.log(' measuring next request latency (preventLeaks / redactLogs)...'); + for (const [label, preventLeaks, redactLogs, port, path] of [ + ['preventLeaks.on', true, true, 3451, '/api/echo'], + ['preventLeaks.off', false, true, 3452, '/api/echo'], + ['redactLogs.on', true, true, 3453, '/api/log'], + ['redactLogs.off', true, false, 3454, '/api/log'], + ] as const) { + fixture.prepareFiles({ + templateFiles: { '.env.dev': 'schemas/.env.dev' }, + files: [ + { path: '.env.schema', content: withSchemaFlags(NEXT_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, + { path: 'next.config.mjs', content: VARLOCK_NEXT_CONFIG }, + { path: 'app/page.tsx', content: VARLOCK_PAGE }, + { path: 'app/api/echo/route.js', content: ECHO_ROUTE }, + { path: 'app/api/log/route.js', content: LOG_ROUTE }, + ], + }); + + rmSync(join(fixture.dir, '.next'), { recursive: true, force: true }); + const buildResult = await measureCommand(['npx', 'next', 'build'], { + cwd: fixture.dir, + timeoutMs: 300_000, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + }); + if (buildResult.exitCode !== 0) { + throw new Error(`next build failed for ${label}:\n${buildResult.stderr}\n${buildResult.stdout}`); + } + + const latency = await withServer( + ['npx', 'next', 'start', '-H', '127.0.0.1', '-p', String(port)], + { + cwd: fixture.dir, + env: { + ...telemetryEnv('off'), + APP_ENV: 'dev', + PORT: String(port), + HOSTNAME: '127.0.0.1', + }, + readyUrl: `http://127.0.0.1:${port}${path}`, + }, + () => measurePathLatency(`http://127.0.0.1:${port}`, path, requestIterations, 3), + ); + + results.push({ + id: `integration.next.request.${label}`, + facet: 'integration-next', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: latency, + notes: path === '/api/echo' + ? `${(LEAK_SCAN_BODY_BYTES / 1024 / 1024).toFixed(0)}MiB safe body; preventLeaks scan cost` + : `${REDACT_LOG_LINES} console.log lines with secret; redactLogs cost`, + }); + } + } finally { + await fixture.teardown(); + } + + return results; +} diff --git a/benchmarks/src/scenarios/integration-vite.ts b/benchmarks/src/scenarios/integration-vite.ts new file mode 100644 index 000000000..1bbe3fcc3 --- /dev/null +++ b/benchmarks/src/scenarios/integration-vite.ts @@ -0,0 +1,255 @@ +import { rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { FrameworkTestEnv } from '../../../framework-tests/harness/fixture-env.ts'; +import type { BenchContext, ScenarioResult, TelemetryMode } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { measurePathLatency, withServer } from '../server.ts'; +import { VITE_MANY_SECRETS_SCHEMA, withSchemaFlags } from '../many-secrets-schema.ts'; +import { LEAK_SCAN_BODY_BYTES, REDACT_LOG_LINES } from './util.ts'; + +const VITE_TEST_DIR = resolve(import.meta.dirname, '../../../framework-tests/frameworks/vite'); + +const BUILD_ENV = { + APP_ENV: 'dev', + PUBLIC_VAR: 'public-test-value', + API_URL: 'https://api.example.com', + ENV_SPECIFIC_VAR: 'env-specific-dev', + VITE_PUBLIC_VAR: 'public-test-value', + VITE_API_URL: 'https://api.example.com', + VITE_ENV_SPECIFIC_VAR: 'env-specific-dev', +}; + +/** Identical apart from the plugin — see the note in integration-next.ts. */ +const BASELINE_VITE_CONFIG = `import { defineConfig } from 'vite'; + +export default defineConfig({}); +`; + +const VARLOCK_VITE_CONFIG = `import { defineConfig } from 'vite'; +import { varlockVitePlugin } from '@varlock/vite-integration'; + +export default defineConfig({ + plugins: [varlockVitePlugin()], +}); +`; + +function mainSource(read: (key: string) => string): string { + return `document.getElementById('app')!.innerHTML = \` +

bench

+

\${${read('PUBLIC_VAR')}}

+

\${${read('API_URL')}}

+

\${${read('ENV_SPECIFIC_VAR')}}

+\`; +`; +} + +const BASELINE_MAIN = mainSource((key) => `import.meta.env.VITE_${key}`); +const VARLOCK_MAIN = `import { ENV } from 'varlock/env'; + +${mainSource((key) => `ENV.${key}`)}`; + +/** + * Dev-server middleware used for latency benches: + * - /api/echo: large body without secrets (preventLeaks still scans) + * - /api/log: many console.log lines containing the secret (redactLogs cost) + */ +const LATENCY_VITE_CONFIG = `import { defineConfig } from 'vite'; +import { varlockVitePlugin } from '@varlock/vite-integration'; +import { ENV } from 'varlock/env'; + +const SECRET_KEYS = [ + 'SECRET_KEY', + 'SECRET_TOKEN', + 'SECRET_API_KEY', + 'SECRET_DB_PASSWORD', + 'SECRET_JWT', + 'SECRET_STRIPE', + 'SECRET_AWS_ACCESS', + 'SECRET_AWS_SECRET', + 'SECRET_REDIS', + 'SECRET_SMTP', + 'SECRET_OAUTH', + 'SECRET_WEBHOOK', + 'SECRET_ENCRYPTION', + 'SECRET_SESSION', + 'SECRET_GITHUB', + 'SECRET_SLACK', + 'SECRET_OPENAI', + 'SECRET_SENTRY', +]; + +export default defineConfig({ + plugins: [ + varlockVitePlugin(), + { + name: 'bench-latency-middleware', + configureServer(server) { + server.middlewares.use('/api/echo', (_req, res) => { + res.setHeader('content-type', 'text/plain'); + res.end(\`ok padding=\${'x'.repeat(${LEAK_SCAN_BODY_BYTES})}\`); + }); + server.middlewares.use('/api/log', (_req, res) => { + for (let i = 0; i < ${REDACT_LOG_LINES}; i++) { + const key = SECRET_KEYS[i % SECRET_KEYS.length]; + console.log(\`bench-log-\${i}:\`, ENV[key]); + } + res.setHeader('content-type', 'text/plain'); + res.end('ok'); + }); + }, + }, + ], +}); +`; + +function createViteEnv(ctx: BenchContext, mode: 'baseline' | 'varlock'): FrameworkTestEnv { + const withVarlock = mode === 'varlock'; + const integrationVersion = ctx.integrationVersions.vite ?? 'latest'; + return new FrameworkTestEnv({ + testDir: VITE_TEST_DIR, + framework: `bench-vite-${mode}`, + packageManager: 'npm', + usePublished: true, + installTimeout: 180_000, + dependencies: { + vite: '^6', + ...(withVarlock + ? { + varlock: ctx.version, + '@varlock/vite-integration': integrationVersion, + } + : {}), + }, + templateFiles: { + '.env.schema': 'schemas/.env.schema', + '.env.dev': 'schemas/.env.dev', + '.env.prod': 'schemas/.env.prod', + }, + }); +} + +function prepareBuildFiles(env: FrameworkTestEnv, mode: 'baseline' | 'varlock'): void { + env.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'index.html': 'html/basic.html', + }, + files: [ + { path: '.env.schema', content: VITE_MANY_SECRETS_SCHEMA }, + { + path: 'vite.config.ts', + content: mode === 'baseline' ? BASELINE_VITE_CONFIG : VARLOCK_VITE_CONFIG, + }, + { path: 'src/main.ts', content: mode === 'baseline' ? BASELINE_MAIN : VARLOCK_MAIN }, + ], + }); +} + +async function measureBuild( + fixture: FrameworkTestEnv, + iterations: number, + telemetry: TelemetryMode, + mockEnv: Record, +) { + return repeatMeasure( + async () => { + rmSync(join(fixture.dir, 'dist'), { recursive: true, force: true }); + return measureCommand(['npx', 'vite', 'build'], { + cwd: fixture.dir, + timeoutMs: 180_000, + env: { ...telemetryEnv(telemetry, mockEnv), ...BUILD_ENV, CI: '1' }, + }); + }, + { iterations, warmup: 0 }, + ); +} + +export async function runViteScenarios(ctx: BenchContext): Promise> { + const results: Array = []; + const buildIterations = Math.max(2, Math.min(4, ctx.iterations)); + const requestIterations = Math.max(10, ctx.iterations * 2); + + console.log(' preparing vite baseline (framework-tests)...'); + const baseline = createViteEnv(ctx, 'baseline'); + await baseline.setup(); + try { + prepareBuildFiles(baseline, 'baseline'); + results.push({ + id: 'integration.vite.build.baseline', + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: await measureBuild(baseline, buildIterations, 'off', ctx.telemetryMockEnv), + notes: 'Cold vite build, no varlock installed', + }); + } finally { + await baseline.teardown(); + } + + console.log(' preparing vite varlock (framework-tests)...'); + const fixture = createViteEnv(ctx, 'varlock'); + await fixture.setup(); + try { + prepareBuildFiles(fixture, 'varlock'); + for (const telemetry of ctx.telemetryModes) { + console.log(` vite varlock build, telemetry=${telemetry}...`); + results.push({ + id: `integration.vite.build.varlock.telemetry.${telemetry}`, + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry, + metrics: await measureBuild(fixture, buildIterations, telemetry, ctx.telemetryMockEnv), + notes: 'Cold vite build; telemetry affects sync varlock load spawn', + }); + } + + console.log(' measuring vite request latency (preventLeaks / redactLogs)...'); + for (const [label, preventLeaks, redactLogs, port, path] of [ + ['preventLeaks.on', true, true, 3461, '/api/echo'], + ['preventLeaks.off', false, true, 3462, '/api/echo'], + ['redactLogs.on', true, true, 3463, '/api/log'], + ['redactLogs.off', true, false, 3464, '/api/log'], + ] as const) { + fixture.prepareFiles({ + templateFiles: { + '.env.dev': 'schemas/.env.dev', + 'index.html': 'html/basic.html', + 'src/main.ts': 'pages/minimal-page.ts', + }, + files: [ + { path: '.env.schema', content: withSchemaFlags(VITE_MANY_SECRETS_SCHEMA, preventLeaks, redactLogs) }, + { path: 'vite.config.ts', content: LATENCY_VITE_CONFIG }, + ], + }); + + const latency = await withServer( + ['npx', 'vite', 'dev', '--host', '127.0.0.1', '--port', String(port)], + { + cwd: fixture.dir, + env: { ...telemetryEnv('off'), APP_ENV: 'dev', CI: '1' }, + readyUrl: `http://127.0.0.1:${port}${path}`, + }, + () => measurePathLatency(`http://127.0.0.1:${port}`, path, requestIterations, 3), + ); + + results.push({ + id: `integration.vite.request.${label}`, + facet: 'integration-vite', + installMethod: 'npm', + packageManager: 'npm', + telemetry: 'off', + metrics: latency, + notes: path === '/api/echo' + ? `${(LEAK_SCAN_BODY_BYTES / 1024 / 1024).toFixed(0)}MiB safe body; preventLeaks scan cost` + : `${REDACT_LOG_LINES} console.log lines with secret; redactLogs cost`, + }); + } + } finally { + await fixture.teardown(); + } + + return results; +} diff --git a/benchmarks/src/scenarios/lang-go.ts b/benchmarks/src/scenarios/lang-go.ts new file mode 100644 index 000000000..ba70c333b --- /dev/null +++ b/benchmarks/src/scenarios/lang-go.ts @@ -0,0 +1,76 @@ +import { rmSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; + +function hasBinary(name: string): boolean { + const result = spawnSync(name, ['version'], { encoding: 'utf8' }); + return result.status === 0; +} + +export async function runGoScenarios(ctx: BenchContext): Promise> { + if (!hasBinary('go')) { + ctx.note('lang-go skipped: go not found on the runner'); + console.log(' skipping go: go not found'); + return []; + } + + const results: Array = []; + const cli = ctx.clis.find((c) => c.label === 'npm') ?? ctx.clis[0]; + if (!cli) { + ctx.note('lang-go skipped: no usable varlock CLI'); + return []; + } + const env = telemetryEnv('off'); + + const dest = fixtureWorkDir(ctx, 'lang-go'); + + const codegen = await repeatMeasure( + async () => { + const genDir = join(dest, 'env'); + if (existsSync(genDir)) { + rmSync(genDir, { recursive: true, force: true }); + } + return measureCommand([...cli.command, 'load', '--clear-cache'], { cwd: dest, env }); + }, + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId('lang.go.load-codegen', cli), + facet: 'lang-go', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: codegen, + notes: 'load triggers @generateGoEnv', + }); + + await measureCommand([...cli.command, 'load'], { cwd: dest, env }); + + // Build once so `go run` is not dominated by compile in every sample + const build = spawnSync('go', ['build', '-o', 'main.bin', '.'], { + cwd: dest, + encoding: 'utf8', + }); + if (build.status !== 0) { + throw new Error(`go build failed:\n${build.stderr}\n${build.stdout}`); + } + + const wrapped = await repeatMeasure( + async () => measureCommand([...cli.command, 'run', '--', join(dest, 'main.bin')], { cwd: dest, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId('lang.go.run', cli), + facet: 'lang-go', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: wrapped, + }); + + return results; +} diff --git a/benchmarks/src/scenarios/lang-python.ts b/benchmarks/src/scenarios/lang-python.ts new file mode 100644 index 000000000..c62342dd6 --- /dev/null +++ b/benchmarks/src/scenarios/lang-python.ts @@ -0,0 +1,70 @@ +import { rmSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { BenchContext, ScenarioResult } from '../types.ts'; +import { measureCommand, repeatMeasure } from '../measure.ts'; +import { telemetryEnv } from '../telemetry.ts'; +import { cliScenarioId, fixtureWorkDir } from './util.ts'; + +function hasBinary(name: string): boolean { + const result = spawnSync(name, ['--version'], { encoding: 'utf8' }); + return result.status === 0; +} + +export async function runPythonScenarios(ctx: BenchContext): Promise> { + if (!hasBinary('python3')) { + ctx.note('lang-python skipped: python3 not found on the runner'); + console.log(' skipping python: python3 not found'); + return []; + } + + const results: Array = []; + // Use npm-installed CLI for lang scenarios (one representative install method) + const cli = ctx.clis.find((c) => c.label === 'npm') ?? ctx.clis[0]; + if (!cli) { + ctx.note('lang-python skipped: no usable varlock CLI'); + return []; + } + const env = telemetryEnv('off'); + + const dest = fixtureWorkDir(ctx, 'lang-python'); + + const codegen = await repeatMeasure( + async () => { + // Remove generated file so codegen has work each time + const gen = join(dest, 'env.py'); + if (existsSync(gen)) { + rmSync(gen); + } + return measureCommand([...cli.command, 'load', '--clear-cache'], { cwd: dest, env }); + }, + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId('lang.python.load-codegen', cli), + facet: 'lang-python', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: codegen, + notes: 'load triggers @generatePythonEnv', + }); + + // Ensure generated file exists for run + await measureCommand([...cli.command, 'load'], { cwd: dest, env }); + + const wrapped = await repeatMeasure( + async () => measureCommand([...cli.command, 'run', '--', 'python3', 'main.py'], { cwd: dest, env }), + { iterations: ctx.iterations, warmup: ctx.warmup }, + ); + results.push({ + id: cliScenarioId('lang.python.run', cli), + facet: 'lang-python', + installMethod: cli.label, + packageManager: cli.packageManager, + telemetry: 'off', + metrics: wrapped, + }); + + return results; +} diff --git a/benchmarks/src/scenarios/util.ts b/benchmarks/src/scenarios/util.ts new file mode 100644 index 000000000..f6da61240 --- /dev/null +++ b/benchmarks/src/scenarios/util.ts @@ -0,0 +1,33 @@ +import { join } from 'node:path'; +import { copyFixture } from '../measure.ts'; +import type { BenchContext, CliInvocation } from '../types.ts'; + +/** + * Workload sizes for the request-latency benches. + * + * Both preventLeaks and redactLogs cost scales with how much data they inspect, + * while the request itself costs a fixed fraction of a millisecond. At 16KB of + * body / 200 log lines the on-vs-off delta sat entirely inside run-to-run noise, + * so the scenarios could not answer the question they exist to answer. These + * sizes put the scanning work above the noise floor. + */ +export const LEAK_SCAN_BODY_BYTES = 2 * 1024 * 1024; +export const REDACT_LOG_LINES = 2_000; + +/** + * Scenario ids must be unique within a run — the same scenario is measured once + * per install method, and anything reading the committed JSON keys off the id. + */ +export function cliScenarioId(base: string, cli: CliInvocation): string { + return `${base}.install.${cli.label}`; +} + +/** + * Copy a fixture into the work dir. Scenarios run against the copy so codegen + * output, caches and build artifacts never land in the checked-in fixtures. + */ +export function fixtureWorkDir(ctx: BenchContext, fixture: string, suffix = ''): string { + const dest = join(ctx.workDir, 'fixtures', `${fixture}${suffix}`); + copyFixture(join(ctx.fixturesDir, fixture), dest); + return dest; +} diff --git a/benchmarks/src/server.ts b/benchmarks/src/server.ts new file mode 100644 index 000000000..9b86dc001 --- /dev/null +++ b/benchmarks/src/server.ts @@ -0,0 +1,136 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { once } from 'node:events'; +import { repeatMeasure } from './measure.ts'; +import type { ScenarioMetrics } from './types.ts'; + +/** Poll until the URL answers with a non-error status. */ +export async function waitForUrl(url: string, timeoutMs: number): Promise { + const start = Date.now(); + let lastStatus: number | string = 'no response'; + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + // A 5xx means the server is up but the route is broken — keep polling so a + // slow boot still succeeds, but report the status if we time out. + if (res.ok) return; + lastStatus = res.status; + } catch (err) { + lastStatus = (err as Error).message; + } + await new Promise((r) => { + setTimeout(r, 250); + }); + } + throw new Error(`Timed out waiting for ${url} (last: ${lastStatus})`); +} + +/** Measure request latency against an already-running server. */ +export async function measurePathLatency( + baseUrl: string, + path: string, + iterations: number, + warmup: number, +): Promise { + return repeatMeasure( + async () => { + const start = performance.now(); + const res = await fetch(`${baseUrl}${path}`); + const body = await res.text(); + const wallMs = performance.now() - start; + if (!res.ok) { + throw new Error(`Request failed: ${res.status} ${body.slice(0, 500)}`); + } + return { wallMs, rssPeakBytes: null, exitCode: 0 }; + }, + { iterations, warmup }, + ); +} + +export type ServerOptions = { + cwd: string; + env: Record; + /** URL polled until it answers before `fn` runs. */ + readyUrl: string; + readyTimeoutMs?: number; +}; + +/** + * Run a dev/prod server for the duration of `fn`, then tear down the whole + * process group. + * + * Two things here are load-bearing: + * - stdout AND stderr are drained. The redactLogs benches make the server write + * hundreds of log lines per request; an unread pipe fills at ~64KB and the + * server then blocks on write(), which would turn a latency measurement into a + * measurement of pipe backpressure. + * - the child is detached and killed by process group. `npx next start` and + * `npx vite dev` both exec the real server as a grandchild, so signalling only + * the direct child leaves an orphan holding the port and burning CPU during + * later scenarios. + */ +export async function withServer( + command: Array, + options: ServerOptions, + fn: () => Promise, +): Promise { + const [bin, ...args] = command; + const env: NodeJS.ProcessEnv = { ...process.env }; + for (const [key, value] of Object.entries(options.env)) { + if (value === undefined) delete env[key]; + else env[key] = value; + } + + const server: ChildProcess = spawn(bin!, args, { + cwd: options.cwd, + env, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + + let output = ''; + const capture = (chunk: Buffer) => { + output += chunk.toString(); + // Bound the buffer — these servers can be very chatty under the log benches. + if (output.length > 200_000) output = output.slice(-100_000); + }; + server.stdout?.on('data', capture); + server.stderr?.on('data', capture); + + const exited = once(server, 'exit'); + let exitedEarly = false; + exited.then( + () => { + exitedEarly = true; + }, + () => { + exitedEarly = true; + }, + ); + + try { + await waitForUrl(options.readyUrl, options.readyTimeoutMs ?? 90_000); + return await fn(); + } catch (err) { + throw new Error(`${String(err)}\nserver output:\n${output.slice(-8_000)}`); + } finally { + if (!exitedEarly && server.pid) { + try { + process.kill(-server.pid, 'SIGTERM'); + } catch { + server.kill('SIGTERM'); + } + const timer = new Promise<'timeout'>((r) => { + setTimeout(() => r('timeout'), 5_000); + }); + const result = await Promise.race([exited.then(() => 'exited' as const), timer]); + if (result === 'timeout') { + try { + process.kill(-server.pid, 'SIGKILL'); + } catch { + server.kill('SIGKILL'); + } + await Promise.race([exited, timer]).catch(() => undefined); + } + } + } +} diff --git a/benchmarks/src/telemetry-mock.ts b/benchmarks/src/telemetry-mock.ts new file mode 100644 index 000000000..c143cbe5c --- /dev/null +++ b/benchmarks/src/telemetry-mock.ts @@ -0,0 +1,51 @@ +import { createServer, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; + +export type TelemetryMock = { + /** Base URL to hand to varlock via VARLOCK_POSTHOG_HOST. */ + url: string; + /** Number of capture requests received so far. */ + requestCount: () => number; + close: () => Promise; +}; + +/** + * Local stand-in for the telemetry collector. + * + * Telemetry-on scenarios exist to measure what the telemetry code path costs + * (payload building plus the exit hook that waits on the in-flight request). + * Pointing that at the real collector would inject hundreds of synthetic events + * into product analytics on every run, and would make the committed timings a + * function of runner-to-collector network latency. A local mock keeps the code + * path intact and the numbers comparable between runs. + */ +export async function startTelemetryMock(): Promise { + let count = 0; + + const server: Server = createServer((req, res) => { + // Drain the body so the client sees a complete request/response cycle. + req.resume(); + req.on('end', () => { + count += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + // Shape of a real PostHog capture response. + res.end('{"status":1}'); + }); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }); + + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}`, + requestCount: () => count, + close: () => new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + }; +} diff --git a/benchmarks/src/telemetry.ts b/benchmarks/src/telemetry.ts new file mode 100644 index 000000000..ccf60442f --- /dev/null +++ b/benchmarks/src/telemetry.ts @@ -0,0 +1,35 @@ +import type { TelemetryMode } from './types.ts'; + +export type { TelemetryMode }; + +/** + * Env overlay for telemetry on/off. Pass through measureCommand. + * + * `mockEnv` points the collector at the local mock (see telemetry-mock.ts) and is + * required for 'on' — benchmarks never send real telemetry. Callers get it from + * `ctx.telemetryMockEnv`, which is empty when the mock is unavailable, in which + * case 'on' has already been dropped from `ctx.telemetryModes`. + */ +export function telemetryEnv( + mode: TelemetryMode, + mockEnv: Record = {}, +): Record { + if (mode === 'off') { + return { + VARLOCK_TELEMETRY_DISABLED: '1', + // Clear legacy opt-out so "off" is unambiguous + PH_OPT_OUT: undefined, + }; + } + if (!mockEnv.VARLOCK_POSTHOG_HOST) { + throw new Error('telemetryEnv("on") requires the local telemetry mock — refusing to emit real telemetry'); + } + // Explicitly clear disable flags so a parent-shell opt-out does not leak in + return { + VARLOCK_TELEMETRY_DISABLED: undefined, + PH_OPT_OUT: undefined, + ...mockEnv, + }; +} + +export const ALL_TELEMETRY_MODES: Array = ['off', 'on']; diff --git a/benchmarks/src/types.ts b/benchmarks/src/types.ts new file mode 100644 index 000000000..c55b16470 --- /dev/null +++ b/benchmarks/src/types.ts @@ -0,0 +1,114 @@ +/** + * How the measured CLI was installed and invoked: + * - `npm` — installed with npm, run with node + * - `bun` — installed with bun, run with the bun runtime + * - `sea` — the standalone compiled binary + */ +export type InstallMethod = 'npm' | 'bun' | 'sea'; + +export type TelemetryMode = 'on' | 'off'; + +export type ScenarioFacet = | 'cli-load' + | 'cli-run' + | 'cli-scan' + | 'cli-audit' + | 'integration-next' + | 'integration-vite' + | 'lang-python' + | 'lang-go'; + +export type TriggerKind = 'release' | 'workflow_dispatch' | 'local'; + +export type Sample = { + wallMs: number; + rssPeakBytes: number | null; + exitCode: number; +}; + +export type ScenarioMetrics = { + /** Number of measured (non-warmup) iterations behind these numbers. */ + iterations: number; + wallMsMin: number; + wallMsMedian: number; + /** Collapses onto the max at low iteration counts — read with wallMsStdDev. */ + wallMsP95: number; + wallMsStdDev: number; + rssPeakBytesMedian: number | null; + /** How many iterations produced an RSS reading (0 means RSS was never sampled). */ + rssSampleCount: number; + samples: Array; +}; + +export type ScenarioResult = { + /** Unique within a run — includes the install method for CLI scenarios. */ + id: string; + facet: ScenarioFacet; + installMethod: InstallMethod; + packageManager?: 'npm' | 'bun'; + /** Whether VARLOCK_TELEMETRY_DISABLED was cleared (on) or set (off). */ + telemetry: TelemetryMode; + metrics: ScenarioMetrics; + notes?: string; +}; + +export type BenchRunMeta = { + timestamp: string; + gitSha: string | null; + githubRunId: string | null; + runnerOs: string; + runnerArch: string; + nodeVersion: string; + versions: { + varlock: string; + nextjsIntegration?: string; + viteIntegration?: string; + '@env-spec/parser'?: string; + }; + trigger: TriggerKind; + /** + * True when telemetry-on scenarios were pointed at a local mock collector. + * When false those scenarios are skipped — benchmarks never emit real telemetry. + */ + telemetryMocked: boolean; + /** Anything skipped, degraded, or otherwise worth knowing when reading the numbers. */ + notes: Array; +}; + +export type BenchRunResult = { + meta: BenchRunMeta; + scenarios: Array; +}; + +export type CliInvocation = { + /** Executable + args that invoke varlock (without the subcommand). */ + command: Array; + label: InstallMethod; + packageManager?: 'npm' | 'bun'; +}; + +export type BenchContext = { + version: string; + /** + * Resolved (not floating) integration versions, so a run records the exact pair + * of packages it measured. + */ + integrationVersions: { + nextjs?: string; + vite?: string; + }; + rootDir: string; + fixturesDir: string; + workDir: string; + iterations: number; + warmup: number; + clis: Array; + seaPath: string | null; + /** Telemetry modes to measure — 'on' is dropped when the mock is unavailable. */ + telemetryModes: Array; + /** Env overlay that points telemetry at the local mock (empty when unavailable). */ + telemetryMockEnv: Record; + /** 64-char hex key that enables the on-disk resolver cache, including in CI. */ + cacheKey: string; + /** Record a skip / degradation so it shows up in the committed results. */ + note: (message: string) => void; +}; diff --git a/benchmarks/tsconfig.json b/benchmarks/tsconfig.json new file mode 100644 index 000000000..6359a2536 --- /dev/null +++ b/benchmarks/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2023"], + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/eslint.config.mjs b/eslint.config.mjs index f0bb8f985..498323524 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -58,6 +58,8 @@ export default tseslint.config( '.claude', 'framework-tests/.test-projects', 'framework-tests/.packed', + 'benchmarks/.work', + 'benchmarks/results/*.json', ], }, @@ -171,6 +173,7 @@ export default tseslint.config( 'packages/varlock/scripts/**', 'smoke-tests/**', 'framework-tests/**', + 'benchmarks/**', 'packages/encryption-binary-swift/scripts/**', 'packages/encryption-binary-rust/scripts/**', 'packages/vscode-plugin/scripts/**', diff --git a/framework-tests/README.md b/framework-tests/README.md index 6a39bf1fd..43252e1f5 100644 --- a/framework-tests/README.md +++ b/framework-tests/README.md @@ -35,11 +35,13 @@ bun run --filter varlock-framework-tests test The shared harness provides `FrameworkTestEnv`, which manages the full lifecycle: -1. **Pack** — varlock packages are built and packed into `.tgz` tarballs (cached in `.packed/`; run `bun run repack` to refresh after source changes) +1. **Pack** — varlock packages are built and packed into `.tgz` tarballs (cached in `.packed/`; run `bun run repack` to refresh after source changes). Set `usePublished: true` on the fixture config to skip packing and install declared versions from npm instead (used by the release benchmarking suite). 2. **Setup** — a temp project is created in `.test-projects/`, deps are installed via pnpm 3. **Scenario** — template files are copied, a build command runs, and output is asserted 4. **Teardown** — temp project is removed (set `KEEP_TEST_DIRS=1` to preserve for debugging) +Imperative APIs (`setup`, `prepareFiles`, `runScenario`, `teardown`) live in `harness/fixture-env.ts` and can be imported without Vitest. Vitest helpers (`describeScenario`, `describeDevScenario`, `runTest`) are layered in `harness/test-fixture.ts`. + ### Adding a new framework Create a directory under `frameworks//` with: diff --git a/framework-tests/harness/fixture-env.ts b/framework-tests/harness/fixture-env.ts new file mode 100644 index 000000000..dd12ba032 --- /dev/null +++ b/framework-tests/harness/fixture-env.ts @@ -0,0 +1,364 @@ +import { + cpSync, writeFileSync, readFileSync, readdirSync, + rmSync, existsSync, mkdirSync, +} from 'node:fs'; +import { + join, dirname, basename, resolve, +} from 'node:path'; +import { runCommand } from './command-runner.js'; +import { runDevServer as runDevServerProcess } from './dev-server.js'; +import { packPackages, getPackedDeps } from './pack.js'; +import type { + TestFixtureConfig, TestScenario, DevServerScenario, DevServerResult, + BuildResult, TemplateFileSource, +} from './types.js'; + +const FRAMEWORK_TESTS_DIR = resolve(import.meta.dirname, '..'); + +/** Insert text after JS directives ('use client', 'use server') at the top of a file */ +export function insertAfterDirectives(content: string, text: string): string { + const lines = content.split('\n'); + let insertIdx = 0; + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].trim(); + if (trimmed === '') { + continue; + } + if (/^['"]use (client|server)['"];?\s*$/.test(trimmed)) { + insertIdx = i + 1; + continue; + } + break; + } + lines.splice(insertIdx, 0, text); + return lines.join('\n'); +} + +/** Get the exec prefix for running binaries from node_modules/.bin */ +export function execPrefix(pm: string): string { + if (pm === 'bun') return 'bunx'; + if (pm === 'yarn') return 'yarn exec'; + if (pm === 'npm') return 'npx'; + return `${pm} exec`; +} + +/** + * Imperative fixture environment (no Vitest dependency). + * Used by framework tests and by the release benchmarking suite. + */ +export class FrameworkTestEnv { + dir!: string; + protected filesDir: string; + protected label: string; + /** Files written by the previous scenario, restored before the next one runs */ + private prevScenarioFiles = new Set(); + + constructor(public config: TestFixtureConfig) { + this.filesDir = join(config.testDir, 'files'); + this.label = config.framework ?? basename(config.testDir); + } + + /** + * Create temp dir, copy base template, build package.json, install deps. + */ + async setup(): Promise { + this.dir = join(FRAMEWORK_TESTS_DIR, '.test-projects', this.label); + if (existsSync(this.dir)) { + rmSync(this.dir, { recursive: true, force: true }); + } + mkdirSync(this.dir, { recursive: true }); + console.log(`[${this.label}] Setting up fixture in ${this.dir}`); + + // Isolate from parent workspaces so the test project + // is treated as its own independent root + + // Prevent bun from inheriting the repo root's bunfig.toml which has + // minimumReleaseAge and a security scanner that can block/hang installs + writeFileSync(join(this.dir, 'bunfig.toml'), [ + '[install]', + 'minimumReleaseAge = 0', + '', + ].join('\n')); + + // pnpm v11 defaults: block all build scripts, and error on packages missing time metadata. + // Allow known native packages that need postinstall scripts. + // minimumReleaseAge is 0 here (matching bunfig.toml) so framework tests can use + // recently published versions (e.g. new Astro majors) without waiting 72h. + writeFileSync(join(this.dir, 'pnpm-workspace.yaml'), [ + 'minimumReleaseAge: 0', + 'onlyBuiltDependencies:', + ' - esbuild', + ' - sharp', + ' - lightningcss', + ' - workerd', + '', + ].join('\n')); + writeFileSync(join(this.dir, '.npmrc'), 'ignore-workspace-root-check=true\n'); + + // Copy _base/ skeleton into the project + const baseDir = join(this.filesDir, '_base'); + if (existsSync(baseDir)) { + cpSync(baseDir, this.dir, { recursive: true }); + } + + // Pack required varlock packages (unless measuring published registry versions) + const varlockPackageNames = Object.keys(this.config.dependencies) + .filter((dep) => dep === 'varlock' || dep.startsWith('@varlock/')); + const usePublished = this.config.usePublished === true; + let packedDeps: Record = {}; + if (!usePublished) { + packPackages(varlockPackageNames); + packedDeps = getPackedDeps(varlockPackageNames); + } + + // Build package.json + const templatePkgPath = join(this.dir, 'package.json'); + const templatePkg = existsSync(templatePkgPath) + ? JSON.parse(readFileSync(templatePkgPath, 'utf-8')) + : {}; + + const pm = this.config.packageManager ?? 'pnpm'; + + const pkg = { + name: 'framework-test-project', + version: '0.0.0', + private: true, + type: 'module', + ...templatePkg, + dependencies: { + ...templatePkg.dependencies, + ...this.config.dependencies, + ...packedDeps, // override varlock deps with packed file: paths (skipped when usePublished) + }, + ...(this.config.devDependencies ? { + devDependencies: { + ...templatePkg.devDependencies, + ...this.config.devDependencies, + }, + } : {}), + ...(this.config.scripts ? { + scripts: { ...templatePkg.scripts, ...this.config.scripts }, + } : {}), + }; + + // Apply overrides — pnpm nests them under `pnpm.overrides` + if (this.config.overrides) { + if (pm === 'pnpm') { + pkg.pnpm = { ...templatePkg.pnpm, overrides: { ...templatePkg.pnpm?.overrides, ...this.config.overrides } }; + } else { + pkg.overrides = { ...templatePkg.overrides, ...this.config.overrides }; + } + } + + // Apply packageJsonMerge (deep merge one level) + if (this.config.packageJsonMerge) { + for (const [key, value] of Object.entries(this.config.packageJsonMerge)) { + if (typeof value === 'object' && !Array.isArray(value) && value !== null) { + pkg[key] = { ...pkg[key], ...value }; + } else { + pkg[key] = value; + } + } + } + + // Replace placeholders in any overrides sections + const overrideSections = [ + pkg.overrides, // npm/bun top-level overrides + pkg.pnpm?.overrides, // pnpm overrides + ]; + for (const overrides of overrideSections) { + if (!overrides) continue; + for (const [key, val] of Object.entries(overrides)) { + if (typeof val === 'string' && val.startsWith('): void { + for (const dest of this.prevScenarioFiles) { + const basePath = join(this.filesDir, '_base', dest); + const destPath = join(this.dir, dest); + if (existsSync(basePath)) { + cpSync(basePath, destPath); + } else { + rmSync(destPath, { force: true }); + } + } + this.prevScenarioFiles = new Set(); + + // Copy template files: fixture defaults merged with scenario overrides + const templateFiles = { + ...this.config.templateFiles, + ...scenario.templateFiles, + }; + for (const [dest, source] of Object.entries(templateFiles)) { + this.prevScenarioFiles.add(dest); + const srcRef: Exclude = typeof source === 'string' + ? { path: source } + : source; + const srcPath = join(this.filesDir, srcRef.path); + const destPath = join(this.dir, dest); + mkdirSync(dirname(destPath), { recursive: true }); + cpSync(srcPath, destPath); + + // Apply transformations (replacements, prepend, append) + if (typeof source !== 'string') { + let content = readFileSync(destPath, 'utf-8'); + if (srcRef.replacements) { + for (const [find, replace] of Object.entries(srcRef.replacements)) { + content = content.replaceAll(find, replace); + } + } + if (srcRef.prepend) { + content = `${srcRef.prepend}\n${content}`; + } + if (srcRef.insertAfterDirectives) { + content = insertAfterDirectives(content, srcRef.insertAfterDirectives); + } + if (srcRef.append) { + content = `${content}\n${srcRef.append}`; + } + writeFileSync(destPath, content); + } + } + + // Write inline files + if (scenario.files) { + for (const file of scenario.files) { + this.prevScenarioFiles.add(file.path); + const destPath = join(this.dir, file.path); + mkdirSync(dirname(destPath), { recursive: true }); + writeFileSync(destPath, file.content); + } + } + } + + /** + * Apply template / inline files without running a command. + * Useful for benchmarks that measure with their own timers. + */ + prepareFiles(scenario: Pick): void { + this.applyFiles(scenario); + } + + /** + * Run a test scenario: write files, build, return result. + */ + async runScenario(scenario: TestScenario): Promise { + // Clean previous build artifacts + this.cleanBuildArtifacts(); + + this.applyFiles(scenario); + + // Run command, auto-prefixed with package manager exec + const pm = this.config.packageManager ?? 'pnpm'; + const buildCmd = `${execPrefix(pm)} ${scenario.command}`; + + return runCommand(this.dir, buildCmd, { + env: scenario.env, + timeout: scenario.timeout ?? 120_000, + killAfterPattern: scenario.killAfterPattern, + }); + } + + /** + * Run a dev server scenario: write files, start server, make requests, return result. + */ + async runDevServer(scenario: DevServerScenario): Promise { + this.cleanBuildArtifacts(); + this.applyFiles(scenario); + + const pm = this.config.packageManager ?? 'pnpm'; + const command = `${execPrefix(pm)} ${scenario.command}`; + + return runDevServerProcess(this.dir, command, scenario); + } + + /** + * Clean build artifacts between scenarios. + */ + cleanBuildArtifacts(): void { + // Clean .next but preserve the cache directory so turbopack/webpack + // compilation cache speeds up subsequent builds within the same fixture + const nextDir = join(this.dir, '.next'); + if (existsSync(nextDir)) { + for (const entry of readdirSync(nextDir)) { + if (entry === 'cache') continue; + rmSync(join(nextDir, entry), { recursive: true, force: true }); + } + } + + const artifactDirs = ['out', 'dist', '.turbo', '.wrangler']; + for (const dir of artifactDirs) { + const fullPath = join(this.dir, dir); + if (existsSync(fullPath)) { + rmSync(fullPath, { recursive: true, force: true }); + } + } + } + + /** + * Remove the entire temp directory. + */ + async teardown(): Promise { + if (this.dir && existsSync(this.dir)) { + if (process.env.KEEP_TEST_DIRS) { + console.log(`[${this.label}] Preserving test dir: ${this.dir}`); + } else { + // retry removal — on CI, child processes (e.g. wrangler) may still be + // releasing file handles when teardown runs, causing ENOTEMPTY + for (let attempt = 0; attempt < 3; attempt++) { + try { + rmSync(this.dir, { recursive: true, force: true }); + break; + } catch (err) { + if (attempt < 2) { + await new Promise((r) => { + setTimeout(r, 500); + }); + } else { + console.warn(`[${this.label}] Failed to clean up ${this.dir}: ${(err as Error).message}`); + } + } + } + console.log(`[${this.label}] Cleaned up ${this.dir}`); + } + } + } +} diff --git a/framework-tests/harness/test-fixture.ts b/framework-tests/harness/test-fixture.ts index 4c799be69..94c50cee3 100644 --- a/framework-tests/harness/test-fixture.ts +++ b/framework-tests/harness/test-fixture.ts @@ -1,23 +1,14 @@ -import { - cpSync, writeFileSync, readFileSync, readdirSync, - rmSync, existsSync, mkdirSync, -} from 'node:fs'; -import { - join, dirname, basename, resolve, -} from 'node:path'; - -const FRAMEWORK_TESTS_DIR = resolve(import.meta.dirname, '..'); import { describe, test, beforeAll, expect, } from 'vitest'; -import { runCommand } from './command-runner.js'; -import { runDevServer as runDevServerProcess } from './dev-server.js'; -import { packPackages, getPackedDeps } from './pack.js'; import { assertBuildResult, assertOutput, assertFiles, } from './assertions.js'; +import { + FrameworkTestEnv as BaseFrameworkTestEnv, +} from './fixture-env.js'; import type { - TestFixtureConfig, TestScenario, DevServerScenario, DevServerResult, + TestScenario, DevServerScenario, DevServerResult, BuildResult, TemplateFileMap, TemplateFileSource, } from './types.js'; @@ -40,25 +31,6 @@ function addEdgeRuntimeToTemplateFiles(templateFiles?: TemplateFileMap): Templat return result; } -/** Insert text after JS directives ('use client', 'use server') at the top of a file */ -function insertAfterDirectives(content: string, text: string): string { - const lines = content.split('\n'); - let insertIdx = 0; - for (let i = 0; i < lines.length; i++) { - const trimmed = lines[i].trim(); - if (trimmed === '') { - continue; - } - if (/^['"]use (client|server)['"];?\s*$/.test(trimmed)) { - insertIdx = i + 1; - continue; - } - break; - } - lines.splice(insertIdx, 0, text); - return lines.join('\n'); -} - /** Apply .only or .skip modifier to a vitest describe/test function based on flags */ function withSkipOrOnly( fn: T, @@ -69,243 +41,12 @@ function withSkipOrOnly( return fn; } -/** Get the exec prefix for running binaries from node_modules/.bin */ -function execPrefix(pm: string): string { - if (pm === 'bun') return 'bunx'; - if (pm === 'yarn') return 'yarn exec'; - if (pm === 'npm') return 'npx'; - return `${pm} exec`; -} - -export class FrameworkTestEnv { - dir!: string; - private filesDir: string; - private label: string; - /** Files written by the previous scenario, restored before the next one runs */ - private prevScenarioFiles = new Set(); - - constructor(public config: TestFixtureConfig) { - this.filesDir = join(config.testDir, 'files'); - this.label = config.framework ?? basename(config.testDir); - } - - /** - * Create temp dir, copy base template, build package.json, install deps. - */ - async setup(): Promise { - this.dir = join(FRAMEWORK_TESTS_DIR, '.test-projects', this.label); - if (existsSync(this.dir)) { - rmSync(this.dir, { recursive: true, force: true }); - } - mkdirSync(this.dir, { recursive: true }); - console.log(`[${this.label}] Setting up fixture in ${this.dir}`); - - // Isolate from parent workspaces so the test project - // is treated as its own independent root - - // Prevent bun from inheriting the repo root's bunfig.toml which has - // minimumReleaseAge and a security scanner that can block/hang installs - writeFileSync(join(this.dir, 'bunfig.toml'), [ - '[install]', - 'minimumReleaseAge = 0', - '', - ].join('\n')); - - // pnpm v11 defaults: block all build scripts, and error on packages missing time metadata. - // Allow known native packages that need postinstall scripts. - // minimumReleaseAge is 0 here (matching bunfig.toml) so framework tests can use - // recently published versions (e.g. new Astro majors) without waiting 72h. - writeFileSync(join(this.dir, 'pnpm-workspace.yaml'), [ - 'minimumReleaseAge: 0', - 'onlyBuiltDependencies:', - ' - esbuild', - ' - sharp', - ' - lightningcss', - ' - workerd', - '', - ].join('\n')); - writeFileSync(join(this.dir, '.npmrc'), 'ignore-workspace-root-check=true\n'); - - // Copy _base/ skeleton into the project - const baseDir = join(this.filesDir, '_base'); - if (existsSync(baseDir)) { - cpSync(baseDir, this.dir, { recursive: true }); - } - - // Pack required varlock packages - const varlockPackageNames = Object.keys(this.config.dependencies) - .filter((dep) => dep === 'varlock' || dep.startsWith('@varlock/')); - packPackages(varlockPackageNames); - const packedDeps = getPackedDeps(varlockPackageNames); - - // Build package.json - const templatePkgPath = join(this.dir, 'package.json'); - const templatePkg = existsSync(templatePkgPath) - ? JSON.parse(readFileSync(templatePkgPath, 'utf-8')) - : {}; - - const pm = this.config.packageManager ?? 'pnpm'; - - const pkg = { - name: 'framework-test-project', - version: '0.0.0', - private: true, - type: 'module', - ...templatePkg, - dependencies: { - ...templatePkg.dependencies, - ...this.config.dependencies, - ...packedDeps, // override varlock deps with packed file: paths - }, - ...(this.config.devDependencies ? { - devDependencies: { - ...templatePkg.devDependencies, - ...this.config.devDependencies, - }, - } : {}), - ...(this.config.scripts ? { - scripts: { ...templatePkg.scripts, ...this.config.scripts }, - } : {}), - }; - - // Apply overrides — pnpm nests them under `pnpm.overrides` - if (this.config.overrides) { - if (pm === 'pnpm') { - pkg.pnpm = { ...templatePkg.pnpm, overrides: { ...templatePkg.pnpm?.overrides, ...this.config.overrides } }; - } else { - pkg.overrides = { ...templatePkg.overrides, ...this.config.overrides }; - } - } - - // Apply packageJsonMerge (deep merge one level) - if (this.config.packageJsonMerge) { - for (const [key, value] of Object.entries(this.config.packageJsonMerge)) { - if (typeof value === 'object' && !Array.isArray(value) && value !== null) { - pkg[key] = { ...pkg[key], ...value }; - } else { - pkg[key] = value; - } - } - } - - // Replace placeholders in any overrides sections - const overrideSections = [ - pkg.overrides, // npm/bun top-level overrides - pkg.pnpm?.overrides, // pnpm overrides - ]; - for (const overrides of overrideSections) { - if (!overrides) continue; - for (const [key, val] of Object.entries(overrides)) { - if (typeof val === 'string' && val.startsWith('): void { - for (const dest of this.prevScenarioFiles) { - const basePath = join(this.filesDir, '_base', dest); - const destPath = join(this.dir, dest); - if (existsSync(basePath)) { - cpSync(basePath, destPath); - } else { - rmSync(destPath, { force: true }); - } - } - this.prevScenarioFiles = new Set(); - - // Copy template files: fixture defaults merged with scenario overrides - const templateFiles = { - ...this.config.templateFiles, - ...scenario.templateFiles, - }; - for (const [dest, source] of Object.entries(templateFiles)) { - this.prevScenarioFiles.add(dest); - const srcRef = typeof source === 'string' ? { path: source } : source; - const srcPath = join(this.filesDir, srcRef.path); - const destPath = join(this.dir, dest); - mkdirSync(dirname(destPath), { recursive: true }); - cpSync(srcPath, destPath); - - // Apply transformations (replacements, prepend, append) - if (typeof source !== 'string') { - let content = readFileSync(destPath, 'utf-8'); - if (srcRef.replacements) { - for (const [find, replace] of Object.entries(srcRef.replacements)) { - content = content.replaceAll(find, replace); - } - } - if (srcRef.prepend) { - content = `${srcRef.prepend}\n${content}`; - } - if (srcRef.insertAfterDirectives) { - content = insertAfterDirectives(content, srcRef.insertAfterDirectives); - } - if (srcRef.append) { - content = `${content}\n${srcRef.append}`; - } - writeFileSync(destPath, content); - } - } - - // Write inline files - if (scenario.files) { - for (const file of scenario.files) { - this.prevScenarioFiles.add(file.path); - const destPath = join(this.dir, file.path); - mkdirSync(dirname(destPath), { recursive: true }); - writeFileSync(destPath, file.content); - } - } - } - - /** - * Run a test scenario: write files, build, return result. - */ - async runScenario(scenario: TestScenario): Promise { - // Clean previous build artifacts - this.cleanBuildArtifacts(); - - this.applyFiles(scenario); - - // Run command, auto-prefixed with package manager exec - const pm = this.config.packageManager ?? 'pnpm'; - const buildCmd = `${execPrefix(pm)} ${scenario.command}`; - - return runCommand(this.dir, buildCmd, { - env: scenario.env, - timeout: scenario.timeout ?? 120_000, - killAfterPattern: scenario.killAfterPattern, - }); - } - +/** + * Vitest-aware fixture env used by framework test suites. + * Benchmarks should import the base class from `./fixture-env.js` instead, + * so Vitest is not loaded outside a test runner. + */ +export class FrameworkTestEnv extends BaseFrameworkTestEnv { /** * Run a scenario and assert results in a single test. */ @@ -371,19 +112,6 @@ export class FrameworkTestEnv { }); } - /** - * Run a dev server scenario: write files, start server, make requests, return result. - */ - async runDevServer(scenario: DevServerScenario): Promise { - this.cleanBuildArtifacts(); - this.applyFiles(scenario); - - const pm = this.config.packageManager ?? 'pnpm'; - const command = `${execPrefix(pm)} ${scenario.command}`; - - return runDevServerProcess(this.dir, command, scenario); - } - /** * Create a describe block that starts a dev server once and runs each assertion as a separate test. */ @@ -442,56 +170,4 @@ export class FrameworkTestEnv { } }); } - - /** - * Clean build artifacts between scenarios. - */ - cleanBuildArtifacts(): void { - // Clean .next but preserve the cache directory so turbopack/webpack - // compilation cache speeds up subsequent builds within the same fixture - const nextDir = join(this.dir, '.next'); - if (existsSync(nextDir)) { - for (const entry of readdirSync(nextDir)) { - if (entry === 'cache') continue; - rmSync(join(nextDir, entry), { recursive: true, force: true }); - } - } - - const artifactDirs = ['out', 'dist', '.turbo', '.wrangler']; - for (const dir of artifactDirs) { - const fullPath = join(this.dir, dir); - if (existsSync(fullPath)) { - rmSync(fullPath, { recursive: true, force: true }); - } - } - } - - /** - * Remove the entire temp directory. - */ - async teardown(): Promise { - if (this.dir && existsSync(this.dir)) { - if (process.env.KEEP_TEST_DIRS) { - console.log(`[${this.label}] Preserving test dir: ${this.dir}`); - } else { - // retry removal — on CI, child processes (e.g. wrangler) may still be - // releasing file handles when teardown runs, causing ENOTEMPTY - for (let attempt = 0; attempt < 3; attempt++) { - try { - rmSync(this.dir, { recursive: true, force: true }); - break; - } catch (err) { - if (attempt < 2) { - await new Promise((r) => { - setTimeout(r, 500); - }); - } else { - console.warn(`[${this.label}] Failed to clean up ${this.dir}: ${(err as Error).message}`); - } - } - } - console.log(`[${this.label}] Cleaned up ${this.dir}`); - } - } - } } diff --git a/framework-tests/harness/types.ts b/framework-tests/harness/types.ts index 0237da874..995604f03 100644 --- a/framework-tests/harness/types.ts +++ b/framework-tests/harness/types.ts @@ -74,6 +74,13 @@ export interface TestFixtureConfig { templateFiles?: TemplateFileMap; /** Timeout for dependency installation in ms (default: 120_000) */ installTimeout?: number; + /** + * Install varlock / `@varlock/*` from the npm registry using the versions in + * `dependencies`, instead of packing workspace tarballs. Also resolves + * `` override placeholders to `npm:pkg`. + * Used by release benchmarks that measure published packages. + */ + usePublished?: boolean; } /** diff --git a/package.json b/package.json index 49c3aee1e..dddd47dbd 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "test:ci": "turbo test:ci --filter=\"!smoke-test-*\"", "smoke-test": "cd smoke-tests && bun run test", "test:frameworks": "cd framework-tests && bun run test", + "bench": "bun run --cwd benchmarks src/run.ts", "typecheck": "tsc --noEmit", "typecheck:all": "turbo typecheck --filter=\"!@varlock/website\" --filter=\"!smoke-test-*\" --filter=\"!varlock-docs-mcp\"", "check": "bun run lint && bun run typecheck:all && bun run build:libs && bun run test:ci", diff --git a/packages/varlock/src/config.ts b/packages/varlock/src/config.ts index 0d5bf6f03..42a89aa09 100644 --- a/packages/varlock/src/config.ts +++ b/packages/varlock/src/config.ts @@ -7,5 +7,11 @@ export const CONFIG = { VARLOCK_API_URL: 'https://api.varlock.dev', GITHUB_APP_CLIENT_ID: 'Iv23li50gB8bMxLauiJQ', // varlock.dev app POSTHOG_API_KEY: 'phc_bfzH97VIta8yQa8HrsgmitqS6rTydjMISs0m8aqJTnq', - POSTHOG_HOST: 'https://ph.varlock.dev', + /** + * Telemetry collector endpoint. Overridable so tooling that needs the telemetry + * code path to actually run (the benchmark suite measures its cost) can point it + * at a local mock instead of the real collector. This is not an opt-out knob: + * use VARLOCK_TELEMETRY_DISABLED to disable telemetry entirely. + */ + POSTHOG_HOST: process.env.VARLOCK_POSTHOG_HOST || 'https://ph.varlock.dev', };