Skip to content

Commit 06f2d40

Browse files
claudeclaude
committed
Add a scan benchmark suite and a CI performance gate
New crate crates/socket-patch-bench (publish = false, no new third-party dependencies) that benchmarks `socket-patch scan` end to end: - Synthetic, seed-deterministic projects for every package manager in its native lockfile format and install layout: npm, pnpm (isolated store), yarn classic, yarn berry, bun, vlt, pip requirements, uv, PEP 751 pylock, poetry, pipenv, pdm, bundler, composer, cargo, go, nuget and maven. Per-user caches live under the fixture's own HOME. - A std-only mock of the patch API, proxy and artifact host that answers from the scenario's catalog and counts requests per endpoint. - Each PM runs a hosted scan of a fresh project and a rescan of an already-redirected one; npm also runs --dry-run, the public proxy and 40 ms simulated latency (request concurrency). - Every run is validated (scanned/lockfile-only/patched counts, redirected patches, exact rewritten files that really changed on disk, allowed warnings, no unexpected requests, dry runs write nothing), so a scan that skips work fails instead of looking fast. Runs start from the pristine tree via snapshot diff/restore, with a cleared env. - `compare` interleaves base and head runs on one machine and fails on a wall/CPU slowdown (median paired ratio past 10% with a 95% sign-test interval above 1, confirmed by a second round), peak RSS growth, more API requests, or a head that fails validation. .github/workflows/bench.yml builds the PR's base and head with the new `perf` profile (release codegen, thin LTO) and runs the comparison on every PR touching Rust code; pushes to main are recorded without gating. The `performance-regression-accepted` label turns a regression into a report. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017hKPGJkJfyhsQrzXYf7RJZ
1 parent 2463257 commit 06f2d40

21 files changed

Lines changed: 5861 additions & 1 deletion

File tree

‎.github/workflows/bench.yml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
name: Benchmarks
2+
3+
# Performance gate for `socket-patch scan` (crates/socket-patch-bench; its
4+
# README has the details). Builds the pull request's base and head with the
5+
# same profile on one runner, times both on a synthetic project per package
6+
# manager against a local patch API (interleaved: base, head, head, base,
7+
# ...), and fails when the head
8+
# - is significantly slower (wall or CPU time; median of the paired
9+
# head/base ratios past the threshold with its 95% interval above 1,
10+
# confirmed by a second round of pairs),
11+
# - uses significantly more memory,
12+
# - makes more API requests, or
13+
# - no longer does a scenario's work (scan counts, redirected patches,
14+
# rewritten files).
15+
# The job summary carries the full table; results.json is uploaded.
16+
#
17+
# An intentional slowdown: label the PR `performance-regression-accepted`.
18+
# The comparison still runs and reports, but a regression no longer fails
19+
# the job (a head that fails a scenario's validation still does).
20+
#
21+
# On main, each push is compared against the commit before it and recorded
22+
# (an artifact per push) without failing on regressions.
23+
24+
on:
25+
pull_request:
26+
types: [opened, synchronize, reopened, labeled, unlabeled]
27+
paths:
28+
- '.github/workflows/bench.yml'
29+
- '.cargo/**'
30+
- 'Cargo.lock'
31+
- 'Cargo.toml'
32+
- 'rust-toolchain.toml'
33+
- 'crates/socket-patch-bench/**'
34+
- 'crates/socket-patch-cli/Cargo.toml'
35+
- 'crates/socket-patch-cli/build.rs'
36+
- 'crates/socket-patch-cli/src/**'
37+
- 'crates/socket-patch-core/Cargo.toml'
38+
- 'crates/socket-patch-core/src/**'
39+
push:
40+
branches: [main]
41+
paths:
42+
- '.github/workflows/bench.yml'
43+
- '.cargo/**'
44+
- 'Cargo.lock'
45+
- 'Cargo.toml'
46+
- 'rust-toolchain.toml'
47+
- 'crates/socket-patch-bench/**'
48+
- 'crates/socket-patch-cli/Cargo.toml'
49+
- 'crates/socket-patch-cli/build.rs'
50+
- 'crates/socket-patch-cli/src/**'
51+
- 'crates/socket-patch-core/Cargo.toml'
52+
- 'crates/socket-patch-core/src/**'
53+
workflow_dispatch:
54+
inputs:
55+
base:
56+
description: 'Commit or ref to compare HEAD against (default: the parent commit)'
57+
required: false
58+
default: ''
59+
filter:
60+
description: 'Only scenarios matching this regex (see `socket-patch-bench list`)'
61+
required: false
62+
default: ''
63+
64+
permissions:
65+
contents: read
66+
67+
jobs:
68+
scan:
69+
name: scan performance
70+
# Labels re-trigger the workflow; only this one changes the outcome.
71+
if: >-
72+
(github.event.action != 'labeled' && github.event.action != 'unlabeled')
73+
|| github.event.label.name == 'performance-regression-accepted'
74+
# Job-level, so an unrelated label (skipped above) cannot cancel a run
75+
# in progress. A newer push to the same PR supersedes the older run.
76+
concurrency:
77+
group: bench-${{ github.event.pull_request.number || github.ref }}
78+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
79+
runs-on: ubuntu-latest
80+
timeout-minutes: 60
81+
env:
82+
# Both sides build with these, so a base that predates the `perf`
83+
# profile (or carries an older copy of it) builds the same way.
84+
CARGO_PROFILE_PERF_INHERITS: release
85+
CARGO_PROFILE_PERF_LTO: thin
86+
CARGO_PROFILE_PERF_STRIP: none
87+
steps:
88+
- name: Checkout
89+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
90+
with:
91+
persist-credentials: false
92+
# The parent is the baseline: for a pull request, the merge
93+
# commit's first parent is exactly the base it was merged onto.
94+
fetch-depth: 2
95+
96+
- name: Install Rust
97+
# rustup is pre-installed; `rustup show` installs the
98+
# rust-toolchain.toml channel.
99+
run: rustup show
100+
101+
- name: Cache cargo
102+
# Swatinem/rust-cache, main-only saves: see ci.yml.
103+
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
104+
with:
105+
save-if: ${{ github.ref == 'refs/heads/main' }}
106+
107+
- name: Resolve the baseline commit
108+
id: base
109+
env:
110+
EVENT_NAME: ${{ github.event_name }}
111+
PUSH_BEFORE: ${{ github.event.before }}
112+
INPUT_BASE: ${{ inputs.base }}
113+
run: |
114+
set -euo pipefail
115+
case "$EVENT_NAME" in
116+
push) ref="$PUSH_BEFORE" ;;
117+
workflow_dispatch) ref="${INPUT_BASE:-HEAD^1}" ;;
118+
*) ref="HEAD^1" ;;
119+
esac
120+
if ! git rev-parse --verify --quiet "$ref^{commit}" >/dev/null; then
121+
git fetch --quiet --depth 1 origin "$ref"
122+
ref=FETCH_HEAD
123+
fi
124+
sha="$(git rev-parse "$ref^{commit}")"
125+
echo "Baseline: $sha"
126+
echo "sha=$sha" >> "$GITHUB_OUTPUT"
127+
128+
- name: Build head
129+
run: |
130+
set -euo pipefail
131+
BENCH="$RUNNER_TEMP/bench"
132+
cargo build --locked --profile perf -p socket-patch-cli -p socket-patch-bench
133+
mkdir -p "$BENCH/head"
134+
cp target/perf/socket-patch "$BENCH/head/"
135+
cp target/perf/socket-patch-bench "$BENCH/"
136+
137+
- name: Build base
138+
env:
139+
BASE_SHA: ${{ steps.base.outputs.sha }}
140+
run: |
141+
set -euo pipefail
142+
BENCH="$RUNNER_TEMP/bench"
143+
git worktree add --detach "$BENCH/base-src" "$BASE_SHA"
144+
# Same target directory: the dependencies are already built, only
145+
# the workspace crates compile again (unless the base pins another
146+
# toolchain, which `rustup show` installs first).
147+
(cd "$BENCH/base-src" && rustup show >/dev/null && \
148+
CARGO_TARGET_DIR="$GITHUB_WORKSPACE/target" \
149+
cargo build --locked --profile perf -p socket-patch-cli)
150+
mkdir -p "$BENCH/base"
151+
cp target/perf/socket-patch "$BENCH/base/"
152+
"$BENCH/base/socket-patch" --version
153+
"$BENCH/head/socket-patch" --version
154+
155+
- name: Compare
156+
env:
157+
EVENT_NAME: ${{ github.event_name }}
158+
ACCEPTED: ${{ contains(github.event.pull_request.labels.*.name, 'performance-regression-accepted') }}
159+
FILTER: ${{ inputs.filter }}
160+
run: |
161+
set -euo pipefail
162+
BENCH="$RUNNER_TEMP/bench"
163+
args=()
164+
if [ "$EVENT_NAME" != pull_request ] || [ "$ACCEPTED" = true ]; then
165+
args+=(--no-fail)
166+
fi
167+
if [ -n "${FILTER:-}" ]; then
168+
args+=(--filter "$FILTER")
169+
fi
170+
"$BENCH/socket-patch-bench" compare \
171+
--base "$BENCH/base/socket-patch" \
172+
--head "$BENCH/head/socket-patch" \
173+
--work-dir "$BENCH/work" \
174+
--out "$BENCH/out" \
175+
${args[@]+"${args[@]}"}
176+
177+
- name: Job summary
178+
if: ${{ !cancelled() }}
179+
run: |
180+
if [ -f "$RUNNER_TEMP/bench/out/summary.md" ]; then
181+
cat "$RUNNER_TEMP/bench/out/summary.md" >> "$GITHUB_STEP_SUMMARY"
182+
fi
183+
184+
- name: Upload results
185+
if: ${{ !cancelled() }}
186+
uses: ./.github/actions/upload-artifact
187+
with:
188+
name: bench-scan
189+
path: ${{ runner.temp }}/bench/out/
190+
if-no-files-found: warn
191+
retention-days: 90

‎Cargo.lock‎

Lines changed: 17 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ members = [
33
"crates/socket-patch-core",
44
"crates/socket-patch-cli",
55
"crates/socket-patch-node",
6+
"crates/socket-patch-bench",
67
]
78
# Bare `cargo build` (release.yml's per-target builds) skips the Node addon,
89
# which no release artifact ships; `--workspace` and `-p` still build it.
@@ -78,6 +79,16 @@ inherits = "release"
7879
lto = false
7980
strip = "none" # unstripped test binaries => usable backtraces on failure
8081

82+
# The benchmark gate's build (crates/socket-patch-bench, .github/workflows/
83+
# bench.yml): the shipped profile's codegen (opt-level = "s", no debug
84+
# assertions) so timings track what users run, with thin instead of full
85+
# LTO so CI can afford to build both sides of a comparison. Unstripped, so
86+
# a profiler can symbolize it.
87+
[profile.perf]
88+
inherits = "release"
89+
lto = "thin"
90+
strip = "none"
91+
8192
# The Node addon (crates/socket-patch-node) for hosts that load it
8293
# in-process: release semantics without the slow full-LTO link.
8394
[profile.addon]

‎crates/socket-patch-bench/Cargo.toml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
[package]
2+
name = "socket-patch-bench"
3+
description = "Benchmark harness for `socket-patch scan`: synthetic projects, a local patch API, and A/B regression checks"
4+
version.workspace = true
5+
edition.workspace = true
6+
license.workspace = true
7+
repository.workspace = true
8+
publish = false
9+
10+
[[bin]]
11+
name = "socket-patch-bench"
12+
path = "src/main.rs"
13+
14+
# Only crates the workspace already pins: the harness adds nothing to
15+
# Cargo.lock that the CLI's own tests do not already pull in.
16+
[dependencies]
17+
clap = { workspace = true }
18+
serde = { workspace = true }
19+
serde_json = { workspace = true }
20+
sha2 = { workspace = true }
21+
sha1 = { workspace = true }
22+
hex = { workspace = true }
23+
base64 = { workspace = true }
24+
tempfile = { workspace = true }
25+
regex = { workspace = true }
26+
# uv's hosted rewrite reads the patched wheel's METADATA, so the mock
27+
# serves real (tiny) wheels.
28+
zip = { workspace = true }
29+
30+
[target.'cfg(unix)'.dependencies]
31+
libc = { workspace = true }

0 commit comments

Comments
 (0)