From 7003450aa9350aa5a280f4068e76c494c47e863a Mon Sep 17 00:00:00 2001 From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:51:03 +0800 Subject: [PATCH 1/4] ci(nvim): measure wall-clock time around the startup benchmark nvim's own --startuptime only instruments Lua/vimscript execution, so it can't explain why CI's numbers (up to 500ms+ on macos-14) are far higher than the same builds run locally (~25ms) -- that gap has to be happening before Lua even starts: process spawn, dynamic linker resolving nvim's own dylib/so deps, or macOS Gatekeeper/AMFI verifying freshly-downloaded nix-built binaries on a runner that's never seen them before. Wrap the benchmark with the shell's `time` builtin for an outside-in wall-clock figure to compare against nvim's internal mean, and print each run's individual time (not just aggregate stats) to see whether the cost is front-loaded (cold-start/Gatekeeper tax amortizing across the 10 runs) or uniform across every run. --- .github/workflows/ci.yml | 12 +++++++++++- scripts/benchmark-startup | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a38b4..ca50940 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,11 +116,21 @@ jobs: # only ~2-3 minutes wall clock with plenty of concurrency headroom # (public repo, at most 3 test-matrix jobs running at once), so there's # no runtime pressure pushing this out into its own workflow. + # nvim's own --startuptime only instruments Lua/vimscript execution -- + # it can't see anything before that (process spawn, dynamic linker + # resolving nvim's own .dylib/.so deps, macOS Gatekeeper/AMFI + # verification of freshly-downloaded nix-built binaries on a runner + # that's never seen them before). Wrapping the whole run with the + # shell's `time` builtin gives an outside-in wall-clock figure to + # compare against: if `wall clock` here is far above 10x nvim's own + # reported mean, the gap is process-launch overhead outside our + # config's control, not something the Lua-level breakdown can show. - name: Benchmark startup time run: | { echo "### Startup time — ${{ matrix.name }}" echo '```' - ./scripts/benchmark-startup 10 + TIMEFORMAT='wall clock for all 10 runs: %R s (user %U s, sys %S s)' + { time ./scripts/benchmark-startup 10; } 2>&1 echo '```' } >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/benchmark-startup b/scripts/benchmark-startup index 58b1315..1cd66af 100755 --- a/scripts/benchmark-startup +++ b/scripts/benchmark-startup @@ -17,6 +17,8 @@ for ((i = 1; i <= RUNS; i++)); do times+=("$(awk 'NF { last = $1 } END { print last }' "$LOG")") done +echo "per run (chronological, ms): ${times[*]}" + printf '%s\n' "${times[@]}" | sort -n | awk ' { a[NR] = $1; sum += $1 } END { From 7ff856dd01f4e83ff7c3486179b85255e29fc931 Mon Sep 17 00:00:00 2001 From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:02:13 +0800 Subject: [PATCH 2/4] ci(nvim): cache lazy.nvim plugins between runs Without a cache, "Restore plugins" does a full network git clone of all ~40 plugins on every single run, so the benchmark that follows always starts from a cold filesystem. Cache ~/.local/share/nvim/lazy keyed on runs-on (native build output like telescope-fzf-native's is arch-specific) and lazy-lock.json's hash, so a cache hit skips the clone entirely. Also corrects the comment above the benchmark step: the wall-clock measurement added in the previous commit shows the pre-Lua gap is small (~5-40ms/run) on both platforms, not the dominant factor -- the 100-600ms CI readings are real Lua/vimscript execution time, most likely from lazy.nvim's spec-resolution work getting hit by contention on weak/shared runners (worst on macos-14), evidenced by erratic non-monotonic per-run timings rather than a clean cold-cache warm-up curve. --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca50940..97ecf73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,16 @@ jobs: run: echo "out=$(nix build --impure .#${{ matrix.target }} --print-build-logs --no-link --print-out-paths)" >> "$GITHUB_OUTPUT" - name: Put bin on PATH run: echo "${{ steps.build.outputs.out }}/bin" >> "$GITHUB_PATH" + # Plugin files land fresh from a network git clone every run below + # (no cache = no OS page-cache warmth, and telescope-fzf-native's + # native build output is arch-specific, hence keying on runs-on), + # which was feeding straight into cold require()/sourcing timings + # in the benchmark step further down. + - name: Cache lazy.nvim plugins + uses: actions/cache@v4 + with: + path: ~/.local/share/nvim/lazy + key: lazy-${{ matrix.runs-on }}-${{ hashFiles('lazy-lock.json') }} - name: Restore plugins run: nvim --headless "+Lazy! restore" +qa - name: Run smoke test @@ -116,15 +126,20 @@ jobs: # only ~2-3 minutes wall clock with plenty of concurrency headroom # (public repo, at most 3 test-matrix jobs running at once), so there's # no runtime pressure pushing this out into its own workflow. + # # nvim's own --startuptime only instruments Lua/vimscript execution -- - # it can't see anything before that (process spawn, dynamic linker - # resolving nvim's own .dylib/.so deps, macOS Gatekeeper/AMFI - # verification of freshly-downloaded nix-built binaries on a runner - # that's never seen them before). Wrapping the whole run with the - # shell's `time` builtin gives an outside-in wall-clock figure to - # compare against: if `wall clock` here is far above 10x nvim's own - # reported mean, the gap is process-launch overhead outside our - # config's control, not something the Lua-level breakdown can show. + # it can't see process spawn or dynamic-linker/codesign overhead before + # that. Measured (see PR #44): that pre-Lua gap is small in practice + # (wall clock vs. nvim's own reported mean differs by only ~5-40ms/run, + # largest on macOS as expected from Gatekeeper/AMFI, but nowhere near + # what explains CI's 100-600ms readings vs. ~25ms locally for the same + # build). The real driver is CI hardware itself: per-run numbers are + # erratic and non-monotonic (not a clean cold-cache warm-up curve), + # consistent with lazy.nvim's CPU-bound spec-resolution work (~40 + # plugins) getting hit by contention on weak/shared/virtualized + # runners -- worst on macos-14. The `time` wrapping stays so future + # runs keep confirming that split; it's not solely diagnosing a + # one-off. - name: Benchmark startup time run: | { From 0b16f44c62112f22e7e57861d952719cecca523a Mon Sep 17 00:00:00 2001 From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:10:42 +0800 Subject: [PATCH 3/4] ci(nvim): trigger CI to verify lazy.nvim plugin cache hits on a real PR run From 75c3e20b7f7ba7b1b38477e777e961bf9f05626f Mon Sep 17 00:00:00 2001 From: "Thing-han, Lim" <15379156+potsrevennil@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:21:45 +0800 Subject: [PATCH 4/4] ci(nvim): correct startup-time root-cause comment now that it's confirmed Verified on a real pull_request-scoped run: caching lazy.nvim plugins alone took every matrix entry from 100-600ms down to ~20-27ms, matching local numbers almost exactly, with the erratic per-run spread gone too. That confirms the actual cause was cold disk reads for freshly-cloned plugin files landing inside nvim's own require()/sourcing timings -- not CPU contention on shared runners, which was our working theory before this data came in. --- .github/workflows/ci.yml | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97ecf73..328c5ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,11 +105,17 @@ jobs: run: echo "out=$(nix build --impure .#${{ matrix.target }} --print-build-logs --no-link --print-out-paths)" >> "$GITHUB_OUTPUT" - name: Put bin on PATH run: echo "${{ steps.build.outputs.out }}/bin" >> "$GITHUB_PATH" - # Plugin files land fresh from a network git clone every run below - # (no cache = no OS page-cache warmth, and telescope-fzf-native's - # native build output is arch-specific, hence keying on runs-on), - # which was feeding straight into cold require()/sourcing timings - # in the benchmark step further down. + # Confirmed root cause of CI's inflated startup numbers below (see + # PR #44): without this cache, plugin files land fresh from a network + # git clone on every run, and each cold file read shows up inside + # nvim's own require()/sourcing timings (--startuptime measures the + # whole call including the read, not just execution). That's what was + # producing 100-600ms readings vs. ~25ms locally for the identical + # build -- not a slow plugin, not CPU contention, not codesign/ + # Gatekeeper overhead (all measured and ruled out, see below). Adding + # this cache alone took every matrix entry down to ~20-27ms, matching + # local numbers almost exactly. Keyed on runs-on because + # telescope-fzf-native's native build output is arch-specific. - name: Cache lazy.nvim plugins uses: actions/cache@v4 with: @@ -129,17 +135,13 @@ jobs: # # nvim's own --startuptime only instruments Lua/vimscript execution -- # it can't see process spawn or dynamic-linker/codesign overhead before - # that. Measured (see PR #44): that pre-Lua gap is small in practice - # (wall clock vs. nvim's own reported mean differs by only ~5-40ms/run, - # largest on macOS as expected from Gatekeeper/AMFI, but nowhere near - # what explains CI's 100-600ms readings vs. ~25ms locally for the same - # build). The real driver is CI hardware itself: per-run numbers are - # erratic and non-monotonic (not a clean cold-cache warm-up curve), - # consistent with lazy.nvim's CPU-bound spec-resolution work (~40 - # plugins) getting hit by contention on weak/shared/virtualized - # runners -- worst on macos-14. The `time` wrapping stays so future - # runs keep confirming that split; it's not solely diagnosing a - # one-off. + # that. Wrapping with `time` for an outside-in wall-clock figure ruled + # that out as the main story (gap vs. nvim's own reported mean was only + # ~5-40ms/run, largest on macOS as expected from Gatekeeper/AMFI, but + # nowhere near what was inflating CI's numbers before the plugin cache + # above). Kept as a standing signal: if a future regression shows up + # here instead of in nvim's own mean, that's process-launch overhead, + # not the config's fault. - name: Benchmark startup time run: | {