From 45a3a8dafc763f32b43f25d63dada9613dabe6de Mon Sep 17 00:00:00 2001 From: t Date: Tue, 4 Aug 2026 02:44:01 -0700 Subject: [PATCH 1/2] ci: fail regeneration shards that report failures but rewrite nothing continue-on-error is required on the update-snapshots run because a rewritten baseline surfaces as a test failure, but it also swallowed genuine breakage: if a shard died (server never came up, wrong container, Playwright crashed) no snapshots changed, merge-and-commit logged 'No changes to commit', and the run went green having done nothing. Keep the JSON report and check it. No report at all means the process never completed. Failures with zero rewritten files is a broken run, not an absence of drift. Zero matched tests stays legitimate - Playwright exits non-zero and records a top-level error for it, so neither the exit code nor the error list can be trusted alone. Parsed with node rather than jq: this step runs inside the CI container, where node is guaranteed but jq is not. --- .../pr-update-playwright-expectations.yaml | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index 01bab7f89d3..b6cfbe56e66 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -172,9 +172,14 @@ jobs: run: pnpm install --frozen-lockfile # Run tests with snapshot updates (browsers pre-installed in container) + # continue-on-error is required: --update-snapshots reports a rewritten + # baseline as a test failure. It also swallows genuine breakage, so the + # JSON report is kept and checked by the next step. - name: Update snapshots (${{ matrix.shard == 'cloud' && 'cloud' || format('Shard {0}/4', matrix.shard) }}) id: playwright-tests - run: pnpm exec playwright test --update-snapshots --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' || format('--grep-invert @cloud --shard={0}/4', matrix.shard) }} + env: + PLAYWRIGHT_JSON_OUTPUT_NAME: /tmp/playwright-results.json + run: pnpm exec playwright test --update-snapshots --reporter=json,html --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' || format('--grep-invert @cloud --shard={0}/4', matrix.shard) }} continue-on-error: true - name: Stage changed snapshot files @@ -212,6 +217,71 @@ jobs: cp "$file" "/tmp/changed_snapshots_shard/$file_without_prefix" done <<< "$changed_files" + # A shard that reported test failures but rewrote nothing did not + # "find no drift" - it broke (server never came up, container wrong, + # Playwright died). Without this the shard is green, merge-and-commit + # logs "No changes to commit", and the whole run reports success having + # done nothing. + - name: Verify shard did the work it claims + shell: bash + env: + SHARD: ${{ matrix.shard }} + HAS_CHANGES: ${{ steps.changed-snapshots.outputs.has-changes }} + run: | + set -uo pipefail + + results='/tmp/playwright-results.json' + + # No JSON at all means the process died before reporting. Playwright + # still writes the report when zero tests match, so absence is fatal. + if [ ! -f "$results" ]; then + echo "::error::Shard ${SHARD}: Playwright wrote no JSON report, so it never completed a run. Refusing to report success." + exit 1 + fi + + # Parsed with node, not jq: this step runs inside the CI container, + # where node is guaranteed (it just ran pnpm/playwright) but jq is not. + if ! summary=$(node -e ' + const fs = require("fs"); + const r = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + const s = r.stats; + if (!s) throw new Error("no stats in report"); + const n = (v) => (typeof v === "number" ? v : 0); + const notPassing = n(s.unexpected) + n(s.flaky); + const ran = n(s.expected) + notPassing; + const errs = Array.isArray(r.errors) ? r.errors.length : 0; + console.log(ran + " " + notPassing + " " + errs); + console.log(JSON.stringify(s)); + ' "$results" 2>&1); then + echo "::error::Shard ${SHARD}: Playwright JSON report is unreadable or has no stats. Refusing to report success." + echo "$summary" + exit 1 + fi + + read -r ran not_passing error_count <<< "$(printf '%s\n' "$summary" | head -1)" + echo "Shard ${SHARD} stats: $(printf '%s\n' "$summary" | tail -1)" + + # Zero tests matched is legitimate for a leg whose grep selects + # nothing (Playwright exits non-zero AND records a top-level error + # for it, which is why neither signal can be trusted on its own). + if [ "$ran" -eq 0 ]; then + echo "::notice::Shard ${SHARD}: no tests matched this shard's filters; nothing to regenerate." + exit 0 + fi + + if [ "$error_count" -gt 0 ]; then + echo "::warning::Shard ${SHARD}: Playwright reported ${error_count} top-level error(s); the regenerated baselines for this shard may be incomplete." + fi + + # A rewritten baseline surfaces as unexpected/flaky, so failures with + # zero rewritten files is the broken-run signature. + if [ "$not_passing" -gt 0 ] && [ "${HAS_CHANGES:-false}" != 'true' ]; then + echo "::error::Shard ${SHARD}: ${not_passing} test(s) did not pass yet no snapshot was rewritten. That is a broken run, not an absence of drift. Refusing to report success." + exit 1 + fi + + echo "Shard ${SHARD}: ${ran} test(s) ran, ${not_passing} not passing, snapshot changes=${HAS_CHANGES:-false}" + # Upload ONLY the changed files from this shard - name: Upload changed snapshots uses: actions/upload-artifact@v6 @@ -351,6 +421,7 @@ jobs: echo "has-changes=true" >> $GITHUB_OUTPUT git add browser_tests/ + git commit -m "[automated] Update test expectations" echo "Pushing to ${{ needs.setup.outputs.branch }}..." From 5ebca2835ca6faf4a64d379bd74587793e5b2a98 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 4 Aug 2026 02:44:19 -0700 Subject: [PATCH 2/2] ci: report per-file pixel diff magnitude for rewritten baselines Regeneration has no notion of whether a baseline actually drifted: it rewrites whatever it captured. Screenshot capture is not bit-deterministic - repeated runs against the same container and dist produce images that differ in text glyph rasterization with layout byte-identical. Normal e2e absorbs this via retries; regeneration captures once under --update-snapshots and commits the result. So a baseline that never changed can be rewritten, and a baseline failing from a genuine visual regression can be laundered into an accepted one. The only guard was a human eyeballing binary files, which is the review binary diffs defeat. Measure each rewritten snapshot against HEAD with ImageMagick and publish the table to the step summary and the commit body, so there is a number to review instead of a PNG. Warn on rewrites under 2000 pixels, which is the rasterization-noise signature rather than an intended change. Degrades to byte sizes if compare is unavailable and never fails the job. The commit body is assembled with cat, never a heredoc: the table contains backticks and file names that must not be evaluated. --- .../pr-update-playwright-expectations.yaml | 100 +++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index b6cfbe56e66..8d63b45e688 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -402,6 +402,88 @@ jobs: echo "$CHANGES" | wc -l fi + # Regeneration has no notion of *whether* a baseline drifted: it rewrites + # whatever it captured. Screenshot capture is not bit-deterministic (text + # glyph rasterization varies run to run), so a baseline that never + # actually changed can be rewritten, and a baseline failing from a real + # visual regression can be laundered into an accepted one. Reviewing that + # means eyeballing binary diffs, which is exactly the review binary files + # defeat. Publish a per-file magnitude so there is a number to review. + - name: Report snapshot diff magnitudes + id: diff-report + shell: bash + run: | + set -uo pipefail + + report='/tmp/diff-report.md' + : > "$report" + + mapfile -t changed < <( + git status --porcelain=v1 --untracked-files=all -- browser_tests/ | + sed -e 's/^...//' -e 's/^"//' -e 's/"$//' + ) + if [ "${#changed[@]}" -eq 0 ]; then + echo "No snapshot changes to measure" + exit 0 + fi + + if command -v compare >/dev/null 2>&1; then + echo "Measuring ${#changed[@]} rewritten snapshot(s) with ImageMagick" + else + echo "::warning::ImageMagick 'compare' unavailable; falling back to byte sizes, which cannot distinguish a real visual change from rasterization noise." + fi + + small=0 + { + echo "| snapshot | status | changed pixels |" + echo "| --- | --- | --- |" + } >> "$report" + + tmp_old="$(mktemp)" + for f in "${changed[@]}"; do + [ -f "$f" ] || continue + name="${f#browser_tests/}" + + if ! git show "HEAD:$f" > "$tmp_old" 2>/dev/null; then + echo "| \`${name}\` | new | n/a |" >> "$report" + continue + fi + + px='' + if command -v compare >/dev/null 2>&1; then + px=$(compare -metric AE "$tmp_old" "$f" null: 2>&1 >/dev/null) + # Non-numeric means differing geometry or a decode failure. + case "$px" in + ''|*[!0-9]*) px='' ;; + esac + fi + + if [ -n "$px" ]; then + echo "| \`${name}\` | modified | ${px} |" >> "$report" + # Sub-threshold rewrites match the glyph-rasterization signature + # rather than an intended visual change. Advisory only. + if [ "$px" -gt 0 ] && [ "$px" -le 2000 ]; then + small=$((small + 1)) + fi + else + old_b=$(wc -c < "$tmp_old" | tr -d ' ') + new_b=$(wc -c < "$f" | tr -d ' ') + echo "| \`${name}\` | modified | ${old_b}B -> ${new_b}B |" >> "$report" + fi + done + rm -f "$tmp_old" + + if [ "$small" -gt 0 ]; then + echo "::warning::${small} rewritten baseline(s) changed by <=2000 pixels. Screenshot capture is not bit-deterministic - small diffs are usually text rasterization noise, not an intended change. Confirm each rewrite corresponds to a test you expected to change before merging." + fi + + cat "$report" + { + echo "### Rewritten snapshots (${#changed[@]})" + echo + cat "$report" + } >> "$GITHUB_STEP_SUMMARY" + - name: Commit updated expectations id: commit run: | @@ -422,7 +504,23 @@ jobs: git add browser_tests/ - git commit -m "[automated] Update test expectations" + # Carry the magnitude table into the commit body so the numbers live + # with the binary diff a reviewer would otherwise have to eyeball. + # Built with cat, never a heredoc: the table contains backticks and + # arbitrary file names that must not be evaluated by the shell. + { + echo '[automated] Update test expectations' + if [ -s /tmp/diff-report.md ]; then + echo + cat /tmp/diff-report.md + echo + echo 'Screenshot capture is not bit-deterministic; small pixel' + echo 'deltas are usually text rasterization noise. Confirm each' + echo 'rewrite corresponds to a test expected to change.' + fi + } > /tmp/commit-msg.txt + + git commit -F /tmp/commit-msg.txt echo "Pushing to ${{ needs.setup.outputs.branch }}..." git push origin ${{ needs.setup.outputs.branch }}