diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index 01bab7f89d3..8d63b45e688 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -172,9 +172,14 @@ jobs: run: pnpm install --frozen-lockfile # Run tests with snapshot updates (browsers pre-installed in container) + # continue-on-error is required: --update-snapshots reports a rewritten + # baseline as a test failure. It also swallows genuine breakage, so the + # JSON report is kept and checked by the next step. - name: Update snapshots (${{ matrix.shard == 'cloud' && 'cloud' || format('Shard {0}/4', matrix.shard) }}) id: playwright-tests - run: pnpm exec playwright test --update-snapshots --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' || format('--grep-invert @cloud --shard={0}/4', matrix.shard) }} + env: + PLAYWRIGHT_JSON_OUTPUT_NAME: /tmp/playwright-results.json + run: pnpm exec playwright test --update-snapshots --reporter=json,html --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' || format('--grep-invert @cloud --shard={0}/4', matrix.shard) }} continue-on-error: true - name: Stage changed snapshot files @@ -212,6 +217,71 @@ jobs: cp "$file" "/tmp/changed_snapshots_shard/$file_without_prefix" done <<< "$changed_files" + # A shard that reported test failures but rewrote nothing did not + # "find no drift" - it broke (server never came up, container wrong, + # Playwright died). Without this the shard is green, merge-and-commit + # logs "No changes to commit", and the whole run reports success having + # done nothing. + - name: Verify shard did the work it claims + shell: bash + env: + SHARD: ${{ matrix.shard }} + HAS_CHANGES: ${{ steps.changed-snapshots.outputs.has-changes }} + run: | + set -uo pipefail + + results='/tmp/playwright-results.json' + + # No JSON at all means the process died before reporting. Playwright + # still writes the report when zero tests match, so absence is fatal. + if [ ! -f "$results" ]; then + echo "::error::Shard ${SHARD}: Playwright wrote no JSON report, so it never completed a run. Refusing to report success." + exit 1 + fi + + # Parsed with node, not jq: this step runs inside the CI container, + # where node is guaranteed (it just ran pnpm/playwright) but jq is not. + if ! summary=$(node -e ' + const fs = require("fs"); + const r = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + const s = r.stats; + if (!s) throw new Error("no stats in report"); + const n = (v) => (typeof v === "number" ? v : 0); + const notPassing = n(s.unexpected) + n(s.flaky); + const ran = n(s.expected) + notPassing; + const errs = Array.isArray(r.errors) ? r.errors.length : 0; + console.log(ran + " " + notPassing + " " + errs); + console.log(JSON.stringify(s)); + ' "$results" 2>&1); then + echo "::error::Shard ${SHARD}: Playwright JSON report is unreadable or has no stats. Refusing to report success." + echo "$summary" + exit 1 + fi + + read -r ran not_passing error_count <<< "$(printf '%s\n' "$summary" | head -1)" + echo "Shard ${SHARD} stats: $(printf '%s\n' "$summary" | tail -1)" + + # Zero tests matched is legitimate for a leg whose grep selects + # nothing (Playwright exits non-zero AND records a top-level error + # for it, which is why neither signal can be trusted on its own). + if [ "$ran" -eq 0 ]; then + echo "::notice::Shard ${SHARD}: no tests matched this shard's filters; nothing to regenerate." + exit 0 + fi + + if [ "$error_count" -gt 0 ]; then + echo "::warning::Shard ${SHARD}: Playwright reported ${error_count} top-level error(s); the regenerated baselines for this shard may be incomplete." + fi + + # A rewritten baseline surfaces as unexpected/flaky, so failures with + # zero rewritten files is the broken-run signature. + if [ "$not_passing" -gt 0 ] && [ "${HAS_CHANGES:-false}" != 'true' ]; then + echo "::error::Shard ${SHARD}: ${not_passing} test(s) did not pass yet no snapshot was rewritten. That is a broken run, not an absence of drift. Refusing to report success." + exit 1 + fi + + echo "Shard ${SHARD}: ${ran} test(s) ran, ${not_passing} not passing, snapshot changes=${HAS_CHANGES:-false}" + # Upload ONLY the changed files from this shard - name: Upload changed snapshots uses: actions/upload-artifact@v6 @@ -332,6 +402,88 @@ jobs: echo "$CHANGES" | wc -l fi + # Regeneration has no notion of *whether* a baseline drifted: it rewrites + # whatever it captured. Screenshot capture is not bit-deterministic (text + # glyph rasterization varies run to run), so a baseline that never + # actually changed can be rewritten, and a baseline failing from a real + # visual regression can be laundered into an accepted one. Reviewing that + # means eyeballing binary diffs, which is exactly the review binary files + # defeat. Publish a per-file magnitude so there is a number to review. + - name: Report snapshot diff magnitudes + id: diff-report + shell: bash + run: | + set -uo pipefail + + report='/tmp/diff-report.md' + : > "$report" + + mapfile -t changed < <( + git status --porcelain=v1 --untracked-files=all -- browser_tests/ | + sed -e 's/^...//' -e 's/^"//' -e 's/"$//' + ) + if [ "${#changed[@]}" -eq 0 ]; then + echo "No snapshot changes to measure" + exit 0 + fi + + if command -v compare >/dev/null 2>&1; then + echo "Measuring ${#changed[@]} rewritten snapshot(s) with ImageMagick" + else + echo "::warning::ImageMagick 'compare' unavailable; falling back to byte sizes, which cannot distinguish a real visual change from rasterization noise." + fi + + small=0 + { + echo "| snapshot | status | changed pixels |" + echo "| --- | --- | --- |" + } >> "$report" + + tmp_old="$(mktemp)" + for f in "${changed[@]}"; do + [ -f "$f" ] || continue + name="${f#browser_tests/}" + + if ! git show "HEAD:$f" > "$tmp_old" 2>/dev/null; then + echo "| \`${name}\` | new | n/a |" >> "$report" + continue + fi + + px='' + if command -v compare >/dev/null 2>&1; then + px=$(compare -metric AE "$tmp_old" "$f" null: 2>&1 >/dev/null) + # Non-numeric means differing geometry or a decode failure. + case "$px" in + ''|*[!0-9]*) px='' ;; + esac + fi + + if [ -n "$px" ]; then + echo "| \`${name}\` | modified | ${px} |" >> "$report" + # Sub-threshold rewrites match the glyph-rasterization signature + # rather than an intended visual change. Advisory only. + if [ "$px" -gt 0 ] && [ "$px" -le 2000 ]; then + small=$((small + 1)) + fi + else + old_b=$(wc -c < "$tmp_old" | tr -d ' ') + new_b=$(wc -c < "$f" | tr -d ' ') + echo "| \`${name}\` | modified | ${old_b}B -> ${new_b}B |" >> "$report" + fi + done + rm -f "$tmp_old" + + if [ "$small" -gt 0 ]; then + echo "::warning::${small} rewritten baseline(s) changed by <=2000 pixels. Screenshot capture is not bit-deterministic - small diffs are usually text rasterization noise, not an intended change. Confirm each rewrite corresponds to a test you expected to change before merging." + fi + + cat "$report" + { + echo "### Rewritten snapshots (${#changed[@]})" + echo + cat "$report" + } >> "$GITHUB_STEP_SUMMARY" + - name: Commit updated expectations id: commit run: | @@ -351,7 +503,24 @@ jobs: echo "has-changes=true" >> $GITHUB_OUTPUT git add browser_tests/ - git commit -m "[automated] Update test expectations" + + # Carry the magnitude table into the commit body so the numbers live + # with the binary diff a reviewer would otherwise have to eyeball. + # Built with cat, never a heredoc: the table contains backticks and + # arbitrary file names that must not be evaluated by the shell. + { + echo '[automated] Update test expectations' + if [ -s /tmp/diff-report.md ]; then + echo + cat /tmp/diff-report.md + echo + echo 'Screenshot capture is not bit-deterministic; small pixel' + echo 'deltas are usually text rasterization noise. Confirm each' + echo 'rewrite corresponds to a test expected to change.' + fi + } > /tmp/commit-msg.txt + + git commit -F /tmp/commit-msg.txt echo "Pushing to ${{ needs.setup.outputs.branch }}..." git push origin ${{ needs.setup.outputs.branch }}