Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 171 additions & 2 deletions .github/workflows/pr-update-playwright-expectations.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -172,9 +172,14 @@ jobs:
run: pnpm install --frozen-lockfile

# Run tests with snapshot updates (browsers pre-installed in container)
# continue-on-error is required: --update-snapshots reports a rewritten
# baseline as a test failure. It also swallows genuine breakage, so the
# JSON report is kept and checked by the next step.
- name: Update snapshots (${{ matrix.shard == 'cloud' && 'cloud' || format('Shard {0}/4', matrix.shard) }})
id: playwright-tests
run: pnpm exec playwright test --update-snapshots --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' || format('--grep-invert @cloud --shard={0}/4', matrix.shard) }}
env:
PLAYWRIGHT_JSON_OUTPUT_NAME: /tmp/playwright-results.json
run: pnpm exec playwright test --update-snapshots --reporter=json,html --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' || format('--grep-invert @cloud --shard={0}/4', matrix.shard) }}
continue-on-error: true

- name: Stage changed snapshot files
Expand Down Expand Up @@ -212,6 +217,71 @@ jobs:
cp "$file" "/tmp/changed_snapshots_shard/$file_without_prefix"
done <<< "$changed_files"

# A shard that reported test failures but rewrote nothing did not
# "find no drift" - it broke (server never came up, container wrong,
# Playwright died). Without this the shard is green, merge-and-commit
# logs "No changes to commit", and the whole run reports success having
# done nothing.
- name: Verify shard did the work it claims
shell: bash
env:
SHARD: ${{ matrix.shard }}
HAS_CHANGES: ${{ steps.changed-snapshots.outputs.has-changes }}
run: |
set -uo pipefail

results='/tmp/playwright-results.json'

# No JSON at all means the process died before reporting. Playwright
# still writes the report when zero tests match, so absence is fatal.
if [ ! -f "$results" ]; then
echo "::error::Shard ${SHARD}: Playwright wrote no JSON report, so it never completed a run. Refusing to report success."
exit 1
fi

# Parsed with node, not jq: this step runs inside the CI container,
# where node is guaranteed (it just ran pnpm/playwright) but jq is not.
if ! summary=$(node -e '
const fs = require("fs");
const r = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
const s = r.stats;
if (!s) throw new Error("no stats in report");
const n = (v) => (typeof v === "number" ? v : 0);
const notPassing = n(s.unexpected) + n(s.flaky);
const ran = n(s.expected) + notPassing;
const errs = Array.isArray(r.errors) ? r.errors.length : 0;
console.log(ran + " " + notPassing + " " + errs);
console.log(JSON.stringify(s));
' "$results" 2>&1); then
echo "::error::Shard ${SHARD}: Playwright JSON report is unreadable or has no stats. Refusing to report success."
echo "$summary"
exit 1
fi

read -r ran not_passing error_count <<< "$(printf '%s\n' "$summary" | head -1)"
echo "Shard ${SHARD} stats: $(printf '%s\n' "$summary" | tail -1)"

# Zero tests matched is legitimate for a leg whose grep selects
# nothing (Playwright exits non-zero AND records a top-level error
# for it, which is why neither signal can be trusted on its own).
if [ "$ran" -eq 0 ]; then
echo "::notice::Shard ${SHARD}: no tests matched this shard's filters; nothing to regenerate."
exit 0
fi

if [ "$error_count" -gt 0 ]; then
echo "::warning::Shard ${SHARD}: Playwright reported ${error_count} top-level error(s); the regenerated baselines for this shard may be incomplete."
fi

# A rewritten baseline surfaces as unexpected/flaky, so failures with
# zero rewritten files is the broken-run signature.
if [ "$not_passing" -gt 0 ] && [ "${HAS_CHANGES:-false}" != 'true' ]; then
echo "::error::Shard ${SHARD}: ${not_passing} test(s) did not pass yet no snapshot was rewritten. That is a broken run, not an absence of drift. Refusing to report success."
exit 1
fi

echo "Shard ${SHARD}: ${ran} test(s) ran, ${not_passing} not passing, snapshot changes=${HAS_CHANGES:-false}"

# Upload ONLY the changed files from this shard
- name: Upload changed snapshots
uses: actions/upload-artifact@v6
Expand Down Expand Up @@ -332,6 +402,88 @@ jobs:
echo "$CHANGES" | wc -l
fi

# Regeneration has no notion of *whether* a baseline drifted: it rewrites
# whatever it captured. Screenshot capture is not bit-deterministic (text
# glyph rasterization varies run to run), so a baseline that never
# actually changed can be rewritten, and a baseline failing from a real
# visual regression can be laundered into an accepted one. Reviewing that
# means eyeballing binary diffs, which is exactly the review binary files
# defeat. Publish a per-file magnitude so there is a number to review.
- name: Report snapshot diff magnitudes
id: diff-report
shell: bash
run: |
set -uo pipefail

report='/tmp/diff-report.md'
: > "$report"

mapfile -t changed < <(
git status --porcelain=v1 --untracked-files=all -- browser_tests/ |
sed -e 's/^...//' -e 's/^"//' -e 's/"$//'
)
if [ "${#changed[@]}" -eq 0 ]; then
echo "No snapshot changes to measure"
exit 0
fi

if command -v compare >/dev/null 2>&1; then
echo "Measuring ${#changed[@]} rewritten snapshot(s) with ImageMagick"
else
echo "::warning::ImageMagick 'compare' unavailable; falling back to byte sizes, which cannot distinguish a real visual change from rasterization noise."
fi

small=0
{
echo "| snapshot | status | changed pixels |"
echo "| --- | --- | --- |"
} >> "$report"

tmp_old="$(mktemp)"
for f in "${changed[@]}"; do
[ -f "$f" ] || continue
name="${f#browser_tests/}"

if ! git show "HEAD:$f" > "$tmp_old" 2>/dev/null; then
echo "| \`${name}\` | new | n/a |" >> "$report"
continue
fi

px=''
if command -v compare >/dev/null 2>&1; then
px=$(compare -metric AE "$tmp_old" "$f" null: 2>&1 >/dev/null)
# Non-numeric means differing geometry or a decode failure.
case "$px" in
''|*[!0-9]*) px='' ;;
esac
fi

if [ -n "$px" ]; then
echo "| \`${name}\` | modified | ${px} |" >> "$report"
# Sub-threshold rewrites match the glyph-rasterization signature
# rather than an intended visual change. Advisory only.
if [ "$px" -gt 0 ] && [ "$px" -le 2000 ]; then
small=$((small + 1))
fi
else
old_b=$(wc -c < "$tmp_old" | tr -d ' ')
new_b=$(wc -c < "$f" | tr -d ' ')
echo "| \`${name}\` | modified | ${old_b}B -> ${new_b}B |" >> "$report"
fi
done
rm -f "$tmp_old"

if [ "$small" -gt 0 ]; then
echo "::warning::${small} rewritten baseline(s) changed by <=2000 pixels. Screenshot capture is not bit-deterministic - small diffs are usually text rasterization noise, not an intended change. Confirm each rewrite corresponds to a test you expected to change before merging."
fi

cat "$report"
{
echo "### Rewritten snapshots (${#changed[@]})"
echo
cat "$report"
} >> "$GITHUB_STEP_SUMMARY"

- name: Commit updated expectations
id: commit
run: |
Expand All @@ -351,7 +503,24 @@ jobs:
echo "has-changes=true" >> $GITHUB_OUTPUT

git add browser_tests/
git commit -m "[automated] Update test expectations"

# Carry the magnitude table into the commit body so the numbers live
# with the binary diff a reviewer would otherwise have to eyeball.
# Built with cat, never a heredoc: the table contains backticks and
# arbitrary file names that must not be evaluated by the shell.
{
echo '[automated] Update test expectations'
if [ -s /tmp/diff-report.md ]; then
echo
cat /tmp/diff-report.md
echo
echo 'Screenshot capture is not bit-deterministic; small pixel'
echo 'deltas are usually text rasterization noise. Confirm each'
echo 'rewrite corresponds to a test expected to change.'
fi
} > /tmp/commit-msg.txt

git commit -F /tmp/commit-msg.txt

echo "Pushing to ${{ needs.setup.outputs.branch }}..."
git push origin ${{ needs.setup.outputs.branch }}
Expand Down
Loading