From 6a250543636ec41c8ddd1fba0d0582089ce32f2a Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Mon, 21 Sep 2026 22:56:26 +0000 Subject: [PATCH 01/19] ci: give every unified PR report section one status line The Storybook and Playwright sections already render as a single `##` heading carrying the verdict with everything else behind a `
`. The rest did not: the perf and coverage sections had no verdict on the heading at all, and the matrix, fallow, preview, website E2E, screenshot and release sections dumped tables and prose straight into the comment. The result grew past a screen of scrolling before a reviewer could tell which check was red. Normalize them all on the Storybook shape. renderPrReportSection() is the single renderer for the three TypeScript producers; explain-check keeps its job summary expanded, because a fork PR's read-only token means the summary is the only place those findings are readable. --- .claude/skills/perf-fix-with-proof/SKILL.md | 7 +- .../actions/comment-release-links/action.yaml | 11 +++ .github/actions/explain-check/action.yaml | 32 +++++-- .github/workflows/ci-tests-storybook.yaml | 9 +- .../ci-vercel-billing-web-preview.yaml | 11 ++- .github/workflows/ci-website-e2e.yaml | 45 ++++++---- .../pr-update-website-screenshots.yaml | 54 +++++++----- .../workflows/pr-vercel-website-preview.yaml | 13 ++- .../workflows/publish-package-on-merge.yaml | 9 ++ .../cicd/pr-playwright-deploy-and-comment.sh | 8 +- scripts/cicd/prReportSection.test.ts | 87 +++++++++++++++++++ scripts/cicd/prReportSection.ts | 45 ++++++++++ scripts/coverage-report.ts | 24 ++++- scripts/perf-report.ts | 82 ++++++++++------- scripts/unified-report.ts | 60 ++++++++----- 15 files changed, 384 insertions(+), 113 deletions(-) create mode 100644 scripts/cicd/prReportSection.test.ts create mode 100644 scripts/cicd/prReportSection.ts diff --git a/.claude/skills/perf-fix-with-proof/SKILL.md b/.claude/skills/perf-fix-with-proof/SKILL.md index 6ffba72aa5f..25f4d957077 100644 --- a/.claude/skills/perf-fix-with-proof/SKILL.md +++ b/.claude/skills/perf-fix-with-proof/SKILL.md @@ -118,11 +118,16 @@ Implement the fix. The `@perf` test from PR1 is now on main and will run automat The `ci-perf-report.yaml` posts a comment like: ```markdown -## ⚑ Performance Report +## ⚑ Performance: βœ… No regressions detected + +
+Details | Metric | Baseline | PR (n=3) | Ξ” | Sig | | --------------------- | -------- | -------- | ---- | --- | | : style recalcs | 450 | 12 | -97% | 🟒 | + +
``` If Ξ” is negative for the target metric, the fix is proven. diff --git a/.github/actions/comment-release-links/action.yaml b/.github/actions/comment-release-links/action.yaml index 222867a771b..31591217a9f 100644 --- a/.github/actions/comment-release-links/action.yaml +++ b/.github/actions/comment-release-links/action.yaml @@ -53,6 +53,7 @@ runs: fi SECTION_NAME='release' + SECTION_TITLE='Release' MESSAGE='Publish jobs finished successfully:' LINKS_VALUE='' @@ -64,18 +65,22 @@ runs: ;; packages/design-system/package.json) SECTION_NAME='design-system-release' + SECTION_TITLE='Design System Release' LINKS_VALUE='npm design-system|https://www.npmjs.com/package/@comfyorg/design-system/v/{{version}}' ;; packages/account-core/package.json) SECTION_NAME='account-core-release' + SECTION_TITLE='Account Core Release' LINKS_VALUE='npm account-core|https://www.npmjs.com/package/@comfyorg/account-core/v/{{version}}' ;; packages/billing-contract/package.json) SECTION_NAME='billing-contract-release' + SECTION_TITLE='Billing Contract Release' LINKS_VALUE='npm billing-contract|https://www.npmjs.com/package/@comfyorg/billing-contract/v/{{version}}' ;; packages/ingest-types/package.json) SECTION_NAME='ingest-types-release' + SECTION_TITLE='Ingest Types Release' LINKS_VALUE='npm ingest-types|https://www.npmjs.com/package/@comfyorg/ingest-types/v/{{version}}' ;; esac @@ -86,6 +91,11 @@ runs: COMMENT_FILE=$(mktemp) { + echo "## πŸš€ $SECTION_TITLE: βœ… Published \`$NEW_VERSION\`" + echo "" + echo "
" + echo "Details" + echo "" echo "$MESSAGE" echo "" echo "- $DIFF_LABEL: [\`$DIFF_PREFIX$PREV_VERSION...$DIFF_PREFIX$NEW_VERSION\`]($DIFF_URL)" @@ -105,6 +115,7 @@ runs: done <<< "$LINKS_VALUE" echo "" + echo "
" } > "$COMMENT_FILE" { diff --git a/.github/actions/explain-check/action.yaml b/.github/actions/explain-check/action.yaml index 1f25ecbfbcd..f8cc17da22d 100644 --- a/.github/actions/explain-check/action.yaml +++ b/.github/actions/explain-check/action.yaml @@ -84,12 +84,8 @@ runs: ;; esac - # Must live inside the workspace: read-and-upsert-comment-section guards - # on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and - # would silently no-op on a RUNNER_TEMP path. - out="$GITHUB_WORKSPACE/.explain-check-${INPUT_SLUG}.md" + body="${RUNNER_TEMP:-/tmp}/explain-check-${INPUT_SLUG}-body.md" { - printf '## %s %s: %s\n\n' "$INPUT_ICON" "$INPUT_TITLE" "$badge" printf '%s\n' "$INPUT_HEADLINE" if [ -n "$INPUT_DETAILS_FILE" ] && [ -s "$INPUT_DETAILS_FILE" ]; then @@ -105,13 +101,35 @@ runs: if [ -n "$INPUT_DOCS" ]; then printf '\nFull docs: %s\n' "$INPUT_DOCS" fi + } > "$body" + + heading=$(printf '## %s %s: %s' "$INPUT_ICON" "$INPUT_TITLE" "$badge") + + # Must live inside the workspace: read-and-upsert-comment-section guards + # on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and + # would silently no-op on a RUNNER_TEMP path. + out="$GITHUB_WORKSPACE/.explain-check-${INPUT_SLUG}.md" + # Every section of the PR report reads as one status line with the rest + # folded away, so the comment stays scannable as checks accumulate. + { + printf '%s\n\n' "$heading" + printf '
\nDetails\n\n' + cat "$body" + printf '\n
\n' } > "$out" echo "file=$out" >> "$GITHUB_OUTPUT" # The summary is the fallback that always works β€” fork PRs carry a - # read-only token, so the comment path below is skipped for them. - cat "$out" >> "$GITHUB_STEP_SUMMARY" + # read-only token, so the comment path below is skipped for them. It + # stays expanded because for those runs it is the only place the + # findings are readable at all. + { + printf '%s\n\n' "$heading" + cat "$body" + } >> "$GITHUB_STEP_SUMMARY" + + rm -f "$body" - name: Upsert section into the PR report if: ${{ inputs.pr-number != '' && inputs.token != '' }} diff --git a/.github/workflows/ci-tests-storybook.yaml b/.github/workflows/ci-tests-storybook.yaml index 8f28f91bd41..41425bab00d 100644 --- a/.github/workflows/ci-tests-storybook.yaml +++ b/.github/workflows/ci-tests-storybook.yaml @@ -236,8 +236,15 @@ jobs: pr-number: ${{ github.event.pull_request.number }} section-name: chromatic section-content: | - ### 🎨 Chromatic Visual Tests + ## 🎨 Chromatic: βœ… Published β€” [View Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }}) + +
+ Details + + **Links** - πŸ“Š [View Chromatic Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }}) - πŸ“š [View Chromatic Storybook](${{ needs.chromatic-deployment.outputs.chromatic-storybook-url }}) + +
comment-marker: '' token: ${{ github.token }} diff --git a/.github/workflows/ci-vercel-billing-web-preview.yaml b/.github/workflows/ci-vercel-billing-web-preview.yaml index 58d377d93c1..4b01fd56095 100644 --- a/.github/workflows/ci-vercel-billing-web-preview.yaml +++ b/.github/workflows/ci-vercel-billing-web-preview.yaml @@ -123,11 +123,16 @@ jobs: section-name: billing-web-preview token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## πŸ’³ Billing Web Preview + ## πŸ’³ Billing Web Preview: βœ… Deployed β€” [View Preview](${{ steps.urls.outputs.stable-url }}) - **Billing Web Preview:** ${{ steps.urls.outputs.stable-url }} +
+ Details - This commit: ${{ steps.deploy.outputs.url }} + **Links** + - πŸ”— Stable: ${{ steps.urls.outputs.stable-url }} + - πŸ“Œ This commit: ${{ steps.deploy.outputs.url }} + +
# Production never follows a merge. Someone dispatches this workflow against # main when the hosted app should change for customers. diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 68f728e12d8..f747cb6e767 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -275,11 +275,15 @@ jobs: comment-marker: '' token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## 🌐 Website E2E + ## 🌐 Website E2E: ⏳ Running… - > [!NOTE] - > Tests are running… [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) +
+ Details + + [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + +
post-result-comment: needs: website-e2e-report @@ -311,21 +315,29 @@ jobs: const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0 const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0 - const lines = ['## 🌐 Website E2E', '', ''] + const verdict = passed + ? 'βœ… Passed' + : infrastructure + ? 'πŸ’₯ Infrastructure failure' + : '❌ Failed' + const status = reportUrl ? `${verdict} β€” [View Report](${reportUrl})` : verdict - if (passed) { - lines.push('> [!TIP]', '> All tests passed.') - } else if (infrastructure) { - lines.push( + // pr-update-website-screenshots keys its comment trigger off this + // marker, so it has to survive every reformat of this section. + const lines = [`## 🌐 Website E2E: ${status}`, '', ''] + + const details = [] + + if (infrastructure) { + details.push( '> [!CAUTION]', - '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.', + '' ) - } else { - lines.push('> [!CAUTION]', '> Some tests failed.') } const rows = [ - ['Status', passed ? 'βœ… Passed' : infrastructure ? 'πŸ’₯ Infrastructure failure' : '❌ Failed'], + ['Status', verdict], ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_'] ] if (!passed && !infrastructure) { @@ -334,8 +346,7 @@ jobs: ['Other failures', String(otherFailures)] ) } - lines.push( - '', + details.push( '| | |', '|---|---|', ...rows.map(([k, v]) => `| **${k}** | ${v} |`) @@ -343,16 +354,18 @@ jobs: if (screenshotFailures > 0) { const s = screenshotFailures === 1 ? '' : 's' - lines.push('', `- [ ] Update website screenshots (${screenshotFailures} screenshot diff${s})`) + details.push('', `- [ ] Update website screenshots (${screenshotFailures} screenshot diff${s})`) } if (otherFailures > 0) { - lines.push( + details.push( '', '> [!WARNING]', `> ${otherFailures} non-screenshot failure${otherFailures === 1 ? '' : 's'} β€” these require manual review.` ) } + lines.push('
', 'Details', '', ...details, '', '
') + core.setOutput('section-content', lines.join('\n')) - uses: ./.github/actions/upsert-comment-section diff --git a/.github/workflows/pr-update-website-screenshots.yaml b/.github/workflows/pr-update-website-screenshots.yaml index 7adf79bccb2..6dcf4351213 100644 --- a/.github/workflows/pr-update-website-screenshots.yaml +++ b/.github/workflows/pr-update-website-screenshots.yaml @@ -191,10 +191,14 @@ jobs: comment-marker: '' token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## πŸ“Έ Screenshot Update + ## πŸ“Έ Screenshot Update: ⏳ Running… - > [!NOTE] - > Updating screenshots… [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) +
+ Details + + [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + +
post-result-comment: needs: update-screenshots @@ -223,25 +227,31 @@ jobs: const changedCount = parseInt(process.env.CHANGED_COUNT) || 0 const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` - const lines = ['## πŸ“Έ Screenshot Update', ''] - - if (outcome !== 'success') { - lines.push( - '> [!CAUTION]', - `> Screenshot update failed. [View workflow run](${runUrl})` - ) - } else if (!hasChanges) { - lines.push( - '> [!TIP]', - '> All screenshots are already up to date.' - ) - } else { - const s = changedCount === 1 ? '' : 's' - lines.push( - '> [!TIP]', - `> Updated ${changedCount} screenshot${s} and pushed to the branch.` - ) - } + const s = changedCount === 1 ? '' : 's' + const status = + outcome !== 'success' + ? '❌ Failed' + : !hasChanges + ? 'βœ… Already up to date' + : `βœ… Updated ${changedCount} screenshot${s}` + + const detail = + outcome !== 'success' + ? 'Screenshot update failed.' + : !hasChanges + ? 'All screenshots are already up to date.' + : `Updated ${changedCount} screenshot${s} and pushed to the branch.` + + const lines = [ + `## πŸ“Έ Screenshot Update: ${status}`, + '', + '
', + 'Details', + '', + `${detail} [View workflow run](${runUrl})`, + '', + '
' + ] core.setOutput('section-content', lines.join('\n')) diff --git a/.github/workflows/pr-vercel-website-preview.yaml b/.github/workflows/pr-vercel-website-preview.yaml index bff99009b13..2e0c98aad1f 100644 --- a/.github/workflows/pr-vercel-website-preview.yaml +++ b/.github/workflows/pr-vercel-website-preview.yaml @@ -69,12 +69,17 @@ jobs: comment-marker: '' token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## πŸ”— Website Preview + ## πŸ”— Website Preview: βœ… Deployed β€” [View Preview](${{ steps.urls.outputs.stable-url }}) - **Website Preview:** ${{ steps.urls.outputs.stable-url }} +
+ Details - ${{ steps.urls.outputs.build-note }} + **Links** + - πŸ”— Stable: ${{ steps.urls.outputs.stable-url }} + - πŸ“Œ This commit: ${{ steps.urls.outputs.unique-url }} - This commit: ${{ steps.urls.outputs.unique-url }} + ${{ steps.urls.outputs.build-note }} Last updated: ${{ github.event.workflow_run.updated_at }} for `${{ steps.urls.outputs.short-sha }}` + +
diff --git a/.github/workflows/publish-package-on-merge.yaml b/.github/workflows/publish-package-on-merge.yaml index 09ca8ecf634..baa6a70b5b0 100644 --- a/.github/workflows/publish-package-on-merge.yaml +++ b/.github/workflows/publish-package-on-merge.yaml @@ -124,7 +124,15 @@ jobs: run: | set -euo pipefail mkdir -p publish-report + PACKAGE_COUNT=$(jq -r '.include | length' <<<"$MATRIX") { + printf '## πŸš€ Release: βœ… Published %s package' "$PACKAGE_COUNT" + [ "$PACKAGE_COUNT" = 1 ] || printf 's' + echo '' + echo '' + echo '
' + echo 'Details' + echo '' echo 'Publish jobs finished successfully:' echo '' jq -r ' @@ -133,6 +141,7 @@ jobs: ' <<<"$MATRIX" echo "- Published from [\`${MERGE_SHA:0:10}\`](https://github.com/${REPO}/commit/${MERGE_SHA})" echo '' + echo '
' } > publish-report/report.md printf '%s' "$PR_NUMBER" > publish-report/pr-number.txt diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index 8a9e035007f..cf93119ce12 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -340,7 +340,9 @@ else if [ $total_failed -gt 0 ]; then comment="$comment -### ❌ Failed Tests" +
+❌ Failed Tests +" for counts_json in "${agg_counts_array[@]}"; do [ -z "$counts_json" ] || [ "$counts_json" = "{}" ] && continue @@ -369,6 +371,10 @@ $test_line" fi fi done + + comment="$comment + +
" fi # Add browser reports in collapsible section diff --git a/scripts/cicd/prReportSection.test.ts b/scripts/cicd/prReportSection.test.ts new file mode 100644 index 00000000000..eef841a023f --- /dev/null +++ b/scripts/cicd/prReportSection.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest' + +import { renderPrReportSection } from './prReportSection' + +describe('renderPrReportSection', () => { + it('renders the verdict on the heading', () => { + expect( + renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: 'βœ… No regressions' + }) + ).toBe('## ⚑ Performance: βœ… No regressions') + }) + + it.for([ + { name: 'omitted', body: undefined }, + { name: 'empty', body: '' }, + { name: 'whitespace only', body: '\n \n' } + ])('emits no collapsed box when the body is $name', ({ body }) => { + const rendered = renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '🟑 66.3%', + body + }) + + expect(rendered).toBe('## πŸ”¬ E2E Coverage: 🟑 66.3%') + }) + + it('folds the body into a collapsed box behind the heading', () => { + expect( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '🟑 66.3% of lines', + body: '| Metric | Pct |\n|---|---|\n| Lines | 66.3% |' + }) + ).toBe( + `## πŸ”¬ E2E Coverage: 🟑 66.3% of lines + +
+Details + +| Metric | Pct | +|---|---| +| Lines | 66.3% | + +
` + ) + }) + + it('labels the disclosure triangle when asked', () => { + expect( + renderPrReportSection({ + icon: '🎭', + title: 'Playwright', + status: 'βœ… 2076 passed', + body: '- chromium', + detailsSummary: 'πŸ“Š Browser Reports' + }) + ).toContain('πŸ“Š Browser Reports') + }) + + it('keeps a blank line between the tags and the body so GitHub parses it', () => { + const lines = renderPrReportSection({ + icon: 'πŸ“¦', + title: 'Bundle', + status: '9.96 MB gzip', + body: '**Summary**' + }).split('\n') + + expect(lines[lines.indexOf('Details') + 1]).toBe('') + expect(lines[lines.indexOf('
') - 1]).toBe('') + }) + + it('strips body padding so nested sections do not drift apart', () => { + expect( + renderPrReportSection({ + icon: '🌾', + title: 'Fallow', + status: 'βœ… Passed', + body: '\n\nNo new findings.\n\n\n' + }) + ).toContain('\n\nNo new findings.\n\n') + }) +}) diff --git a/scripts/cicd/prReportSection.ts b/scripts/cicd/prReportSection.ts new file mode 100644 index 00000000000..40652a430a7 --- /dev/null +++ b/scripts/cicd/prReportSection.ts @@ -0,0 +1,45 @@ +/** + * Shared shape for every section of the unified PR report comment. + * + * A reader scanning the comment should get each check's verdict from one line + * without scrolling: the heading carries the status, and everything else is + * folded away. The Storybook and Playwright sections established the shape and + * this renderer is how the rest of them keep it. + */ +export interface PrReportSection { + /** Emoji shown beside the title, e.g. `⚑`. */ + icon: string + /** Human name of the check, e.g. `Performance`. */ + title: string + /** Verdict for the heading, e.g. `βœ… No regressions`. */ + status: string + /** Markdown folded into the collapsed box. Omit for a status-only section. */ + body?: string + /** Text on the disclosure triangle. */ + detailsSummary?: string +} + +export function renderPrReportSection({ + icon, + title, + status, + body, + detailsSummary = 'Details' +}: PrReportSection): string { + const heading = `## ${icon} ${title}: ${status}` + const trimmedBody = body?.trim() + if (!trimmedBody) return heading + + // The blank lines around the body are load-bearing: GitHub only parses + // Markdown inside an HTML block when it is separated from the tags. + return [ + heading, + '', + '
', + `${detailsSummary}`, + '', + trimmedBody, + '', + '
' + ].join('\n') +} diff --git a/scripts/coverage-report.ts b/scripts/coverage-report.ts index 27e2359cb46..1c7a4190782 100644 --- a/scripts/coverage-report.ts +++ b/scripts/coverage-report.ts @@ -1,5 +1,7 @@ import { existsSync, readFileSync } from 'node:fs' +import { renderPrReportSection } from './cicd/prReportSection' + interface FileStats { lines: number covered: number @@ -15,7 +17,11 @@ const lcovPath = process.argv[2] || 'coverage/playwright/coverage.lcov' if (!existsSync(lcovPath)) { process.stdout.write( - '## πŸ”¬ E2E Coverage\n\n> ⚠️ No coverage data found. Check the CI workflow logs.\n' + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '⚠️ No coverage data found β€” check the CI workflow logs' + }) + '\n' ) process.exit(0) } @@ -112,8 +118,6 @@ function bar(covered: number, total: number): string { } const lines: string[] = [] -lines.push('## πŸ”¬ E2E Coverage') -lines.push('') lines.push('| Metric | Covered | Total | Pct | |') lines.push('|---|--:|--:|--:|---|') lines.push( @@ -151,4 +155,16 @@ if (uncovered.length > 0) { lines.push('') } -process.stdout.write(lines.join('\n') + '\n') +const linesStatus = + totalLines === 0 + ? '⚠️ No line coverage recorded' + : `${bar(coveredLines, totalLines)} ${pct(coveredLines, totalLines)} of lines` + +process.stdout.write( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: linesStatus, + body: lines.join('\n') + }) + '\n' +) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index 2a2ae088830..3f33337d702 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, readdirSync } from 'node:fs' import { join } from 'node:path' +import { renderPrReportSection } from './cicd/prReportSection' import type { MetricStats } from './perf-stats' import { classifyChange, @@ -254,11 +255,16 @@ function renderHeadlineSummary( return lines } +interface RenderedReport { + status: string + lines: string[] +} + function renderFullReport( prGroups: Map, baseline: PerfReport, historical: PerfReport[] -): string[] { +): RenderedReport { const lines: string[] = [] const baselineGroups = groupByName(baseline.measurements) const tableHeader = [ @@ -311,10 +317,13 @@ function renderFullReport( } } + const status = + flaggedRows.length > 0 + ? `⚠️ ${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected` + : 'βœ… No regressions detected' + if (flaggedRows.length > 0) { lines.push( - `⚠️ **${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected**`, - '', '
Show regressions', '', ...tableHeader, @@ -323,8 +332,6 @@ function renderFullReport( '
', '' ) - } else { - lines.push('βœ… No regressions detected.', '') } lines.push( @@ -383,14 +390,14 @@ function renderFullReport( ) } - return lines + return { status, lines } } function renderColdStartReport( prGroups: Map, baseline: PerfReport, historicalCount: number -): string[] { +): RenderedReport { const lines: string[] = [] const baselineGroups = groupByName(baseline.measurements) lines.push( @@ -436,16 +443,17 @@ function renderColdStartReport( } lines.push('', '') - return lines + return { + status: `ℹ️ Collecting baseline variance (${historicalCount}/15 runs)`, + lines + } } function renderNoBaselineReport( prGroups: Map -): string[] { +): RenderedReport { const lines: string[] = [] lines.push( - '> ℹ️ No baseline found β€” significance unavailable.', - '', '
Absolute values', '', '| Metric | Value |', @@ -459,13 +467,17 @@ function renderNoBaselineReport( } } lines.push('', '
') - return lines + return { status: 'ℹ️ No baseline β€” significance unavailable', lines } } function main() { if (!existsSync(CURRENT_PATH)) { process.stdout.write( - '## ⚑ Performance Report\n\nNo perf metrics found. Perf tests may not have run.\n' + renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: '⏭️ No metrics found β€” perf tests may not have run' + }) + '\n' ) process.exit(0) } @@ -479,17 +491,12 @@ function main() { const historical = loadHistoricalReports() const prGroups = groupByName(current.measurements) - const lines: string[] = [] - lines.push('## ⚑ Performance Report\n') - lines.push(...renderHeadlineSummary(prGroups)) - - if (baseline && historical.length >= 2) { - lines.push(...renderFullReport(prGroups, baseline, historical)) - } else if (baseline) { - lines.push(...renderColdStartReport(prGroups, baseline, historical.length)) - } else { - lines.push(...renderNoBaselineReport(prGroups)) - } + const report = + baseline && historical.length >= 2 + ? renderFullReport(prGroups, baseline, historical) + : baseline + ? renderColdStartReport(prGroups, baseline, historical.length) + : renderNoBaselineReport(prGroups) const rawData = { ...current, @@ -497,13 +504,28 @@ function main() { ({ allFrameDurationsMs: _, ...rest }) => rest ) } - lines.push('\n
Raw data\n') - lines.push('```json') - lines.push(JSON.stringify(rawData, null, 2)) - lines.push('```') - lines.push('\n
') - process.stdout.write(lines.join('\n') + '\n') + const body = [ + ...renderHeadlineSummary(prGroups), + ...report.lines, + '', + '
Raw data', + '', + '```json', + JSON.stringify(rawData, null, 2), + '```', + '', + '
' + ] + + process.stdout.write( + renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: report.status, + body: body.join('\n') + }) + '\n' + ) } main() diff --git a/scripts/unified-report.ts b/scripts/unified-report.ts index f3b0b4b5008..860436e42bd 100644 --- a/scripts/unified-report.ts +++ b/scripts/unified-report.ts @@ -1,6 +1,8 @@ import { execFileSync } from 'node:child_process' import { existsSync } from 'node:fs' +import { renderPrReportSection } from './cicd/prReportSection' + const args: string[] = process.argv.slice(2) function getArg(name: string): string | undefined { @@ -13,6 +15,18 @@ const sizeStatus = getArg('size-status') ?? 'pending' const perfStatus = getArg('perf-status') ?? 'pending' const coverageStatus = getArg('coverage-status') ?? 'skip' +function bundleStatusLine(status: string): string { + return renderPrReportSection({ icon: 'πŸ“¦', title: 'Bundle', status }) +} + +function perfStatusLine(status: string): string { + return renderPrReportSection({ icon: '⚑', title: 'Performance', status }) +} + +function coverageStatusLine(status: string): string { + return renderPrReportSection({ icon: 'πŸ”¬', title: 'E2E Coverage', status }) +} + const lines: string[] = [] const hasSizeData = existsSync('temp/size') @@ -24,20 +38,20 @@ if (sizeStatus === 'ready' && hasSizeData) { }).trimEnd() lines.push(sizeReport) } catch { - lines.push('## πŸ“¦ Bundle Size') - lines.push('') lines.push( - '> ⚠️ Failed to render bundle size report. Check the CI workflow logs.' + bundleStatusLine( + '⚠️ Failed to render report β€” check the CI workflow logs' + ) ) } } else if (sizeStatus === 'failed') { - lines.push('## πŸ“¦ Bundle Size') - lines.push('') - lines.push('> ⚠️ Size data collection failed. Check the CI workflow logs.') + lines.push( + bundleStatusLine( + '⚠️ Size data collection failed β€” check the CI workflow logs' + ) + ) } else if (sizeStatus !== 'ready') { - lines.push('## πŸ“¦ Bundle Size') - lines.push('') - lines.push('> ⏳ Size data collection in progress…') + lines.push(bundleStatusLine('⏳ Size data collection in progress…')) } if (lines.length > 0) lines.push('') @@ -51,23 +65,19 @@ if (perfStatus === 'ready' && existsSync('test-results/perf-metrics.json')) { ).trimEnd() lines.push(perfReport) } catch { - lines.push('## ⚑ Performance') - lines.push('') lines.push( - '> ⚠️ Failed to render performance report. Check the CI workflow logs.' + perfStatusLine('⚠️ Failed to render report β€” check the CI workflow logs') ) } } else if ( perfStatus === 'failed' || (perfStatus === 'ready' && !existsSync('test-results/perf-metrics.json')) ) { - lines.push('## ⚑ Performance') - lines.push('') - lines.push('> ⚠️ Performance tests failed. Check the CI workflow logs.') + lines.push( + perfStatusLine('⚠️ Performance tests failed β€” check the CI workflow logs') + ) } else if (perfStatus !== 'skip') { - lines.push('## ⚑ Performance') - lines.push('') - lines.push('> ⏳ Performance tests in progress…') + lines.push(perfStatusLine('⏳ Performance tests in progress…')) } if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) { @@ -85,18 +95,20 @@ if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) { lines.push('') lines.push(coverageReport) } catch { - lines.push('') - lines.push('## πŸ”¬ E2E Coverage') lines.push('') lines.push( - '> ⚠️ Failed to render coverage report. Check the CI workflow logs.' + coverageStatusLine( + '⚠️ Failed to render report β€” check the CI workflow logs' + ) ) } } else if (coverageStatus === 'failed') { lines.push('') - lines.push('## πŸ”¬ E2E Coverage') - lines.push('') - lines.push('> ⚠️ Coverage collection failed. Check the CI workflow logs.') + lines.push( + coverageStatusLine( + '⚠️ Coverage collection failed β€” check the CI workflow logs' + ) + ) } process.stdout.write(lines.join('\n') + '\n') From 2bf6883b60843299a4bd23443f2ef203b9ece7c4 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Mon, 21 Sep 2026 23:26:26 +0000 Subject: [PATCH 02/19] ci: tighten the normalized PR report status lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups. The release count is the resolve matrix, not the number of npm pushes β€” a version already published is skipped and still lands in it β€” so say "N packages on npm" rather than claiming the run published them. Put the website E2E failure breakdown on the heading: the checkbox that acts on a screenshot diff lives inside the box, so nothing hinted the self-service fix existed. Drop the two "Running…" boxes that wrapped a single link, matching what renderPrReportSection already does for an empty body, and the two lines that just restated their own heading. --- .github/workflows/ci-tests-storybook.yaml | 1 - .github/workflows/ci-website-e2e.yaml | 29 +++++++++++------ .../pr-update-website-screenshots.yaml | 32 ++++--------------- .../workflows/publish-package-on-merge.yaml | 5 ++- scripts/perf-report.ts | 2 +- 5 files changed, 31 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci-tests-storybook.yaml b/.github/workflows/ci-tests-storybook.yaml index 41425bab00d..1e15342d788 100644 --- a/.github/workflows/ci-tests-storybook.yaml +++ b/.github/workflows/ci-tests-storybook.yaml @@ -242,7 +242,6 @@ jobs: Details **Links** - - πŸ“Š [View Chromatic Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }}) - πŸ“š [View Chromatic Storybook](${{ needs.chromatic-deployment.outputs.chromatic-storybook-url }}) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index f747cb6e767..18e8320b75a 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -275,16 +275,9 @@ jobs: comment-marker: '' token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## 🌐 Website E2E: ⏳ Running… + ## 🌐 Website E2E: ⏳ Running… β€” [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) -
- Details - - [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) - -
- post-result-comment: needs: website-e2e-report if: always() && !cancelled() && needs.website-e2e-report.outputs.is-pr == 'true' @@ -320,7 +313,25 @@ jobs: : infrastructure ? 'πŸ’₯ Infrastructure failure' : '❌ Failed' - const status = reportUrl ? `${verdict} β€” [View Report](${reportUrl})` : verdict + + // The screenshot-diff count belongs on the heading: the checkbox + // that acts on it sits inside the box, so nothing would hint the + // self-service fix exists without expanding. + const breakdown = [] + if (screenshotFailures > 0) { + breakdown.push(`${screenshotFailures} screenshot diff${screenshotFailures === 1 ? '' : 's'}`) + } + if (otherFailures > 0) { + breakdown.push(`${otherFailures} other failure${otherFailures === 1 ? '' : 's'}`) + } + + const status = [ + verdict, + breakdown.length > 0 ? breakdown.join(', ') : '', + reportUrl ? `[View Report](${reportUrl})` : '' + ] + .filter(Boolean) + .join(' β€” ') // pr-update-website-screenshots keys its comment trigger off this // marker, so it has to survive every reformat of this section. diff --git a/.github/workflows/pr-update-website-screenshots.yaml b/.github/workflows/pr-update-website-screenshots.yaml index 6dcf4351213..933e4513b51 100644 --- a/.github/workflows/pr-update-website-screenshots.yaml +++ b/.github/workflows/pr-update-website-screenshots.yaml @@ -191,14 +191,7 @@ jobs: comment-marker: '' token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## πŸ“Έ Screenshot Update: ⏳ Running… - -
- Details - - [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) - -
+ ## πŸ“Έ Screenshot Update: ⏳ Running… β€” [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) post-result-comment: needs: update-screenshots @@ -235,25 +228,12 @@ jobs: ? 'βœ… Already up to date' : `βœ… Updated ${changedCount} screenshot${s}` - const detail = - outcome !== 'success' - ? 'Screenshot update failed.' - : !hasChanges - ? 'All screenshots are already up to date.' - : `Updated ${changedCount} screenshot${s} and pushed to the branch.` - - const lines = [ - `## πŸ“Έ Screenshot Update: ${status}`, - '', - '
', - 'Details', - '', - `${detail} [View workflow run](${runUrl})`, - '', - '
' - ] + const pushed = outcome === 'success' && hasChanges ? ', pushed to the branch' : '' - core.setOutput('section-content', lines.join('\n')) + core.setOutput( + 'section-content', + `## πŸ“Έ Screenshot Update: ${status}${pushed} β€” [View workflow run](${runUrl})` + ) - uses: ./.github/actions/upsert-comment-section with: diff --git a/.github/workflows/publish-package-on-merge.yaml b/.github/workflows/publish-package-on-merge.yaml index baa6a70b5b0..79474dab729 100644 --- a/.github/workflows/publish-package-on-merge.yaml +++ b/.github/workflows/publish-package-on-merge.yaml @@ -124,10 +124,13 @@ jobs: run: | set -euo pipefail mkdir -p publish-report + # The matrix counts packages the release resolved, not pushes: a + # version already on npm is skipped and still lands here. PACKAGE_COUNT=$(jq -r '.include | length' <<<"$MATRIX") { - printf '## πŸš€ Release: βœ… Published %s package' "$PACKAGE_COUNT" + printf '## πŸš€ Release: βœ… %s package' "$PACKAGE_COUNT" [ "$PACKAGE_COUNT" = 1 ] || printf 's' + printf ' on npm' echo '' echo '' echo '
' diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index 3f33337d702..1b5e049e6fa 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -401,7 +401,7 @@ function renderColdStartReport( const lines: string[] = [] const baselineGroups = groupByName(baseline.measurements) lines.push( - `> ℹ️ Collecting baseline variance data (${historicalCount}/15 runs). Significance will appear after 2 main branch runs.`, + '> Significance will appear after 2 main branch runs.', '', '
All metrics (cold start)', '', From 4dcbd37ce4e3a0a4e5e19dc0bc70acf526d233f1 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Tue, 22 Sep 2026 00:00:33 +0000 Subject: [PATCH 03/19] ci: drop the report boxes that reveal nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the second review. renderPrReportSection's detailsSummary had no TypeScript caller and could not have one β€” the two sections wanting a custom triangle label come from a shell script. The explain-check wrapper now guards on a non-empty body, matching the renderer. A clean website E2E run no longer opens a box containing only the verdict already on its heading; a missing report says so there instead of in a table row. --- .github/actions/explain-check/action.yaml | 21 ++++--- .github/workflows/ci-website-e2e.yaml | 58 +++++++++---------- .../pr-update-website-screenshots.yaml | 6 +- scripts/cicd/prReportSection.test.ts | 12 ---- scripts/cicd/prReportSection.ts | 7 +-- 5 files changed, 43 insertions(+), 61 deletions(-) diff --git a/.github/actions/explain-check/action.yaml b/.github/actions/explain-check/action.yaml index f8cc17da22d..ac0e358f1cc 100644 --- a/.github/actions/explain-check/action.yaml +++ b/.github/actions/explain-check/action.yaml @@ -104,6 +104,11 @@ runs: } > "$body" heading=$(printf '## %s %s: %s' "$INPUT_ICON" "$INPUT_TITLE" "$badge") + # Drop leading blank lines here and trailing ones via the substitution, + # so a check that supplies nothing but a verdict renders as a bare + # heading rather than an empty box. Mirrors renderPrReportSection. + body_text=$(sed -e '/[^[:space:]]/,$!d' "$body") + rm -f "$body" # Must live inside the workspace: read-and-upsert-comment-section guards # on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and @@ -112,10 +117,10 @@ runs: # Every section of the PR report reads as one status line with the rest # folded away, so the comment stays scannable as checks accumulate. { - printf '%s\n\n' "$heading" - printf '
\nDetails\n\n' - cat "$body" - printf '\n
\n' + printf '%s\n' "$heading" + if [ -n "$body_text" ]; then + printf '\n
\nDetails\n\n%s\n\n
\n' "$body_text" + fi } > "$out" echo "file=$out" >> "$GITHUB_OUTPUT" @@ -125,12 +130,12 @@ runs: # stays expanded because for those runs it is the only place the # findings are readable at all. { - printf '%s\n\n' "$heading" - cat "$body" + printf '%s\n' "$heading" + if [ -n "$body_text" ]; then + printf '\n%s\n' "$body_text" + fi } >> "$GITHUB_STEP_SUMMARY" - rm -f "$body" - - name: Upsert section into the PR report if: ${{ inputs.pr-number != '' && inputs.token != '' }} # A comment failure must not turn a green check red, nor mask a red one. diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 18e8320b75a..b27a3ac6a87 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -327,55 +327,49 @@ jobs: const status = [ verdict, - breakdown.length > 0 ? breakdown.join(', ') : '', - reportUrl ? `[View Report](${reportUrl})` : '' + breakdown.join(', '), + reportUrl ? `[View Report](${reportUrl})` : '_report unavailable_' ] .filter(Boolean) .join(' β€” ') - // pr-update-website-screenshots keys its comment trigger off this - // marker, so it has to survive every reformat of this section. - const lines = [`## 🌐 Website E2E: ${status}`, '', ''] - - const details = [] + // A clean run has nothing the heading does not already say, so it + // gets no box to expand. + const blocks = [] if (infrastructure) { - details.push( + blocks.push([ '> [!CAUTION]', - '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.', - '' - ) - } - - const rows = [ - ['Status', verdict], - ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_'] - ] - if (!passed && !infrastructure) { - rows.push( - ['Screenshot diffs', String(screenshotFailures)], - ['Other failures', String(otherFailures)] - ) + '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + ]) } - details.push( - '| | |', - '|---|---|', - ...rows.map(([k, v]) => `| **${k}** | ${v} |`) - ) if (screenshotFailures > 0) { const s = screenshotFailures === 1 ? '' : 's' - details.push('', `- [ ] Update website screenshots (${screenshotFailures} screenshot diff${s})`) + blocks.push([`- [ ] Update website screenshots (${screenshotFailures} screenshot diff${s})`]) } + if (otherFailures > 0) { - details.push( - '', + blocks.push([ '> [!WARNING]', `> ${otherFailures} non-screenshot failure${otherFailures === 1 ? '' : 's'} β€” these require manual review.` - ) + ]) } - lines.push('
', 'Details', '', ...details, '', '
') + // pr-update-website-screenshots keys its comment trigger off this + // marker, so it has to survive every reformat of this section. + const lines = [`## 🌐 Website E2E: ${status}`, ''] + if (blocks.length > 0) { + lines.push( + '', + '
', + 'Details', + '', + blocks.map((block) => block.join('\n')).join('\n\n'), + '', + '
' + ) + } core.setOutput('section-content', lines.join('\n')) diff --git a/.github/workflows/pr-update-website-screenshots.yaml b/.github/workflows/pr-update-website-screenshots.yaml index 933e4513b51..9ee08fe6d80 100644 --- a/.github/workflows/pr-update-website-screenshots.yaml +++ b/.github/workflows/pr-update-website-screenshots.yaml @@ -226,13 +226,11 @@ jobs: ? '❌ Failed' : !hasChanges ? 'βœ… Already up to date' - : `βœ… Updated ${changedCount} screenshot${s}` - - const pushed = outcome === 'success' && hasChanges ? ', pushed to the branch' : '' + : `βœ… Updated ${changedCount} screenshot${s}, pushed to the branch` core.setOutput( 'section-content', - `## πŸ“Έ Screenshot Update: ${status}${pushed} β€” [View workflow run](${runUrl})` + `## πŸ“Έ Screenshot Update: ${status} β€” [View workflow run](${runUrl})` ) - uses: ./.github/actions/upsert-comment-section diff --git a/scripts/cicd/prReportSection.test.ts b/scripts/cicd/prReportSection.test.ts index eef841a023f..89b9b534abb 100644 --- a/scripts/cicd/prReportSection.test.ts +++ b/scripts/cicd/prReportSection.test.ts @@ -50,18 +50,6 @@ describe('renderPrReportSection', () => { ) }) - it('labels the disclosure triangle when asked', () => { - expect( - renderPrReportSection({ - icon: '🎭', - title: 'Playwright', - status: 'βœ… 2076 passed', - body: '- chromium', - detailsSummary: 'πŸ“Š Browser Reports' - }) - ).toContain('πŸ“Š Browser Reports') - }) - it('keeps a blank line between the tags and the body so GitHub parses it', () => { const lines = renderPrReportSection({ icon: 'πŸ“¦', diff --git a/scripts/cicd/prReportSection.ts b/scripts/cicd/prReportSection.ts index 40652a430a7..5f395dfc6b4 100644 --- a/scripts/cicd/prReportSection.ts +++ b/scripts/cicd/prReportSection.ts @@ -15,16 +15,13 @@ export interface PrReportSection { status: string /** Markdown folded into the collapsed box. Omit for a status-only section. */ body?: string - /** Text on the disclosure triangle. */ - detailsSummary?: string } export function renderPrReportSection({ icon, title, status, - body, - detailsSummary = 'Details' + body }: PrReportSection): string { const heading = `## ${icon} ${title}: ${status}` const trimmedBody = body?.trim() @@ -36,7 +33,7 @@ export function renderPrReportSection({ heading, '', '
', - `${detailsSummary}`, + 'Details', '', trimmedBody, '', From b11189a7561c80e0ce247aca71324ea2dce5b153 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Tue, 22 Sep 2026 00:34:05 +0000 Subject: [PATCH 04/19] ci: fix the report content that broke on being nested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub's alert extension does not fire inside an HTML block, so the two `> [!CAUTION]` / `> [!WARNING]` blocks the website E2E section moved into its
rendered as quotes reading literally "[!CAUTION]" β€” on the two failure states the box exists to explain. Carry the emphasis inline instead. Two more boxes that could open onto nothing: the Playwright failure count and the failure names come from different fields, and the index.html fallback in extract-playwright-counts.ts reports a count with no names, so collect the lines before opening the box. And the perf heading derived its verdict only from baseline-relative regressions while the P5 FPS target verdict sat in the collapsed summary, so a run could read βœ… over a hidden ❌. --- .github/workflows/ci-website-e2e.yaml | 11 +++++++---- .../cicd/pr-playwright-deploy-and-comment.sh | 19 ++++++++++++------- scripts/perf-report.ts | 19 ++++++++++++++++++- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index b27a3ac6a87..f9912377721 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -335,12 +335,15 @@ jobs: // A clean run has nothing the heading does not already say, so it // gets no box to expand. + // + // Emphasis is inline rather than a `> [!CAUTION]` alert: GitHub's + // alert extension does not fire inside an HTML block, so nested in + //
it degrades to a quote reading literally "[!CAUTION]". const blocks = [] if (infrastructure) { blocks.push([ - '> [!CAUTION]', - '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + '**πŸ’₯ A shard failed before producing a report.** The suite did not fully run, so no pass/fail counts are available.' ]) } @@ -350,9 +353,9 @@ jobs: } if (otherFailures > 0) { + const s = otherFailures === 1 ? '' : 's' blocks.push([ - '> [!WARNING]', - `> ${otherFailures} non-screenshot failure${otherFailures === 1 ? '' : 's'} β€” these require manual review.` + `**⚠️ ${otherFailures} non-screenshot failure${s}.** These require manual review.` ]) } diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index cf93119ce12..cd6da8466b3 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -337,12 +337,11 @@ else fi # Extract and display failed tests from all browsers (flaky tests are treated as passing) + # The count and the names come from different fields, and the index.html + # fallback in extract-playwright-counts.ts reports a count with no names β€” + # so collect first and only open the box if there is something to show. if [ $total_failed -gt 0 ]; then - comment="$comment - -
-❌ Failed Tests -" + failed_test_lines="" for counts_json in "${agg_counts_array[@]}"; do [ -z "$counts_json" ] || [ "$counts_json" = "{}" ] && continue @@ -365,16 +364,22 @@ else test_line="$test_line: [View trace]($trace_url)" fi - comment="$comment + failed_test_lines="$failed_test_lines $test_line" done <<< "$failures" fi fi done - comment="$comment + if [ -n "$failed_test_lines" ]; then + comment="$comment + +
+❌ Failed Tests +$failed_test_lines
" + fi fi # Add browser reports in collapsible section diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index 1b5e049e6fa..f2486cbfc94 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -219,6 +219,15 @@ function frameTimeToFps(ms: number): number { return ms > 0 ? 1000 / ms : 0 } +function countBelowFpsTarget(prGroups: Map): number { + let below = 0 + for (const prSamples of prGroups.values()) { + const p95Frame = medianMetric(prSamples, 'p95FrameDurationMs') + if (p95Frame !== null && frameTimeToFps(p95Frame) < TARGET_P5_FPS) below++ + } + return below +} + function renderHeadlineSummary( prGroups: Map ): string[] { @@ -518,11 +527,19 @@ function main() { '
' ] + // The per-test target verdict lives in the collapsed summary, so without + // this the heading can read βœ… over a hidden ❌. + const belowTarget = countBelowFpsTarget(prGroups) + const status = + belowTarget > 0 + ? `${report.status} Β· ❌ ${belowTarget} below ${TARGET_P5_FPS} FPS target` + : report.status + process.stdout.write( renderPrReportSection({ icon: '⚑', title: 'Performance', - status: report.status, + status, body: body.join('\n') }) + '\n' ) From 4c8f86883e98ba524846778fbf29e61d08bb7795 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Tue, 22 Sep 2026 01:16:44 +0000 Subject: [PATCH 05/19] ci: flag the website E2E couplings that outlive this change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verdict wording now exists in two shapes β€” the job summary keeps GitHub alerts because it renders expanded, the PR comment cannot β€” so say so at both sites, the way this repo already does for the duplicated trigger condition in pr-update-website-screenshots. Same for the screenshot checkbox label, which is the literal that workflow matches on. --- .github/workflows/ci-website-e2e.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index f9912377721..66d7ef26808 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -186,6 +186,10 @@ jobs: core.setOutput('screenshot', screenshotFailures) core.setOutput('other', failed.length - screenshotFailures) + # ⚠️ The verdict wording here is duplicated on `post-result-comment` β€” + # keep them in sync. The shapes deliberately differ: this surface stays + # expanded, so it can use `> [!TIP]`-style alerts, which the comment's + # collapsed box cannot. - name: Write job summary if: always() && needs.website-e2e-shard.result != 'cancelled' uses: actions/github-script@v9 @@ -291,6 +295,8 @@ jobs: steps: - uses: actions/checkout@v7 + # ⚠️ The verdict wording here is duplicated from `Write job summary` in + # `website-e2e-report` β€” keep them in sync. - name: Build e2e section content id: content uses: actions/github-script@v9 @@ -347,6 +353,9 @@ jobs: ]) } + // Ticking this box is what fires pr-update-website-screenshots, so + // the `- [x] Update website screenshots` prefix it matches on has + // to survive verbatim. if (screenshotFailures > 0) { const s = screenshotFailures === 1 ? '' : 's' blocks.push([`- [ ] Update website screenshots (${screenshotFailures} screenshot diff${s})`]) From 8f733ff718f1334dd1387bd57fc8a8e722f1cb74 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Tue, 22 Sep 2026 01:53:08 +0000 Subject: [PATCH 06/19] ci: lead the perf heading with the failing verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The target-miss note was appended after the regression verdict, so a run that moved nothing but sits under 52 FPS rendered a heading opening with βœ… and closing with ❌ β€” the exact confusion the note was added to prevent. Put the ❌ first. Pin the two literals pr-update-website-screenshots matches on, and the blank line between concatenated section blocks that every heading after the first now depends on:
holds an HTML block open until one appears, so losing it turns the next into plain text. Rename the renderer to the kebab-case this directory uses. --- ...tion.test.ts => pr-report-section.test.ts} | 22 ++++++++++++++++++- ...rReportSection.ts => pr-report-section.ts} | 0 scripts/cicd/website-e2e-workflow.test.ts | 18 +++++++++++++++ scripts/coverage-report.ts | 2 +- scripts/perf-report.ts | 8 ++++--- scripts/unified-report.ts | 2 +- 6 files changed, 46 insertions(+), 6 deletions(-) rename scripts/cicd/{prReportSection.test.ts => pr-report-section.test.ts} (71%) rename scripts/cicd/{prReportSection.ts => pr-report-section.ts} (100%) diff --git a/scripts/cicd/prReportSection.test.ts b/scripts/cicd/pr-report-section.test.ts similarity index 71% rename from scripts/cicd/prReportSection.test.ts rename to scripts/cicd/pr-report-section.test.ts index 89b9b534abb..7dbf833b96c 100644 --- a/scripts/cicd/prReportSection.test.ts +++ b/scripts/cicd/pr-report-section.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { renderPrReportSection } from './prReportSection' +import { renderPrReportSection } from './pr-report-section' describe('renderPrReportSection', () => { it('renders the verdict on the heading', () => { @@ -62,6 +62,26 @@ describe('renderPrReportSection', () => { expect(lines[lines.indexOf('
') - 1]).toBe('') }) + it('leaves a blank line before the next section once blocks are joined', () => { + // upsert-comment-section concatenates section blocks with a blank line. + // Without one, the HTML block opened by
runs on and swallows + // the following `##` into plain text. + const bundle = renderPrReportSection({ + icon: 'πŸ“¦', + title: 'Bundle', + status: '9.96 MB gzip', + body: '**Summary**' + }) + const perf = renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: 'βœ… No regressions' + }) + + expect(bundle.endsWith('
')).toBe(true) + expect(`${bundle}\n\n${perf}`).toContain('
\n\n## ⚑') + }) + it('strips body padding so nested sections do not drift apart', () => { expect( renderPrReportSection({ diff --git a/scripts/cicd/prReportSection.ts b/scripts/cicd/pr-report-section.ts similarity index 100% rename from scripts/cicd/prReportSection.ts rename to scripts/cicd/pr-report-section.ts diff --git a/scripts/cicd/website-e2e-workflow.test.ts b/scripts/cicd/website-e2e-workflow.test.ts index 09dd7cfdb0c..0577155bebe 100644 --- a/scripts/cicd/website-e2e-workflow.test.ts +++ b/scripts/cicd/website-e2e-workflow.test.ts @@ -141,6 +141,24 @@ describe('website E2E workflow', () => { ) }) + it('keeps the screenshot-update trigger literals synchronized', () => { + const producer = readFileSync( + join(ROOT, '.github/workflows/ci-website-e2e.yaml'), + 'utf8' + ) + const consumer = readFileSync( + join(ROOT, '.github/workflows/pr-update-website-screenshots.yaml'), + 'utf8' + ) + + // The consumer fires on a comment body containing both of these, so the + // E2E section has to keep emitting them verbatim however it is reshaped. + expect(producer).toContain('') + expect(consumer).toContain("''") + expect(producer).toContain('- [ ] Update website screenshots') + expect(consumer).toContain("'- [x] Update website screenshots'") + }) + it('passes only skipped or fully successful runs', () => { expect( checkResult({ diff --git a/scripts/coverage-report.ts b/scripts/coverage-report.ts index 1c7a4190782..9b7ec140320 100644 --- a/scripts/coverage-report.ts +++ b/scripts/coverage-report.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs' -import { renderPrReportSection } from './cicd/prReportSection' +import { renderPrReportSection } from './cicd/pr-report-section' interface FileStats { lines: number diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index f2486cbfc94..5c783024bb2 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync, readdirSync } from 'node:fs' import { join } from 'node:path' -import { renderPrReportSection } from './cicd/prReportSection' +import { renderPrReportSection } from './cicd/pr-report-section' import type { MetricStats } from './perf-stats' import { classifyChange, @@ -528,11 +528,13 @@ function main() { ] // The per-test target verdict lives in the collapsed summary, so without - // this the heading can read βœ… over a hidden ❌. + // this the heading can read βœ… over a hidden ❌. It leads, because a reader + // scanning the comment takes the verdict from the first glyph and a target + // miss outranks "nothing moved since the baseline". const belowTarget = countBelowFpsTarget(prGroups) const status = belowTarget > 0 - ? `${report.status} Β· ❌ ${belowTarget} below ${TARGET_P5_FPS} FPS target` + ? `❌ ${belowTarget} below ${TARGET_P5_FPS} FPS target Β· ${report.status}` : report.status process.stdout.write( diff --git a/scripts/unified-report.ts b/scripts/unified-report.ts index 860436e42bd..86ee0322314 100644 --- a/scripts/unified-report.ts +++ b/scripts/unified-report.ts @@ -1,7 +1,7 @@ import { execFileSync } from 'node:child_process' import { existsSync } from 'node:fs' -import { renderPrReportSection } from './cicd/prReportSection' +import { renderPrReportSection } from './cicd/pr-report-section' const args: string[] = process.argv.slice(2) From 1d7985911d4c5f8548bdc9e898e4851ffb128ca6 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 28 Oct 2026 06:48:12 +0000 Subject: [PATCH 07/19] ci: address review on the one-line PR report sections - Perf: a scenario with no frame samples (p95 frame duration recorded as 0) no longer counts as missing the FPS target; it is reported separately. A run with no measurements no longer reads as clean. - Release headings say the publish jobs finished rather than claiming the release is published, since prereleases stay drafts and already-published packages are counted. - Keep failure calls to action and coverage caveats visible under the heading (the shape main adopted in #21215) via a notice field on the shared renderer. - explain-check: validate the slug before it becomes part of a file path. --- .../actions/comment-release-links/action.yaml | 4 +- .github/actions/explain-check/action.yaml | 45 +++------ .../workflows/publish-package-on-merge.yaml | 4 +- scripts/cicd/pr-report-section.test.ts | 44 +++++++++ scripts/cicd/pr-report-section.ts | 44 +++++---- scripts/coverage-report.test.ts | 2 +- scripts/coverage-report.ts | 25 +++-- scripts/perf-report.test.ts | 68 ++++++++++++++ scripts/perf-report.ts | 94 ++++++++++++++----- 9 files changed, 239 insertions(+), 91 deletions(-) create mode 100644 scripts/perf-report.test.ts diff --git a/.github/actions/comment-release-links/action.yaml b/.github/actions/comment-release-links/action.yaml index 31591217a9f..8a1b25d69d4 100644 --- a/.github/actions/comment-release-links/action.yaml +++ b/.github/actions/comment-release-links/action.yaml @@ -91,7 +91,9 @@ runs: COMMENT_FILE=$(mktemp) { - echo "## πŸš€ $SECTION_TITLE: βœ… Published \`$NEW_VERSION\`" + # Neutral on purpose: release-draft-create.yaml calls this for + # prereleases too, whose GitHub release is still an unpublished draft. + echo "## πŸš€ $SECTION_TITLE: βœ… Publish jobs finished for \`$NEW_VERSION\`" echo "" echo "
" echo "Details" diff --git a/.github/actions/explain-check/action.yaml b/.github/actions/explain-check/action.yaml index f936785bd53..072f17d0fb7 100644 --- a/.github/actions/explain-check/action.yaml +++ b/.github/actions/explain-check/action.yaml @@ -85,7 +85,7 @@ runs: ;; esac - # The slug is the upsert key and part of a file path below; refuse + # The slug is the upsert key and part of the file path below; refuse # anything that could escape the workspace. Same rule as # upsert-comment-section. if ! [[ "$INPUT_SLUG" =~ ^[a-z0-9-]+$ ]]; then @@ -93,11 +93,15 @@ runs: exit 1 fi - body=$(mktemp) - trap 'rm -f "$body"' EXIT + # Must live inside the workspace: read-and-upsert-comment-section guards + # on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and + # would silently no-op on a RUNNER_TEMP path. + out="$GITHUB_WORKSPACE/.explain-check-${INPUT_SLUG}.md" { + printf '## %s %s: %s\n' "$INPUT_ICON" "$INPUT_TITLE" "$badge" + if [ -n "$INPUT_HEADLINE" ]; then - printf '%s\n' "$INPUT_HEADLINE" + printf '\n%s\n' "$INPUT_HEADLINE" fi run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" @@ -111,8 +115,9 @@ runs: esac if [ -n "$INPUT_DETAILS_FILE" ] && [ -s "$INPUT_DETAILS_FILE" ]; then - printf '\n' + printf '\n
\nFindings and check details\n\n' cat "$INPUT_DETAILS_FILE" + printf '\n\n
\n' fi if [ -n "$INPUT_REPRODUCE" ]; then @@ -123,39 +128,13 @@ runs: if [ -n "$INPUT_DOCS" ]; then printf '\nFull docs: %s\n' "$INPUT_DOCS" fi - } > "$body" - - heading=$(printf '## %s %s: %s' "$INPUT_ICON" "$INPUT_TITLE" "$badge") - # Drop leading blank lines here and trailing ones via the substitution, - # so a check that supplies nothing but a verdict renders as a bare - # heading rather than an empty box. Mirrors renderPrReportSection. - body_text=$(sed -e '/[^[:space:]]/,$!d' "$body") - - # Must live inside the workspace: read-and-upsert-comment-section guards - # on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and - # would silently no-op on a RUNNER_TEMP path. - out="$GITHUB_WORKSPACE/.explain-check-${INPUT_SLUG}.md" - # Every section of the PR report reads as one status line with the rest - # folded away, so the comment stays scannable as checks accumulate. - { - printf '%s\n' "$heading" - if [ -n "$body_text" ]; then - printf '\n
\nDetails\n\n%s\n\n
\n' "$body_text" - fi } > "$out" echo "file=$out" >> "$GITHUB_OUTPUT" # The summary is the fallback that always works β€” fork PRs carry a - # read-only token, so the comment path below is skipped for them. It - # stays expanded because for those runs it is the only place the - # findings are readable at all. - { - printf '%s\n' "$heading" - if [ -n "$body_text" ]; then - printf '\n%s\n' "$body_text" - fi - } >> "$GITHUB_STEP_SUMMARY" + # read-only token, so the comment path below is skipped for them. + cat "$out" >> "$GITHUB_STEP_SUMMARY" - name: Upsert section into the PR report if: ${{ inputs.pr-number != '' && inputs.token != '' }} diff --git a/.github/workflows/publish-package-on-merge.yaml b/.github/workflows/publish-package-on-merge.yaml index 79474dab729..6bc7b3f0ae0 100644 --- a/.github/workflows/publish-package-on-merge.yaml +++ b/.github/workflows/publish-package-on-merge.yaml @@ -128,9 +128,9 @@ jobs: # version already on npm is skipped and still lands here. PACKAGE_COUNT=$(jq -r '.include | length' <<<"$MATRIX") { - printf '## πŸš€ Release: βœ… %s package' "$PACKAGE_COUNT" + # Resolved, not pushed: say what finished, not what landed on npm. + printf '## πŸš€ Release: βœ… Publish jobs finished for %s package' "$PACKAGE_COUNT" [ "$PACKAGE_COUNT" = 1 ] || printf 's' - printf ' on npm' echo '' echo '' echo '
' diff --git a/scripts/cicd/pr-report-section.test.ts b/scripts/cicd/pr-report-section.test.ts index 7dbf833b96c..378d3acda93 100644 --- a/scripts/cicd/pr-report-section.test.ts +++ b/scripts/cicd/pr-report-section.test.ts @@ -92,4 +92,48 @@ describe('renderPrReportSection', () => { }) ).toContain('\n\nNo new findings.\n\n
') }) + + // A failure call to action has to be readable without expanding anything: + // the heading says what is wrong, the notice says what to do about it. + it('keeps the notice visible between the heading and the collapsed box', () => { + const rendered = renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: '❌ 1 below 52 FPS target', + notice: '> ❌ **1 scenario below the target.** Profile it.', + body: '| Metric | Value |' + }) + + expect(rendered.split('
')[0]).toBe( + '## ⚑ Performance: ❌ 1 below 52 FPS target\n\n> ❌ **1 scenario below the target.** Profile it.\n\n' + ) + expect(rendered).toContain('Details\n\n| Metric') + }) + + it('renders a notice without a box when there is no body', () => { + expect( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '⚠️ No line coverage recorded', + notice: '\n> [!NOTE]\n> Check the logs.\n' + }) + ).toBe( + '## πŸ”¬ E2E Coverage: ⚠️ No line coverage recorded\n\n> [!NOTE]\n> Check the logs.' + ) + }) + + it('names the collapsed box when a summary is given', () => { + expect( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '🟒 90.0% of lines', + summary: 'Coverage totals', + body: '| Metric |' + }) + ).toContain( + 'Coverage totals\n\n| Metric |\n\n
' + ) + }) }) diff --git a/scripts/cicd/pr-report-section.ts b/scripts/cicd/pr-report-section.ts index 5f395dfc6b4..343f056b26c 100644 --- a/scripts/cicd/pr-report-section.ts +++ b/scripts/cicd/pr-report-section.ts @@ -2,9 +2,8 @@ * Shared shape for every section of the unified PR report comment. * * A reader scanning the comment should get each check's verdict from one line - * without scrolling: the heading carries the status, and everything else is - * folded away. The Storybook and Playwright sections established the shape and - * this renderer is how the rest of them keep it. + * without scrolling: the heading carries the status, a failure's call to + * action stays visible beneath it, and everything else is folded away. */ export interface PrReportSection { /** Emoji shown beside the title, e.g. `⚑`. */ @@ -13,30 +12,41 @@ export interface PrReportSection { title: string /** Verdict for the heading, e.g. `βœ… No regressions`. */ status: string + /** + * Markdown kept visible under the heading: what to do about a failure, or a + * caveat on the verdict. Lives outside the box, so GitHub alerts still render. + */ + notice?: string /** Markdown folded into the collapsed box. Omit for a status-only section. */ body?: string + /** Label of the collapsed box. Defaults to `Details`. */ + summary?: string } export function renderPrReportSection({ icon, title, status, - body + notice, + body, + summary = 'Details' }: PrReportSection): string { - const heading = `## ${icon} ${title}: ${status}` - const trimmedBody = body?.trim() - if (!trimmedBody) return heading + const blocks = [`## ${icon} ${title}: ${status}`] + + const trimmedNotice = notice?.trim() + if (trimmedNotice) blocks.push(trimmedNotice) // The blank lines around the body are load-bearing: GitHub only parses // Markdown inside an HTML block when it is separated from the tags. - return [ - heading, - '', - '
', - 'Details', - '', - trimmedBody, - '', - '
' - ].join('\n') + const trimmedBody = body?.trim() + if (trimmedBody) { + blocks.push( + ['
', `${summary}`, '', trimmedBody].join( + '\n' + ), + '
' + ) + } + + return blocks.join('\n\n') } diff --git a/scripts/coverage-report.test.ts b/scripts/coverage-report.test.ts index 9d87cabaf3d..a7ee9dd08b2 100644 --- a/scripts/coverage-report.test.ts +++ b/scripts/coverage-report.test.ts @@ -49,7 +49,7 @@ describe('coverage-report shard banner', () => { expect(output).not.toContain('[!WARNING]') expect(output).not.toContain('[!NOTE]') expect(output).toContain('| Lines | 3 | 6 | 50.0%') - expect(output.split('
')[0]).toContain('Line coverage: **50.0%**') + expect(output.split('\n')[0]).toBe('## πŸ”¬ E2E Coverage: 🟑 50.0% of lines') expect(output.split('
')[0]).not.toContain('| Metric |') expect(output).toMatch( /Coverage totals<\/summary>\n\n\| Metric[\s\S]*\| Lines \| 3 \| 6 \| 50.0%[\s\S]*\n\n<\/details>/ diff --git a/scripts/coverage-report.ts b/scripts/coverage-report.ts index 069c7b8bd5b..4d228c67cca 100644 --- a/scripts/coverage-report.ts +++ b/scripts/coverage-report.ts @@ -124,24 +124,21 @@ function bar(covered: number, total: number): string { } const lines: string[] = [] -// GitHub alerts (`> [!NOTE]`) do not render inside the collapsed box, so the -// shard-completeness caveat is plain bold text in the body and a short -// qualifier on the heading, where a reader sees it without expanding. +// The shard caveat qualifies the verdict, so it stays visible under the +// heading rather than folded into the box with the totals. +const notice: string[] = [] const metadata = readCoverageMetadata( join(dirname(lcovPath), COVERAGE_METADATA_FILE) ) -let completenessQualifier = '' if (!metadata) { - completenessQualifier = ' (shards unverified)' - lines.push( - '**Note:** Shard completeness could not be verified for this run, so these totals may not be comparable with a whole merge. Check the coverage workflow logs and re-run collection before using these totals for comparison.', - '' + notice.push( + '> [!NOTE]', + '> Shard completeness could not be verified for this run, so these totals may not be comparable with a whole merge. Check the coverage workflow logs and re-run collection before using these totals for comparison.' ) } else if (!metadata.complete) { - completenessQualifier = ' (incomplete shards)' - lines.push( - '**Note:** The shard matrix did not pass, so these totals cannot be confirmed comparable with a whole merge. Fix the failing shards and re-run coverage before using these totals for comparison.', - '' + notice.push( + '> [!NOTE]', + '> The shard matrix did not pass, so these totals cannot be confirmed comparable with a whole merge. Fix the failing shards and re-run coverage before using these totals for comparison.' ) } @@ -185,13 +182,15 @@ if (uncovered.length > 0) { const linesStatus = totalLines === 0 ? '⚠️ No line coverage recorded' - : `${bar(coveredLines, totalLines)} ${pct(coveredLines, totalLines)} of lines${completenessQualifier}` + : `${bar(coveredLines, totalLines)} ${pct(coveredLines, totalLines)} of lines` process.stdout.write( renderPrReportSection({ icon: 'πŸ”¬', title: 'E2E Coverage', status: linesStatus, + notice: notice.join('\n'), + summary: 'Coverage totals', body: lines.join('\n') }) + '\n' ) diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts new file mode 100644 index 00000000000..c9b4834246c --- /dev/null +++ b/scripts/perf-report.test.ts @@ -0,0 +1,68 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const ROOT = join(import.meta.dirname, '..') +const TSX = join(ROOT, 'node_modules/.bin/tsx') +const SCRIPT = join(import.meta.dirname, 'perf-report.ts') + +function render(measurements: Record[]): string { + const dir = mkdtempSync(join(tmpdir(), 'perf-report-')) + try { + mkdirSync(join(dir, 'test-results')) + writeFileSync( + join(dir, 'test-results/perf-metrics.json'), + JSON.stringify({ measurements }) + ) + const result = spawnSync(TSX, [SCRIPT], { cwd: dir, encoding: 'utf8' }) + if (result.status !== 0) { + throw new Error( + `perf-report.ts exited ${result.status}: ${result.stderr}` + ) + } + return result.stdout + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +const heading = (output: string) => output.split('\n')[0] + +describe('perf-report heading', () => { + // PerformanceHelper records p95FrameDurationMs as 0 when frame sampling + // times out. That is missing data, not a scenario running at 0 FPS. + it('does not count a scenario without frame samples as missing the FPS target', () => { + const output = render([ + { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 }, + { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 } + ]) + + expect(heading(output)).not.toContain('below 52 FPS target') + expect(output).not.toContain('0.0 P5 FPS ❌') + expect(output.split('
')[0]).toContain( + '1 scenario recorded no frame samples' + ) + }) + + it('still leads with a real FPS target miss', () => { + const output = render([ + { name: 'slow', frameDurationMs: 25, p95FrameDurationMs: 40 }, + { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 } + ]) + + expect(heading(output)).toMatch( + /^## ⚑ Performance: ❌ 1 below 52 FPS target/ + ) + }) + + it('does not report a run that measured nothing as clean', () => { + const output = render([]) + + expect(heading(output)).toBe( + '## ⚑ Performance: ⚠️ No measurements recorded β€” check the perf test logs' + ) + }) +}) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index 453ee7fbf07..56ad527467c 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -219,25 +219,56 @@ function frameTimeToFps(ms: number): number { return ms > 0 ? 1000 / ms : 0 } -function countBelowFpsTarget(prGroups: Map): number { - let below = 0 +/** + * PerformanceHelper records a frame duration of 0 when sampling times out or + * collects fewer than two timestamps. That is missing data, not a scenario + * running at 0 FPS, so it must never count as a target miss. + */ +function sampledFrameMetric( + samples: PerfMeasurement[], + key: 'frameDurationMs' | 'p95FrameDurationMs' +): number | null { + const value = medianMetric(samples, key) + return value !== null && value > 0 ? value : null +} + +interface FpsTargetTally { + below: number + unsampled: number +} + +function tallyFpsTarget( + prGroups: Map +): FpsTargetTally { + const tally: FpsTargetTally = { below: 0, unsampled: 0 } for (const prSamples of prGroups.values()) { - const p95Frame = medianMetric(prSamples, 'p95FrameDurationMs') - if (p95Frame !== null && frameTimeToFps(p95Frame) < TARGET_P5_FPS) below++ + const p95Frame = sampledFrameMetric(prSamples, 'p95FrameDurationMs') + if (p95Frame === null) { + if (medianMetric(prSamples, 'p95FrameDurationMs') !== null) + tally.unsampled++ + continue + } + if (frameTimeToFps(p95Frame) < TARGET_P5_FPS) tally.below++ } - return below + return tally } -function renderHeadlineSummary( - prGroups: Map -): string[] { +function plural(count: number, noun: string): string { + return `${count} ${noun}${count === 1 ? '' : 's'}` +} + +function renderHeadlineSummary(prGroups: Map): { + notice: string[] + lines: string[] +} { + const notice: string[] = [] const lines: string[] = [] const summaries: string[] = [] - let belowTarget = 0 + const { below: belowTarget, unsampled } = tallyFpsTarget(prGroups) for (const [testName, prSamples] of prGroups) { - const avgFrame = medianMetric(prSamples, 'frameDurationMs') - const p95Frame = medianMetric(prSamples, 'p95FrameDurationMs') + const avgFrame = sampledFrameMetric(prSamples, 'frameDurationMs') + const p95Frame = sampledFrameMetric(prSamples, 'p95FrameDurationMs') const tbt = medianMetric(prSamples, 'totalBlockingTimeMs') const heap = medianMetric(prSamples, 'heapUsedBytes') @@ -248,7 +279,6 @@ function renderHeadlineSummary( if (avgFps !== null) parts.push(`${avgFps.toFixed(1)} avg FPS`) if (p5Fps !== null) { const pass = p5Fps >= TARGET_P5_FPS - if (!pass) belowTarget++ parts.push( `${p5Fps.toFixed(1)} P5 FPS ${pass ? 'βœ…' : '❌'} (target: β‰₯${TARGET_P5_FPS})` ) @@ -260,8 +290,15 @@ function renderHeadlineSummary( } if (belowTarget > 0) { - lines.push( - `> ❌ **${belowTarget} scenario${belowTarget === 1 ? '' : 's'} below the β‰₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`, + notice.push( + `> ❌ **${plural(belowTarget, 'scenario')} below the β‰₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`, + '' + ) + } + + if (unsampled > 0) { + notice.push( + `> ⚠️ ${plural(unsampled, 'scenario')} recorded no frame samples, so ${unsampled === 1 ? 'its' : 'their'} FPS could not be checked against the target. Check the perf test logs for a frame sampling timeout.`, '' ) } @@ -277,11 +314,12 @@ function renderHeadlineSummary( ) } - return lines + return { notice, lines } } interface RenderedReport { status: string + notice?: string[] lines: string[] } @@ -346,10 +384,13 @@ function renderFullReport( ? `⚠️ ${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected` : 'βœ… No regressions detected' + const notice: string[] = [] if (flaggedRows.length > 0) { - lines.push( + notice.push( `⚠️ **${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected**. Review the regressions below and profile the affected scenarios before merging.`, - '', + '' + ) + lines.push( '
Show regressions', '', ...tableHeader, @@ -416,7 +457,7 @@ function renderFullReport( ) } - return { status, lines } + return { status, notice, lines } } function renderColdStartReport( @@ -531,8 +572,9 @@ function main() { ) } + const headline = renderHeadlineSummary(prGroups) const body = [ - ...renderHeadlineSummary(prGroups), + ...headline.lines, ...report.lines, '', '
Raw data', @@ -547,18 +589,22 @@ function main() { // The per-test target verdict lives in the collapsed summary, so without // this the heading can read βœ… over a hidden ❌. It leads, because a reader // scanning the comment takes the verdict from the first glyph and a target - // miss outranks "nothing moved since the baseline". - const belowTarget = countBelowFpsTarget(prGroups) + // miss outranks "nothing moved since the baseline". A run that measured + // nothing has no verdict at all, and must not read as a clean one. + const { below: belowTarget } = tallyFpsTarget(prGroups) const status = - belowTarget > 0 - ? `❌ ${belowTarget} below ${TARGET_P5_FPS} FPS target Β· ${report.status}` - : report.status + current.measurements.length === 0 + ? '⚠️ No measurements recorded β€” check the perf test logs' + : belowTarget > 0 + ? `❌ ${belowTarget} below ${TARGET_P5_FPS} FPS target Β· ${report.status}` + : report.status process.stdout.write( renderPrReportSection({ icon: '⚑', title: 'Performance', status, + notice: [...headline.notice, ...(report.notice ?? [])].join('\n'), body: body.join('\n') }) + '\n' ) From a72772a200817af69b1637418567a85ea028977f Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 28 Oct 2026 06:57:11 +0000 Subject: [PATCH 08/19] ci: keep unsampled perf runs out of the FPS verdict - Drop runs that recorded no frames before taking the median frame time, so a 0 cannot halve the frame time and turn a real FPS miss into a pass. - When no scenario recorded frame samples, the heading says the FPS target was not checked instead of reading clean. - Chromatic's heading says the build was uploaded: the job exits once the upload finishes, before the visual tests have run. - Replace a renderer test that asserted its own string join with a unified-report test that checks the real section separation. --- .github/workflows/ci-tests-storybook.yaml | 2 +- scripts/cicd/pr-report-section.test.ts | 20 --------- scripts/perf-report.test.ts | 26 ++++++++++++ scripts/perf-report.ts | 22 ++++++---- scripts/unified-report.test.ts | 51 +++++++++++++++++++++++ 5 files changed, 93 insertions(+), 28 deletions(-) create mode 100644 scripts/unified-report.test.ts diff --git a/.github/workflows/ci-tests-storybook.yaml b/.github/workflows/ci-tests-storybook.yaml index 1e15342d788..96b787a7b30 100644 --- a/.github/workflows/ci-tests-storybook.yaml +++ b/.github/workflows/ci-tests-storybook.yaml @@ -236,7 +236,7 @@ jobs: pr-number: ${{ github.event.pull_request.number }} section-name: chromatic section-content: | - ## 🎨 Chromatic: βœ… Published β€” [View Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }}) + ## 🎨 Chromatic: πŸ“€ Uploaded β€” [View Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }})
Details diff --git a/scripts/cicd/pr-report-section.test.ts b/scripts/cicd/pr-report-section.test.ts index 378d3acda93..c2ac2f7311c 100644 --- a/scripts/cicd/pr-report-section.test.ts +++ b/scripts/cicd/pr-report-section.test.ts @@ -62,26 +62,6 @@ describe('renderPrReportSection', () => { expect(lines[lines.indexOf('
') - 1]).toBe('') }) - it('leaves a blank line before the next section once blocks are joined', () => { - // upsert-comment-section concatenates section blocks with a blank line. - // Without one, the HTML block opened by
runs on and swallows - // the following `##` into plain text. - const bundle = renderPrReportSection({ - icon: 'πŸ“¦', - title: 'Bundle', - status: '9.96 MB gzip', - body: '**Summary**' - }) - const perf = renderPrReportSection({ - icon: '⚑', - title: 'Performance', - status: 'βœ… No regressions' - }) - - expect(bundle.endsWith('
')).toBe(true) - expect(`${bundle}\n\n${perf}`).toContain('
\n\n## ⚑') - }) - it('strips body padding so nested sections do not drift apart', () => { expect( renderPrReportSection({ diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts index c9b4834246c..430a24bcfef 100644 --- a/scripts/perf-report.test.ts +++ b/scripts/perf-report.test.ts @@ -58,6 +58,32 @@ describe('perf-report heading', () => { ) }) + // A run that recorded no frames stores 0 alongside the real samples. It must + // be dropped before the median, or it halves the frame time and turns a + // real miss into a pass. + it('ignores unsampled runs when taking the median frame time', () => { + const output = render([ + { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, + { name: 'slow', frameDurationMs: 16, p95FrameDurationMs: 18 }, + { name: 'slow', frameDurationMs: 16, p95FrameDurationMs: 21 } + ]) + + expect(heading(output)).toMatch( + /^## ⚑ Performance: ❌ 1 below 52 FPS target/ + ) + expect(output).not.toContain('recorded no frame samples') + }) + + it('does not read as clean when no scenario recorded frame samples', () => { + const output = render([ + { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 } + ]) + + expect(heading(output)).toMatch( + /^## ⚑ Performance: ⚠️ No frame samples β€” FPS target not checked/ + ) + }) + it('does not report a run that measured nothing as clean', () => { const output = render([]) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index 56ad527467c..12e8d7a501b 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -228,19 +228,22 @@ function sampledFrameMetric( samples: PerfMeasurement[], key: 'frameDurationMs' | 'p95FrameDurationMs' ): number | null { - const value = medianMetric(samples, key) - return value !== null && value > 0 ? value : null + // Drop the unsampled runs before the median: left in, a 0 drags the median + // frame time down and reports a scenario faster than it ran. + const sampled = samples.filter((s) => (getMetricValue(s, key) ?? 0) > 0) + return medianMetric(sampled, key) } interface FpsTargetTally { below: number + sampled: number unsampled: number } function tallyFpsTarget( prGroups: Map ): FpsTargetTally { - const tally: FpsTargetTally = { below: 0, unsampled: 0 } + const tally: FpsTargetTally = { below: 0, sampled: 0, unsampled: 0 } for (const prSamples of prGroups.values()) { const p95Frame = sampledFrameMetric(prSamples, 'p95FrameDurationMs') if (p95Frame === null) { @@ -248,6 +251,7 @@ function tallyFpsTarget( tally.unsampled++ continue } + tally.sampled++ if (frameTimeToFps(p95Frame) < TARGET_P5_FPS) tally.below++ } return tally @@ -591,13 +595,17 @@ function main() { // scanning the comment takes the verdict from the first glyph and a target // miss outranks "nothing moved since the baseline". A run that measured // nothing has no verdict at all, and must not read as a clean one. - const { below: belowTarget } = tallyFpsTarget(prGroups) + // Frame data that never arrived is not a pass either: with nothing sampled + // the target was not checked at all. + const fps = tallyFpsTarget(prGroups) const status = current.measurements.length === 0 ? '⚠️ No measurements recorded β€” check the perf test logs' - : belowTarget > 0 - ? `❌ ${belowTarget} below ${TARGET_P5_FPS} FPS target Β· ${report.status}` - : report.status + : fps.below > 0 + ? `❌ ${fps.below} below ${TARGET_P5_FPS} FPS target Β· ${report.status}` + : fps.unsampled > 0 && fps.sampled === 0 + ? `⚠️ No frame samples β€” FPS target not checked Β· ${report.status}` + : report.status process.stdout.write( renderPrReportSection({ diff --git a/scripts/unified-report.test.ts b/scripts/unified-report.test.ts new file mode 100644 index 00000000000..71eb0457725 --- /dev/null +++ b/scripts/unified-report.test.ts @@ -0,0 +1,51 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const ROOT = join(import.meta.dirname, '..') +const TSX = join(ROOT, 'node_modules/.bin/tsx') +const SCRIPT = join(import.meta.dirname, 'unified-report.ts') + +function render(args: string[]): string { + const dir = mkdtempSync(join(tmpdir(), 'unified-report-')) + try { + const result = spawnSync(TSX, [SCRIPT, ...args], { + cwd: dir, + encoding: 'utf8' + }) + if (result.status !== 0) { + throw new Error( + `unified-report.ts exited ${result.status}: ${result.stderr}` + ) + } + return result.stdout + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +describe('unified-report', () => { + // A heading that follows another section without a blank line is swallowed + // into the previous block (a `
` keeps an HTML block open), so every + // section after the first must start on a fresh paragraph. + it('separates each section from the one before it with a blank line', () => { + const output = render([ + '--size-status=failed', + '--perf-status=pending', + '--coverage-status=failed' + ]) + + const headings = output.match(/^## .*$/gm) ?? [] + expect(headings).toEqual([ + '## πŸ“¦ Bundle: ⚠️ Size data collection failed β€” check the CI workflow logs', + '## ⚑ Performance: ⏳ Performance tests in progress…', + '## πŸ”¬ E2E Coverage: ⚠️ Coverage collection failed β€” check the CI workflow logs' + ]) + for (const heading of headings.slice(1)) { + expect(output).toContain(`\n\n${heading}`) + } + }) +}) From 00145030e6882d65e74c9f370cf3221f07acb0c3 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 28 Oct 2026 07:42:51 +0000 Subject: [PATCH 09/19] refactor(perf-report): split the report into small helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verdict work made renderFullReport, renderHeadlineSummary and main exceed the fallow complexity gate. Pull the per-metric comparison, the scenario summary, the FPS notice, the baseline selection, the variance and trend tables and the heading status into helpers, and share the percent-delta and baseline-median logic with the cold-start table. Output is byte-identical for the full, cold-start and no-baseline paths. Also drop the stray separator after the scenario name in the timing lines (`**a**: Β· 62.5 avg FPS` -> `**a**: 62.5 avg FPS`). --- scripts/perf-report.test.ts | 8 + scripts/perf-report.ts | 376 ++++++++++++++++++++---------------- 2 files changed, 220 insertions(+), 164 deletions(-) diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts index 430a24bcfef..6f3ddd8555d 100644 --- a/scripts/perf-report.test.ts +++ b/scripts/perf-report.test.ts @@ -84,6 +84,14 @@ describe('perf-report heading', () => { ) }) + it('starts a scenario timing line with its first metric, not a separator', () => { + const output = render([ + { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 } + ]) + + expect(output).toContain('> **fast**: 62.5 avg FPS Β· 58.8 P5 FPS βœ…') + }) + it('does not report a run that measured nothing as clean', () => { const output = render([]) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index 12e8d7a501b..d2e1fd39de8 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -261,51 +261,83 @@ function plural(count: number, noun: string): string { return `${count} ${noun}${count === 1 ? '' : 's'}` } -function renderHeadlineSummary(prGroups: Map): { - notice: string[] - lines: string[] -} { - const notice: string[] = [] - const lines: string[] = [] - const summaries: string[] = [] - const { below: belowTarget, unsampled } = tallyFpsTarget(prGroups) +function percentDelta(prVal: number, baseVal: number): number | null { + if (baseVal === 0) return prVal === 0 ? 0 : null + return ((prVal - baseVal) / baseVal) * 100 +} - for (const [testName, prSamples] of prGroups) { - const avgFrame = sampledFrameMetric(prSamples, 'frameDurationMs') - const p95Frame = sampledFrameMetric(prSamples, 'p95FrameDurationMs') - const tbt = medianMetric(prSamples, 'totalBlockingTimeMs') - const heap = medianMetric(prSamples, 'heapUsedBytes') - - const avgFps = avgFrame !== null ? frameTimeToFps(avgFrame) : null - const p5Fps = p95Frame !== null ? frameTimeToFps(p95Frame) : null - - const parts: string[] = [`**${testName}**:`] - if (avgFps !== null) parts.push(`${avgFps.toFixed(1)} avg FPS`) - if (p5Fps !== null) { - const pass = p5Fps >= TARGET_P5_FPS - parts.push( - `${p5Fps.toFixed(1)} P5 FPS ${pass ? 'βœ…' : '❌'} (target: β‰₯${TARGET_P5_FPS})` - ) - } - if (tbt !== null) parts.push(`${tbt.toFixed(0)}ms TBT`) - if (heap !== null) parts.push(`${formatBytes(heap)} heap`) +function baselineMedian( + baseSamples: PerfMeasurement[] | undefined, + key: MetricKey +): number | null { + return baseSamples?.length ? medianMetric(baseSamples, key) : null +} - if (parts.length > 1) summaries.push(parts.join(' Β· ')) - } +function formatIfPresent( + value: number | null, + format: (value: number) => string +): string | null { + return value === null ? null : format(value) +} - if (belowTarget > 0) { +function p5FpsPart(p95Frame: number): string { + const p5Fps = frameTimeToFps(p95Frame) + const verdict = p5Fps >= TARGET_P5_FPS ? 'βœ…' : '❌' + return `${p5Fps.toFixed(1)} P5 FPS ${verdict} (target: β‰₯${TARGET_P5_FPS})` +} + +function scenarioSummary( + testName: string, + prSamples: PerfMeasurement[] +): string | null { + const parts = [ + formatIfPresent( + sampledFrameMetric(prSamples, 'frameDurationMs'), + (ms) => `${frameTimeToFps(ms).toFixed(1)} avg FPS` + ), + formatIfPresent( + sampledFrameMetric(prSamples, 'p95FrameDurationMs'), + p5FpsPart + ), + formatIfPresent( + medianMetric(prSamples, 'totalBlockingTimeMs'), + (ms) => `${ms.toFixed(0)}ms TBT` + ), + formatIfPresent( + medianMetric(prSamples, 'heapUsedBytes'), + (bytes) => `${formatBytes(bytes)} heap` + ) + ].filter((part): part is string => part !== null) + + return parts.length > 0 ? `**${testName}**: ${parts.join(' Β· ')}` : null +} + +function fpsTargetNotice({ below, unsampled }: FpsTargetTally): string[] { + const notice: string[] = [] + if (below > 0) { notice.push( - `> ❌ **${plural(belowTarget, 'scenario')} below the β‰₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`, + `> ❌ **${plural(below, 'scenario')} below the β‰₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`, '' ) } - if (unsampled > 0) { notice.push( `> ⚠️ ${plural(unsampled, 'scenario')} recorded no frame samples, so ${unsampled === 1 ? 'its' : 'their'} FPS could not be checked against the target. Check the perf test logs for a frame sampling timeout.`, '' ) } + return notice +} + +function renderHeadlineSummary(prGroups: Map): { + notice: string[] + lines: string[] +} { + const notice = fpsTargetNotice(tallyFpsTarget(prGroups)) + const lines: string[] = [] + const summaries = [...prGroups] + .map(([testName, prSamples]) => scenarioSummary(testName, prSamples)) + .filter((summary): summary is string => summary !== null) if (summaries.length > 0) { lines.push( @@ -327,6 +359,49 @@ interface RenderedReport { lines: string[] } +interface ComparedRow { + row: string + flagged: boolean +} + +function compareMetric( + testName: string, + metric: MetricDef, + prSamples: PerfMeasurement[], + baseSamples: PerfMeasurement[] | undefined, + historical: PerfReport[] +): ComparedRow | null { + const { key, label, unit, minAbsDelta } = metric + const prVal = medianMetric(prSamples, key) + if (prVal === null) return null + + const baseVal = baselineMedian(baseSamples, key) + if (baseVal === null) { + return { + row: `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new | β€” |`, + flagged: false + } + } + + const histStats = getHistoricalStats(historical, testName, key) + const z = zScore(prVal, histStats) + const sig = classifyChange( + z, + computeCV(histStats), + prVal - baseVal, + minAbsDelta + ) + + return { + row: `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(percentDelta(prVal, baseVal))} | ${formatSignificance(sig, z)} |`, + flagged: isNoteworthy(sig) + } +} + +function regressionVerdict(count: number): string { + return `${count} regression${count > 1 ? 's' : ''} detected` +} + function renderFullReport( prGroups: Map, baseline: PerfReport, @@ -339,59 +414,31 @@ function renderFullReport( '|--------|----------|----------|---|-----|' ] - const flaggedRows: string[] = [] - const allRows: string[] = [] - - for (const [testName, prSamples] of prGroups) { - const baseSamples = baselineGroups.get(testName) - - for (const { key, label, unit, minAbsDelta } of REPORTED_METRICS) { - const prVal = medianMetric(prSamples, key) - if (prVal === null) continue - const histStats = getHistoricalStats(historical, testName, key) - const cv = computeCV(histStats) - - if (!baseSamples?.length) { - allRows.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new | β€” |` - ) - continue - } - - const baseVal = medianMetric(baseSamples, key) - if (baseVal === null) { - allRows.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new | β€” |` - ) - continue - } - const absDelta = prVal - baseVal - const deltaPct = - baseVal === 0 - ? prVal === 0 - ? 0 - : null - : ((prVal - baseVal) / baseVal) * 100 - const z = zScore(prVal, histStats) - const sig = classifyChange(z, cv, absDelta, minAbsDelta) - - const row = `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(deltaPct)} | ${formatSignificance(sig, z)} |` - allRows.push(row) - if (isNoteworthy(sig)) { - flaggedRows.push(row) - } - } - } + const compared = [...prGroups].flatMap(([testName, prSamples]) => + REPORTED_METRICS.map((metric) => + compareMetric( + testName, + metric, + prSamples, + baselineGroups.get(testName), + historical + ) + ).filter((row): row is ComparedRow => row !== null) + ) + const allRows = compared.map(({ row }) => row) + const flaggedRows = compared + .filter(({ flagged }) => flagged) + .map(({ row }) => row) const status = flaggedRows.length > 0 - ? `⚠️ ${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected` + ? `⚠️ ${regressionVerdict(flaggedRows.length)}` : 'βœ… No regressions detected' const notice: string[] = [] if (flaggedRows.length > 0) { notice.push( - `⚠️ **${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected**. Review the regressions below and profile the affected scenarios before merging.`, + `⚠️ **${regressionVerdict(flaggedRows.length)}**. Review the regressions below and profile the affected scenarios before merging.`, '' ) lines.push( @@ -416,11 +463,23 @@ function renderFullReport( ) lines.push( + ...renderHistoricalVariance(prGroups, historical), + ...renderTrend(prGroups, historical) + ) + + return { status, notice, lines } +} + +function renderHistoricalVariance( + prGroups: Map, + historical: PerfReport[] +): string[] { + const lines = [ `
Historical variance (last ${historical.length} runs)`, '', '| Metric | ΞΌ | Οƒ | CV |', '|--------|---|---|-----|' - ) + ] for (const [testName] of prGroups) { for (const { key, label, unit } of REPORTED_METRICS) { const stats = getHistoricalStats(historical, testName, key) @@ -432,36 +491,36 @@ function renderFullReport( } } lines.push('', '
') + return lines +} +function renderTrend( + prGroups: Map, + historical: PerfReport[] +): string[] { const trendRows: string[] = [] for (const [testName] of prGroups) { for (const { key, label, unit } of REPORTED_METRICS) { const series = getHistoricalTimeSeries(historical, testName, key) if (series.length < 3) continue - const dir = trendDirection(series) - const arrow = trendArrow(dir) - const spark = sparkline(series) + const arrow = trendArrow(trendDirection(series)) const last = series[series.length - 1] trendRows.push( - `| ${testName}: ${label} | ${spark} | ${arrow} | ${formatValue(last, unit)} |` + `| ${testName}: ${label} | ${sparkline(series)} | ${arrow} | ${formatValue(last, unit)} |` ) } } - - if (trendRows.length > 0) { - lines.push( - '', - `
Trend (last ${historical.length} commits on main)`, - '', - '| Metric | Trend | Dir | Latest |', - '|--------|-------|-----|--------|', - ...trendRows, - '', - '
' - ) - } - - return { status, notice, lines } + if (trendRows.length === 0) return [] + return [ + '', + `
Trend (last ${historical.length} commits on main)`, + '', + '| Metric | Trend | Dir | Latest |', + '|--------|-------|-----|--------|', + ...trendRows, + '', + '
' + ] } function renderColdStartReport( @@ -487,28 +546,11 @@ function renderColdStartReport( const prVal = medianMetric(prSamples, key) if (prVal === null) continue - if (!baseSamples?.length) { - lines.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new |` - ) - continue - } - - const baseVal = medianMetric(baseSamples, key) - if (baseVal === null) { - lines.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new |` - ) - continue - } - const deltaPct = - baseVal === 0 - ? prVal === 0 - ? 0 - : null - : ((prVal - baseVal) / baseVal) * 100 + const baseVal = baselineMedian(baseSamples, key) lines.push( - `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(deltaPct)} |` + baseVal === null + ? `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new |` + : `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(percentDelta(prVal, baseVal))} |` ) } } @@ -541,45 +583,25 @@ function renderNoBaselineReport( return { status: 'ℹ️ No baseline β€” significance unavailable', lines } } -function main() { - if (!existsSync(CURRENT_PATH)) { - process.stdout.write( - renderPrReportSection({ - icon: '⚑', - title: 'Performance', - status: '⏭️ No metrics found β€” perf tests may not have run' - }) + '\n' - ) - process.exit(0) - } - - const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8')) - - const baseline: PerfReport | null = existsSync(BASELINE_PATH) - ? JSON.parse(readFileSync(BASELINE_PATH, 'utf-8')) - : null - +function renderComparison( + prGroups: Map +): RenderedReport { + if (!existsSync(BASELINE_PATH)) return renderNoBaselineReport(prGroups) + const baseline: PerfReport = JSON.parse(readFileSync(BASELINE_PATH, 'utf-8')) const historical = loadHistoricalReports() - const prGroups = groupByName(current.measurements) - - const report = - baseline && historical.length >= 2 - ? renderFullReport(prGroups, baseline, historical) - : baseline - ? renderColdStartReport(prGroups, baseline, historical.length) - : renderNoBaselineReport(prGroups) + return historical.length >= 2 + ? renderFullReport(prGroups, baseline, historical) + : renderColdStartReport(prGroups, baseline, historical.length) +} +function renderRawData(current: PerfReport): string[] { const rawData = { ...current, measurements: current.measurements.map( ({ allFrameDurationsMs: _, ...rest }) => rest ) } - - const headline = renderHeadlineSummary(prGroups) - const body = [ - ...headline.lines, - ...report.lines, + return [ '', '
Raw data', '', @@ -589,23 +611,49 @@ function main() { '', '
' ] +} - // The per-test target verdict lives in the collapsed summary, so without - // this the heading can read βœ… over a hidden ❌. It leads, because a reader - // scanning the comment takes the verdict from the first glyph and a target - // miss outranks "nothing moved since the baseline". A run that measured - // nothing has no verdict at all, and must not read as a clean one. - // Frame data that never arrived is not a pass either: with nothing sampled - // the target was not checked at all. +/** + * The per-test target verdict lives in the collapsed summary, so without this + * the heading can read βœ… over a hidden ❌. It leads, because a reader scanning + * the comment takes the verdict from the first glyph and a target miss + * outranks "nothing moved since the baseline". A run that measured nothing, or + * sampled no frames, has not checked the target and must not read as clean. + */ +function perfHeadingStatus( + current: PerfReport, + prGroups: Map, + reportStatus: string +): string { + if (current.measurements.length === 0) + return '⚠️ No measurements recorded β€” check the perf test logs' const fps = tallyFpsTarget(prGroups) - const status = - current.measurements.length === 0 - ? '⚠️ No measurements recorded β€” check the perf test logs' - : fps.below > 0 - ? `❌ ${fps.below} below ${TARGET_P5_FPS} FPS target Β· ${report.status}` - : fps.unsampled > 0 && fps.sampled === 0 - ? `⚠️ No frame samples β€” FPS target not checked Β· ${report.status}` - : report.status + if (fps.below > 0) + return `❌ ${fps.below} below ${TARGET_P5_FPS} FPS target Β· ${reportStatus}` + if (fps.unsampled > 0 && fps.sampled === 0) + return `⚠️ No frame samples β€” FPS target not checked Β· ${reportStatus}` + return reportStatus +} + +function main() { + if (!existsSync(CURRENT_PATH)) { + process.stdout.write( + renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: '⏭️ No metrics found β€” perf tests may not have run' + }) + '\n' + ) + process.exit(0) + } + + const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8')) + const prGroups = groupByName(current.measurements) + const report = renderComparison(prGroups) + + const headline = renderHeadlineSummary(prGroups) + const body = [...headline.lines, ...report.lines, ...renderRawData(current)] + const status = perfHeadingStatus(current, prGroups, report.status) process.stdout.write( renderPrReportSection({ From b9954594c9640e1278b130b0171415e1a1dce742 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 28 Oct 2026 08:18:42 +0000 Subject: [PATCH 10/19] ci: keep unsampled runs out of the perf tables; drop added comments - Frame-time medians in the comparison, cold-start and no-baseline tables skip runs that recorded no frames, the same as the headline and the FPS tally, so a 0 cannot hide a regression as a -100% change. - Remove the comments this branch added, per the repository's no-comment rule; keep one-line JSDoc on the exported section shape. - unified-report test asserts the whole output instead of looping. --- .../actions/comment-release-links/action.yaml | 2 -- .github/actions/explain-check/action.yaml | 3 -- .../workflows/publish-package-on-merge.yaml | 3 -- .../cicd/pr-playwright-deploy-and-comment.sh | 3 -- scripts/cicd/pr-report-section.test.ts | 2 -- scripts/cicd/pr-report-section.ts | 15 ++------- scripts/coverage-report.ts | 2 -- scripts/perf-report.test.ts | 20 ++++++++---- scripts/perf-report.ts | 31 ++++++++----------- scripts/unified-report.test.ts | 22 ++++++------- 10 files changed, 39 insertions(+), 64 deletions(-) diff --git a/.github/actions/comment-release-links/action.yaml b/.github/actions/comment-release-links/action.yaml index 8a1b25d69d4..e38e8749532 100644 --- a/.github/actions/comment-release-links/action.yaml +++ b/.github/actions/comment-release-links/action.yaml @@ -91,8 +91,6 @@ runs: COMMENT_FILE=$(mktemp) { - # Neutral on purpose: release-draft-create.yaml calls this for - # prereleases too, whose GitHub release is still an unpublished draft. echo "## πŸš€ $SECTION_TITLE: βœ… Publish jobs finished for \`$NEW_VERSION\`" echo "" echo "
" diff --git a/.github/actions/explain-check/action.yaml b/.github/actions/explain-check/action.yaml index 072f17d0fb7..c6b5db5af00 100644 --- a/.github/actions/explain-check/action.yaml +++ b/.github/actions/explain-check/action.yaml @@ -85,9 +85,6 @@ runs: ;; esac - # The slug is the upsert key and part of the file path below; refuse - # anything that could escape the workspace. Same rule as - # upsert-comment-section. if ! [[ "$INPUT_SLUG" =~ ^[a-z0-9-]+$ ]]; then echo "::error::explain-check: slug '$INPUT_SLUG' must match ^[a-z0-9-]+$" exit 1 diff --git a/.github/workflows/publish-package-on-merge.yaml b/.github/workflows/publish-package-on-merge.yaml index 6bc7b3f0ae0..cb72673a394 100644 --- a/.github/workflows/publish-package-on-merge.yaml +++ b/.github/workflows/publish-package-on-merge.yaml @@ -124,11 +124,8 @@ jobs: run: | set -euo pipefail mkdir -p publish-report - # The matrix counts packages the release resolved, not pushes: a - # version already on npm is skipped and still lands here. PACKAGE_COUNT=$(jq -r '.include | length' <<<"$MATRIX") { - # Resolved, not pushed: say what finished, not what landed on npm. printf '## πŸš€ Release: βœ… Publish jobs finished for %s package' "$PACKAGE_COUNT" [ "$PACKAGE_COUNT" = 1 ] || printf 's' echo '' diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index e8a9a33afc5..548ba7d2efd 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -327,9 +327,6 @@ else fi # Extract and display failed tests from all browsers (flaky tests are treated as passing) - # The count and the names come from different fields, and the index.html - # fallback in extract-playwright-counts.ts reports a count with no names β€” - # so collect first and only open the box if there is something to show. if [ $total_failed -gt 0 ]; then failed_test_lines="" diff --git a/scripts/cicd/pr-report-section.test.ts b/scripts/cicd/pr-report-section.test.ts index c2ac2f7311c..8ad901818be 100644 --- a/scripts/cicd/pr-report-section.test.ts +++ b/scripts/cicd/pr-report-section.test.ts @@ -73,8 +73,6 @@ describe('renderPrReportSection', () => { ).toContain('\n\nNo new findings.\n\n
') }) - // A failure call to action has to be readable without expanding anything: - // the heading says what is wrong, the notice says what to do about it. it('keeps the notice visible between the heading and the collapsed box', () => { const rendered = renderPrReportSection({ icon: '⚑', diff --git a/scripts/cicd/pr-report-section.ts b/scripts/cicd/pr-report-section.ts index 343f056b26c..bb6ff03512b 100644 --- a/scripts/cicd/pr-report-section.ts +++ b/scripts/cicd/pr-report-section.ts @@ -1,10 +1,4 @@ -/** - * Shared shape for every section of the unified PR report comment. - * - * A reader scanning the comment should get each check's verdict from one line - * without scrolling: the heading carries the status, a failure's call to - * action stays visible beneath it, and everything else is folded away. - */ +/** One section of the unified PR report comment. */ export interface PrReportSection { /** Emoji shown beside the title, e.g. `⚑`. */ icon: string @@ -12,10 +6,7 @@ export interface PrReportSection { title: string /** Verdict for the heading, e.g. `βœ… No regressions`. */ status: string - /** - * Markdown kept visible under the heading: what to do about a failure, or a - * caveat on the verdict. Lives outside the box, so GitHub alerts still render. - */ + /** Markdown kept visible under the heading, e.g. a failure's next step. */ notice?: string /** Markdown folded into the collapsed box. Omit for a status-only section. */ body?: string @@ -36,8 +27,6 @@ export function renderPrReportSection({ const trimmedNotice = notice?.trim() if (trimmedNotice) blocks.push(trimmedNotice) - // The blank lines around the body are load-bearing: GitHub only parses - // Markdown inside an HTML block when it is separated from the tags. const trimmedBody = body?.trim() if (trimmedBody) { blocks.push( diff --git a/scripts/coverage-report.ts b/scripts/coverage-report.ts index 4d228c67cca..517c6d5f376 100644 --- a/scripts/coverage-report.ts +++ b/scripts/coverage-report.ts @@ -124,8 +124,6 @@ function bar(covered: number, total: number): string { } const lines: string[] = [] -// The shard caveat qualifies the verdict, so it stays visible under the -// heading rather than folded into the box with the totals. const notice: string[] = [] const metadata = readCoverageMetadata( join(dirname(lcovPath), COVERAGE_METADATA_FILE) diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts index 6f3ddd8555d..49ab9104efe 100644 --- a/scripts/perf-report.test.ts +++ b/scripts/perf-report.test.ts @@ -29,11 +29,11 @@ function render(measurements: Record[]): string { } } -const heading = (output: string) => output.split('\n')[0] +function heading(output: string): string { + return output.split('\n')[0] +} describe('perf-report heading', () => { - // PerformanceHelper records p95FrameDurationMs as 0 when frame sampling - // times out. That is missing data, not a scenario running at 0 FPS. it('does not count a scenario without frame samples as missing the FPS target', () => { const output = render([ { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 }, @@ -58,9 +58,6 @@ describe('perf-report heading', () => { ) }) - // A run that recorded no frames stores 0 alongside the real samples. It must - // be dropped before the median, or it halves the frame time and turns a - // real miss into a pass. it('ignores unsampled runs when taking the median frame time', () => { const output = render([ { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, @@ -92,6 +89,17 @@ describe('perf-report heading', () => { expect(output).toContain('> **fast**: 62.5 avg FPS Β· 58.8 P5 FPS βœ…') }) + it('leaves unsampled runs out of the frame time tables', () => { + const output = render([ + { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, + { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, + { name: 'slow', frameDurationMs: 30, p95FrameDurationMs: 40 } + ]) + + expect(output).toContain('| slow: p95 frame time | 40ms |') + expect(output).toContain('| slow: avg frame time | 30ms |') + }) + it('does not report a run that measured nothing as clean', () => { const output = render([]) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index d2e1fd39de8..e8bb2ae5757 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -219,21 +219,23 @@ function frameTimeToFps(ms: number): number { return ms > 0 ? 1000 / ms : 0 } -/** - * PerformanceHelper records a frame duration of 0 when sampling times out or - * collects fewer than two timestamps. That is missing data, not a scenario - * running at 0 FPS, so it must never count as a target miss. - */ function sampledFrameMetric( samples: PerfMeasurement[], key: 'frameDurationMs' | 'p95FrameDurationMs' ): number | null { - // Drop the unsampled runs before the median: left in, a 0 drags the median - // frame time down and reports a scenario faster than it ran. const sampled = samples.filter((s) => (getMetricValue(s, key) ?? 0) > 0) return medianMetric(sampled, key) } +function reportedMedian( + samples: PerfMeasurement[], + key: MetricKey +): number | null { + return key === 'frameDurationMs' || key === 'p95FrameDurationMs' + ? sampledFrameMetric(samples, key) + : medianMetric(samples, key) +} + interface FpsTargetTally { below: number sampled: number @@ -270,7 +272,7 @@ function baselineMedian( baseSamples: PerfMeasurement[] | undefined, key: MetricKey ): number | null { - return baseSamples?.length ? medianMetric(baseSamples, key) : null + return baseSamples?.length ? reportedMedian(baseSamples, key) : null } function formatIfPresent( @@ -372,7 +374,7 @@ function compareMetric( historical: PerfReport[] ): ComparedRow | null { const { key, label, unit, minAbsDelta } = metric - const prVal = medianMetric(prSamples, key) + const prVal = reportedMedian(prSamples, key) if (prVal === null) return null const baseVal = baselineMedian(baseSamples, key) @@ -543,7 +545,7 @@ function renderColdStartReport( const baseSamples = baselineGroups.get(testName) for (const { key, label, unit } of REPORTED_METRICS) { - const prVal = medianMetric(prSamples, key) + const prVal = reportedMedian(prSamples, key) if (prVal === null) continue const baseVal = baselineMedian(baseSamples, key) @@ -574,7 +576,7 @@ function renderNoBaselineReport( ) for (const [testName, prSamples] of prGroups) { for (const { key, label, unit } of REPORTED_METRICS) { - const prVal = medianMetric(prSamples, key) + const prVal = reportedMedian(prSamples, key) if (prVal === null) continue lines.push(`| ${testName}: ${label} | ${formatValue(prVal, unit)} |`) } @@ -613,13 +615,6 @@ function renderRawData(current: PerfReport): string[] { ] } -/** - * The per-test target verdict lives in the collapsed summary, so without this - * the heading can read βœ… over a hidden ❌. It leads, because a reader scanning - * the comment takes the verdict from the first glyph and a target miss - * outranks "nothing moved since the baseline". A run that measured nothing, or - * sampled no frames, has not checked the target and must not read as clean. - */ function perfHeadingStatus( current: PerfReport, prGroups: Map, diff --git a/scripts/unified-report.test.ts b/scripts/unified-report.test.ts index 71eb0457725..ec50f973163 100644 --- a/scripts/unified-report.test.ts +++ b/scripts/unified-report.test.ts @@ -28,9 +28,6 @@ function render(args: string[]): string { } describe('unified-report', () => { - // A heading that follows another section without a blank line is swallowed - // into the previous block (a `
` keeps an HTML block open), so every - // section after the first must start on a fresh paragraph. it('separates each section from the one before it with a blank line', () => { const output = render([ '--size-status=failed', @@ -38,14 +35,15 @@ describe('unified-report', () => { '--coverage-status=failed' ]) - const headings = output.match(/^## .*$/gm) ?? [] - expect(headings).toEqual([ - '## πŸ“¦ Bundle: ⚠️ Size data collection failed β€” check the CI workflow logs', - '## ⚑ Performance: ⏳ Performance tests in progress…', - '## πŸ”¬ E2E Coverage: ⚠️ Coverage collection failed β€” check the CI workflow logs' - ]) - for (const heading of headings.slice(1)) { - expect(output).toContain(`\n\n${heading}`) - } + expect(output).toBe( + [ + '## πŸ“¦ Bundle: ⚠️ Size data collection failed β€” check the CI workflow logs', + '', + '## ⚑ Performance: ⏳ Performance tests in progress…', + '', + '## πŸ”¬ E2E Coverage: ⚠️ Coverage collection failed β€” check the CI workflow logs', + '' + ].join('\n') + ) }) }) From 96b906cf00e9f2ad65973e5d783ac7ac835f39e4 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:10:38 +0000 Subject: [PATCH 11/19] test: drop duplicate section renderer cases and assert exact padded output Addresses review threads: https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570622 https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570651 https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570668 https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570674 --- scripts/cicd/pr-report-section.test.ts | 26 +++----------------------- scripts/cicd/pr-report-section.ts | 1 - 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/scripts/cicd/pr-report-section.test.ts b/scripts/cicd/pr-report-section.test.ts index 8ad901818be..ab2aa822f71 100644 --- a/scripts/cicd/pr-report-section.test.ts +++ b/scripts/cicd/pr-report-section.test.ts @@ -3,16 +3,6 @@ import { describe, expect, it } from 'vitest' import { renderPrReportSection } from './pr-report-section' describe('renderPrReportSection', () => { - it('renders the verdict on the heading', () => { - expect( - renderPrReportSection({ - icon: '⚑', - title: 'Performance', - status: 'βœ… No regressions' - }) - ).toBe('## ⚑ Performance: βœ… No regressions') - }) - it.for([ { name: 'omitted', body: undefined }, { name: 'empty', body: '' }, @@ -50,18 +40,6 @@ describe('renderPrReportSection', () => { ) }) - it('keeps a blank line between the tags and the body so GitHub parses it', () => { - const lines = renderPrReportSection({ - icon: 'πŸ“¦', - title: 'Bundle', - status: '9.96 MB gzip', - body: '**Summary**' - }).split('\n') - - expect(lines[lines.indexOf('Details') + 1]).toBe('') - expect(lines[lines.indexOf('
') - 1]).toBe('') - }) - it('strips body padding so nested sections do not drift apart', () => { expect( renderPrReportSection({ @@ -70,7 +48,9 @@ describe('renderPrReportSection', () => { status: 'βœ… Passed', body: '\n\nNo new findings.\n\n\n' }) - ).toContain('\n\nNo new findings.\n\n') + ).toBe( + '## 🌾 Fallow: βœ… Passed\n\n
\nDetails\n\nNo new findings.\n\n
' + ) }) it('keeps the notice visible between the heading and the collapsed box', () => { diff --git a/scripts/cicd/pr-report-section.ts b/scripts/cicd/pr-report-section.ts index bb6ff03512b..ce0a15cfbc6 100644 --- a/scripts/cicd/pr-report-section.ts +++ b/scripts/cicd/pr-report-section.ts @@ -1,4 +1,3 @@ -/** One section of the unified PR report comment. */ export interface PrReportSection { /** Emoji shown beside the title, e.g. `⚑`. */ icon: string From e8f41fa998b5af460118535f5b83334710aa479d Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:10:38 +0000 Subject: [PATCH 12/19] fix: leave unsampled runs out of historical frame time stats Historical means and trend values still averaged the zero frame sentinels the sampler writes when it records no frames, so unchanged frame times were flagged as regressions once current and baseline medians dropped those zeros. Apply the same positive-sample rule to historical frame aggregates, and cover baseline, cold-start and full-comparison reports with typed fixtures. Addresses review threads: https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570614 https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570636 https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570629 --- scripts/perf-report.test.ts | 115 ++++++++++++++++++++++++++++++++++-- scripts/perf-report.ts | 37 +++++++++--- 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts index 49ab9104efe..78a85163cf7 100644 --- a/scripts/perf-report.test.ts +++ b/scripts/perf-report.test.ts @@ -5,18 +5,51 @@ import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import type { PerfMeasurement } from '../browser_tests/fixtures/helpers/PerformanceHelper' + const ROOT = join(import.meta.dirname, '..') const TSX = join(ROOT, 'node_modules/.bin/tsx') const SCRIPT = join(import.meta.dirname, 'perf-report.ts') -function render(measurements: Record[]): string { +type Sample = Pick< + PerfMeasurement, + 'name' | 'frameDurationMs' | 'p95FrameDurationMs' +> & + Partial> + +interface ReportInputs { + baseline?: Sample[] + history?: Sample[][] +} + +function writeReport(path: string, measurements: Sample[], index = 0) { + writeFileSync( + path, + JSON.stringify({ + timestamp: new Date(Date.UTC(2026, 0, index + 1)).toISOString(), + measurements + }) + ) +} + +function render( + measurements: Sample[], + { baseline, history = [] }: ReportInputs = {} +): string { const dir = mkdtempSync(join(tmpdir(), 'perf-report-')) try { mkdirSync(join(dir, 'test-results')) - writeFileSync( - join(dir, 'test-results/perf-metrics.json'), - JSON.stringify({ measurements }) - ) + writeReport(join(dir, 'test-results/perf-metrics.json'), measurements) + if (baseline) { + mkdirSync(join(dir, 'temp/perf-baseline'), { recursive: true }) + writeReport(join(dir, 'temp/perf-baseline/perf-metrics.json'), baseline) + } + if (history.length > 0) { + mkdirSync(join(dir, 'temp/perf-history'), { recursive: true }) + history.forEach((run, index) => + writeReport(join(dir, `temp/perf-history/${index}.json`), run, index) + ) + } const result = spawnSync(TSX, [SCRIPT], { cwd: dir, encoding: 'utf8' }) if (result.status !== 0) { throw new Error( @@ -29,6 +62,15 @@ function render(measurements: Record[]): string { } } +function frames(name: string, ...frameMs: number[]): Sample[] { + return frameMs.map((ms) => ({ + name, + frameDurationMs: ms, + p95FrameDurationMs: ms, + totalBlockingTimeMs: 0 + })) +} + function heading(output: string): string { return output.split('\n')[0] } @@ -108,3 +150,66 @@ describe('perf-report heading', () => { ) }) }) + +describe('perf-report baseline comparison', () => { + it('compares sampled frame medians against the baseline on a cold start', () => { + const output = render(frames('grow', 0, 40), { + baseline: frames('grow', 0, 20) + }) + + expect(output).toContain('| grow: p95 frame time | 20ms | 40ms | +100% |') + expect(output).toContain('| grow: avg frame time | 20ms | 40ms | +100% |') + expect(output).toContain('| grow: TBT | 0ms | 0ms | +0% |') + }) + + it('does not flag unchanged frame times when history has unsampled runs', () => { + const output = render(frames('steady', 0, 0, 17), { + baseline: frames('steady', 0, 0, 17), + history: [ + frames('steady', 0, 0, 16), + frames('steady', 0, 0, 17), + frames('steady', 0, 0, 17) + ] + }) + + expect(heading(output)).toBe( + '## ⚑ Performance: βœ… No regressions detected' + ) + expect(output).toContain('| steady: p95 frame time | 17ms | 17ms | +0% |') + expect(output).toContain('| steady: TBT | 0ms | 0ms | +0% |') + expect(output).toMatch(/\| steady: p95 frame time \| 17ms \| 1ms \|/) + expect(output).toMatch( + /\| steady: p95 frame time \| \S+ \| [^|]+ \| 17ms \|/ + ) + }) + + it('leaves wholly unsampled history runs out of the frame time variance', () => { + const output = render(frames('steady', 0, 17), { + baseline: frames('steady', 0, 17), + history: [ + frames('steady', 0, 0, 16), + frames('steady', 0, 0, 18), + frames('steady', 0, 0) + ] + }) + + expect(heading(output)).toBe( + '## ⚑ Performance: βœ… No regressions detected' + ) + expect(output).toContain('| steady: p95 frame time | 17ms | 1ms |') + expect(output).toContain('| steady: TBT | 0ms | 0ms |') + }) + + it('flags a real frame time regression against sampled history', () => { + const output = render(frames('grow', 0, 40), { + baseline: frames('grow', 0, 20), + history: [frames('grow', 0, 20), frames('grow', 0, 21)] + }) + + expect(heading(output)).toContain('2 regressions detected') + expect(output.split('
')[0]).toContain( + 'Review the regressions below and profile the affected scenarios before merging.' + ) + expect(output).toContain('| grow: p95 frame time | 20ms | 40ms | +100% |') + }) +}) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index e8bb2ae5757..f9f0a191ba5 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -134,7 +134,7 @@ function getHistoricalStats( const group = groupByName(r.measurements) const samples = group.get(testName) if (samples) { - const mean = meanMetric(samples, metric) + const mean = reportedMean(samples, metric) if (mean !== null) values.push(mean) } } @@ -154,9 +154,8 @@ function getHistoricalTimeSeries( const group = groupByName(r.measurements) const samples = group.get(testName) if (samples) { - values.push( - samples.reduce((sum, s) => sum + s[metric], 0) / samples.length - ) + const mean = reportedMean(samples, metric) + if (mean !== null) values.push(mean) } } return values @@ -219,23 +218,45 @@ function frameTimeToFps(ms: number): number { return ms > 0 ? 1000 / ms : 0 } +type FrameMetricKey = 'frameDurationMs' | 'p95FrameDurationMs' + +function isFrameMetric(key: MetricKey): key is FrameMetricKey { + return key === 'frameDurationMs' || key === 'p95FrameDurationMs' +} + +function frameSampled( + samples: PerfMeasurement[], + key: FrameMetricKey +): PerfMeasurement[] { + return samples.filter((s) => (getMetricValue(s, key) ?? 0) > 0) +} + function sampledFrameMetric( samples: PerfMeasurement[], - key: 'frameDurationMs' | 'p95FrameDurationMs' + key: FrameMetricKey ): number | null { - const sampled = samples.filter((s) => (getMetricValue(s, key) ?? 0) > 0) - return medianMetric(sampled, key) + return medianMetric(frameSampled(samples, key), key) } function reportedMedian( samples: PerfMeasurement[], key: MetricKey ): number | null { - return key === 'frameDurationMs' || key === 'p95FrameDurationMs' + return isFrameMetric(key) ? sampledFrameMetric(samples, key) : medianMetric(samples, key) } +function reportedMean( + samples: PerfMeasurement[], + key: MetricKey +): number | null { + return meanMetric( + isFrameMetric(key) ? frameSampled(samples, key) : samples, + key + ) +} + interface FpsTargetTally { below: number sampled: number From 6e6b6e472f51a9db6b7a3b678545c527834dade2 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:10:39 +0000 Subject: [PATCH 13/19] test: cover both failed-test outcomes of the Playwright report section Runs the real script with a failed count and a named test, and with a failed count but no extracted names. Also says "1 test failed" instead of "1 tests failed". Addresses review thread: https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570646 --- .../cicd/pr-playwright-deploy-and-comment.sh | 6 +- .../pr-playwright-deploy-and-comment.test.ts | 66 +++++++++++++++++-- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index 548ba7d2efd..56e3e3adba4 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -329,6 +329,8 @@ else # Extract and display failed tests from all browsers (flaky tests are treated as passing) if [ $total_failed -gt 0 ]; then failed_test_lines="" + failed_tests_label="$total_failed tests" + [ "$total_failed" -eq 1 ] && failed_tests_label="1 test" for counts_json in "${counts_array[@]}"; do [ -z "$counts_json" ] || [ "$counts_json" = "{}" ] && continue @@ -361,7 +363,7 @@ $test_line" if [ -n "$failed_test_lines" ]; then comment="$comment -> ❌ $total_failed tests failed. Expand the failed tests below, review their traces and fix the failures before merging. +> ❌ $failed_tests_label failed. Expand the failed tests below, review their traces and fix the failures before merging.
❌ Failed Tests @@ -372,7 +374,7 @@ ${failed_test_lines#$'\n'} else comment="$comment -> ❌ $total_failed tests failed. Open the browser reports below, review the traces and fix the failures before merging." +> ❌ $failed_tests_label failed. Open the browser reports below, review the traces and fix the failures before merging." fi fi diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts index a9bc6efdbf1..ad98e368229 100644 --- a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts +++ b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts @@ -42,7 +42,12 @@ const uploadedProjects = [ ) ] -function runWithReports(projects: string[]) { +const passingCounts = { passed: 2, failed: 0, flaky: 0, skipped: 0, total: 2 } + +function runWithReports( + projects: string[], + countsByProject: Record = {} +) { const root = mkdtempSync(join(tmpdir(), 'playwright-pr-report-')) const bin = join(root, 'bin') const wranglerLog = join(root, 'wrangler.log') @@ -59,15 +64,19 @@ echo "https://abc.$project.pages.dev" writeFileSync( join(bin, 'tsx'), `#!/bin/bash -echo '{"passed":2,"failed":0,"flaky":0,"skipped":0,"total":2}' +cat "$2/counts.json" ` ) chmodSync(join(bin, 'wrangler'), 0o755) chmodSync(join(bin, 'tsx'), 0o755) - for (const project of projects) - mkdirSync(join(root, 'reports', `playwright-report-${project}`), { - recursive: true - }) + for (const project of projects) { + const reportDir = join(root, 'reports', `playwright-report-${project}`) + mkdirSync(reportDir, { recursive: true }) + writeFileSync( + join(reportDir, 'counts.json'), + JSON.stringify(countsByProject[project] ?? passingCounts) + ) + } const result = spawnSync('bash', [script, '1', 'feature', 'completed'], { cwd: root, @@ -117,4 +126,49 @@ describe('PR Playwright report section', () => { expect(summary).toMatch(/^## 🎭 Playwright: ❌ .*, 1 missing/) expect(summary).toContain('- **cloud**: ❌ Report missing') }) + + it('lists each named failed test with its trace in a collapsed box', () => { + const { status, summary } = runWithReports(uploadedProjects, { + chromium: { + passed: 1, + failed: 1, + flaky: 0, + skipped: 0, + total: 2, + failures: [ + { + name: 'saves a workflow', + file: 'browser_tests/tests/save.spec.ts', + traceUrl: 'https://abc.chromium.pages.dev/trace/1' + } + ] + } + }) + + expect(status).toBe(0) + expect(summary).toMatch(/^## 🎭 Playwright: ❌ .*, 1 failed/) + expect(summary).toContain( + `> ❌ 1 test failed. Expand the failed tests below, review their traces and fix the failures before merging. + +
+❌ Failed Tests + +- [saves a workflow](https://github.com/Comfy-Org/ComfyUI_frontend/blob/sha/browser_tests/tests/save.spec.ts): [View trace](https://abc.chromium.pages.dev/trace/1) + +
` + ) + }) + + it('points to the browser reports when failed tests have no extracted names', () => { + const { status, summary } = runWithReports(uploadedProjects, { + chromium: { passed: 1, failed: 1, flaky: 0, skipped: 0, total: 2 } + }) + + expect(status).toBe(0) + expect(summary).toMatch(/^## 🎭 Playwright: ❌ .*, 1 failed/) + expect(summary).toContain( + '> ❌ 1 test failed. Open the browser reports below, review the traces and fix the failures before merging.' + ) + expect(summary).not.toContain('Failed Tests') + }) }) From 486f96cf24918c636e666416ec9185e5c9953d84 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:10:39 +0000 Subject: [PATCH 14/19] test: cover the no-line-coverage heading at the coverage producer Addresses review thread: https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570683 --- scripts/coverage-report.test.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/coverage-report.test.ts b/scripts/coverage-report.test.ts index a7ee9dd08b2..7332ba28aa4 100644 --- a/scripts/coverage-report.test.ts +++ b/scripts/coverage-report.test.ts @@ -16,10 +16,13 @@ const TRACEFILE = Array.from( ).join('') /** A string is written verbatim, so cases can supply unparseable metadata. */ -function render(metadata: Record | string | null): string { +function render( + metadata: Record | string | null, + tracefile = TRACEFILE +): string { const dir = mkdtempSync(join(tmpdir(), 'coverage-report-')) const lcov = join(dir, 'coverage.lcov') - writeFileSync(lcov, TRACEFILE) + writeFileSync(lcov, tracefile) if (metadata !== null) { writeFileSync( join(dir, 'coverage-metadata.json'), @@ -93,3 +96,13 @@ describe('coverage-report shard banner', () => { expect(render(metadata)).toContain('| Lines | 3 | 6 | 50.0%') }) }) + +describe('coverage-report line totals', () => { + it('warns instead of showing a percentage when no lines were recorded', () => { + const output = render({ complete: true }, '') + + expect(output.split('\n')[0]).toBe( + '## πŸ”¬ E2E Coverage: ⚠️ No line coverage recorded' + ) + }) +}) From 3d7217bf871b91972b51477984b96fa3322c711d Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:10:39 +0000 Subject: [PATCH 15/19] test: reject invalid explain-check slugs before any report file is written Addresses review thread: https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4398570694 --- scripts/cicd/pr-report-disclosure.test.ts | 89 ++++++++++++++--------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/scripts/cicd/pr-report-disclosure.test.ts b/scripts/cicd/pr-report-disclosure.test.ts index 12e05008cc0..e7df34f9871 100644 --- a/scripts/cicd/pr-report-disclosure.test.ts +++ b/scripts/cicd/pr-report-disclosure.test.ts @@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process' import { mkdirSync, mkdtempSync, + readdirSync, readFileSync, rmSync, writeFileSync @@ -25,6 +26,45 @@ function fixture() { } } +function runExplainCheck( + workspace: string, + inputs: { slug: string; status: string } +) { + const actionFile = parseDocument( + readFileSync( + join(ROOT, '.github/actions/explain-check/action.yaml'), + 'utf8' + ) + ) + const script = actionFile.getIn(['runs', 'steps', 0, 'run']) + expect(typeof script).toBe('string') + const details = join(workspace, 'findings.md') + writeFileSync( + details, + '| Finding | Remedy |\n| --- | --- |\n| Clone | Extract |' + ) + return spawnSync('bash', ['-c', String(script)], { + encoding: 'utf8', + env: { + ...process.env, + INPUT_SLUG: inputs.slug, + INPUT_TITLE: 'Test check', + INPUT_ICON: '', + INPUT_STATUS: inputs.status, + INPUT_HEADLINE: 'Check summary.', + INPUT_DETAILS_FILE: details, + INPUT_REPRODUCE: 'pnpm check', + INPUT_DOCS: 'docs/check.md', + GITHUB_WORKSPACE: workspace, + GITHUB_OUTPUT: join(workspace, 'output'), + GITHUB_STEP_SUMMARY: join(workspace, 'summary'), + GITHUB_SERVER_URL: 'https://github.com', + GITHUB_REPOSITORY: 'owner/repo', + GITHUB_RUN_ID: '123' + } + }) +} + describe('shared PR report disclosure', () => { it.for([ ['pass', ''], @@ -36,39 +76,7 @@ describe('shared PR report disclosure', () => { 'collapses check details for %s with a visible action when needed', ([status, action]) => { using dir = fixture() - const actionFile = parseDocument( - readFileSync( - join(ROOT, '.github/actions/explain-check/action.yaml'), - 'utf8' - ) - ) - const script = actionFile.getIn(['runs', 'steps', 0, 'run']) - expect(typeof script).toBe('string') - const details = join(dir.root, 'findings.md') - writeFileSync( - details, - '| Finding | Remedy |\n| --- | --- |\n| Clone | Extract |' - ) - const result = spawnSync('bash', ['-c', String(script)], { - encoding: 'utf8', - env: { - ...process.env, - INPUT_SLUG: 'test', - INPUT_TITLE: 'Test check', - INPUT_ICON: '', - INPUT_STATUS: status, - INPUT_HEADLINE: 'Check summary.', - INPUT_DETAILS_FILE: details, - INPUT_REPRODUCE: 'pnpm check', - INPUT_DOCS: 'docs/check.md', - GITHUB_WORKSPACE: dir.root, - GITHUB_OUTPUT: join(dir.root, 'output'), - GITHUB_STEP_SUMMARY: join(dir.root, 'summary'), - GITHUB_SERVER_URL: 'https://github.com', - GITHUB_REPOSITORY: 'owner/repo', - GITHUB_RUN_ID: '123' - } - }) + const result = runExplainCheck(dir.root, { slug: 'test', status }) expect(result.status).toBe(0) const output = readFileSync(join(dir.root, 'summary'), 'utf8') const visible = output.split('
')[0] @@ -82,6 +90,21 @@ describe('shared PR report disclosure', () => { } ) + it.for(['../escape', 'Upper', 'a/b', '', 'a;b', 'a\nb'])( + 'rejects slug %j before writing any report file', + (slug) => { + using dir = fixture() + const workspace = join(dir.root, 'workspace') + mkdirSync(workspace) + const result = runExplainCheck(workspace, { slug, status: 'pass' }) + + expect(result.status).not.toBe(0) + expect(result.stdout).toContain('explain-check: slug') + expect(readdirSync(workspace)).toEqual(['findings.md']) + expect(readdirSync(dir.root)).toEqual(['workspace']) + } + ) + it('keeps failed FPS targets visible while collapsing passing scenario timings', () => { using dir = fixture() mkdirSync(join(dir.root, 'test-results')) From ed286a86ebb37c480fcebe11a5528761fa1740d8 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:20:41 +0000 Subject: [PATCH 16/19] fix: stop history table headings overstating frame sample counts Frame rows now skip history runs with no frame samples, so the variance and trend headings say "up to N" runs per metric instead of "last N". --- scripts/perf-report.test.ts | 3 +++ scripts/perf-report.ts | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts index 78a85163cf7..05069fbc0d7 100644 --- a/scripts/perf-report.test.ts +++ b/scripts/perf-report.test.ts @@ -196,6 +196,9 @@ describe('perf-report baseline comparison', () => { expect(heading(output)).toBe( '## ⚑ Performance: βœ… No regressions detected' ) + expect(output).toContain( + 'Historical variance (up to 3 runs per metric)' + ) expect(output).toContain('| steady: p95 frame time | 17ms | 1ms |') expect(output).toContain('| steady: TBT | 0ms | 0ms |') }) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index f9f0a191ba5..02e5cb172dc 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -498,7 +498,7 @@ function renderHistoricalVariance( historical: PerfReport[] ): string[] { const lines = [ - `
Historical variance (last ${historical.length} runs)`, + `
Historical variance (up to ${historical.length} runs per metric)`, '', '| Metric | ΞΌ | Οƒ | CV |', '|--------|---|---|-----|' @@ -536,7 +536,7 @@ function renderTrend( if (trendRows.length === 0) return [] return [ '', - `
Trend (last ${historical.length} commits on main)`, + `
Trend (up to ${historical.length} commits on main per metric)`, '', '| Metric | Trend | Dir | Latest |', '|--------|-------|-----|--------|', From b2e0b9bd599ef1783cec2c9445ea5912af64482b Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 16:44:52 +0000 Subject: [PATCH 17/19] test: accept a hyphen-and-digit explain-check slug and write its section Addresses review thread: https://github.com/Comfy-Org/ComfyUI_frontend/pull/18331#discussion_r4415085689 --- scripts/cicd/pr-report-disclosure.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/cicd/pr-report-disclosure.test.ts b/scripts/cicd/pr-report-disclosure.test.ts index e7df34f9871..06917f80dd3 100644 --- a/scripts/cicd/pr-report-disclosure.test.ts +++ b/scripts/cicd/pr-report-disclosure.test.ts @@ -105,6 +105,23 @@ describe('shared PR report disclosure', () => { } ) + it('accepts a slug with hyphens and digits and writes its section file', () => { + using dir = fixture() + const result = runExplainCheck(dir.root, { + slug: 'custom-nodes-2', + status: 'pass' + }) + + expect(result.status).toBe(0) + const sectionFile = join(dir.root, '.explain-check-custom-nodes-2.md') + expect(readFileSync(join(dir.root, 'output'), 'utf8')).toBe( + `file=${sectionFile}\n` + ) + expect(readFileSync(sectionFile, 'utf8')).toMatch( + /^## {2}Test check: βœ… Passed\n/ + ) + }) + it('keeps failed FPS targets visible while collapsing passing scenario timings', () => { using dir = fixture() mkdirSync(join(dir.root, 'test-results')) From 746a36225e0d3d5333016d8a14060a33f0875dc8 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 31 Oct 2026 06:21:31 +0000 Subject: [PATCH 18/19] ci: address final report review findings Assert no frame trend from two sampled runs, label the preview URL without promising the alias succeeded, and drop the private helper's JSDoc. --- .github/workflows/ci-vercel-billing-web-preview.yaml | 2 +- scripts/coverage-report.test.ts | 1 - scripts/perf-report.test.ts | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-vercel-billing-web-preview.yaml b/.github/workflows/ci-vercel-billing-web-preview.yaml index 30d8510f566..9228803eb72 100644 --- a/.github/workflows/ci-vercel-billing-web-preview.yaml +++ b/.github/workflows/ci-vercel-billing-web-preview.yaml @@ -133,7 +133,7 @@ jobs: Details **Links** - - πŸ”— Stable: ${{ steps.urls.outputs.stable-url }} + - πŸ”— Preview: ${{ steps.urls.outputs.stable-url }} - πŸ“Œ This commit: ${{ steps.deploy.outputs.url }}
diff --git a/scripts/coverage-report.test.ts b/scripts/coverage-report.test.ts index 7332ba28aa4..ad089ee0171 100644 --- a/scripts/coverage-report.test.ts +++ b/scripts/coverage-report.test.ts @@ -15,7 +15,6 @@ const TRACEFILE = Array.from( `SF:src/components/Component${index}.vue\nDA:1,1\nDA:2,0\nLF:2\nLH:1\nend_of_record\n` ).join('') -/** A string is written verbatim, so cases can supply unparseable metadata. */ function render( metadata: Record | string | null, tracefile = TRACEFILE diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts index 05069fbc0d7..256c2910b50 100644 --- a/scripts/perf-report.test.ts +++ b/scripts/perf-report.test.ts @@ -201,6 +201,7 @@ describe('perf-report baseline comparison', () => { ) expect(output).toContain('| steady: p95 frame time | 17ms | 1ms |') expect(output).toContain('| steady: TBT | 0ms | 0ms |') + expect(output).not.toMatch(/\| steady: p95 frame time \| [β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ]+ \|/) }) it('flags a real frame time regression against sampled history', () => { From 3654782194f816885569bd276e04be05eb029e71 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 31 Oct 2026 06:22:30 +0000 Subject: [PATCH 19/19] ci: format report comment test --- .../pr-playwright-deploy-and-comment.test.ts | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts index 01a5ce83a49..06fe400ddf2 100644 --- a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts +++ b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts @@ -46,7 +46,13 @@ const uploadedProjects = [ ) ] -const passingCounts: TestCounts = { passed: 2, failed: 0, flaky: 0, skipped: 0, total: 2 } +const passingCounts: TestCounts = { + passed: 2, + failed: 0, + flaky: 0, + skipped: 0, + total: 2 +} function runWithReports( projects: string[], @@ -126,25 +132,22 @@ describe('PR Playwright report section', () => { }) it('links a failed test at its path in the repository and in the report', () => { - const { summary, extractorCalls } = runWithReports( - ['chromium'], - { - chromium: { - passed: 0, - failed: 1, - flaky: 0, - skipped: 0, - total: 1, - failures: [ - { - name: 'saves a workflow', - file: 'browser_tests/tests/save.spec.ts', - reportUrl: 'https://abc.chromium.pages.dev/#?testId=fail-1' - } - ] - } + const { summary, extractorCalls } = runWithReports(['chromium'], { + chromium: { + passed: 0, + failed: 1, + flaky: 0, + skipped: 0, + total: 1, + failures: [ + { + name: 'saves a workflow', + file: 'browser_tests/tests/save.spec.ts', + reportUrl: 'https://abc.chromium.pages.dev/#?testId=fail-1' + } + ] } - ) + }) expect(extractorCalls).toContain('browser_tests') expect(summary).toContain(