diff --git a/.claude/skills/perf-fix-with-proof/SKILL.md b/.claude/skills/perf-fix-with-proof/SKILL.md index 6ffba72aa5f..25f4d957077 100644 --- a/.claude/skills/perf-fix-with-proof/SKILL.md +++ b/.claude/skills/perf-fix-with-proof/SKILL.md @@ -118,11 +118,16 @@ Implement the fix. The `@perf` test from PR1 is now on main and will run automat The `ci-perf-report.yaml` posts a comment like: ```markdown -## ⚑ Performance Report +## ⚑ Performance: βœ… No regressions detected + +
+Details | Metric | Baseline | PR (n=3) | Ξ” | Sig | | --------------------- | -------- | -------- | ---- | --- | | : style recalcs | 450 | 12 | -97% | 🟒 | + +
``` If Ξ” is negative for the target metric, the fix is proven. diff --git a/.github/actions/comment-release-links/action.yaml b/.github/actions/comment-release-links/action.yaml index 222867a771b..e38e8749532 100644 --- a/.github/actions/comment-release-links/action.yaml +++ b/.github/actions/comment-release-links/action.yaml @@ -53,6 +53,7 @@ runs: fi SECTION_NAME='release' + SECTION_TITLE='Release' MESSAGE='Publish jobs finished successfully:' LINKS_VALUE='' @@ -64,18 +65,22 @@ runs: ;; packages/design-system/package.json) SECTION_NAME='design-system-release' + SECTION_TITLE='Design System Release' LINKS_VALUE='npm design-system|https://www.npmjs.com/package/@comfyorg/design-system/v/{{version}}' ;; packages/account-core/package.json) SECTION_NAME='account-core-release' + SECTION_TITLE='Account Core Release' LINKS_VALUE='npm account-core|https://www.npmjs.com/package/@comfyorg/account-core/v/{{version}}' ;; packages/billing-contract/package.json) SECTION_NAME='billing-contract-release' + SECTION_TITLE='Billing Contract Release' LINKS_VALUE='npm billing-contract|https://www.npmjs.com/package/@comfyorg/billing-contract/v/{{version}}' ;; packages/ingest-types/package.json) SECTION_NAME='ingest-types-release' + SECTION_TITLE='Ingest Types Release' LINKS_VALUE='npm ingest-types|https://www.npmjs.com/package/@comfyorg/ingest-types/v/{{version}}' ;; esac @@ -86,6 +91,11 @@ runs: COMMENT_FILE=$(mktemp) { + echo "## πŸš€ $SECTION_TITLE: βœ… Publish jobs finished for \`$NEW_VERSION\`" + echo "" + echo "
" + echo "Details" + echo "" echo "$MESSAGE" echo "" echo "- $DIFF_LABEL: [\`$DIFF_PREFIX$PREV_VERSION...$DIFF_PREFIX$NEW_VERSION\`]($DIFF_URL)" @@ -105,6 +115,7 @@ runs: done <<< "$LINKS_VALUE" echo "" + echo "
" } > "$COMMENT_FILE" { diff --git a/.github/actions/explain-check/action.yaml b/.github/actions/explain-check/action.yaml index 841e3d69368..c6b5db5af00 100644 --- a/.github/actions/explain-check/action.yaml +++ b/.github/actions/explain-check/action.yaml @@ -85,6 +85,11 @@ runs: ;; esac + if ! [[ "$INPUT_SLUG" =~ ^[a-z0-9-]+$ ]]; then + echo "::error::explain-check: slug '$INPUT_SLUG' must match ^[a-z0-9-]+$" + exit 1 + fi + # Must live inside the workspace: read-and-upsert-comment-section guards # on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and # would silently no-op on a RUNNER_TEMP path. diff --git a/.github/workflows/ci-tests-storybook.yaml b/.github/workflows/ci-tests-storybook.yaml index 8f28f91bd41..96b787a7b30 100644 --- a/.github/workflows/ci-tests-storybook.yaml +++ b/.github/workflows/ci-tests-storybook.yaml @@ -236,8 +236,14 @@ jobs: pr-number: ${{ github.event.pull_request.number }} section-name: chromatic section-content: | - ### 🎨 Chromatic Visual Tests - - πŸ“Š [View Chromatic Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }}) + ## 🎨 Chromatic: πŸ“€ Uploaded β€” [View Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }}) + +
+ Details + + **Links** - πŸ“š [View Chromatic Storybook](${{ needs.chromatic-deployment.outputs.chromatic-storybook-url }}) + +
comment-marker: '' token: ${{ github.token }} diff --git a/.github/workflows/ci-vercel-billing-web-preview.yaml b/.github/workflows/ci-vercel-billing-web-preview.yaml index c904956cf2a..9228803eb72 100644 --- a/.github/workflows/ci-vercel-billing-web-preview.yaml +++ b/.github/workflows/ci-vercel-billing-web-preview.yaml @@ -127,13 +127,13 @@ jobs: section-name: billing-web-preview token: ${{ secrets.GITHUB_TOKEN }} section-content: |- - ## πŸ’³ Billing Web Preview - - **Billing Web Preview:** ${{ steps.urls.outputs.stable-url }} + ## πŸ’³ Billing Web Preview: βœ… Deployed β€” [View Preview](${{ steps.urls.outputs.stable-url }})
- Deployment details + Details - This commit: ${{ steps.deploy.outputs.url }} + **Links** + - πŸ”— Preview: ${{ steps.urls.outputs.stable-url }} + - πŸ“Œ This commit: ${{ steps.deploy.outputs.url }}
diff --git a/.github/workflows/publish-package-on-merge.yaml b/.github/workflows/publish-package-on-merge.yaml index 09ca8ecf634..cb72673a394 100644 --- a/.github/workflows/publish-package-on-merge.yaml +++ b/.github/workflows/publish-package-on-merge.yaml @@ -124,7 +124,15 @@ jobs: run: | set -euo pipefail mkdir -p publish-report + PACKAGE_COUNT=$(jq -r '.include | length' <<<"$MATRIX") { + printf '## πŸš€ Release: βœ… Publish jobs finished for %s package' "$PACKAGE_COUNT" + [ "$PACKAGE_COUNT" = 1 ] || printf 's' + echo '' + echo '' + echo '
' + echo 'Details' + echo '' echo 'Publish jobs finished successfully:' echo '' jq -r ' @@ -133,6 +141,7 @@ jobs: ' <<<"$MATRIX" echo "- Published from [\`${MERGE_SHA:0:10}\`](https://github.com/${REPO}/commit/${MERGE_SHA})" echo '' + echo '
' } > publish-report/report.md printf '%s' "$PR_NUMBER" > publish-report/pr-number.txt diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index 6c5a490a681..f64420f70b6 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -285,14 +285,9 @@ else fi if [ $total_failed -gt 0 ]; then - comment="$comment - -> ❌ $total_failed tests failed. Expand the failed tests below, review their reports and fix the failures before merging. - -
-Failed tests - -" + failed_test_lines="" + failed_tests_label="$total_failed tests" + [ "$total_failed" -eq 1 ] && failed_tests_label="1 test" for counts_json in "${counts_array[@]}"; do [ -z "$counts_json" ] || [ "$counts_json" = "{}" ] && continue @@ -312,15 +307,29 @@ else test_line="$test_line: [View test report]($report_url)" fi - comment="$comment + failed_test_lines="$failed_test_lines $test_line" done <<< "$failures" fi fi done - comment="$comment + + if [ -n "$failed_test_lines" ]; then + comment="$comment + +> ❌ $failed_tests_label failed. Expand the failed tests below, review their reports and fix the failures before merging. + +
+Failed tests + +${failed_test_lines#$'\n'}
" + else + comment="$comment + +> ❌ $failed_tests_label failed. Open the browser reports below, review the failed tests and fix the failures before merging." + fi fi screenshot_diffs=$(count_screenshot_diffs) || screenshot_diffs=0 diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts index 48f19a9334f..06fe400ddf2 100644 --- a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts +++ b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts @@ -14,6 +14,10 @@ import { describe, expect, it, onTestFinished } from 'vitest' import { parse } from 'yaml' import { z } from 'zod' +import type { extractTestCounts } from './extract-playwright-counts' + +type TestCounts = ReturnType + const script = join(import.meta.dirname, 'pr-playwright-deploy-and-comment.sh') const e2eWorkflow = z @@ -42,9 +46,18 @@ const uploadedProjects = [ ) ] -const passingCounts = '{"passed":2,"failed":0,"flaky":0,"skipped":0,"total":2}' +const passingCounts: TestCounts = { + passed: 2, + failed: 0, + flaky: 0, + skipped: 0, + total: 2 +} -function runWithReports(projects: string[], counts = passingCounts) { +function runWithReports( + projects: string[], + countsByProject: Record = {} +) { const root = mkdtempSync(join(tmpdir(), 'playwright-pr-report-')) const bin = join(root, 'bin') const wranglerLog = join(root, 'wrangler.log') @@ -64,15 +77,19 @@ echo "https://abc.$project.pages.dev" `#!/bin/bash echo "$*" >> "${extractorLog}" echo 'Could not list failed tests: diagnostic' >&2 -echo '${counts}' +cat "$2/counts.json" ` ) chmodSync(join(bin, 'wrangler'), 0o755) chmodSync(join(bin, 'tsx'), 0o755) - for (const project of projects) - mkdirSync(join(root, 'reports', `playwright-report-${project}`), { - recursive: true - }) + for (const project of projects) { + const reportDir = join(root, 'reports', `playwright-report-${project}`) + mkdirSync(reportDir, { recursive: true }) + writeFileSync( + join(reportDir, 'counts.json'), + JSON.stringify(countsByProject[project] ?? passingCounts) + ) + } const result = spawnSync('bash', [script, '1', 'feature', 'completed'], { cwd: root, @@ -115,10 +132,22 @@ describe('PR Playwright report section', () => { }) it('links a failed test at its path in the repository and in the report', () => { - const { summary, extractorCalls } = runWithReports( - ['chromium'], - '{"passed":0,"failed":1,"flaky":0,"skipped":0,"total":1,"failures":[{"name":"saves a workflow","file":"browser_tests/tests/save.spec.ts","reportUrl":"https://abc.chromium.pages.dev/#?testId=fail-1"}]}' - ) + const { summary, extractorCalls } = runWithReports(['chromium'], { + chromium: { + passed: 0, + failed: 1, + flaky: 0, + skipped: 0, + total: 1, + failures: [ + { + name: 'saves a workflow', + file: 'browser_tests/tests/save.spec.ts', + reportUrl: 'https://abc.chromium.pages.dev/#?testId=fail-1' + } + ] + } + }) expect(extractorCalls).toContain('browser_tests') expect(summary).toContain( @@ -135,4 +164,49 @@ describe('PR Playwright report section', () => { expect(summary).toMatch(/^## 🎭 Playwright: ❌ .*, 1 missing/) expect(summary).toContain('- **cloud**: ❌ Report missing') }) + + it('lists each named failed test with its report in a collapsed box', () => { + const { status, summary } = runWithReports(uploadedProjects, { + chromium: { + passed: 1, + failed: 1, + flaky: 0, + skipped: 0, + total: 2, + failures: [ + { + name: 'saves a workflow', + file: 'browser_tests/tests/save.spec.ts', + reportUrl: 'https://abc.chromium.pages.dev/#?testId=fail-1' + } + ] + } + }) + + expect(status).toBe(0) + expect(summary).toMatch(/^## 🎭 Playwright: ❌ .*, 1 failed/) + expect(summary).toContain( + `> ❌ 1 test failed. Expand the failed tests below, review their reports and fix the failures before merging. + +
+Failed tests + +- [saves a workflow](https://github.com/Comfy-Org/ComfyUI_frontend/blob/sha/browser_tests/tests/save.spec.ts): [View test report](https://abc.chromium.pages.dev/#?testId=fail-1) + +
` + ) + }) + + it('points to the browser reports when failed tests have no extracted names', () => { + const { status, summary } = runWithReports(uploadedProjects, { + chromium: { passed: 1, failed: 1, flaky: 0, skipped: 0, total: 2 } + }) + + expect(status).toBe(0) + expect(summary).toMatch(/^## 🎭 Playwright: ❌ .*, 1 failed/) + expect(summary).toContain( + '> ❌ 1 test failed. Open the browser reports below, review the failed tests and fix the failures before merging.' + ) + expect(summary).not.toContain('Failed tests') + }) }) diff --git a/scripts/cicd/pr-report-disclosure.test.ts b/scripts/cicd/pr-report-disclosure.test.ts index 12e05008cc0..06917f80dd3 100644 --- a/scripts/cicd/pr-report-disclosure.test.ts +++ b/scripts/cicd/pr-report-disclosure.test.ts @@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process' import { mkdirSync, mkdtempSync, + readdirSync, readFileSync, rmSync, writeFileSync @@ -25,6 +26,45 @@ function fixture() { } } +function runExplainCheck( + workspace: string, + inputs: { slug: string; status: string } +) { + const actionFile = parseDocument( + readFileSync( + join(ROOT, '.github/actions/explain-check/action.yaml'), + 'utf8' + ) + ) + const script = actionFile.getIn(['runs', 'steps', 0, 'run']) + expect(typeof script).toBe('string') + const details = join(workspace, 'findings.md') + writeFileSync( + details, + '| Finding | Remedy |\n| --- | --- |\n| Clone | Extract |' + ) + return spawnSync('bash', ['-c', String(script)], { + encoding: 'utf8', + env: { + ...process.env, + INPUT_SLUG: inputs.slug, + INPUT_TITLE: 'Test check', + INPUT_ICON: '', + INPUT_STATUS: inputs.status, + INPUT_HEADLINE: 'Check summary.', + INPUT_DETAILS_FILE: details, + INPUT_REPRODUCE: 'pnpm check', + INPUT_DOCS: 'docs/check.md', + GITHUB_WORKSPACE: workspace, + GITHUB_OUTPUT: join(workspace, 'output'), + GITHUB_STEP_SUMMARY: join(workspace, 'summary'), + GITHUB_SERVER_URL: 'https://github.com', + GITHUB_REPOSITORY: 'owner/repo', + GITHUB_RUN_ID: '123' + } + }) +} + describe('shared PR report disclosure', () => { it.for([ ['pass', ''], @@ -36,39 +76,7 @@ describe('shared PR report disclosure', () => { 'collapses check details for %s with a visible action when needed', ([status, action]) => { using dir = fixture() - const actionFile = parseDocument( - readFileSync( - join(ROOT, '.github/actions/explain-check/action.yaml'), - 'utf8' - ) - ) - const script = actionFile.getIn(['runs', 'steps', 0, 'run']) - expect(typeof script).toBe('string') - const details = join(dir.root, 'findings.md') - writeFileSync( - details, - '| Finding | Remedy |\n| --- | --- |\n| Clone | Extract |' - ) - const result = spawnSync('bash', ['-c', String(script)], { - encoding: 'utf8', - env: { - ...process.env, - INPUT_SLUG: 'test', - INPUT_TITLE: 'Test check', - INPUT_ICON: '', - INPUT_STATUS: status, - INPUT_HEADLINE: 'Check summary.', - INPUT_DETAILS_FILE: details, - INPUT_REPRODUCE: 'pnpm check', - INPUT_DOCS: 'docs/check.md', - GITHUB_WORKSPACE: dir.root, - GITHUB_OUTPUT: join(dir.root, 'output'), - GITHUB_STEP_SUMMARY: join(dir.root, 'summary'), - GITHUB_SERVER_URL: 'https://github.com', - GITHUB_REPOSITORY: 'owner/repo', - GITHUB_RUN_ID: '123' - } - }) + const result = runExplainCheck(dir.root, { slug: 'test', status }) expect(result.status).toBe(0) const output = readFileSync(join(dir.root, 'summary'), 'utf8') const visible = output.split('
')[0] @@ -82,6 +90,38 @@ describe('shared PR report disclosure', () => { } ) + it.for(['../escape', 'Upper', 'a/b', '', 'a;b', 'a\nb'])( + 'rejects slug %j before writing any report file', + (slug) => { + using dir = fixture() + const workspace = join(dir.root, 'workspace') + mkdirSync(workspace) + const result = runExplainCheck(workspace, { slug, status: 'pass' }) + + expect(result.status).not.toBe(0) + expect(result.stdout).toContain('explain-check: slug') + expect(readdirSync(workspace)).toEqual(['findings.md']) + expect(readdirSync(dir.root)).toEqual(['workspace']) + } + ) + + it('accepts a slug with hyphens and digits and writes its section file', () => { + using dir = fixture() + const result = runExplainCheck(dir.root, { + slug: 'custom-nodes-2', + status: 'pass' + }) + + expect(result.status).toBe(0) + const sectionFile = join(dir.root, '.explain-check-custom-nodes-2.md') + expect(readFileSync(join(dir.root, 'output'), 'utf8')).toBe( + `file=${sectionFile}\n` + ) + expect(readFileSync(sectionFile, 'utf8')).toMatch( + /^## {2}Test check: βœ… Passed\n/ + ) + }) + it('keeps failed FPS targets visible while collapsing passing scenario timings', () => { using dir = fixture() mkdirSync(join(dir.root, 'test-results')) diff --git a/scripts/cicd/pr-report-section.test.ts b/scripts/cicd/pr-report-section.test.ts new file mode 100644 index 00000000000..ab2aa822f71 --- /dev/null +++ b/scripts/cicd/pr-report-section.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest' + +import { renderPrReportSection } from './pr-report-section' + +describe('renderPrReportSection', () => { + it.for([ + { name: 'omitted', body: undefined }, + { name: 'empty', body: '' }, + { name: 'whitespace only', body: '\n \n' } + ])('emits no collapsed box when the body is $name', ({ body }) => { + const rendered = renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '🟑 66.3%', + body + }) + + expect(rendered).toBe('## πŸ”¬ E2E Coverage: 🟑 66.3%') + }) + + it('folds the body into a collapsed box behind the heading', () => { + expect( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '🟑 66.3% of lines', + body: '| Metric | Pct |\n|---|---|\n| Lines | 66.3% |' + }) + ).toBe( + `## πŸ”¬ E2E Coverage: 🟑 66.3% of lines + +
+Details + +| Metric | Pct | +|---|---| +| Lines | 66.3% | + +
` + ) + }) + + it('strips body padding so nested sections do not drift apart', () => { + expect( + renderPrReportSection({ + icon: '🌾', + title: 'Fallow', + status: 'βœ… Passed', + body: '\n\nNo new findings.\n\n\n' + }) + ).toBe( + '## 🌾 Fallow: βœ… Passed\n\n
\nDetails\n\nNo new findings.\n\n
' + ) + }) + + it('keeps the notice visible between the heading and the collapsed box', () => { + const rendered = renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: '❌ 1 below 52 FPS target', + notice: '> ❌ **1 scenario below the target.** Profile it.', + body: '| Metric | Value |' + }) + + expect(rendered.split('
')[0]).toBe( + '## ⚑ Performance: ❌ 1 below 52 FPS target\n\n> ❌ **1 scenario below the target.** Profile it.\n\n' + ) + expect(rendered).toContain('Details\n\n| Metric') + }) + + it('renders a notice without a box when there is no body', () => { + expect( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '⚠️ No line coverage recorded', + notice: '\n> [!NOTE]\n> Check the logs.\n' + }) + ).toBe( + '## πŸ”¬ E2E Coverage: ⚠️ No line coverage recorded\n\n> [!NOTE]\n> Check the logs.' + ) + }) + + it('names the collapsed box when a summary is given', () => { + expect( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '🟒 90.0% of lines', + summary: 'Coverage totals', + body: '| Metric |' + }) + ).toContain( + 'Coverage totals\n\n| Metric |\n\n
' + ) + }) +}) diff --git a/scripts/cicd/pr-report-section.ts b/scripts/cicd/pr-report-section.ts new file mode 100644 index 00000000000..ce0a15cfbc6 --- /dev/null +++ b/scripts/cicd/pr-report-section.ts @@ -0,0 +1,40 @@ +export interface PrReportSection { + /** Emoji shown beside the title, e.g. `⚑`. */ + icon: string + /** Human name of the check, e.g. `Performance`. */ + title: string + /** Verdict for the heading, e.g. `βœ… No regressions`. */ + status: string + /** Markdown kept visible under the heading, e.g. a failure's next step. */ + notice?: string + /** Markdown folded into the collapsed box. Omit for a status-only section. */ + body?: string + /** Label of the collapsed box. Defaults to `Details`. */ + summary?: string +} + +export function renderPrReportSection({ + icon, + title, + status, + notice, + body, + summary = 'Details' +}: PrReportSection): string { + const blocks = [`## ${icon} ${title}: ${status}`] + + const trimmedNotice = notice?.trim() + if (trimmedNotice) blocks.push(trimmedNotice) + + const trimmedBody = body?.trim() + if (trimmedBody) { + blocks.push( + ['
', `${summary}`, '', trimmedBody].join( + '\n' + ), + '
' + ) + } + + return blocks.join('\n\n') +} diff --git a/scripts/coverage-report.test.ts b/scripts/coverage-report.test.ts index 9d87cabaf3d..ad089ee0171 100644 --- a/scripts/coverage-report.test.ts +++ b/scripts/coverage-report.test.ts @@ -15,11 +15,13 @@ const TRACEFILE = Array.from( `SF:src/components/Component${index}.vue\nDA:1,1\nDA:2,0\nLF:2\nLH:1\nend_of_record\n` ).join('') -/** A string is written verbatim, so cases can supply unparseable metadata. */ -function render(metadata: Record | string | null): string { +function render( + metadata: Record | string | null, + tracefile = TRACEFILE +): string { const dir = mkdtempSync(join(tmpdir(), 'coverage-report-')) const lcov = join(dir, 'coverage.lcov') - writeFileSync(lcov, TRACEFILE) + writeFileSync(lcov, tracefile) if (metadata !== null) { writeFileSync( join(dir, 'coverage-metadata.json'), @@ -49,7 +51,7 @@ describe('coverage-report shard banner', () => { expect(output).not.toContain('[!WARNING]') expect(output).not.toContain('[!NOTE]') expect(output).toContain('| Lines | 3 | 6 | 50.0%') - expect(output.split('
')[0]).toContain('Line coverage: **50.0%**') + expect(output.split('\n')[0]).toBe('## πŸ”¬ E2E Coverage: 🟑 50.0% of lines') expect(output.split('
')[0]).not.toContain('| Metric |') expect(output).toMatch( /Coverage totals<\/summary>\n\n\| Metric[\s\S]*\| Lines \| 3 \| 6 \| 50.0%[\s\S]*\n\n<\/details>/ @@ -93,3 +95,13 @@ describe('coverage-report shard banner', () => { expect(render(metadata)).toContain('| Lines | 3 | 6 | 50.0%') }) }) + +describe('coverage-report line totals', () => { + it('warns instead of showing a percentage when no lines were recorded', () => { + const output = render({ complete: true }, '') + + expect(output.split('\n')[0]).toBe( + '## πŸ”¬ E2E Coverage: ⚠️ No line coverage recorded' + ) + }) +}) diff --git a/scripts/coverage-report.ts b/scripts/coverage-report.ts index ff8751d6640..517c6d5f376 100644 --- a/scripts/coverage-report.ts +++ b/scripts/coverage-report.ts @@ -6,6 +6,8 @@ import { readCoverageMetadata } from './coverage-metadata' +import { renderPrReportSection } from './cicd/pr-report-section' + interface FileStats { lines: number covered: number @@ -21,7 +23,11 @@ const lcovPath = process.argv[2] || 'coverage/playwright/coverage.lcov' if (!existsSync(lcovPath)) { process.stdout.write( - '## πŸ”¬ E2E Coverage\n\n> ⚠️ No coverage data found. Check the CI workflow logs.\n' + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: '⚠️ No coverage data found β€” check the CI workflow logs' + }) + '\n' ) process.exit(0) } @@ -118,28 +124,22 @@ function bar(covered: number, total: number): string { } const lines: string[] = [] -lines.push('## πŸ”¬ E2E Coverage') -lines.push('') - +const notice: string[] = [] const metadata = readCoverageMetadata( join(dirname(lcovPath), COVERAGE_METADATA_FILE) ) if (!metadata) { - lines.push( + notice.push( '> [!NOTE]', - '> Shard completeness could not be verified for this run, so these totals may not be comparable with a whole merge. Check the coverage workflow logs and re-run collection before using these totals for comparison.', - '' + '> Shard completeness could not be verified for this run, so these totals may not be comparable with a whole merge. Check the coverage workflow logs and re-run collection before using these totals for comparison.' ) } else if (!metadata.complete) { - lines.push( + notice.push( '> [!NOTE]', - '> The shard matrix did not pass, so these totals cannot be confirmed comparable with a whole merge. Fix the failing shards and re-run coverage before using these totals for comparison.', - '' + '> The shard matrix did not pass, so these totals cannot be confirmed comparable with a whole merge. Fix the failing shards and re-run coverage before using these totals for comparison.' ) } -lines.push(`Line coverage: **${pct(coveredLines, totalLines)}**`, '') -lines.push('
', 'Coverage totals', '') lines.push('| Metric | Covered | Total | Pct | |') lines.push('|---|--:|--:|--:|---|') lines.push( @@ -151,7 +151,6 @@ lines.push( lines.push( `| Branches | ${coveredBranches} | ${totalBranches} | ${pct(coveredBranches, totalBranches)} | ${bar(coveredBranches, totalBranches)} |` ) -lines.push('', '
') const uncovered: UncoveredFile[] = [...fileStats.entries()] .filter(([, s]) => s.lines > 0) @@ -178,4 +177,18 @@ if (uncovered.length > 0) { lines.push('
') } -process.stdout.write(lines.join('\n') + '\n') +const linesStatus = + totalLines === 0 + ? '⚠️ No line coverage recorded' + : `${bar(coveredLines, totalLines)} ${pct(coveredLines, totalLines)} of lines` + +process.stdout.write( + renderPrReportSection({ + icon: 'πŸ”¬', + title: 'E2E Coverage', + status: linesStatus, + notice: notice.join('\n'), + summary: 'Coverage totals', + body: lines.join('\n') + }) + '\n' +) diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts new file mode 100644 index 00000000000..256c2910b50 --- /dev/null +++ b/scripts/perf-report.test.ts @@ -0,0 +1,219 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import type { PerfMeasurement } from '../browser_tests/fixtures/helpers/PerformanceHelper' + +const ROOT = join(import.meta.dirname, '..') +const TSX = join(ROOT, 'node_modules/.bin/tsx') +const SCRIPT = join(import.meta.dirname, 'perf-report.ts') + +type Sample = Pick< + PerfMeasurement, + 'name' | 'frameDurationMs' | 'p95FrameDurationMs' +> & + Partial> + +interface ReportInputs { + baseline?: Sample[] + history?: Sample[][] +} + +function writeReport(path: string, measurements: Sample[], index = 0) { + writeFileSync( + path, + JSON.stringify({ + timestamp: new Date(Date.UTC(2026, 0, index + 1)).toISOString(), + measurements + }) + ) +} + +function render( + measurements: Sample[], + { baseline, history = [] }: ReportInputs = {} +): string { + const dir = mkdtempSync(join(tmpdir(), 'perf-report-')) + try { + mkdirSync(join(dir, 'test-results')) + writeReport(join(dir, 'test-results/perf-metrics.json'), measurements) + if (baseline) { + mkdirSync(join(dir, 'temp/perf-baseline'), { recursive: true }) + writeReport(join(dir, 'temp/perf-baseline/perf-metrics.json'), baseline) + } + if (history.length > 0) { + mkdirSync(join(dir, 'temp/perf-history'), { recursive: true }) + history.forEach((run, index) => + writeReport(join(dir, `temp/perf-history/${index}.json`), run, index) + ) + } + const result = spawnSync(TSX, [SCRIPT], { cwd: dir, encoding: 'utf8' }) + if (result.status !== 0) { + throw new Error( + `perf-report.ts exited ${result.status}: ${result.stderr}` + ) + } + return result.stdout + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +function frames(name: string, ...frameMs: number[]): Sample[] { + return frameMs.map((ms) => ({ + name, + frameDurationMs: ms, + p95FrameDurationMs: ms, + totalBlockingTimeMs: 0 + })) +} + +function heading(output: string): string { + return output.split('\n')[0] +} + +describe('perf-report heading', () => { + it('does not count a scenario without frame samples as missing the FPS target', () => { + const output = render([ + { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 }, + { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 } + ]) + + expect(heading(output)).not.toContain('below 52 FPS target') + expect(output).not.toContain('0.0 P5 FPS ❌') + expect(output.split('
')[0]).toContain( + '1 scenario recorded no frame samples' + ) + }) + + it('still leads with a real FPS target miss', () => { + const output = render([ + { name: 'slow', frameDurationMs: 25, p95FrameDurationMs: 40 }, + { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 } + ]) + + expect(heading(output)).toMatch( + /^## ⚑ Performance: ❌ 1 below 52 FPS target/ + ) + }) + + it('ignores unsampled runs when taking the median frame time', () => { + const output = render([ + { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, + { name: 'slow', frameDurationMs: 16, p95FrameDurationMs: 18 }, + { name: 'slow', frameDurationMs: 16, p95FrameDurationMs: 21 } + ]) + + expect(heading(output)).toMatch( + /^## ⚑ Performance: ❌ 1 below 52 FPS target/ + ) + expect(output).not.toContain('recorded no frame samples') + }) + + it('does not read as clean when no scenario recorded frame samples', () => { + const output = render([ + { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 } + ]) + + expect(heading(output)).toMatch( + /^## ⚑ Performance: ⚠️ No frame samples β€” FPS target not checked/ + ) + }) + + it('starts a scenario timing line with its first metric, not a separator', () => { + const output = render([ + { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 } + ]) + + expect(output).toContain('> **fast**: 62.5 avg FPS Β· 58.8 P5 FPS βœ…') + }) + + it('leaves unsampled runs out of the frame time tables', () => { + const output = render([ + { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, + { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 }, + { name: 'slow', frameDurationMs: 30, p95FrameDurationMs: 40 } + ]) + + expect(output).toContain('| slow: p95 frame time | 40ms |') + expect(output).toContain('| slow: avg frame time | 30ms |') + }) + + it('does not report a run that measured nothing as clean', () => { + const output = render([]) + + expect(heading(output)).toBe( + '## ⚑ Performance: ⚠️ No measurements recorded β€” check the perf test logs' + ) + }) +}) + +describe('perf-report baseline comparison', () => { + it('compares sampled frame medians against the baseline on a cold start', () => { + const output = render(frames('grow', 0, 40), { + baseline: frames('grow', 0, 20) + }) + + expect(output).toContain('| grow: p95 frame time | 20ms | 40ms | +100% |') + expect(output).toContain('| grow: avg frame time | 20ms | 40ms | +100% |') + expect(output).toContain('| grow: TBT | 0ms | 0ms | +0% |') + }) + + it('does not flag unchanged frame times when history has unsampled runs', () => { + const output = render(frames('steady', 0, 0, 17), { + baseline: frames('steady', 0, 0, 17), + history: [ + frames('steady', 0, 0, 16), + frames('steady', 0, 0, 17), + frames('steady', 0, 0, 17) + ] + }) + + expect(heading(output)).toBe( + '## ⚑ Performance: βœ… No regressions detected' + ) + expect(output).toContain('| steady: p95 frame time | 17ms | 17ms | +0% |') + expect(output).toContain('| steady: TBT | 0ms | 0ms | +0% |') + expect(output).toMatch(/\| steady: p95 frame time \| 17ms \| 1ms \|/) + expect(output).toMatch( + /\| steady: p95 frame time \| \S+ \| [^|]+ \| 17ms \|/ + ) + }) + + it('leaves wholly unsampled history runs out of the frame time variance', () => { + const output = render(frames('steady', 0, 17), { + baseline: frames('steady', 0, 17), + history: [ + frames('steady', 0, 0, 16), + frames('steady', 0, 0, 18), + frames('steady', 0, 0) + ] + }) + + expect(heading(output)).toBe( + '## ⚑ Performance: βœ… No regressions detected' + ) + expect(output).toContain( + 'Historical variance (up to 3 runs per metric)' + ) + expect(output).toContain('| steady: p95 frame time | 17ms | 1ms |') + expect(output).toContain('| steady: TBT | 0ms | 0ms |') + expect(output).not.toMatch(/\| steady: p95 frame time \| [β–β–‚β–ƒβ–„β–…β–†β–‡β–ˆ]+ \|/) + }) + + it('flags a real frame time regression against sampled history', () => { + const output = render(frames('grow', 0, 40), { + baseline: frames('grow', 0, 20), + history: [frames('grow', 0, 20), frames('grow', 0, 21)] + }) + + expect(heading(output)).toContain('2 regressions detected') + expect(output.split('
')[0]).toContain( + 'Review the regressions below and profile the affected scenarios before merging.' + ) + expect(output).toContain('| grow: p95 frame time | 20ms | 40ms | +100% |') + }) +}) diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts index c3659ab9a94..02e5cb172dc 100644 --- a/scripts/perf-report.ts +++ b/scripts/perf-report.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, readdirSync } from 'node:fs' import { join } from 'node:path' +import { renderPrReportSection } from './cicd/pr-report-section' import type { MetricStats } from './perf-stats' import { classifyChange, @@ -133,7 +134,7 @@ function getHistoricalStats( const group = groupByName(r.measurements) const samples = group.get(testName) if (samples) { - const mean = meanMetric(samples, metric) + const mean = reportedMean(samples, metric) if (mean !== null) values.push(mean) } } @@ -153,9 +154,8 @@ function getHistoricalTimeSeries( const group = groupByName(r.measurements) const samples = group.get(testName) if (samples) { - values.push( - samples.reduce((sum, s) => sum + s[metric], 0) / samples.length - ) + const mean = reportedMean(samples, metric) + if (mean !== null) values.push(mean) } } return values @@ -218,43 +218,149 @@ function frameTimeToFps(ms: number): number { return ms > 0 ? 1000 / ms : 0 } -function renderHeadlineSummary( - prGroups: Map -): string[] { - const lines: string[] = [] - const summaries: string[] = [] - let belowTarget = 0 +type FrameMetricKey = 'frameDurationMs' | 'p95FrameDurationMs' - for (const [testName, prSamples] of prGroups) { - const avgFrame = medianMetric(prSamples, 'frameDurationMs') - const p95Frame = medianMetric(prSamples, 'p95FrameDurationMs') - const tbt = medianMetric(prSamples, 'totalBlockingTimeMs') - const heap = medianMetric(prSamples, 'heapUsedBytes') - - const avgFps = avgFrame !== null ? frameTimeToFps(avgFrame) : null - const p5Fps = p95Frame !== null ? frameTimeToFps(p95Frame) : null - - const parts: string[] = [`**${testName}**:`] - if (avgFps !== null) parts.push(`${avgFps.toFixed(1)} avg FPS`) - if (p5Fps !== null) { - const pass = p5Fps >= TARGET_P5_FPS - if (!pass) belowTarget++ - parts.push( - `${p5Fps.toFixed(1)} P5 FPS ${pass ? 'βœ…' : '❌'} (target: β‰₯${TARGET_P5_FPS})` - ) - } - if (tbt !== null) parts.push(`${tbt.toFixed(0)}ms TBT`) - if (heap !== null) parts.push(`${formatBytes(heap)} heap`) +function isFrameMetric(key: MetricKey): key is FrameMetricKey { + return key === 'frameDurationMs' || key === 'p95FrameDurationMs' +} + +function frameSampled( + samples: PerfMeasurement[], + key: FrameMetricKey +): PerfMeasurement[] { + return samples.filter((s) => (getMetricValue(s, key) ?? 0) > 0) +} + +function sampledFrameMetric( + samples: PerfMeasurement[], + key: FrameMetricKey +): number | null { + return medianMetric(frameSampled(samples, key), key) +} + +function reportedMedian( + samples: PerfMeasurement[], + key: MetricKey +): number | null { + return isFrameMetric(key) + ? sampledFrameMetric(samples, key) + : medianMetric(samples, key) +} + +function reportedMean( + samples: PerfMeasurement[], + key: MetricKey +): number | null { + return meanMetric( + isFrameMetric(key) ? frameSampled(samples, key) : samples, + key + ) +} + +interface FpsTargetTally { + below: number + sampled: number + unsampled: number +} - if (parts.length > 1) summaries.push(parts.join(' Β· ')) +function tallyFpsTarget( + prGroups: Map +): FpsTargetTally { + const tally: FpsTargetTally = { below: 0, sampled: 0, unsampled: 0 } + for (const prSamples of prGroups.values()) { + const p95Frame = sampledFrameMetric(prSamples, 'p95FrameDurationMs') + if (p95Frame === null) { + if (medianMetric(prSamples, 'p95FrameDurationMs') !== null) + tally.unsampled++ + continue + } + tally.sampled++ + if (frameTimeToFps(p95Frame) < TARGET_P5_FPS) tally.below++ } + return tally +} - if (belowTarget > 0) { - lines.push( - `> ❌ **${belowTarget} scenario${belowTarget === 1 ? '' : 's'} below the β‰₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`, +function plural(count: number, noun: string): string { + return `${count} ${noun}${count === 1 ? '' : 's'}` +} + +function percentDelta(prVal: number, baseVal: number): number | null { + if (baseVal === 0) return prVal === 0 ? 0 : null + return ((prVal - baseVal) / baseVal) * 100 +} + +function baselineMedian( + baseSamples: PerfMeasurement[] | undefined, + key: MetricKey +): number | null { + return baseSamples?.length ? reportedMedian(baseSamples, key) : null +} + +function formatIfPresent( + value: number | null, + format: (value: number) => string +): string | null { + return value === null ? null : format(value) +} + +function p5FpsPart(p95Frame: number): string { + const p5Fps = frameTimeToFps(p95Frame) + const verdict = p5Fps >= TARGET_P5_FPS ? 'βœ…' : '❌' + return `${p5Fps.toFixed(1)} P5 FPS ${verdict} (target: β‰₯${TARGET_P5_FPS})` +} + +function scenarioSummary( + testName: string, + prSamples: PerfMeasurement[] +): string | null { + const parts = [ + formatIfPresent( + sampledFrameMetric(prSamples, 'frameDurationMs'), + (ms) => `${frameTimeToFps(ms).toFixed(1)} avg FPS` + ), + formatIfPresent( + sampledFrameMetric(prSamples, 'p95FrameDurationMs'), + p5FpsPart + ), + formatIfPresent( + medianMetric(prSamples, 'totalBlockingTimeMs'), + (ms) => `${ms.toFixed(0)}ms TBT` + ), + formatIfPresent( + medianMetric(prSamples, 'heapUsedBytes'), + (bytes) => `${formatBytes(bytes)} heap` + ) + ].filter((part): part is string => part !== null) + + return parts.length > 0 ? `**${testName}**: ${parts.join(' Β· ')}` : null +} + +function fpsTargetNotice({ below, unsampled }: FpsTargetTally): string[] { + const notice: string[] = [] + if (below > 0) { + notice.push( + `> ❌ **${plural(below, 'scenario')} below the β‰₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`, + '' + ) + } + if (unsampled > 0) { + notice.push( + `> ⚠️ ${plural(unsampled, 'scenario')} recorded no frame samples, so ${unsampled === 1 ? 'its' : 'their'} FPS could not be checked against the target. Check the perf test logs for a frame sampling timeout.`, '' ) } + return notice +} + +function renderHeadlineSummary(prGroups: Map): { + notice: string[] + lines: string[] +} { + const notice = fpsTargetNotice(tallyFpsTarget(prGroups)) + const lines: string[] = [] + const summaries = [...prGroups] + .map(([testName, prSamples]) => scenarioSummary(testName, prSamples)) + .filter((summary): summary is string => summary !== null) if (summaries.length > 0) { lines.push( @@ -267,14 +373,63 @@ function renderHeadlineSummary( ) } - return lines + return { notice, lines } +} + +interface RenderedReport { + status: string + notice?: string[] + lines: string[] +} + +interface ComparedRow { + row: string + flagged: boolean +} + +function compareMetric( + testName: string, + metric: MetricDef, + prSamples: PerfMeasurement[], + baseSamples: PerfMeasurement[] | undefined, + historical: PerfReport[] +): ComparedRow | null { + const { key, label, unit, minAbsDelta } = metric + const prVal = reportedMedian(prSamples, key) + if (prVal === null) return null + + const baseVal = baselineMedian(baseSamples, key) + if (baseVal === null) { + return { + row: `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new | β€” |`, + flagged: false + } + } + + const histStats = getHistoricalStats(historical, testName, key) + const z = zScore(prVal, histStats) + const sig = classifyChange( + z, + computeCV(histStats), + prVal - baseVal, + minAbsDelta + ) + + return { + row: `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(percentDelta(prVal, baseVal))} | ${formatSignificance(sig, z)} |`, + flagged: isNoteworthy(sig) + } +} + +function regressionVerdict(count: number): string { + return `${count} regression${count > 1 ? 's' : ''} detected` } function renderFullReport( prGroups: Map, baseline: PerfReport, historical: PerfReport[] -): string[] { +): RenderedReport { const lines: string[] = [] const baselineGroups = groupByName(baseline.measurements) const tableHeader = [ @@ -282,54 +437,34 @@ function renderFullReport( '|--------|----------|----------|---|-----|' ] - const flaggedRows: string[] = [] - const allRows: string[] = [] - - for (const [testName, prSamples] of prGroups) { - const baseSamples = baselineGroups.get(testName) + const compared = [...prGroups].flatMap(([testName, prSamples]) => + REPORTED_METRICS.map((metric) => + compareMetric( + testName, + metric, + prSamples, + baselineGroups.get(testName), + historical + ) + ).filter((row): row is ComparedRow => row !== null) + ) + const allRows = compared.map(({ row }) => row) + const flaggedRows = compared + .filter(({ flagged }) => flagged) + .map(({ row }) => row) - for (const { key, label, unit, minAbsDelta } of REPORTED_METRICS) { - const prVal = medianMetric(prSamples, key) - if (prVal === null) continue - const histStats = getHistoricalStats(historical, testName, key) - const cv = computeCV(histStats) - - if (!baseSamples?.length) { - allRows.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new | β€” |` - ) - continue - } - - const baseVal = medianMetric(baseSamples, key) - if (baseVal === null) { - allRows.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new | β€” |` - ) - continue - } - const absDelta = prVal - baseVal - const deltaPct = - baseVal === 0 - ? prVal === 0 - ? 0 - : null - : ((prVal - baseVal) / baseVal) * 100 - const z = zScore(prVal, histStats) - const sig = classifyChange(z, cv, absDelta, minAbsDelta) - - const row = `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(deltaPct)} | ${formatSignificance(sig, z)} |` - allRows.push(row) - if (isNoteworthy(sig)) { - flaggedRows.push(row) - } - } - } + const status = + flaggedRows.length > 0 + ? `⚠️ ${regressionVerdict(flaggedRows.length)}` + : 'βœ… No regressions detected' + const notice: string[] = [] if (flaggedRows.length > 0) { + notice.push( + `⚠️ **${regressionVerdict(flaggedRows.length)}**. Review the regressions below and profile the affected scenarios before merging.`, + '' + ) lines.push( - `⚠️ **${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected**. Review the regressions below and profile the affected scenarios before merging.`, - '', '
Show regressions', '', ...tableHeader, @@ -338,8 +473,6 @@ function renderFullReport( '
', '' ) - } else { - lines.push('βœ… No regressions detected.', '') } lines.push( @@ -353,11 +486,23 @@ function renderFullReport( ) lines.push( - `
Historical variance (last ${historical.length} runs)`, + ...renderHistoricalVariance(prGroups, historical), + ...renderTrend(prGroups, historical) + ) + + return { status, notice, lines } +} + +function renderHistoricalVariance( + prGroups: Map, + historical: PerfReport[] +): string[] { + const lines = [ + `
Historical variance (up to ${historical.length} runs per metric)`, '', '| Metric | ΞΌ | Οƒ | CV |', '|--------|---|---|-----|' - ) + ] for (const [testName] of prGroups) { for (const { key, label, unit } of REPORTED_METRICS) { const stats = getHistoricalStats(historical, testName, key) @@ -369,47 +514,47 @@ function renderFullReport( } } lines.push('', '
') + return lines +} +function renderTrend( + prGroups: Map, + historical: PerfReport[] +): string[] { const trendRows: string[] = [] for (const [testName] of prGroups) { for (const { key, label, unit } of REPORTED_METRICS) { const series = getHistoricalTimeSeries(historical, testName, key) if (series.length < 3) continue - const dir = trendDirection(series) - const arrow = trendArrow(dir) - const spark = sparkline(series) + const arrow = trendArrow(trendDirection(series)) const last = series[series.length - 1] trendRows.push( - `| ${testName}: ${label} | ${spark} | ${arrow} | ${formatValue(last, unit)} |` + `| ${testName}: ${label} | ${sparkline(series)} | ${arrow} | ${formatValue(last, unit)} |` ) } } - - if (trendRows.length > 0) { - lines.push( - '', - `
Trend (last ${historical.length} commits on main)`, - '', - '| Metric | Trend | Dir | Latest |', - '|--------|-------|-----|--------|', - ...trendRows, - '', - '
' - ) - } - - return lines + if (trendRows.length === 0) return [] + return [ + '', + `
Trend (up to ${historical.length} commits on main per metric)`, + '', + '| Metric | Trend | Dir | Latest |', + '|--------|-------|-----|--------|', + ...trendRows, + '', + '
' + ] } function renderColdStartReport( prGroups: Map, baseline: PerfReport, historicalCount: number -): string[] { +): RenderedReport { const lines: string[] = [] const baselineGroups = groupByName(baseline.measurements) lines.push( - `> ℹ️ Collecting baseline variance data (${historicalCount}/15 runs). Significance will appear after 2 main branch runs.`, + '> Significance will appear after 2 main branch runs.', '', '
All metrics (cold start)', '', @@ -421,46 +566,30 @@ function renderColdStartReport( const baseSamples = baselineGroups.get(testName) for (const { key, label, unit } of REPORTED_METRICS) { - const prVal = medianMetric(prSamples, key) + const prVal = reportedMedian(prSamples, key) if (prVal === null) continue - if (!baseSamples?.length) { - lines.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new |` - ) - continue - } - - const baseVal = medianMetric(baseSamples, key) - if (baseVal === null) { - lines.push( - `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new |` - ) - continue - } - const deltaPct = - baseVal === 0 - ? prVal === 0 - ? 0 - : null - : ((prVal - baseVal) / baseVal) * 100 + const baseVal = baselineMedian(baseSamples, key) lines.push( - `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(deltaPct)} |` + baseVal === null + ? `| ${testName}: ${label} | β€” | ${formatValue(prVal, unit)} | new |` + : `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(percentDelta(prVal, baseVal))} |` ) } } lines.push('', '
') - return lines + return { + status: `ℹ️ Collecting baseline variance (${historicalCount}/15 runs)`, + lines + } } function renderNoBaselineReport( prGroups: Map -): string[] { +): RenderedReport { const lines: string[] = [] lines.push( - '> ℹ️ No baseline found β€” significance unavailable.', - '', '
Absolute values', '', '| Metric | Value |', @@ -468,57 +597,89 @@ function renderNoBaselineReport( ) for (const [testName, prSamples] of prGroups) { for (const { key, label, unit } of REPORTED_METRICS) { - const prVal = medianMetric(prSamples, key) + const prVal = reportedMedian(prSamples, key) if (prVal === null) continue lines.push(`| ${testName}: ${label} | ${formatValue(prVal, unit)} |`) } } lines.push('', '
') - return lines + return { status: 'ℹ️ No baseline β€” significance unavailable', lines } } -function main() { - if (!existsSync(CURRENT_PATH)) { - process.stdout.write( - '## ⚑ Performance Report\n\nNo perf metrics found. Perf tests may not have run.\n' - ) - process.exit(0) - } - - const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8')) - - const baseline: PerfReport | null = existsSync(BASELINE_PATH) - ? JSON.parse(readFileSync(BASELINE_PATH, 'utf-8')) - : null - +function renderComparison( + prGroups: Map +): RenderedReport { + if (!existsSync(BASELINE_PATH)) return renderNoBaselineReport(prGroups) + const baseline: PerfReport = JSON.parse(readFileSync(BASELINE_PATH, 'utf-8')) const historical = loadHistoricalReports() - const prGroups = groupByName(current.measurements) - - const lines: string[] = [] - lines.push('## ⚑ Performance Report\n') - lines.push(...renderHeadlineSummary(prGroups)) - - if (baseline && historical.length >= 2) { - lines.push(...renderFullReport(prGroups, baseline, historical)) - } else if (baseline) { - lines.push(...renderColdStartReport(prGroups, baseline, historical.length)) - } else { - lines.push(...renderNoBaselineReport(prGroups)) - } + return historical.length >= 2 + ? renderFullReport(prGroups, baseline, historical) + : renderColdStartReport(prGroups, baseline, historical.length) +} +function renderRawData(current: PerfReport): string[] { const rawData = { ...current, measurements: current.measurements.map( ({ allFrameDurationsMs: _, ...rest }) => rest ) } - lines.push('\n
Raw data\n') - lines.push('```json') - lines.push(JSON.stringify(rawData, null, 2)) - lines.push('```') - lines.push('\n
') + return [ + '', + '
Raw data', + '', + '```json', + JSON.stringify(rawData, null, 2), + '```', + '', + '
' + ] +} - process.stdout.write(lines.join('\n') + '\n') +function perfHeadingStatus( + current: PerfReport, + prGroups: Map, + reportStatus: string +): string { + if (current.measurements.length === 0) + return '⚠️ No measurements recorded β€” check the perf test logs' + const fps = tallyFpsTarget(prGroups) + if (fps.below > 0) + return `❌ ${fps.below} below ${TARGET_P5_FPS} FPS target Β· ${reportStatus}` + if (fps.unsampled > 0 && fps.sampled === 0) + return `⚠️ No frame samples β€” FPS target not checked Β· ${reportStatus}` + return reportStatus +} + +function main() { + if (!existsSync(CURRENT_PATH)) { + process.stdout.write( + renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status: '⏭️ No metrics found β€” perf tests may not have run' + }) + '\n' + ) + process.exit(0) + } + + const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8')) + const prGroups = groupByName(current.measurements) + const report = renderComparison(prGroups) + + const headline = renderHeadlineSummary(prGroups) + const body = [...headline.lines, ...report.lines, ...renderRawData(current)] + const status = perfHeadingStatus(current, prGroups, report.status) + + process.stdout.write( + renderPrReportSection({ + icon: '⚑', + title: 'Performance', + status, + notice: [...headline.notice, ...(report.notice ?? [])].join('\n'), + body: body.join('\n') + }) + '\n' + ) } main() diff --git a/scripts/unified-report.test.ts b/scripts/unified-report.test.ts new file mode 100644 index 00000000000..ec50f973163 --- /dev/null +++ b/scripts/unified-report.test.ts @@ -0,0 +1,49 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const ROOT = join(import.meta.dirname, '..') +const TSX = join(ROOT, 'node_modules/.bin/tsx') +const SCRIPT = join(import.meta.dirname, 'unified-report.ts') + +function render(args: string[]): string { + const dir = mkdtempSync(join(tmpdir(), 'unified-report-')) + try { + const result = spawnSync(TSX, [SCRIPT, ...args], { + cwd: dir, + encoding: 'utf8' + }) + if (result.status !== 0) { + throw new Error( + `unified-report.ts exited ${result.status}: ${result.stderr}` + ) + } + return result.stdout + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +describe('unified-report', () => { + it('separates each section from the one before it with a blank line', () => { + const output = render([ + '--size-status=failed', + '--perf-status=pending', + '--coverage-status=failed' + ]) + + expect(output).toBe( + [ + '## πŸ“¦ Bundle: ⚠️ Size data collection failed β€” check the CI workflow logs', + '', + '## ⚑ Performance: ⏳ Performance tests in progress…', + '', + '## πŸ”¬ E2E Coverage: ⚠️ Coverage collection failed β€” check the CI workflow logs', + '' + ].join('\n') + ) + }) +}) diff --git a/scripts/unified-report.ts b/scripts/unified-report.ts index edf132abb53..13d6710b652 100644 --- a/scripts/unified-report.ts +++ b/scripts/unified-report.ts @@ -1,6 +1,8 @@ import { execFileSync } from 'node:child_process' import { existsSync } from 'node:fs' +import { renderPrReportSection } from './cicd/pr-report-section' + const args: string[] = process.argv.slice(2) function getArg(name: string): string | undefined { @@ -13,6 +15,18 @@ const sizeStatus = getArg('size-status') ?? 'pending' const perfStatus = getArg('perf-status') ?? 'pending' const coverageStatus = getArg('coverage-status') ?? 'skip' +function bundleStatusLine(status: string): string { + return renderPrReportSection({ icon: 'πŸ“¦', title: 'Bundle', status }) +} + +function perfStatusLine(status: string): string { + return renderPrReportSection({ icon: '⚑', title: 'Performance', status }) +} + +function coverageStatusLine(status: string): string { + return renderPrReportSection({ icon: 'πŸ”¬', title: 'E2E Coverage', status }) +} + const lines: string[] = [] const hasSizeData = existsSync('temp/size') @@ -26,20 +40,20 @@ if (sizeStatus === 'ready' && hasSizeData) { ).trimEnd() lines.push(sizeReport) } catch { - lines.push('## πŸ“¦ Bundle Size') - lines.push('') lines.push( - '> ⚠️ Failed to render bundle size report. Check the CI workflow logs.' + bundleStatusLine( + '⚠️ Failed to render report β€” check the CI workflow logs' + ) ) } } else if (sizeStatus === 'failed') { - lines.push('## πŸ“¦ Bundle Size') - lines.push('') - lines.push('> ⚠️ Size data collection failed. Check the CI workflow logs.') + lines.push( + bundleStatusLine( + '⚠️ Size data collection failed β€” check the CI workflow logs' + ) + ) } else if (sizeStatus !== 'ready') { - lines.push('## πŸ“¦ Bundle Size') - lines.push('') - lines.push('> ⏳ Size data collection in progress…') + lines.push(bundleStatusLine('⏳ Size data collection in progress…')) } if (lines.length > 0) lines.push('') @@ -53,23 +67,19 @@ if (perfStatus === 'ready' && existsSync('test-results/perf-metrics.json')) { ).trimEnd() lines.push(perfReport) } catch { - lines.push('## ⚑ Performance') - lines.push('') lines.push( - '> ⚠️ Failed to render performance report. Check the CI workflow logs.' + perfStatusLine('⚠️ Failed to render report β€” check the CI workflow logs') ) } } else if ( perfStatus === 'failed' || (perfStatus === 'ready' && !existsSync('test-results/perf-metrics.json')) ) { - lines.push('## ⚑ Performance') - lines.push('') - lines.push('> ⚠️ Performance tests failed. Check the CI workflow logs.') + lines.push( + perfStatusLine('⚠️ Performance tests failed β€” check the CI workflow logs') + ) } else if (perfStatus !== 'skip') { - lines.push('## ⚑ Performance') - lines.push('') - lines.push('> ⏳ Performance tests in progress…') + lines.push(perfStatusLine('⏳ Performance tests in progress…')) } if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) { @@ -87,18 +97,20 @@ if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) { lines.push('') lines.push(coverageReport) } catch { - lines.push('') - lines.push('## πŸ”¬ E2E Coverage') lines.push('') lines.push( - '> ⚠️ Failed to render coverage report. Check the CI workflow logs.' + coverageStatusLine( + '⚠️ Failed to render report β€” check the CI workflow logs' + ) ) } } else if (coverageStatus === 'failed') { lines.push('') - lines.push('## πŸ”¬ E2E Coverage') - lines.push('') - lines.push('> ⚠️ Coverage collection failed. Check the CI workflow logs.') + lines.push( + coverageStatusLine( + '⚠️ Coverage collection failed β€” check the CI workflow logs' + ) + ) } process.stdout.write(lines.join('\n') + '\n')