diff --git a/.claude/skills/perf-fix-with-proof/SKILL.md b/.claude/skills/perf-fix-with-proof/SKILL.md
index 6ffba72aa5f..25f4d957077 100644
--- a/.claude/skills/perf-fix-with-proof/SKILL.md
+++ b/.claude/skills/perf-fix-with-proof/SKILL.md
@@ -118,11 +118,16 @@ Implement the fix. The `@perf` test from PR1 is now on main and will run automat
The `ci-perf-report.yaml` posts a comment like:
```markdown
-## β‘ Performance Report
+## β‘ Performance: β
No regressions detected
+
+
+Details
| Metric | Baseline | PR (n=3) | Ξ | Sig |
| --------------------- | -------- | -------- | ---- | --- |
| : style recalcs | 450 | 12 | -97% | π’ |
+
+
```
If Ξ is negative for the target metric, the fix is proven.
diff --git a/.github/actions/comment-release-links/action.yaml b/.github/actions/comment-release-links/action.yaml
index 222867a771b..e38e8749532 100644
--- a/.github/actions/comment-release-links/action.yaml
+++ b/.github/actions/comment-release-links/action.yaml
@@ -53,6 +53,7 @@ runs:
fi
SECTION_NAME='release'
+ SECTION_TITLE='Release'
MESSAGE='Publish jobs finished successfully:'
LINKS_VALUE=''
@@ -64,18 +65,22 @@ runs:
;;
packages/design-system/package.json)
SECTION_NAME='design-system-release'
+ SECTION_TITLE='Design System Release'
LINKS_VALUE='npm design-system|https://www.npmjs.com/package/@comfyorg/design-system/v/{{version}}'
;;
packages/account-core/package.json)
SECTION_NAME='account-core-release'
+ SECTION_TITLE='Account Core Release'
LINKS_VALUE='npm account-core|https://www.npmjs.com/package/@comfyorg/account-core/v/{{version}}'
;;
packages/billing-contract/package.json)
SECTION_NAME='billing-contract-release'
+ SECTION_TITLE='Billing Contract Release'
LINKS_VALUE='npm billing-contract|https://www.npmjs.com/package/@comfyorg/billing-contract/v/{{version}}'
;;
packages/ingest-types/package.json)
SECTION_NAME='ingest-types-release'
+ SECTION_TITLE='Ingest Types Release'
LINKS_VALUE='npm ingest-types|https://www.npmjs.com/package/@comfyorg/ingest-types/v/{{version}}'
;;
esac
@@ -86,6 +91,11 @@ runs:
COMMENT_FILE=$(mktemp)
{
+ echo "## π $SECTION_TITLE: β
Publish jobs finished for \`$NEW_VERSION\`"
+ echo ""
+ echo ""
+ echo "Details
"
+ echo ""
echo "$MESSAGE"
echo ""
echo "- $DIFF_LABEL: [\`$DIFF_PREFIX$PREV_VERSION...$DIFF_PREFIX$NEW_VERSION\`]($DIFF_URL)"
@@ -105,6 +115,7 @@ runs:
done <<< "$LINKS_VALUE"
echo ""
+ echo " "
} > "$COMMENT_FILE"
{
diff --git a/.github/actions/explain-check/action.yaml b/.github/actions/explain-check/action.yaml
index 841e3d69368..c6b5db5af00 100644
--- a/.github/actions/explain-check/action.yaml
+++ b/.github/actions/explain-check/action.yaml
@@ -85,6 +85,11 @@ runs:
;;
esac
+ if ! [[ "$INPUT_SLUG" =~ ^[a-z0-9-]+$ ]]; then
+ echo "::error::explain-check: slug '$INPUT_SLUG' must match ^[a-z0-9-]+$"
+ exit 1
+ fi
+
# Must live inside the workspace: read-and-upsert-comment-section guards
# on hashFiles(), which only resolves paths under GITHUB_WORKSPACE and
# would silently no-op on a RUNNER_TEMP path.
diff --git a/.github/workflows/ci-tests-storybook.yaml b/.github/workflows/ci-tests-storybook.yaml
index 8f28f91bd41..96b787a7b30 100644
--- a/.github/workflows/ci-tests-storybook.yaml
+++ b/.github/workflows/ci-tests-storybook.yaml
@@ -236,8 +236,14 @@ jobs:
pr-number: ${{ github.event.pull_request.number }}
section-name: chromatic
section-content: |
- ### π¨ Chromatic Visual Tests
- - π [View Chromatic Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }})
+ ## π¨ Chromatic: π€ Uploaded β [View Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }})
+
+
+ Details
+
+ **Links**
- π [View Chromatic Storybook](${{ needs.chromatic-deployment.outputs.chromatic-storybook-url }})
+
+
comment-marker: ''
token: ${{ github.token }}
diff --git a/.github/workflows/ci-vercel-billing-web-preview.yaml b/.github/workflows/ci-vercel-billing-web-preview.yaml
index c904956cf2a..9228803eb72 100644
--- a/.github/workflows/ci-vercel-billing-web-preview.yaml
+++ b/.github/workflows/ci-vercel-billing-web-preview.yaml
@@ -127,13 +127,13 @@ jobs:
section-name: billing-web-preview
token: ${{ secrets.GITHUB_TOKEN }}
section-content: |-
- ## π³ Billing Web Preview
-
- **Billing Web Preview:** ${{ steps.urls.outputs.stable-url }}
+ ## π³ Billing Web Preview: β
Deployed β [View Preview](${{ steps.urls.outputs.stable-url }})
- Deployment details
+ Details
- This commit: ${{ steps.deploy.outputs.url }}
+ **Links**
+ - π Preview: ${{ steps.urls.outputs.stable-url }}
+ - π This commit: ${{ steps.deploy.outputs.url }}
diff --git a/.github/workflows/publish-package-on-merge.yaml b/.github/workflows/publish-package-on-merge.yaml
index 09ca8ecf634..cb72673a394 100644
--- a/.github/workflows/publish-package-on-merge.yaml
+++ b/.github/workflows/publish-package-on-merge.yaml
@@ -124,7 +124,15 @@ jobs:
run: |
set -euo pipefail
mkdir -p publish-report
+ PACKAGE_COUNT=$(jq -r '.include | length' <<<"$MATRIX")
{
+ printf '## π Release: β
Publish jobs finished for %s package' "$PACKAGE_COUNT"
+ [ "$PACKAGE_COUNT" = 1 ] || printf 's'
+ echo ''
+ echo ''
+ echo ''
+ echo 'Details
'
+ echo ''
echo 'Publish jobs finished successfully:'
echo ''
jq -r '
@@ -133,6 +141,7 @@ jobs:
' <<<"$MATRIX"
echo "- Published from [\`${MERGE_SHA:0:10}\`](https://github.com/${REPO}/commit/${MERGE_SHA})"
echo ''
+ echo ' '
} > publish-report/report.md
printf '%s' "$PR_NUMBER" > publish-report/pr-number.txt
diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh
index 6c5a490a681..f64420f70b6 100755
--- a/scripts/cicd/pr-playwright-deploy-and-comment.sh
+++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh
@@ -285,14 +285,9 @@ else
fi
if [ $total_failed -gt 0 ]; then
- comment="$comment
-
-> β $total_failed tests failed. Expand the failed tests below, review their reports and fix the failures before merging.
-
-
-Failed tests
-
-"
+ failed_test_lines=""
+ failed_tests_label="$total_failed tests"
+ [ "$total_failed" -eq 1 ] && failed_tests_label="1 test"
for counts_json in "${counts_array[@]}"; do
[ -z "$counts_json" ] || [ "$counts_json" = "{}" ] && continue
@@ -312,15 +307,29 @@ else
test_line="$test_line: [View test report]($report_url)"
fi
- comment="$comment
+ failed_test_lines="$failed_test_lines
$test_line"
done <<< "$failures"
fi
fi
done
- comment="$comment
+
+ if [ -n "$failed_test_lines" ]; then
+ comment="$comment
+
+> β $failed_tests_label failed. Expand the failed tests below, review their reports and fix the failures before merging.
+
+
+Failed tests
+
+${failed_test_lines#$'\n'}
"
+ else
+ comment="$comment
+
+> β $failed_tests_label failed. Open the browser reports below, review the failed tests and fix the failures before merging."
+ fi
fi
screenshot_diffs=$(count_screenshot_diffs) || screenshot_diffs=0
diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts
index 48f19a9334f..06fe400ddf2 100644
--- a/scripts/cicd/pr-playwright-deploy-and-comment.test.ts
+++ b/scripts/cicd/pr-playwright-deploy-and-comment.test.ts
@@ -14,6 +14,10 @@ import { describe, expect, it, onTestFinished } from 'vitest'
import { parse } from 'yaml'
import { z } from 'zod'
+import type { extractTestCounts } from './extract-playwright-counts'
+
+type TestCounts = ReturnType
+
const script = join(import.meta.dirname, 'pr-playwright-deploy-and-comment.sh')
const e2eWorkflow = z
@@ -42,9 +46,18 @@ const uploadedProjects = [
)
]
-const passingCounts = '{"passed":2,"failed":0,"flaky":0,"skipped":0,"total":2}'
+const passingCounts: TestCounts = {
+ passed: 2,
+ failed: 0,
+ flaky: 0,
+ skipped: 0,
+ total: 2
+}
-function runWithReports(projects: string[], counts = passingCounts) {
+function runWithReports(
+ projects: string[],
+ countsByProject: Record = {}
+) {
const root = mkdtempSync(join(tmpdir(), 'playwright-pr-report-'))
const bin = join(root, 'bin')
const wranglerLog = join(root, 'wrangler.log')
@@ -64,15 +77,19 @@ echo "https://abc.$project.pages.dev"
`#!/bin/bash
echo "$*" >> "${extractorLog}"
echo 'Could not list failed tests: diagnostic' >&2
-echo '${counts}'
+cat "$2/counts.json"
`
)
chmodSync(join(bin, 'wrangler'), 0o755)
chmodSync(join(bin, 'tsx'), 0o755)
- for (const project of projects)
- mkdirSync(join(root, 'reports', `playwright-report-${project}`), {
- recursive: true
- })
+ for (const project of projects) {
+ const reportDir = join(root, 'reports', `playwright-report-${project}`)
+ mkdirSync(reportDir, { recursive: true })
+ writeFileSync(
+ join(reportDir, 'counts.json'),
+ JSON.stringify(countsByProject[project] ?? passingCounts)
+ )
+ }
const result = spawnSync('bash', [script, '1', 'feature', 'completed'], {
cwd: root,
@@ -115,10 +132,22 @@ describe('PR Playwright report section', () => {
})
it('links a failed test at its path in the repository and in the report', () => {
- const { summary, extractorCalls } = runWithReports(
- ['chromium'],
- '{"passed":0,"failed":1,"flaky":0,"skipped":0,"total":1,"failures":[{"name":"saves a workflow","file":"browser_tests/tests/save.spec.ts","reportUrl":"https://abc.chromium.pages.dev/#?testId=fail-1"}]}'
- )
+ const { summary, extractorCalls } = runWithReports(['chromium'], {
+ chromium: {
+ passed: 0,
+ failed: 1,
+ flaky: 0,
+ skipped: 0,
+ total: 1,
+ failures: [
+ {
+ name: 'saves a workflow',
+ file: 'browser_tests/tests/save.spec.ts',
+ reportUrl: 'https://abc.chromium.pages.dev/#?testId=fail-1'
+ }
+ ]
+ }
+ })
expect(extractorCalls).toContain('browser_tests')
expect(summary).toContain(
@@ -135,4 +164,49 @@ describe('PR Playwright report section', () => {
expect(summary).toMatch(/^## π Playwright: β .*, 1 missing/)
expect(summary).toContain('- **cloud**: β Report missing')
})
+
+ it('lists each named failed test with its report in a collapsed box', () => {
+ const { status, summary } = runWithReports(uploadedProjects, {
+ chromium: {
+ passed: 1,
+ failed: 1,
+ flaky: 0,
+ skipped: 0,
+ total: 2,
+ failures: [
+ {
+ name: 'saves a workflow',
+ file: 'browser_tests/tests/save.spec.ts',
+ reportUrl: 'https://abc.chromium.pages.dev/#?testId=fail-1'
+ }
+ ]
+ }
+ })
+
+ expect(status).toBe(0)
+ expect(summary).toMatch(/^## π Playwright: β .*, 1 failed/)
+ expect(summary).toContain(
+ `> β 1 test failed. Expand the failed tests below, review their reports and fix the failures before merging.
+
+
+Failed tests
+
+- [saves a workflow](https://github.com/Comfy-Org/ComfyUI_frontend/blob/sha/browser_tests/tests/save.spec.ts): [View test report](https://abc.chromium.pages.dev/#?testId=fail-1)
+
+ `
+ )
+ })
+
+ it('points to the browser reports when failed tests have no extracted names', () => {
+ const { status, summary } = runWithReports(uploadedProjects, {
+ chromium: { passed: 1, failed: 1, flaky: 0, skipped: 0, total: 2 }
+ })
+
+ expect(status).toBe(0)
+ expect(summary).toMatch(/^## π Playwright: β .*, 1 failed/)
+ expect(summary).toContain(
+ '> β 1 test failed. Open the browser reports below, review the failed tests and fix the failures before merging.'
+ )
+ expect(summary).not.toContain('Failed tests')
+ })
})
diff --git a/scripts/cicd/pr-report-disclosure.test.ts b/scripts/cicd/pr-report-disclosure.test.ts
index 12e05008cc0..06917f80dd3 100644
--- a/scripts/cicd/pr-report-disclosure.test.ts
+++ b/scripts/cicd/pr-report-disclosure.test.ts
@@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process'
import {
mkdirSync,
mkdtempSync,
+ readdirSync,
readFileSync,
rmSync,
writeFileSync
@@ -25,6 +26,45 @@ function fixture() {
}
}
+function runExplainCheck(
+ workspace: string,
+ inputs: { slug: string; status: string }
+) {
+ const actionFile = parseDocument(
+ readFileSync(
+ join(ROOT, '.github/actions/explain-check/action.yaml'),
+ 'utf8'
+ )
+ )
+ const script = actionFile.getIn(['runs', 'steps', 0, 'run'])
+ expect(typeof script).toBe('string')
+ const details = join(workspace, 'findings.md')
+ writeFileSync(
+ details,
+ '| Finding | Remedy |\n| --- | --- |\n| Clone | Extract |'
+ )
+ return spawnSync('bash', ['-c', String(script)], {
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ INPUT_SLUG: inputs.slug,
+ INPUT_TITLE: 'Test check',
+ INPUT_ICON: '',
+ INPUT_STATUS: inputs.status,
+ INPUT_HEADLINE: 'Check summary.',
+ INPUT_DETAILS_FILE: details,
+ INPUT_REPRODUCE: 'pnpm check',
+ INPUT_DOCS: 'docs/check.md',
+ GITHUB_WORKSPACE: workspace,
+ GITHUB_OUTPUT: join(workspace, 'output'),
+ GITHUB_STEP_SUMMARY: join(workspace, 'summary'),
+ GITHUB_SERVER_URL: 'https://github.com',
+ GITHUB_REPOSITORY: 'owner/repo',
+ GITHUB_RUN_ID: '123'
+ }
+ })
+}
+
describe('shared PR report disclosure', () => {
it.for([
['pass', ''],
@@ -36,39 +76,7 @@ describe('shared PR report disclosure', () => {
'collapses check details for %s with a visible action when needed',
([status, action]) => {
using dir = fixture()
- const actionFile = parseDocument(
- readFileSync(
- join(ROOT, '.github/actions/explain-check/action.yaml'),
- 'utf8'
- )
- )
- const script = actionFile.getIn(['runs', 'steps', 0, 'run'])
- expect(typeof script).toBe('string')
- const details = join(dir.root, 'findings.md')
- writeFileSync(
- details,
- '| Finding | Remedy |\n| --- | --- |\n| Clone | Extract |'
- )
- const result = spawnSync('bash', ['-c', String(script)], {
- encoding: 'utf8',
- env: {
- ...process.env,
- INPUT_SLUG: 'test',
- INPUT_TITLE: 'Test check',
- INPUT_ICON: '',
- INPUT_STATUS: status,
- INPUT_HEADLINE: 'Check summary.',
- INPUT_DETAILS_FILE: details,
- INPUT_REPRODUCE: 'pnpm check',
- INPUT_DOCS: 'docs/check.md',
- GITHUB_WORKSPACE: dir.root,
- GITHUB_OUTPUT: join(dir.root, 'output'),
- GITHUB_STEP_SUMMARY: join(dir.root, 'summary'),
- GITHUB_SERVER_URL: 'https://github.com',
- GITHUB_REPOSITORY: 'owner/repo',
- GITHUB_RUN_ID: '123'
- }
- })
+ const result = runExplainCheck(dir.root, { slug: 'test', status })
expect(result.status).toBe(0)
const output = readFileSync(join(dir.root, 'summary'), 'utf8')
const visible = output.split('')[0]
@@ -82,6 +90,38 @@ describe('shared PR report disclosure', () => {
}
)
+ it.for(['../escape', 'Upper', 'a/b', '', 'a;b', 'a\nb'])(
+ 'rejects slug %j before writing any report file',
+ (slug) => {
+ using dir = fixture()
+ const workspace = join(dir.root, 'workspace')
+ mkdirSync(workspace)
+ const result = runExplainCheck(workspace, { slug, status: 'pass' })
+
+ expect(result.status).not.toBe(0)
+ expect(result.stdout).toContain('explain-check: slug')
+ expect(readdirSync(workspace)).toEqual(['findings.md'])
+ expect(readdirSync(dir.root)).toEqual(['workspace'])
+ }
+ )
+
+ it('accepts a slug with hyphens and digits and writes its section file', () => {
+ using dir = fixture()
+ const result = runExplainCheck(dir.root, {
+ slug: 'custom-nodes-2',
+ status: 'pass'
+ })
+
+ expect(result.status).toBe(0)
+ const sectionFile = join(dir.root, '.explain-check-custom-nodes-2.md')
+ expect(readFileSync(join(dir.root, 'output'), 'utf8')).toBe(
+ `file=${sectionFile}\n`
+ )
+ expect(readFileSync(sectionFile, 'utf8')).toMatch(
+ /^## {2}Test check: β
Passed\n/
+ )
+ })
+
it('keeps failed FPS targets visible while collapsing passing scenario timings', () => {
using dir = fixture()
mkdirSync(join(dir.root, 'test-results'))
diff --git a/scripts/cicd/pr-report-section.test.ts b/scripts/cicd/pr-report-section.test.ts
new file mode 100644
index 00000000000..ab2aa822f71
--- /dev/null
+++ b/scripts/cicd/pr-report-section.test.ts
@@ -0,0 +1,97 @@
+import { describe, expect, it } from 'vitest'
+
+import { renderPrReportSection } from './pr-report-section'
+
+describe('renderPrReportSection', () => {
+ it.for([
+ { name: 'omitted', body: undefined },
+ { name: 'empty', body: '' },
+ { name: 'whitespace only', body: '\n \n' }
+ ])('emits no collapsed box when the body is $name', ({ body }) => {
+ const rendered = renderPrReportSection({
+ icon: 'π¬',
+ title: 'E2E Coverage',
+ status: 'π‘ 66.3%',
+ body
+ })
+
+ expect(rendered).toBe('## π¬ E2E Coverage: π‘ 66.3%')
+ })
+
+ it('folds the body into a collapsed box behind the heading', () => {
+ expect(
+ renderPrReportSection({
+ icon: 'π¬',
+ title: 'E2E Coverage',
+ status: 'π‘ 66.3% of lines',
+ body: '| Metric | Pct |\n|---|---|\n| Lines | 66.3% |'
+ })
+ ).toBe(
+ `## π¬ E2E Coverage: π‘ 66.3% of lines
+
+
+Details
+
+| Metric | Pct |
+|---|---|
+| Lines | 66.3% |
+
+ `
+ )
+ })
+
+ it('strips body padding so nested sections do not drift apart', () => {
+ expect(
+ renderPrReportSection({
+ icon: 'πΎ',
+ title: 'Fallow',
+ status: 'β
Passed',
+ body: '\n\nNo new findings.\n\n\n'
+ })
+ ).toBe(
+ '## πΎ Fallow: β
Passed\n\n\nDetails
\n\nNo new findings.\n\n '
+ )
+ })
+
+ it('keeps the notice visible between the heading and the collapsed box', () => {
+ const rendered = renderPrReportSection({
+ icon: 'β‘',
+ title: 'Performance',
+ status: 'β 1 below 52 FPS target',
+ notice: '> β **1 scenario below the target.** Profile it.',
+ body: '| Metric | Value |'
+ })
+
+ expect(rendered.split('')[0]).toBe(
+ '## β‘ Performance: β 1 below 52 FPS target\n\n> β **1 scenario below the target.** Profile it.\n\n'
+ )
+ expect(rendered).toContain('Details
\n\n| Metric')
+ })
+
+ it('renders a notice without a box when there is no body', () => {
+ expect(
+ renderPrReportSection({
+ icon: 'π¬',
+ title: 'E2E Coverage',
+ status: 'β οΈ No line coverage recorded',
+ notice: '\n> [!NOTE]\n> Check the logs.\n'
+ })
+ ).toBe(
+ '## π¬ E2E Coverage: β οΈ No line coverage recorded\n\n> [!NOTE]\n> Check the logs.'
+ )
+ })
+
+ it('names the collapsed box when a summary is given', () => {
+ expect(
+ renderPrReportSection({
+ icon: 'π¬',
+ title: 'E2E Coverage',
+ status: 'π’ 90.0% of lines',
+ summary: 'Coverage totals',
+ body: '| Metric |'
+ })
+ ).toContain(
+ 'Coverage totals
\n\n| Metric |\n\n '
+ )
+ })
+})
diff --git a/scripts/cicd/pr-report-section.ts b/scripts/cicd/pr-report-section.ts
new file mode 100644
index 00000000000..ce0a15cfbc6
--- /dev/null
+++ b/scripts/cicd/pr-report-section.ts
@@ -0,0 +1,40 @@
+export interface PrReportSection {
+ /** Emoji shown beside the title, e.g. `β‘`. */
+ icon: string
+ /** Human name of the check, e.g. `Performance`. */
+ title: string
+ /** Verdict for the heading, e.g. `β
No regressions`. */
+ status: string
+ /** Markdown kept visible under the heading, e.g. a failure's next step. */
+ notice?: string
+ /** Markdown folded into the collapsed box. Omit for a status-only section. */
+ body?: string
+ /** Label of the collapsed box. Defaults to `Details`. */
+ summary?: string
+}
+
+export function renderPrReportSection({
+ icon,
+ title,
+ status,
+ notice,
+ body,
+ summary = 'Details'
+}: PrReportSection): string {
+ const blocks = [`## ${icon} ${title}: ${status}`]
+
+ const trimmedNotice = notice?.trim()
+ if (trimmedNotice) blocks.push(trimmedNotice)
+
+ const trimmedBody = body?.trim()
+ if (trimmedBody) {
+ blocks.push(
+ ['', `${summary}
`, '', trimmedBody].join(
+ '\n'
+ ),
+ ' '
+ )
+ }
+
+ return blocks.join('\n\n')
+}
diff --git a/scripts/coverage-report.test.ts b/scripts/coverage-report.test.ts
index 9d87cabaf3d..ad089ee0171 100644
--- a/scripts/coverage-report.test.ts
+++ b/scripts/coverage-report.test.ts
@@ -15,11 +15,13 @@ const TRACEFILE = Array.from(
`SF:src/components/Component${index}.vue\nDA:1,1\nDA:2,0\nLF:2\nLH:1\nend_of_record\n`
).join('')
-/** A string is written verbatim, so cases can supply unparseable metadata. */
-function render(metadata: Record | string | null): string {
+function render(
+ metadata: Record | string | null,
+ tracefile = TRACEFILE
+): string {
const dir = mkdtempSync(join(tmpdir(), 'coverage-report-'))
const lcov = join(dir, 'coverage.lcov')
- writeFileSync(lcov, TRACEFILE)
+ writeFileSync(lcov, tracefile)
if (metadata !== null) {
writeFileSync(
join(dir, 'coverage-metadata.json'),
@@ -49,7 +51,7 @@ describe('coverage-report shard banner', () => {
expect(output).not.toContain('[!WARNING]')
expect(output).not.toContain('[!NOTE]')
expect(output).toContain('| Lines | 3 | 6 | 50.0%')
- expect(output.split('')[0]).toContain('Line coverage: **50.0%**')
+ expect(output.split('\n')[0]).toBe('## π¬ E2E Coverage: π‘ 50.0% of lines')
expect(output.split('')[0]).not.toContain('| Metric |')
expect(output).toMatch(
/Coverage totals<\/summary>\n\n\| Metric[\s\S]*\| Lines \| 3 \| 6 \| 50.0%[\s\S]*\n\n<\/details>/
@@ -93,3 +95,13 @@ describe('coverage-report shard banner', () => {
expect(render(metadata)).toContain('| Lines | 3 | 6 | 50.0%')
})
})
+
+describe('coverage-report line totals', () => {
+ it('warns instead of showing a percentage when no lines were recorded', () => {
+ const output = render({ complete: true }, '')
+
+ expect(output.split('\n')[0]).toBe(
+ '## π¬ E2E Coverage: β οΈ No line coverage recorded'
+ )
+ })
+})
diff --git a/scripts/coverage-report.ts b/scripts/coverage-report.ts
index ff8751d6640..517c6d5f376 100644
--- a/scripts/coverage-report.ts
+++ b/scripts/coverage-report.ts
@@ -6,6 +6,8 @@ import {
readCoverageMetadata
} from './coverage-metadata'
+import { renderPrReportSection } from './cicd/pr-report-section'
+
interface FileStats {
lines: number
covered: number
@@ -21,7 +23,11 @@ const lcovPath = process.argv[2] || 'coverage/playwright/coverage.lcov'
if (!existsSync(lcovPath)) {
process.stdout.write(
- '## π¬ E2E Coverage\n\n> β οΈ No coverage data found. Check the CI workflow logs.\n'
+ renderPrReportSection({
+ icon: 'π¬',
+ title: 'E2E Coverage',
+ status: 'β οΈ No coverage data found β check the CI workflow logs'
+ }) + '\n'
)
process.exit(0)
}
@@ -118,28 +124,22 @@ function bar(covered: number, total: number): string {
}
const lines: string[] = []
-lines.push('## π¬ E2E Coverage')
-lines.push('')
-
+const notice: string[] = []
const metadata = readCoverageMetadata(
join(dirname(lcovPath), COVERAGE_METADATA_FILE)
)
if (!metadata) {
- lines.push(
+ notice.push(
'> [!NOTE]',
- '> Shard completeness could not be verified for this run, so these totals may not be comparable with a whole merge. Check the coverage workflow logs and re-run collection before using these totals for comparison.',
- ''
+ '> Shard completeness could not be verified for this run, so these totals may not be comparable with a whole merge. Check the coverage workflow logs and re-run collection before using these totals for comparison.'
)
} else if (!metadata.complete) {
- lines.push(
+ notice.push(
'> [!NOTE]',
- '> The shard matrix did not pass, so these totals cannot be confirmed comparable with a whole merge. Fix the failing shards and re-run coverage before using these totals for comparison.',
- ''
+ '> The shard matrix did not pass, so these totals cannot be confirmed comparable with a whole merge. Fix the failing shards and re-run coverage before using these totals for comparison.'
)
}
-lines.push(`Line coverage: **${pct(coveredLines, totalLines)}**`, '')
-lines.push('', 'Coverage totals
', '')
lines.push('| Metric | Covered | Total | Pct | |')
lines.push('|---|--:|--:|--:|---|')
lines.push(
@@ -151,7 +151,6 @@ lines.push(
lines.push(
`| Branches | ${coveredBranches} | ${totalBranches} | ${pct(coveredBranches, totalBranches)} | ${bar(coveredBranches, totalBranches)} |`
)
-lines.push('', ' ')
const uncovered: UncoveredFile[] = [...fileStats.entries()]
.filter(([, s]) => s.lines > 0)
@@ -178,4 +177,18 @@ if (uncovered.length > 0) {
lines.push('
')
}
-process.stdout.write(lines.join('\n') + '\n')
+const linesStatus =
+ totalLines === 0
+ ? 'β οΈ No line coverage recorded'
+ : `${bar(coveredLines, totalLines)} ${pct(coveredLines, totalLines)} of lines`
+
+process.stdout.write(
+ renderPrReportSection({
+ icon: 'π¬',
+ title: 'E2E Coverage',
+ status: linesStatus,
+ notice: notice.join('\n'),
+ summary: 'Coverage totals',
+ body: lines.join('\n')
+ }) + '\n'
+)
diff --git a/scripts/perf-report.test.ts b/scripts/perf-report.test.ts
new file mode 100644
index 00000000000..256c2910b50
--- /dev/null
+++ b/scripts/perf-report.test.ts
@@ -0,0 +1,219 @@
+import { spawnSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+
+import type { PerfMeasurement } from '../browser_tests/fixtures/helpers/PerformanceHelper'
+
+const ROOT = join(import.meta.dirname, '..')
+const TSX = join(ROOT, 'node_modules/.bin/tsx')
+const SCRIPT = join(import.meta.dirname, 'perf-report.ts')
+
+type Sample = Pick<
+ PerfMeasurement,
+ 'name' | 'frameDurationMs' | 'p95FrameDurationMs'
+> &
+ Partial>
+
+interface ReportInputs {
+ baseline?: Sample[]
+ history?: Sample[][]
+}
+
+function writeReport(path: string, measurements: Sample[], index = 0) {
+ writeFileSync(
+ path,
+ JSON.stringify({
+ timestamp: new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+ measurements
+ })
+ )
+}
+
+function render(
+ measurements: Sample[],
+ { baseline, history = [] }: ReportInputs = {}
+): string {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-report-'))
+ try {
+ mkdirSync(join(dir, 'test-results'))
+ writeReport(join(dir, 'test-results/perf-metrics.json'), measurements)
+ if (baseline) {
+ mkdirSync(join(dir, 'temp/perf-baseline'), { recursive: true })
+ writeReport(join(dir, 'temp/perf-baseline/perf-metrics.json'), baseline)
+ }
+ if (history.length > 0) {
+ mkdirSync(join(dir, 'temp/perf-history'), { recursive: true })
+ history.forEach((run, index) =>
+ writeReport(join(dir, `temp/perf-history/${index}.json`), run, index)
+ )
+ }
+ const result = spawnSync(TSX, [SCRIPT], { cwd: dir, encoding: 'utf8' })
+ if (result.status !== 0) {
+ throw new Error(
+ `perf-report.ts exited ${result.status}: ${result.stderr}`
+ )
+ }
+ return result.stdout
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+}
+
+function frames(name: string, ...frameMs: number[]): Sample[] {
+ return frameMs.map((ms) => ({
+ name,
+ frameDurationMs: ms,
+ p95FrameDurationMs: ms,
+ totalBlockingTimeMs: 0
+ }))
+}
+
+function heading(output: string): string {
+ return output.split('\n')[0]
+}
+
+describe('perf-report heading', () => {
+ it('does not count a scenario without frame samples as missing the FPS target', () => {
+ const output = render([
+ { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 },
+ { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 }
+ ])
+
+ expect(heading(output)).not.toContain('below 52 FPS target')
+ expect(output).not.toContain('0.0 P5 FPS β')
+ expect(output.split('')[0]).toContain(
+ '1 scenario recorded no frame samples'
+ )
+ })
+
+ it('still leads with a real FPS target miss', () => {
+ const output = render([
+ { name: 'slow', frameDurationMs: 25, p95FrameDurationMs: 40 },
+ { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 }
+ ])
+
+ expect(heading(output)).toMatch(
+ /^## β‘ Performance: β 1 below 52 FPS target/
+ )
+ })
+
+ it('ignores unsampled runs when taking the median frame time', () => {
+ const output = render([
+ { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 },
+ { name: 'slow', frameDurationMs: 16, p95FrameDurationMs: 18 },
+ { name: 'slow', frameDurationMs: 16, p95FrameDurationMs: 21 }
+ ])
+
+ expect(heading(output)).toMatch(
+ /^## β‘ Performance: β 1 below 52 FPS target/
+ )
+ expect(output).not.toContain('recorded no frame samples')
+ })
+
+ it('does not read as clean when no scenario recorded frame samples', () => {
+ const output = render([
+ { name: 'unsampled', frameDurationMs: 0, p95FrameDurationMs: 0 }
+ ])
+
+ expect(heading(output)).toMatch(
+ /^## β‘ Performance: β οΈ No frame samples β FPS target not checked/
+ )
+ })
+
+ it('starts a scenario timing line with its first metric, not a separator', () => {
+ const output = render([
+ { name: 'fast', frameDurationMs: 16, p95FrameDurationMs: 17 }
+ ])
+
+ expect(output).toContain('> **fast**: 62.5 avg FPS Β· 58.8 P5 FPS β
')
+ })
+
+ it('leaves unsampled runs out of the frame time tables', () => {
+ const output = render([
+ { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 },
+ { name: 'slow', frameDurationMs: 0, p95FrameDurationMs: 0 },
+ { name: 'slow', frameDurationMs: 30, p95FrameDurationMs: 40 }
+ ])
+
+ expect(output).toContain('| slow: p95 frame time | 40ms |')
+ expect(output).toContain('| slow: avg frame time | 30ms |')
+ })
+
+ it('does not report a run that measured nothing as clean', () => {
+ const output = render([])
+
+ expect(heading(output)).toBe(
+ '## β‘ Performance: β οΈ No measurements recorded β check the perf test logs'
+ )
+ })
+})
+
+describe('perf-report baseline comparison', () => {
+ it('compares sampled frame medians against the baseline on a cold start', () => {
+ const output = render(frames('grow', 0, 40), {
+ baseline: frames('grow', 0, 20)
+ })
+
+ expect(output).toContain('| grow: p95 frame time | 20ms | 40ms | +100% |')
+ expect(output).toContain('| grow: avg frame time | 20ms | 40ms | +100% |')
+ expect(output).toContain('| grow: TBT | 0ms | 0ms | +0% |')
+ })
+
+ it('does not flag unchanged frame times when history has unsampled runs', () => {
+ const output = render(frames('steady', 0, 0, 17), {
+ baseline: frames('steady', 0, 0, 17),
+ history: [
+ frames('steady', 0, 0, 16),
+ frames('steady', 0, 0, 17),
+ frames('steady', 0, 0, 17)
+ ]
+ })
+
+ expect(heading(output)).toBe(
+ '## β‘ Performance: β
No regressions detected'
+ )
+ expect(output).toContain('| steady: p95 frame time | 17ms | 17ms | +0% |')
+ expect(output).toContain('| steady: TBT | 0ms | 0ms | +0% |')
+ expect(output).toMatch(/\| steady: p95 frame time \| 17ms \| 1ms \|/)
+ expect(output).toMatch(
+ /\| steady: p95 frame time \| \S+ \| [^|]+ \| 17ms \|/
+ )
+ })
+
+ it('leaves wholly unsampled history runs out of the frame time variance', () => {
+ const output = render(frames('steady', 0, 17), {
+ baseline: frames('steady', 0, 17),
+ history: [
+ frames('steady', 0, 0, 16),
+ frames('steady', 0, 0, 18),
+ frames('steady', 0, 0)
+ ]
+ })
+
+ expect(heading(output)).toBe(
+ '## β‘ Performance: β
No regressions detected'
+ )
+ expect(output).toContain(
+ 'Historical variance (up to 3 runs per metric)
'
+ )
+ expect(output).toContain('| steady: p95 frame time | 17ms | 1ms |')
+ expect(output).toContain('| steady: TBT | 0ms | 0ms |')
+ expect(output).not.toMatch(/\| steady: p95 frame time \| [βββββ
βββ]+ \|/)
+ })
+
+ it('flags a real frame time regression against sampled history', () => {
+ const output = render(frames('grow', 0, 40), {
+ baseline: frames('grow', 0, 20),
+ history: [frames('grow', 0, 20), frames('grow', 0, 21)]
+ })
+
+ expect(heading(output)).toContain('2 regressions detected')
+ expect(output.split('')[0]).toContain(
+ 'Review the regressions below and profile the affected scenarios before merging.'
+ )
+ expect(output).toContain('| grow: p95 frame time | 20ms | 40ms | +100% |')
+ })
+})
diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts
index c3659ab9a94..02e5cb172dc 100644
--- a/scripts/perf-report.ts
+++ b/scripts/perf-report.ts
@@ -1,6 +1,7 @@
import { existsSync, readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
+import { renderPrReportSection } from './cicd/pr-report-section'
import type { MetricStats } from './perf-stats'
import {
classifyChange,
@@ -133,7 +134,7 @@ function getHistoricalStats(
const group = groupByName(r.measurements)
const samples = group.get(testName)
if (samples) {
- const mean = meanMetric(samples, metric)
+ const mean = reportedMean(samples, metric)
if (mean !== null) values.push(mean)
}
}
@@ -153,9 +154,8 @@ function getHistoricalTimeSeries(
const group = groupByName(r.measurements)
const samples = group.get(testName)
if (samples) {
- values.push(
- samples.reduce((sum, s) => sum + s[metric], 0) / samples.length
- )
+ const mean = reportedMean(samples, metric)
+ if (mean !== null) values.push(mean)
}
}
return values
@@ -218,43 +218,149 @@ function frameTimeToFps(ms: number): number {
return ms > 0 ? 1000 / ms : 0
}
-function renderHeadlineSummary(
- prGroups: Map
-): string[] {
- const lines: string[] = []
- const summaries: string[] = []
- let belowTarget = 0
+type FrameMetricKey = 'frameDurationMs' | 'p95FrameDurationMs'
- for (const [testName, prSamples] of prGroups) {
- const avgFrame = medianMetric(prSamples, 'frameDurationMs')
- const p95Frame = medianMetric(prSamples, 'p95FrameDurationMs')
- const tbt = medianMetric(prSamples, 'totalBlockingTimeMs')
- const heap = medianMetric(prSamples, 'heapUsedBytes')
-
- const avgFps = avgFrame !== null ? frameTimeToFps(avgFrame) : null
- const p5Fps = p95Frame !== null ? frameTimeToFps(p95Frame) : null
-
- const parts: string[] = [`**${testName}**:`]
- if (avgFps !== null) parts.push(`${avgFps.toFixed(1)} avg FPS`)
- if (p5Fps !== null) {
- const pass = p5Fps >= TARGET_P5_FPS
- if (!pass) belowTarget++
- parts.push(
- `${p5Fps.toFixed(1)} P5 FPS ${pass ? 'β
' : 'β'} (target: β₯${TARGET_P5_FPS})`
- )
- }
- if (tbt !== null) parts.push(`${tbt.toFixed(0)}ms TBT`)
- if (heap !== null) parts.push(`${formatBytes(heap)} heap`)
+function isFrameMetric(key: MetricKey): key is FrameMetricKey {
+ return key === 'frameDurationMs' || key === 'p95FrameDurationMs'
+}
+
+function frameSampled(
+ samples: PerfMeasurement[],
+ key: FrameMetricKey
+): PerfMeasurement[] {
+ return samples.filter((s) => (getMetricValue(s, key) ?? 0) > 0)
+}
+
+function sampledFrameMetric(
+ samples: PerfMeasurement[],
+ key: FrameMetricKey
+): number | null {
+ return medianMetric(frameSampled(samples, key), key)
+}
+
+function reportedMedian(
+ samples: PerfMeasurement[],
+ key: MetricKey
+): number | null {
+ return isFrameMetric(key)
+ ? sampledFrameMetric(samples, key)
+ : medianMetric(samples, key)
+}
+
+function reportedMean(
+ samples: PerfMeasurement[],
+ key: MetricKey
+): number | null {
+ return meanMetric(
+ isFrameMetric(key) ? frameSampled(samples, key) : samples,
+ key
+ )
+}
+
+interface FpsTargetTally {
+ below: number
+ sampled: number
+ unsampled: number
+}
- if (parts.length > 1) summaries.push(parts.join(' Β· '))
+function tallyFpsTarget(
+ prGroups: Map
+): FpsTargetTally {
+ const tally: FpsTargetTally = { below: 0, sampled: 0, unsampled: 0 }
+ for (const prSamples of prGroups.values()) {
+ const p95Frame = sampledFrameMetric(prSamples, 'p95FrameDurationMs')
+ if (p95Frame === null) {
+ if (medianMetric(prSamples, 'p95FrameDurationMs') !== null)
+ tally.unsampled++
+ continue
+ }
+ tally.sampled++
+ if (frameTimeToFps(p95Frame) < TARGET_P5_FPS) tally.below++
}
+ return tally
+}
- if (belowTarget > 0) {
- lines.push(
- `> β **${belowTarget} scenario${belowTarget === 1 ? '' : 's'} below the β₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`,
+function plural(count: number, noun: string): string {
+ return `${count} ${noun}${count === 1 ? '' : 's'}`
+}
+
+function percentDelta(prVal: number, baseVal: number): number | null {
+ if (baseVal === 0) return prVal === 0 ? 0 : null
+ return ((prVal - baseVal) / baseVal) * 100
+}
+
+function baselineMedian(
+ baseSamples: PerfMeasurement[] | undefined,
+ key: MetricKey
+): number | null {
+ return baseSamples?.length ? reportedMedian(baseSamples, key) : null
+}
+
+function formatIfPresent(
+ value: number | null,
+ format: (value: number) => string
+): string | null {
+ return value === null ? null : format(value)
+}
+
+function p5FpsPart(p95Frame: number): string {
+ const p5Fps = frameTimeToFps(p95Frame)
+ const verdict = p5Fps >= TARGET_P5_FPS ? 'β
' : 'β'
+ return `${p5Fps.toFixed(1)} P5 FPS ${verdict} (target: β₯${TARGET_P5_FPS})`
+}
+
+function scenarioSummary(
+ testName: string,
+ prSamples: PerfMeasurement[]
+): string | null {
+ const parts = [
+ formatIfPresent(
+ sampledFrameMetric(prSamples, 'frameDurationMs'),
+ (ms) => `${frameTimeToFps(ms).toFixed(1)} avg FPS`
+ ),
+ formatIfPresent(
+ sampledFrameMetric(prSamples, 'p95FrameDurationMs'),
+ p5FpsPart
+ ),
+ formatIfPresent(
+ medianMetric(prSamples, 'totalBlockingTimeMs'),
+ (ms) => `${ms.toFixed(0)}ms TBT`
+ ),
+ formatIfPresent(
+ medianMetric(prSamples, 'heapUsedBytes'),
+ (bytes) => `${formatBytes(bytes)} heap`
+ )
+ ].filter((part): part is string => part !== null)
+
+ return parts.length > 0 ? `**${testName}**: ${parts.join(' Β· ')}` : null
+}
+
+function fpsTargetNotice({ below, unsampled }: FpsTargetTally): string[] {
+ const notice: string[] = []
+ if (below > 0) {
+ notice.push(
+ `> β **${plural(below, 'scenario')} below the β₯${TARGET_P5_FPS} P5 FPS target.** Expand the scenario timings below and profile the affected scenarios before merging.`,
+ ''
+ )
+ }
+ if (unsampled > 0) {
+ notice.push(
+ `> β οΈ ${plural(unsampled, 'scenario')} recorded no frame samples, so ${unsampled === 1 ? 'its' : 'their'} FPS could not be checked against the target. Check the perf test logs for a frame sampling timeout.`,
''
)
}
+ return notice
+}
+
+function renderHeadlineSummary(prGroups: Map): {
+ notice: string[]
+ lines: string[]
+} {
+ const notice = fpsTargetNotice(tallyFpsTarget(prGroups))
+ const lines: string[] = []
+ const summaries = [...prGroups]
+ .map(([testName, prSamples]) => scenarioSummary(testName, prSamples))
+ .filter((summary): summary is string => summary !== null)
if (summaries.length > 0) {
lines.push(
@@ -267,14 +373,63 @@ function renderHeadlineSummary(
)
}
- return lines
+ return { notice, lines }
+}
+
+interface RenderedReport {
+ status: string
+ notice?: string[]
+ lines: string[]
+}
+
+interface ComparedRow {
+ row: string
+ flagged: boolean
+}
+
+function compareMetric(
+ testName: string,
+ metric: MetricDef,
+ prSamples: PerfMeasurement[],
+ baseSamples: PerfMeasurement[] | undefined,
+ historical: PerfReport[]
+): ComparedRow | null {
+ const { key, label, unit, minAbsDelta } = metric
+ const prVal = reportedMedian(prSamples, key)
+ if (prVal === null) return null
+
+ const baseVal = baselineMedian(baseSamples, key)
+ if (baseVal === null) {
+ return {
+ row: `| ${testName}: ${label} | β | ${formatValue(prVal, unit)} | new | β |`,
+ flagged: false
+ }
+ }
+
+ const histStats = getHistoricalStats(historical, testName, key)
+ const z = zScore(prVal, histStats)
+ const sig = classifyChange(
+ z,
+ computeCV(histStats),
+ prVal - baseVal,
+ minAbsDelta
+ )
+
+ return {
+ row: `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(percentDelta(prVal, baseVal))} | ${formatSignificance(sig, z)} |`,
+ flagged: isNoteworthy(sig)
+ }
+}
+
+function regressionVerdict(count: number): string {
+ return `${count} regression${count > 1 ? 's' : ''} detected`
}
function renderFullReport(
prGroups: Map,
baseline: PerfReport,
historical: PerfReport[]
-): string[] {
+): RenderedReport {
const lines: string[] = []
const baselineGroups = groupByName(baseline.measurements)
const tableHeader = [
@@ -282,54 +437,34 @@ function renderFullReport(
'|--------|----------|----------|---|-----|'
]
- const flaggedRows: string[] = []
- const allRows: string[] = []
-
- for (const [testName, prSamples] of prGroups) {
- const baseSamples = baselineGroups.get(testName)
+ const compared = [...prGroups].flatMap(([testName, prSamples]) =>
+ REPORTED_METRICS.map((metric) =>
+ compareMetric(
+ testName,
+ metric,
+ prSamples,
+ baselineGroups.get(testName),
+ historical
+ )
+ ).filter((row): row is ComparedRow => row !== null)
+ )
+ const allRows = compared.map(({ row }) => row)
+ const flaggedRows = compared
+ .filter(({ flagged }) => flagged)
+ .map(({ row }) => row)
- for (const { key, label, unit, minAbsDelta } of REPORTED_METRICS) {
- const prVal = medianMetric(prSamples, key)
- if (prVal === null) continue
- const histStats = getHistoricalStats(historical, testName, key)
- const cv = computeCV(histStats)
-
- if (!baseSamples?.length) {
- allRows.push(
- `| ${testName}: ${label} | β | ${formatValue(prVal, unit)} | new | β |`
- )
- continue
- }
-
- const baseVal = medianMetric(baseSamples, key)
- if (baseVal === null) {
- allRows.push(
- `| ${testName}: ${label} | β | ${formatValue(prVal, unit)} | new | β |`
- )
- continue
- }
- const absDelta = prVal - baseVal
- const deltaPct =
- baseVal === 0
- ? prVal === 0
- ? 0
- : null
- : ((prVal - baseVal) / baseVal) * 100
- const z = zScore(prVal, histStats)
- const sig = classifyChange(z, cv, absDelta, minAbsDelta)
-
- const row = `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(deltaPct)} | ${formatSignificance(sig, z)} |`
- allRows.push(row)
- if (isNoteworthy(sig)) {
- flaggedRows.push(row)
- }
- }
- }
+ const status =
+ flaggedRows.length > 0
+ ? `β οΈ ${regressionVerdict(flaggedRows.length)}`
+ : 'β
No regressions detected'
+ const notice: string[] = []
if (flaggedRows.length > 0) {
+ notice.push(
+ `β οΈ **${regressionVerdict(flaggedRows.length)}**. Review the regressions below and profile the affected scenarios before merging.`,
+ ''
+ )
lines.push(
- `β οΈ **${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected**. Review the regressions below and profile the affected scenarios before merging.`,
- '',
'Show regressions
',
'',
...tableHeader,
@@ -338,8 +473,6 @@ function renderFullReport(
' ',
''
)
- } else {
- lines.push('β
No regressions detected.', '')
}
lines.push(
@@ -353,11 +486,23 @@ function renderFullReport(
)
lines.push(
- `Historical variance (last ${historical.length} runs)
`,
+ ...renderHistoricalVariance(prGroups, historical),
+ ...renderTrend(prGroups, historical)
+ )
+
+ return { status, notice, lines }
+}
+
+function renderHistoricalVariance(
+ prGroups: Map,
+ historical: PerfReport[]
+): string[] {
+ const lines = [
+ `Historical variance (up to ${historical.length} runs per metric)
`,
'',
'| Metric | ΞΌ | Ο | CV |',
'|--------|---|---|-----|'
- )
+ ]
for (const [testName] of prGroups) {
for (const { key, label, unit } of REPORTED_METRICS) {
const stats = getHistoricalStats(historical, testName, key)
@@ -369,47 +514,47 @@ function renderFullReport(
}
}
lines.push('', ' ')
+ return lines
+}
+function renderTrend(
+ prGroups: Map,
+ historical: PerfReport[]
+): string[] {
const trendRows: string[] = []
for (const [testName] of prGroups) {
for (const { key, label, unit } of REPORTED_METRICS) {
const series = getHistoricalTimeSeries(historical, testName, key)
if (series.length < 3) continue
- const dir = trendDirection(series)
- const arrow = trendArrow(dir)
- const spark = sparkline(series)
+ const arrow = trendArrow(trendDirection(series))
const last = series[series.length - 1]
trendRows.push(
- `| ${testName}: ${label} | ${spark} | ${arrow} | ${formatValue(last, unit)} |`
+ `| ${testName}: ${label} | ${sparkline(series)} | ${arrow} | ${formatValue(last, unit)} |`
)
}
}
-
- if (trendRows.length > 0) {
- lines.push(
- '',
- `Trend (last ${historical.length} commits on main)
`,
- '',
- '| Metric | Trend | Dir | Latest |',
- '|--------|-------|-----|--------|',
- ...trendRows,
- '',
- ' '
- )
- }
-
- return lines
+ if (trendRows.length === 0) return []
+ return [
+ '',
+ `Trend (up to ${historical.length} commits on main per metric)
`,
+ '',
+ '| Metric | Trend | Dir | Latest |',
+ '|--------|-------|-----|--------|',
+ ...trendRows,
+ '',
+ ' '
+ ]
}
function renderColdStartReport(
prGroups: Map,
baseline: PerfReport,
historicalCount: number
-): string[] {
+): RenderedReport {
const lines: string[] = []
const baselineGroups = groupByName(baseline.measurements)
lines.push(
- `> βΉοΈ Collecting baseline variance data (${historicalCount}/15 runs). Significance will appear after 2 main branch runs.`,
+ '> Significance will appear after 2 main branch runs.',
'',
'All metrics (cold start)
',
'',
@@ -421,46 +566,30 @@ function renderColdStartReport(
const baseSamples = baselineGroups.get(testName)
for (const { key, label, unit } of REPORTED_METRICS) {
- const prVal = medianMetric(prSamples, key)
+ const prVal = reportedMedian(prSamples, key)
if (prVal === null) continue
- if (!baseSamples?.length) {
- lines.push(
- `| ${testName}: ${label} | β | ${formatValue(prVal, unit)} | new |`
- )
- continue
- }
-
- const baseVal = medianMetric(baseSamples, key)
- if (baseVal === null) {
- lines.push(
- `| ${testName}: ${label} | β | ${formatValue(prVal, unit)} | new |`
- )
- continue
- }
- const deltaPct =
- baseVal === 0
- ? prVal === 0
- ? 0
- : null
- : ((prVal - baseVal) / baseVal) * 100
+ const baseVal = baselineMedian(baseSamples, key)
lines.push(
- `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(deltaPct)} |`
+ baseVal === null
+ ? `| ${testName}: ${label} | β | ${formatValue(prVal, unit)} | new |`
+ : `| ${testName}: ${label} | ${formatValue(baseVal, unit)} | ${formatValue(prVal, unit)} | ${formatDelta(percentDelta(prVal, baseVal))} |`
)
}
}
lines.push('', ' ')
- return lines
+ return {
+ status: `βΉοΈ Collecting baseline variance (${historicalCount}/15 runs)`,
+ lines
+ }
}
function renderNoBaselineReport(
prGroups: Map
-): string[] {
+): RenderedReport {
const lines: string[] = []
lines.push(
- '> βΉοΈ No baseline found β significance unavailable.',
- '',
'Absolute values
',
'',
'| Metric | Value |',
@@ -468,57 +597,89 @@ function renderNoBaselineReport(
)
for (const [testName, prSamples] of prGroups) {
for (const { key, label, unit } of REPORTED_METRICS) {
- const prVal = medianMetric(prSamples, key)
+ const prVal = reportedMedian(prSamples, key)
if (prVal === null) continue
lines.push(`| ${testName}: ${label} | ${formatValue(prVal, unit)} |`)
}
}
lines.push('', ' ')
- return lines
+ return { status: 'βΉοΈ No baseline β significance unavailable', lines }
}
-function main() {
- if (!existsSync(CURRENT_PATH)) {
- process.stdout.write(
- '## β‘ Performance Report\n\nNo perf metrics found. Perf tests may not have run.\n'
- )
- process.exit(0)
- }
-
- const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8'))
-
- const baseline: PerfReport | null = existsSync(BASELINE_PATH)
- ? JSON.parse(readFileSync(BASELINE_PATH, 'utf-8'))
- : null
-
+function renderComparison(
+ prGroups: Map
+): RenderedReport {
+ if (!existsSync(BASELINE_PATH)) return renderNoBaselineReport(prGroups)
+ const baseline: PerfReport = JSON.parse(readFileSync(BASELINE_PATH, 'utf-8'))
const historical = loadHistoricalReports()
- const prGroups = groupByName(current.measurements)
-
- const lines: string[] = []
- lines.push('## β‘ Performance Report\n')
- lines.push(...renderHeadlineSummary(prGroups))
-
- if (baseline && historical.length >= 2) {
- lines.push(...renderFullReport(prGroups, baseline, historical))
- } else if (baseline) {
- lines.push(...renderColdStartReport(prGroups, baseline, historical.length))
- } else {
- lines.push(...renderNoBaselineReport(prGroups))
- }
+ return historical.length >= 2
+ ? renderFullReport(prGroups, baseline, historical)
+ : renderColdStartReport(prGroups, baseline, historical.length)
+}
+function renderRawData(current: PerfReport): string[] {
const rawData = {
...current,
measurements: current.measurements.map(
({ allFrameDurationsMs: _, ...rest }) => rest
)
}
- lines.push('\nRaw data
\n')
- lines.push('```json')
- lines.push(JSON.stringify(rawData, null, 2))
- lines.push('```')
- lines.push('\n ')
+ return [
+ '',
+ 'Raw data
',
+ '',
+ '```json',
+ JSON.stringify(rawData, null, 2),
+ '```',
+ '',
+ ' '
+ ]
+}
- process.stdout.write(lines.join('\n') + '\n')
+function perfHeadingStatus(
+ current: PerfReport,
+ prGroups: Map,
+ reportStatus: string
+): string {
+ if (current.measurements.length === 0)
+ return 'β οΈ No measurements recorded β check the perf test logs'
+ const fps = tallyFpsTarget(prGroups)
+ if (fps.below > 0)
+ return `β ${fps.below} below ${TARGET_P5_FPS} FPS target Β· ${reportStatus}`
+ if (fps.unsampled > 0 && fps.sampled === 0)
+ return `β οΈ No frame samples β FPS target not checked Β· ${reportStatus}`
+ return reportStatus
+}
+
+function main() {
+ if (!existsSync(CURRENT_PATH)) {
+ process.stdout.write(
+ renderPrReportSection({
+ icon: 'β‘',
+ title: 'Performance',
+ status: 'βοΈ No metrics found β perf tests may not have run'
+ }) + '\n'
+ )
+ process.exit(0)
+ }
+
+ const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8'))
+ const prGroups = groupByName(current.measurements)
+ const report = renderComparison(prGroups)
+
+ const headline = renderHeadlineSummary(prGroups)
+ const body = [...headline.lines, ...report.lines, ...renderRawData(current)]
+ const status = perfHeadingStatus(current, prGroups, report.status)
+
+ process.stdout.write(
+ renderPrReportSection({
+ icon: 'β‘',
+ title: 'Performance',
+ status,
+ notice: [...headline.notice, ...(report.notice ?? [])].join('\n'),
+ body: body.join('\n')
+ }) + '\n'
+ )
}
main()
diff --git a/scripts/unified-report.test.ts b/scripts/unified-report.test.ts
new file mode 100644
index 00000000000..ec50f973163
--- /dev/null
+++ b/scripts/unified-report.test.ts
@@ -0,0 +1,49 @@
+import { spawnSync } from 'node:child_process'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+
+const ROOT = join(import.meta.dirname, '..')
+const TSX = join(ROOT, 'node_modules/.bin/tsx')
+const SCRIPT = join(import.meta.dirname, 'unified-report.ts')
+
+function render(args: string[]): string {
+ const dir = mkdtempSync(join(tmpdir(), 'unified-report-'))
+ try {
+ const result = spawnSync(TSX, [SCRIPT, ...args], {
+ cwd: dir,
+ encoding: 'utf8'
+ })
+ if (result.status !== 0) {
+ throw new Error(
+ `unified-report.ts exited ${result.status}: ${result.stderr}`
+ )
+ }
+ return result.stdout
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+}
+
+describe('unified-report', () => {
+ it('separates each section from the one before it with a blank line', () => {
+ const output = render([
+ '--size-status=failed',
+ '--perf-status=pending',
+ '--coverage-status=failed'
+ ])
+
+ expect(output).toBe(
+ [
+ '## π¦ Bundle: β οΈ Size data collection failed β check the CI workflow logs',
+ '',
+ '## β‘ Performance: β³ Performance tests in progressβ¦',
+ '',
+ '## π¬ E2E Coverage: β οΈ Coverage collection failed β check the CI workflow logs',
+ ''
+ ].join('\n')
+ )
+ })
+})
diff --git a/scripts/unified-report.ts b/scripts/unified-report.ts
index edf132abb53..13d6710b652 100644
--- a/scripts/unified-report.ts
+++ b/scripts/unified-report.ts
@@ -1,6 +1,8 @@
import { execFileSync } from 'node:child_process'
import { existsSync } from 'node:fs'
+import { renderPrReportSection } from './cicd/pr-report-section'
+
const args: string[] = process.argv.slice(2)
function getArg(name: string): string | undefined {
@@ -13,6 +15,18 @@ const sizeStatus = getArg('size-status') ?? 'pending'
const perfStatus = getArg('perf-status') ?? 'pending'
const coverageStatus = getArg('coverage-status') ?? 'skip'
+function bundleStatusLine(status: string): string {
+ return renderPrReportSection({ icon: 'π¦', title: 'Bundle', status })
+}
+
+function perfStatusLine(status: string): string {
+ return renderPrReportSection({ icon: 'β‘', title: 'Performance', status })
+}
+
+function coverageStatusLine(status: string): string {
+ return renderPrReportSection({ icon: 'π¬', title: 'E2E Coverage', status })
+}
+
const lines: string[] = []
const hasSizeData = existsSync('temp/size')
@@ -26,20 +40,20 @@ if (sizeStatus === 'ready' && hasSizeData) {
).trimEnd()
lines.push(sizeReport)
} catch {
- lines.push('## π¦ Bundle Size')
- lines.push('')
lines.push(
- '> β οΈ Failed to render bundle size report. Check the CI workflow logs.'
+ bundleStatusLine(
+ 'β οΈ Failed to render report β check the CI workflow logs'
+ )
)
}
} else if (sizeStatus === 'failed') {
- lines.push('## π¦ Bundle Size')
- lines.push('')
- lines.push('> β οΈ Size data collection failed. Check the CI workflow logs.')
+ lines.push(
+ bundleStatusLine(
+ 'β οΈ Size data collection failed β check the CI workflow logs'
+ )
+ )
} else if (sizeStatus !== 'ready') {
- lines.push('## π¦ Bundle Size')
- lines.push('')
- lines.push('> β³ Size data collection in progressβ¦')
+ lines.push(bundleStatusLine('β³ Size data collection in progressβ¦'))
}
if (lines.length > 0) lines.push('')
@@ -53,23 +67,19 @@ if (perfStatus === 'ready' && existsSync('test-results/perf-metrics.json')) {
).trimEnd()
lines.push(perfReport)
} catch {
- lines.push('## β‘ Performance')
- lines.push('')
lines.push(
- '> β οΈ Failed to render performance report. Check the CI workflow logs.'
+ perfStatusLine('β οΈ Failed to render report β check the CI workflow logs')
)
}
} else if (
perfStatus === 'failed' ||
(perfStatus === 'ready' && !existsSync('test-results/perf-metrics.json'))
) {
- lines.push('## β‘ Performance')
- lines.push('')
- lines.push('> β οΈ Performance tests failed. Check the CI workflow logs.')
+ lines.push(
+ perfStatusLine('β οΈ Performance tests failed β check the CI workflow logs')
+ )
} else if (perfStatus !== 'skip') {
- lines.push('## β‘ Performance')
- lines.push('')
- lines.push('> β³ Performance tests in progressβ¦')
+ lines.push(perfStatusLine('β³ Performance tests in progressβ¦'))
}
if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) {
@@ -87,18 +97,20 @@ if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) {
lines.push('')
lines.push(coverageReport)
} catch {
- lines.push('')
- lines.push('## π¬ E2E Coverage')
lines.push('')
lines.push(
- '> β οΈ Failed to render coverage report. Check the CI workflow logs.'
+ coverageStatusLine(
+ 'β οΈ Failed to render report β check the CI workflow logs'
+ )
)
}
} else if (coverageStatus === 'failed') {
lines.push('')
- lines.push('## π¬ E2E Coverage')
- lines.push('')
- lines.push('> β οΈ Coverage collection failed. Check the CI workflow logs.')
+ lines.push(
+ coverageStatusLine(
+ 'β οΈ Coverage collection failed β check the CI workflow logs'
+ )
+ )
}
process.stdout.write(lines.join('\n') + '\n')