diff --git a/.github/actions/changes-filter/action.yaml b/.github/actions/changes-filter/action.yaml index 06fed9f5640..b7f5925c6a1 100644 --- a/.github/actions/changes-filter/action.yaml +++ b/.github/actions/changes-filter/action.yaml @@ -41,7 +41,7 @@ outputs: candidate's synthetic head. value: ${{ (github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'merge_group' && inputs.evaluate-candidate != 'true') || steps.relevant.outputs.relevant == 'true' }} app-website-changes: - description: 'Shared deps or `apps/website/**` changed.' + description: 'Shared deps, `apps/website/**`, or website e2e workflow support changed.' value: ${{ github.event_name != 'pull_request' || steps.filter.outputs.deps == 'true' || steps.filter.outputs.app_website == 'true' }} app-frontend-changes: description: 'Shared deps or `src/**` changed.' @@ -70,6 +70,9 @@ runs: filters: | app_website: - 'apps/website/**' + - '.github/workflows/ci-website-e2e.yaml' + - 'scripts/cicd/prepare-website-e2e-report.sh' + - 'scripts/cicd/check-website-e2e-result.sh' app_frontend: - 'src/**' packages: diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 98255c57dd3..68f728e12d8 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -24,7 +24,7 @@ jobs: - id: changes uses: ./.github/actions/changes-filter - website-e2e: + website-e2e-shard: needs: changes if: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} runs-on: ubuntu-latest @@ -33,13 +33,11 @@ jobs: timeout-minutes: 30 permissions: contents: read - outputs: - test-outcome: ${{ steps.tests.outcome }} - report-url: ${{ steps.deploy.outputs.url }} - screenshot-failures: ${{ steps.failures.outputs.screenshot }} - other-failures: ${{ steps.failures.outputs.other }} - # Evaluated at job level (not from a step) β€” static expression. - is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }} + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] steps: - uses: actions/checkout@v7 @@ -60,13 +58,62 @@ jobs: PUBLIC_CUSTOMERIO_WRITE_KEY: test-e2e-write-key run: pnpm --filter @comfyorg/website build - - name: Run Playwright tests + - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: tests - run: pnpm --filter @comfyorg/website test:e2e + run: pnpm --filter @comfyorg/website test:e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + env: + PLAYWRIGHT_BLOB_OUTPUT_DIR: ${{ github.workspace }}/apps/website/blob-report - - name: Upload test report + - name: Upload blob report uses: actions/upload-artifact@v6 if: ${{ !cancelled() }} + with: + name: website-blob-report-${{ matrix.shardIndex }} + path: apps/website/blob-report/ + if-no-files-found: error + retention-days: 1 + + website-e2e-report: + needs: [changes, website-e2e-shard] + if: ${{ always() && (needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + test-outcome: ${{ steps.outcome.outputs.result }} + report-url: ${{ steps.deploy.outputs.url }} + screenshot-failures: ${{ steps.failures.outputs.screenshot }} + other-failures: ${{ steps.failures.outputs.other }} + is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }} + steps: + - uses: actions/checkout@v7 + + - name: Install pnpm + run: corepack enable && corepack prepare + + - name: Download blob reports + id: blob-reports + continue-on-error: true + uses: actions/download-artifact@v8 + with: + path: apps/website/all-blob-reports + pattern: website-blob-report-* + merge-multiple: true + + - name: Validate and merge blob reports + id: outcome + working-directory: apps/website + env: + SHARDS_RESULT: ${{ needs.website-e2e-shard.result }} + BLOB_REPORTS_OUTCOME: ${{ steps.blob-reports.outcome }} + run: | + PLAYWRIGHT_VERSION=$(pnpm list @playwright/test --depth 0 --json --lockfile-only | jq -r '.[0].devDependencies["@playwright/test"].version') + ../../scripts/cicd/prepare-website-e2e-report.sh all-blob-reports 4 "$PLAYWRIGHT_VERSION" + + - name: Upload test report + uses: actions/upload-artifact@v6 + if: ${{ !cancelled() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} with: name: website-playwright-report path: | @@ -80,7 +127,8 @@ jobs: if: >- ${{ always() && - !cancelled() && + needs.website-e2e-shard.result != 'cancelled' && + steps.outcome.outputs.result != 'infrastructure' && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false @@ -98,7 +146,7 @@ jobs: OUTPUT=$(pnpx wrangler@^4.0.0 pages deploy apps/website/playwright-report \ --project-name=comfyui-website-e2e \ --branch="$BRANCH" 2>&1) && { DEPLOY_OK=true; break; } || echo "$OUTPUT" - [ $i -lt 3 ] && sleep 10 + [ "$i" -lt 3 ] && sleep 10 done echo "$OUTPUT" if [ "$DEPLOY_OK" != "true" ]; then @@ -106,11 +154,11 @@ jobs: exit 1 fi URL=$(echo "$OUTPUT" | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1) - echo "url=${URL}" >> $GITHUB_OUTPUT + echo "url=${URL}" >> "$GITHUB_OUTPUT" - name: Categorize failures id: failures - if: always() && !cancelled() && steps.tests.outcome != 'success' + if: always() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result == 'failure' uses: actions/github-script@v9 with: script: | @@ -129,8 +177,6 @@ jobs: ] } - // True: Visual - // False: Other const failed = specsOf(report) .flatMap(spec => (spec.tests || []) .filter(isFailed) @@ -141,16 +187,18 @@ jobs: core.setOutput('other', failed.length - screenshotFailures) - name: Write job summary - if: always() && !cancelled() + if: always() && needs.website-e2e-shard.result != 'cancelled' uses: actions/github-script@v9 env: - TEST_OUTCOME: ${{ steps.tests.outcome }} + TEST_OUTCOME: ${{ steps.outcome.outputs.result }} REPORT_URL: ${{ steps.deploy.outputs.url }} SCREENSHOT_FAILURES: ${{ steps.failures.outputs.screenshot }} OTHER_FAILURES: ${{ steps.failures.outputs.other }} with: script: | - const passed = process.env.TEST_OUTCOME === 'success' + const outcome = process.env.TEST_OUTCOME + const passed = outcome === 'success' + const infrastructure = outcome === 'infrastructure' const reportUrl = process.env.REPORT_URL const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0 const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0 @@ -159,15 +207,20 @@ jobs: if (passed) { lines.push('> [!TIP]', '> All tests passed.') + } else if (infrastructure) { + lines.push( + '> [!CAUTION]', + '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + ) } else { lines.push('> [!CAUTION]', '> Some tests failed.') } const rows = [ - ['Status', passed ? 'βœ… Passed' : '❌ Failed'], + ['Status', passed ? 'βœ… Passed' : infrastructure ? 'πŸ’₯ Infrastructure failure' : '❌ Failed'], ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_'] ] - if (!passed) { + if (!passed && !infrastructure) { rows.push( ['Screenshot diffs', String(screenshotFailures)], ['Other failures', String(otherFailures)] @@ -182,9 +235,25 @@ jobs: await core.summary.addRaw(lines.join('\n')).write() + website-e2e: + if: ${{ always() }} + needs: [changes, website-e2e-shard, website-e2e-report] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - name: Check website E2E result + env: + SHOULD_RUN: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} + SHARD_RESULT: ${{ needs.website-e2e-shard.result }} + REPORT_RESULT: ${{ needs.website-e2e-report.result }} + TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }} + run: scripts/cicd/check-website-e2e-result.sh + post-starting-comment: - # Safe to comment from pull_request trigger: fork PRs are excluded by the guard below. - # This avoids a ci-*/pr-* workflow_run split for a comment that must appear immediately. needs: changes if: | github.event_name == 'pull_request' @@ -213,8 +282,8 @@ jobs: > Tests are running… [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) post-result-comment: - needs: website-e2e - if: always() && !cancelled() && needs.website-e2e.outputs.is-pr == 'true' + needs: website-e2e-report + if: always() && !cancelled() && needs.website-e2e-report.outputs.is-pr == 'true' runs-on: ubuntu-latest permissions: pull-requests: write @@ -229,13 +298,15 @@ jobs: id: content uses: actions/github-script@v9 env: - TEST_OUTCOME: ${{ needs.website-e2e.outputs.test-outcome }} - REPORT_URL: ${{ needs.website-e2e.outputs.report-url }} - SCREENSHOT_FAILURES: ${{ needs.website-e2e.outputs.screenshot-failures }} - OTHER_FAILURES: ${{ needs.website-e2e.outputs.other-failures }} + TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }} + REPORT_URL: ${{ needs.website-e2e-report.outputs.report-url }} + SCREENSHOT_FAILURES: ${{ needs.website-e2e-report.outputs.screenshot-failures }} + OTHER_FAILURES: ${{ needs.website-e2e-report.outputs.other-failures }} with: script: | - const passed = process.env.TEST_OUTCOME === 'success' + const outcome = process.env.TEST_OUTCOME + const passed = outcome === 'success' + const infrastructure = outcome === 'infrastructure' const reportUrl = process.env.REPORT_URL const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0 const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0 @@ -244,15 +315,20 @@ jobs: if (passed) { lines.push('> [!TIP]', '> All tests passed.') + } else if (infrastructure) { + lines.push( + '> [!CAUTION]', + '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + ) } else { lines.push('> [!CAUTION]', '> Some tests failed.') } const rows = [ - ['Status', passed ? 'βœ… Passed' : '❌ Failed'], + ['Status', passed ? 'βœ… Passed' : infrastructure ? 'πŸ’₯ Infrastructure failure' : '❌ Failed'], ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_'] ] - if (!passed) { + if (!passed && !infrastructure) { rows.push( ['Screenshot diffs', String(screenshotFailures)], ['Other failures', String(otherFailures)] diff --git a/apps/website/playwright.config.ts b/apps/website/playwright.config.ts index 1845a1010d0..95da727c40a 100644 --- a/apps/website/playwright.config.ts +++ b/apps/website/playwright.config.ts @@ -30,9 +30,11 @@ export default defineConfig({ fullyParallel: true, forbidOnly: !!process.env.CI, globalTimeout: process.env.CI ? 20 * 60_000 : 0, - reporter: process.env.CI - ? [['list'], ['html'], ['json', { outputFile: 'results.json' }]] - : 'html', + reporter: process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR + ? 'blob' + : process.env.CI + ? [['list'], ['html'], ['json', { outputFile: 'results.json' }]] + : 'html', expect: { toHaveScreenshot: { maxDiffPixels: 100 } }, diff --git a/scripts/cicd/check-website-e2e-result.sh b/scripts/cicd/check-website-e2e-result.sh new file mode 100755 index 00000000000..3cd1f10e5c6 --- /dev/null +++ b/scripts/cicd/check-website-e2e-result.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$SHOULD_RUN" != true ]]; then + echo "Website E2E skipped (no relevant changes)" + exit 0 +fi + +if [[ "$SHARD_RESULT" != success || "$REPORT_RESULT" != success || "$TEST_OUTCOME" != success ]]; then + echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT, outcome: $TEST_OUTCOME)" + exit 1 +fi + +echo "Website E2E passed" diff --git a/scripts/cicd/prepare-website-e2e-report.sh b/scripts/cicd/prepare-website-e2e-report.sh new file mode 100755 index 00000000000..df4c5d4ead2 --- /dev/null +++ b/scripts/cicd/prepare-website-e2e-report.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +reports_dir=$1 +expected_reports=$2 +playwright_version=$3 +report_count=0 +if [[ -d "$reports_dir" ]]; then + report_count=$(find "$reports_dir" -type f -name '*.zip' | wc -l) +fi + +infrastructure_failure() { + echo "$1" >&2 + echo "result=infrastructure" >> "$GITHUB_OUTPUT" +} + +if [[ "$BLOB_REPORTS_OUTCOME" != success || "$report_count" -ne "$expected_reports" ]]; then + infrastructure_failure "Incomplete blob reports ($report_count/$expected_reports)" + exit 0 +fi + +if ! pnpm dlx "@playwright/test@$playwright_version" merge-reports --reporter=html "$reports_dir" || + ! PLAYWRIGHT_JSON_OUTPUT_NAME=results.json pnpm dlx "@playwright/test@$playwright_version" merge-reports --reporter=json "$reports_dir"; then + infrastructure_failure "Failed to merge blob reports" + exit 0 +fi + +if [[ "$SHARDS_RESULT" == success ]]; then + echo "result=success" >> "$GITHUB_OUTPUT" +else + echo "result=failure" >> "$GITHUB_OUTPUT" +fi diff --git a/scripts/cicd/website-e2e-workflow.test.ts b/scripts/cicd/website-e2e-workflow.test.ts new file mode 100644 index 00000000000..09dd7cfdb0c --- /dev/null +++ b/scripts/cicd/website-e2e-workflow.test.ts @@ -0,0 +1,170 @@ +import { spawnSync } from 'node:child_process' +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const ROOT = join(import.meta.dirname, '../..') +const PREPARE_REPORT = join( + import.meta.dirname, + 'prepare-website-e2e-report.sh' +) +const CHECK_RESULT = join(import.meta.dirname, 'check-website-e2e-result.sh') + +function reportFixture() { + const root = mkdtempSync(join(tmpdir(), 'website-e2e-report-')) + const reports = join(root, 'reports') + const bin = join(root, 'bin') + const output = join(root, 'github-output') + mkdirSync(reports) + mkdirSync(bin) + writeFileSync( + join(bin, 'pnpm'), + `#!/usr/bin/env bash +set -euo pipefail +echo "$*" >> "$CALLS" +[[ "\${MERGE_REPORTS_OK:-true}" == true ]] +`, + { mode: 0o755 } + ) + + return { + root, + reports, + output, + addReports(count: number) { + for (let index = 0; index < count; index++) + writeFileSync(join(reports, `${index}.zip`), '') + }, + run( + options: { + download?: string + mergeReports?: string + shards?: string + } = {} + ) { + const calls = join(root, 'calls') + writeFileSync(calls, '') + const result = spawnSync( + 'bash', + [PREPARE_REPORT, reports, '4', '1.61.1'], + { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + BLOB_REPORTS_OUTCOME: options.download ?? 'success', + SHARDS_RESULT: options.shards ?? 'success', + MERGE_REPORTS_OK: options.mergeReports ?? 'true', + GITHUB_OUTPUT: output, + CALLS: calls + } + } + ) + return { + status: result.status, + output: readFileSync(output, 'utf8'), + calls: readFileSync(calls, 'utf8') + } + }, + [Symbol.dispose]() { + rmSync(root, { recursive: true, force: true }) + } + } +} + +function checkResult(env: Record) { + return spawnSync('bash', [CHECK_RESULT], { + encoding: 'utf8', + env: { ...process.env, ...env } + }) +} + +describe('website E2E workflow', () => { + it('merges every blob before reporting successful tests', () => { + using fixture = reportFixture() + fixture.addReports(4) + + const result = fixture.run() + + expect(result.status).toBe(0) + expect(result.output).toBe('result=success\n') + expect(result.calls).toContain('@playwright/test@1.61.1') + expect(result.calls).toContain('--reporter=html') + expect(result.calls).toContain('--reporter=json') + }) + + it('preserves a complete failing test run', () => { + using fixture = reportFixture() + fixture.addReports(4) + + expect(fixture.run({ shards: 'failure' }).output).toBe('result=failure\n') + }) + + it('reports infrastructure failure for incomplete artifacts', () => { + using failedDownload = reportFixture() + failedDownload.addReports(4) + using missingReport = reportFixture() + missingReport.addReports(3) + + expect(failedDownload.run({ download: 'failure' }).output).toBe( + 'result=infrastructure\n' + ) + expect(missingReport.run().output).toBe('result=infrastructure\n') + }) + + it('reports infrastructure failure when report generation fails', () => { + using fixture = reportFixture() + fixture.addReports(4) + + expect(fixture.run({ mergeReports: 'false' }).output).toBe( + 'result=infrastructure\n' + ) + }) + + it('keeps the matrix and completeness check synchronized', () => { + const workflow = readFileSync( + join(ROOT, '.github/workflows/ci-website-e2e.yaml'), + 'utf8' + ) + + expect(workflow).toContain('shardTotal: [4]') + expect(workflow).toContain( + 'prepare-website-e2e-report.sh all-blob-reports 4 "$PLAYWRIGHT_VERSION"' + ) + }) + + it('passes only skipped or fully successful runs', () => { + expect( + checkResult({ + SHOULD_RUN: 'false', + SHARD_RESULT: 'skipped', + REPORT_RESULT: 'skipped', + TEST_OUTCOME: '' + }).status + ).toBe(0) + expect( + checkResult({ + SHOULD_RUN: 'true', + SHARD_RESULT: 'success', + REPORT_RESULT: 'success', + TEST_OUTCOME: 'success' + }).status + ).toBe(0) + expect( + checkResult({ + SHOULD_RUN: 'true', + SHARD_RESULT: 'success', + REPORT_RESULT: 'success', + TEST_OUTCOME: 'infrastructure' + }).status + ).toBe(1) + }) +})