From 74bc8df1d0801094edb3d1ca3ee3ccd1d61f3ba3 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 2 Sep 2026 19:04:04 +0000 Subject: [PATCH 01/10] ci(website-e2e): shard merge-group Playwright run to fix timeout cancellations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge-group website-e2e job runs 407 Playwright tests with only 2 workers (ubuntu-latest is 4-core, Playwright defaults to ceil(cpus/2) workers). A passing run takes 8.2m of test wall time alone, leaving almost no slack under the 15m job timeout — any runner slowness pushes it past 15m and the job is cancelled, which silently dequeues an already-approved PR from the merge queue instead of failing it visibly. Evidence (job timing from run logs): - #15753 merge-group 18:13Z: cancelled https://github.com/Comfy-Org/ComfyUI_frontend/actions/runs/33665855796 - #15660 merge-group 18:28Z: ran 18:29:05Z->18:44:09Z (15m04s), cancelled at 'Run Playwright tests' https://github.com/Comfy-Org/ComfyUI_frontend/actions/runs/33667445294 - #16234 dequeued twice today; passing merge-group run 18:44Z measured 'Running 407 tests using 2 workers' / '406 passed (8.2m)', job total 9m47s (run 33669067839) — confirms the near-zero timeout margin even on a clean pass. Fix: shard the Playwright run 4-way (matrix job website-e2e-shard), each shard building the site independently (build is ~13s, not worth artifact-passing) and uploading a blob report. A new website-e2e-report job merges the blobs into the HTML/JSON report, deploys to Cloudflare, categorizes failures, and writes the job summary — the same responsibilities the single job used to have. The required-check context 'website-e2e' (pinned in the branch ruleset) is preserved as a lightweight gate job so the check name doesn't fragment into matrix-expanded contexts. No tests were weakened, skipped, or deleted — same 407 specs run, just parallelized across more workers. Pattern follows the existing sharded-matrix + merge-reports approach already used in ci-tests-e2e.yaml (playwright-tests-chromium-sharded / merge-reports). --- .github/workflows/ci-website-e2e.yaml | 132 ++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index f901c6eff6b..717d57dc0b4 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -24,7 +24,16 @@ jobs: - id: changes uses: ./.github/actions/changes-filter - website-e2e: + # Sharded so 407 specs no longer run in a single 2-worker job (that took + # ~8.2m of Playwright wall time alone, leaving almost no slack under the + # 15m job timeout — any runner slowness pushed it over and the + # merge-group cancellation silently dequeued the PR). Each shard builds + # independently (the site build is ~13s, not worth artifact-passing) and + # uploads a blob report for website-e2e-report to merge. Named + # `-shard` (not `website-e2e`) because the required-check context + # `website-e2e` (branch ruleset) must stay a single, non-matrix-expanded + # check name — see the `website-e2e` gate job below. + website-e2e-shard: needs: changes if: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} runs-on: ubuntu-latest @@ -33,13 +42,11 @@ jobs: timeout-minutes: 15 permissions: contents: read - outputs: - test-outcome: ${{ steps.tests.outcome }} - report-url: ${{ steps.deploy.outputs.url }} - screenshot-failures: ${{ steps.failures.outputs.screenshot }} - other-failures: ${{ steps.failures.outputs.other }} - # Evaluated at job level (not from a step) — static expression. - is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }} + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] steps: - uses: actions/checkout@v7 @@ -57,13 +64,72 @@ jobs: PUBLIC_CUSTOMERIO_WRITE_KEY: test-e2e-write-key run: pnpm --filter @comfyorg/website build - - name: Run Playwright tests + - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: tests - run: pnpm --filter @comfyorg/website test:e2e + run: pnpm --filter @comfyorg/website test:e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + env: + PLAYWRIGHT_BLOB_OUTPUT_DIR: ${{ github.workspace }}/apps/website/blob-report - - name: Upload test report + - name: Upload blob report uses: actions/upload-artifact@v6 if: ${{ !cancelled() }} + with: + name: website-blob-report-${{ matrix.shardIndex }} + path: apps/website/blob-report/ + retention-days: 1 + + # Merges the sharded blob reports into one HTML/JSON report, deploys it, + # categorizes failures, and writes the job summary — the single place + # downstream jobs (post-result-comment) read outcome/report-url from. + website-e2e-report: + needs: [changes, website-e2e-shard] + if: ${{ always() && (needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true') }} + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + test-outcome: ${{ steps.outcome.outputs.result }} + report-url: ${{ steps.deploy.outputs.url }} + screenshot-failures: ${{ steps.failures.outputs.screenshot }} + other-failures: ${{ steps.failures.outputs.other }} + # Evaluated at job level (not from a step) — static expression. + is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }} + steps: + - uses: actions/checkout@v7 + + - name: Install pnpm + run: corepack enable && corepack prepare + + - name: Determine shard outcome + id: outcome + env: + SHARDS_RESULT: ${{ needs.website-e2e-shard.result }} + run: | + if [ "$SHARDS_RESULT" = "success" ]; then + echo "result=success" >> "$GITHUB_OUTPUT" + else + echo "result=failure" >> "$GITHUB_OUTPUT" + fi + + - name: Download blob reports + if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + uses: actions/download-artifact@v6 + with: + path: apps/website/all-blob-reports + pattern: website-blob-report-* + merge-multiple: true + + - name: Merge into HTML and JSON reports + if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + working-directory: apps/website + run: | + pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports + PLAYWRIGHT_JSON_OUTPUT_NAME=results.json \ + pnpm dlx @playwright/test merge-reports --reporter=json ./all-blob-reports + + - name: Upload test report + uses: actions/upload-artifact@v6 + if: ${{ !cancelled() && needs.website-e2e-shard.result != 'cancelled' }} with: name: website-playwright-report path: apps/website/playwright-report/ @@ -74,7 +140,7 @@ jobs: if: >- ${{ always() && - !cancelled() && + needs.website-e2e-shard.result != 'cancelled' && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false @@ -104,7 +170,7 @@ jobs: - name: Categorize failures id: failures - if: always() && !cancelled() && steps.tests.outcome != 'success' + if: always() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'success' uses: actions/github-script@v9 with: script: | @@ -135,10 +201,10 @@ jobs: core.setOutput('other', failed.length - screenshotFailures) - name: Write job summary - if: always() && !cancelled() + if: always() && needs.website-e2e-shard.result != 'cancelled' uses: actions/github-script@v9 env: - TEST_OUTCOME: ${{ steps.tests.outcome }} + TEST_OUTCOME: ${{ steps.outcome.outputs.result }} REPORT_URL: ${{ steps.deploy.outputs.url }} SCREENSHOT_FAILURES: ${{ steps.failures.outputs.screenshot }} OTHER_FAILURES: ${{ steps.failures.outputs.other }} @@ -176,6 +242,30 @@ jobs: await core.summary.addRaw(lines.join('\n')).write() + # Required-check gate: branch ruleset pins the context name `website-e2e`, + # so this stays a single non-matrix job (unlike website-e2e-shard above) + # that passes/fails/skips as one status regardless of shard count. + website-e2e: + if: ${{ always() }} + needs: [changes, website-e2e-shard, website-e2e-report] + runs-on: ubuntu-latest + steps: + - name: Check website E2E result + env: + SHOULD_RUN: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} + SHARD_RESULT: ${{ needs.website-e2e-shard.result }} + REPORT_RESULT: ${{ needs.website-e2e-report.result }} + run: | + if [ "$SHOULD_RUN" != "true" ]; then + echo "Website E2E skipped (no relevant changes)" + exit 0 + fi + if [ "$SHARD_RESULT" != "success" ] || [ "$REPORT_RESULT" != "success" ]; then + echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT)" + exit 1 + fi + echo "Website E2E passed" + post-starting-comment: # Safe to comment from pull_request trigger: fork PRs are excluded by the guard below. # This avoids a ci-*/pr-* workflow_run split for a comment that must appear immediately. @@ -207,8 +297,8 @@ jobs: > Tests are running… [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) post-result-comment: - needs: website-e2e - if: always() && !cancelled() && needs.website-e2e.outputs.is-pr == 'true' + needs: website-e2e-report + if: always() && !cancelled() && needs.website-e2e-report.outputs.is-pr == 'true' runs-on: ubuntu-latest permissions: pull-requests: write @@ -223,10 +313,10 @@ jobs: id: content uses: actions/github-script@v9 env: - TEST_OUTCOME: ${{ needs.website-e2e.outputs.test-outcome }} - REPORT_URL: ${{ needs.website-e2e.outputs.report-url }} - SCREENSHOT_FAILURES: ${{ needs.website-e2e.outputs.screenshot-failures }} - OTHER_FAILURES: ${{ needs.website-e2e.outputs.other-failures }} + TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }} + REPORT_URL: ${{ needs.website-e2e-report.outputs.report-url }} + SCREENSHOT_FAILURES: ${{ needs.website-e2e-report.outputs.screenshot-failures }} + OTHER_FAILURES: ${{ needs.website-e2e-report.outputs.other-failures }} with: script: | const passed = process.env.TEST_OUTCOME === 'success' From 43796dbcba087eabdd338936ed7209e39de87188 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 2 Sep 2026 19:06:07 +0000 Subject: [PATCH 02/10] fix: pin download-artifact to allowlisted v8 (pinact validate-pins) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .pinact.yaml only allowlists actions/download-artifact at ref v8, not v6 (upload-artifact is the one allowlisted at v6) — the validate-pins check caught this on push. --- .github/workflows/ci-website-e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 717d57dc0b4..4c4f3f071a5 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -113,7 +113,7 @@ jobs: - name: Download blob reports if: ${{ needs.website-e2e-shard.result != 'cancelled' }} - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: path: apps/website/all-blob-reports pattern: website-blob-report-* From 26eb767048bcc14a9ae0d62e71c2d2cfc1ab7c97 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 2 Sep 2026 13:53:29 -0700 Subject: [PATCH 03/10] ci(website-e2e): emit blob reports so the sharded run can merge Select the blob reporter when PLAYWRIGHT_BLOB_OUTPUT_DIR is set, matching the root config; the shard jobs were still writing html+json so the merge step had nothing to read. Trigger the website e2e jobs on changes to their own workflow file and bound the report and gate jobs. --- .github/actions/changes-filter/action.yaml | 3 ++- .github/workflows/ci-website-e2e.yaml | 2 ++ apps/website/playwright.config.ts | 8 +++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/actions/changes-filter/action.yaml b/.github/actions/changes-filter/action.yaml index 441296d08c1..596f6d97c14 100644 --- a/.github/actions/changes-filter/action.yaml +++ b/.github/actions/changes-filter/action.yaml @@ -24,7 +24,7 @@ outputs: description: 'Any file outside `apps/`, `docs/`, `.storybook/`, or `**/*.md` changed.' value: ${{ github.event_name != 'pull_request' || steps.relevant.outputs.relevant == 'true' }} app-website-changes: - description: 'Shared deps or `apps/website/**` changed.' + description: 'Shared deps, `apps/website/**`, or the website e2e workflow changed.' value: ${{ github.event_name != 'pull_request' || steps.filter.outputs.deps == 'true' || steps.filter.outputs.app_website == 'true' }} app-frontend-changes: description: 'Shared deps or `src/**` changed.' @@ -53,6 +53,7 @@ runs: filters: | app_website: - 'apps/website/**' + - '.github/workflows/ci-website-e2e.yaml' app_frontend: - 'src/**' packages: diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 4c4f3f071a5..fc57421525d 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -85,6 +85,7 @@ jobs: needs: [changes, website-e2e-shard] if: ${{ always() && (needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true') }} runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read outputs: @@ -249,6 +250,7 @@ jobs: if: ${{ always() }} needs: [changes, website-e2e-shard, website-e2e-report] runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Check website E2E result env: diff --git a/apps/website/playwright.config.ts b/apps/website/playwright.config.ts index b6100286bc0..4cee6d1f12d 100644 --- a/apps/website/playwright.config.ts +++ b/apps/website/playwright.config.ts @@ -24,9 +24,11 @@ export default defineConfig({ testDir: './e2e', fullyParallel: true, forbidOnly: !!process.env.CI, - reporter: process.env.CI - ? [['html'], ['json', { outputFile: 'results.json' }]] - : 'html', + reporter: process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR + ? 'blob' + : process.env.CI + ? [['html'], ['json', { outputFile: 'results.json' }]] + : 'html', expect: { toHaveScreenshot: { maxDiffPixels: 100 } }, From 6fa21c32e41e93ea2ef13abc49585fbaf06ab44b Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 4 Sep 2026 18:31:01 -0700 Subject: [PATCH 04/10] ci(website-e2e): separate infrastructure failures from test failures A shard that dies before its test step (install, build, container) uploads no blob report, but `Determine shard outcome` labelled every non-success shard result a test failure. The merge then ran over the surviving blobs only, and categorization published counts derived from that subset -- so a run where three shards passed and one never started rendered as "Some tests failed" with 0 screenshot diffs and 0 other failures, presenting a partial run as an aggregate result. Each shard now records a `reported`/`missing` status file and uploads it unconditionally, so the report job can tell "this shard reported" from "this shard never got far enough to report" without hardcoding a shard total that could drift from the matrix. The outcome becomes three-state; `infrastructure` skips the merge, the report upload, the Cloudflare deploy and categorization, and renders distinctly in the job summary and the PR comment instead of emitting phantom counts. The gate job also checks `test-outcome` directly, because a shard can succeed and still emit no blob report, which would otherwise leave both job results green while the suite never actually reported. Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/16682#discussion_r3918603362 --- .github/workflows/ci-website-e2e.yaml | 89 +++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index fc57421525d..4c26454a418 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -78,6 +78,32 @@ jobs: path: apps/website/blob-report/ retention-days: 1 + # Self-describing completeness signal. A shard that dies before the + # test step (install, build, container) uploads no blob report, so + # website-e2e-report would otherwise merge a partial set and publish + # its counts as if they were the aggregate. Recording one status file + # per shard lets that job tell "this shard reported" from "this shard + # never got far enough to report" without hardcoding a shard total + # that could drift from the matrix above. + - name: Record shard status + if: ${{ always() }} + run: | + mkdir -p shard-status + if [ -d apps/website/blob-report ] && [ -n "$(ls -A apps/website/blob-report 2>/dev/null)" ]; then + echo reported > "shard-status/${{ matrix.shardIndex }}" + else + echo missing > "shard-status/${{ matrix.shardIndex }}" + fi + + - name: Upload shard status + if: ${{ always() }} + uses: actions/upload-artifact@v6 + with: + name: website-shard-status-${{ matrix.shardIndex }} + path: shard-status/ + if-no-files-found: error + retention-days: 1 + # Merges the sharded blob reports into one HTML/JSON report, deploys it, # categorizes failures, and writes the job summary — the single place # downstream jobs (post-result-comment) read outcome/report-url from. @@ -101,19 +127,37 @@ jobs: - name: Install pnpm run: corepack enable && corepack prepare + - name: Download shard statuses + if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + uses: actions/download-artifact@v8 + with: + path: apps/website/shard-status + pattern: website-shard-status-* + merge-multiple: true + + # Three outcomes, not two. `infrastructure` means at least one shard + # never produced a blob report, so any merged report is partial and + # its pass/fail counts describe a subset of the suite. The gate job + # fails on all three of cancelled/infrastructure/failure; the split + # exists so the summary and PR comment stop rendering a partial run + # as an aggregate test result (previously a phantom `0/0`). - name: Determine shard outcome id: outcome env: SHARDS_RESULT: ${{ needs.website-e2e-shard.result }} run: | - if [ "$SHARDS_RESULT" = "success" ]; then + missing=$(grep -rlx missing apps/website/shard-status 2>/dev/null | wc -l) + if [ "$missing" -gt 0 ]; then + echo "$missing shard(s) produced no blob report" >&2 + echo "result=infrastructure" >> "$GITHUB_OUTPUT" + elif [ "$SHARDS_RESULT" = "success" ]; then echo "result=success" >> "$GITHUB_OUTPUT" else echo "result=failure" >> "$GITHUB_OUTPUT" fi - name: Download blob reports - if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + if: ${{ needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} uses: actions/download-artifact@v8 with: path: apps/website/all-blob-reports @@ -121,7 +165,7 @@ jobs: merge-multiple: true - name: Merge into HTML and JSON reports - if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + if: ${{ needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} working-directory: apps/website run: | pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports @@ -130,7 +174,7 @@ jobs: - name: Upload test report uses: actions/upload-artifact@v6 - if: ${{ !cancelled() && needs.website-e2e-shard.result != 'cancelled' }} + if: ${{ !cancelled() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} with: name: website-playwright-report path: apps/website/playwright-report/ @@ -142,6 +186,7 @@ jobs: ${{ always() && needs.website-e2e-shard.result != 'cancelled' && + steps.outcome.outputs.result != 'infrastructure' && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false @@ -171,7 +216,7 @@ jobs: - name: Categorize failures id: failures - if: always() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'success' + if: always() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result == 'failure' uses: actions/github-script@v9 with: script: | @@ -211,7 +256,9 @@ jobs: OTHER_FAILURES: ${{ steps.failures.outputs.other }} with: script: | - const passed = process.env.TEST_OUTCOME === 'success' + const outcome = process.env.TEST_OUTCOME + const passed = outcome === 'success' + const infrastructure = outcome === 'infrastructure' const reportUrl = process.env.REPORT_URL const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0 const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0 @@ -220,15 +267,20 @@ jobs: if (passed) { lines.push('> [!TIP]', '> All tests passed.') + } else if (infrastructure) { + lines.push( + '> [!CAUTION]', + '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + ) } else { lines.push('> [!CAUTION]', '> Some tests failed.') } const rows = [ - ['Status', passed ? '✅ Passed' : '❌ Failed'], + ['Status', passed ? '✅ Passed' : infrastructure ? '💥 Infrastructure failure' : '❌ Failed'], ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_'] ] - if (!passed) { + if (!passed && !infrastructure) { rows.push( ['Screenshot diffs', String(screenshotFailures)], ['Other failures', String(otherFailures)] @@ -257,13 +309,17 @@ jobs: SHOULD_RUN: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} SHARD_RESULT: ${{ needs.website-e2e-shard.result }} REPORT_RESULT: ${{ needs.website-e2e-report.result }} + TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }} run: | if [ "$SHOULD_RUN" != "true" ]; then echo "Website E2E skipped (no relevant changes)" exit 0 fi - if [ "$SHARD_RESULT" != "success" ] || [ "$REPORT_RESULT" != "success" ]; then - echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT)" + # TEST_OUTCOME is checked separately from the job results: a shard + # can succeed and still emit no blob report, which leaves both job + # results green while the suite never actually reported. + if [ "$SHARD_RESULT" != "success" ] || [ "$REPORT_RESULT" != "success" ] || [ "$TEST_OUTCOME" != "success" ]; then + echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT, outcome: $TEST_OUTCOME)" exit 1 fi echo "Website E2E passed" @@ -321,7 +377,9 @@ jobs: OTHER_FAILURES: ${{ needs.website-e2e-report.outputs.other-failures }} with: script: | - const passed = process.env.TEST_OUTCOME === 'success' + const outcome = process.env.TEST_OUTCOME + const passed = outcome === 'success' + const infrastructure = outcome === 'infrastructure' const reportUrl = process.env.REPORT_URL const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0 const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0 @@ -330,15 +388,20 @@ jobs: if (passed) { lines.push('> [!TIP]', '> All tests passed.') + } else if (infrastructure) { + lines.push( + '> [!CAUTION]', + '> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.' + ) } else { lines.push('> [!CAUTION]', '> Some tests failed.') } const rows = [ - ['Status', passed ? '✅ Passed' : '❌ Failed'], + ['Status', passed ? '✅ Passed' : infrastructure ? '💥 Infrastructure failure' : '❌ Failed'], ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_'] ] - if (!passed) { + if (!passed && !infrastructure) { rows.push( ['Screenshot diffs', String(screenshotFailures)], ['Other failures', String(otherFailures)] From 90a3bd08d658f8a87aadf034576108570d5ad0ea Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 5 Sep 2026 01:33:14 +0000 Subject: [PATCH 05/10] fix(ci): reject incomplete website e2e artifacts --- .github/workflows/ci-website-e2e.yaml | 35 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 4c26454a418..454991487e0 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -129,12 +129,24 @@ jobs: - name: Download shard statuses if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + id: shard-statuses + continue-on-error: true uses: actions/download-artifact@v8 with: path: apps/website/shard-status pattern: website-shard-status-* merge-multiple: true + - name: Download blob reports + if: ${{ needs.website-e2e-shard.result != 'cancelled' }} + id: blob-reports + continue-on-error: true + uses: actions/download-artifact@v8 + with: + path: apps/website/all-blob-reports + pattern: website-blob-report-* + merge-multiple: true + # Three outcomes, not two. `infrastructure` means at least one shard # never produced a blob report, so any merged report is partial and # its pass/fail counts describe a subset of the suite. The gate job @@ -145,10 +157,17 @@ jobs: id: outcome env: SHARDS_RESULT: ${{ needs.website-e2e-shard.result }} + SHARD_STATUSES_OUTCOME: ${{ steps.shard-statuses.outcome }} + BLOB_REPORTS_OUTCOME: ${{ steps.blob-reports.outcome }} run: | + statuses=$(find apps/website/shard-status -maxdepth 1 -type f 2>/dev/null | wc -l) + reports=$(find apps/website/all-blob-reports -type f -name '*.zip' 2>/dev/null | wc -l) missing=$(grep -rlx missing apps/website/shard-status 2>/dev/null | wc -l) - if [ "$missing" -gt 0 ]; then - echo "$missing shard(s) produced no blob report" >&2 + if [ "$SHARD_STATUSES_OUTCOME" != "success" ] || \ + [ "$BLOB_REPORTS_OUTCOME" != "success" ] || \ + [ "$statuses" -ne 4 ] || [ "$reports" -ne 4 ] || \ + [ "$missing" -gt 0 ]; then + echo "Incomplete shard artifacts (statuses: $statuses/4, reports: $reports/4, missing: $missing)" >&2 echo "result=infrastructure" >> "$GITHUB_OUTPUT" elif [ "$SHARDS_RESULT" = "success" ]; then echo "result=success" >> "$GITHUB_OUTPUT" @@ -156,14 +175,6 @@ jobs: echo "result=failure" >> "$GITHUB_OUTPUT" fi - - name: Download blob reports - if: ${{ needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} - uses: actions/download-artifact@v8 - with: - path: apps/website/all-blob-reports - pattern: website-blob-report-* - merge-multiple: true - - name: Merge into HTML and JSON reports if: ${{ needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} working-directory: apps/website @@ -204,7 +215,7 @@ jobs: OUTPUT=$(pnpx wrangler@^4.0.0 pages deploy apps/website/playwright-report \ --project-name=comfyui-website-e2e \ --branch="$BRANCH" 2>&1) && { DEPLOY_OK=true; break; } || echo "$OUTPUT" - [ $i -lt 3 ] && sleep 10 + [ "$i" -lt 3 ] && sleep 10 done echo "$OUTPUT" if [ "$DEPLOY_OK" != "true" ]; then @@ -212,7 +223,7 @@ jobs: exit 1 fi URL=$(echo "$OUTPUT" | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1) - echo "url=${URL}" >> $GITHUB_OUTPUT + echo "url=${URL}" >> "$GITHUB_OUTPUT" - name: Categorize failures id: failures From 86491ffe30362af96e1341c883fc89e678fa408c Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 13 Sep 2026 05:00:15 +0000 Subject: [PATCH 06/10] ci: remove redundant website e2e comments Amp-Thread-ID: https://ampcode.com/threads/T-01a097bd-51a4-74ed-8f02-4f7db2fe8351 --- .github/workflows/ci-website-e2e.yaml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 454991487e0..3098813186d 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -24,15 +24,6 @@ jobs: - id: changes uses: ./.github/actions/changes-filter - # Sharded so 407 specs no longer run in a single 2-worker job (that took - # ~8.2m of Playwright wall time alone, leaving almost no slack under the - # 15m job timeout — any runner slowness pushed it over and the - # merge-group cancellation silently dequeued the PR). Each shard builds - # independently (the site build is ~13s, not worth artifact-passing) and - # uploads a blob report for website-e2e-report to merge. Named - # `-shard` (not `website-e2e`) because the required-check context - # `website-e2e` (branch ruleset) must stay a single, non-matrix-expanded - # check name — see the `website-e2e` gate job below. website-e2e-shard: needs: changes if: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} @@ -78,13 +69,6 @@ jobs: path: apps/website/blob-report/ retention-days: 1 - # Self-describing completeness signal. A shard that dies before the - # test step (install, build, container) uploads no blob report, so - # website-e2e-report would otherwise merge a partial set and publish - # its counts as if they were the aggregate. Recording one status file - # per shard lets that job tell "this shard reported" from "this shard - # never got far enough to report" without hardcoding a shard total - # that could drift from the matrix above. - name: Record shard status if: ${{ always() }} run: | @@ -104,9 +88,6 @@ jobs: if-no-files-found: error retention-days: 1 - # Merges the sharded blob reports into one HTML/JSON report, deploys it, - # categorizes failures, and writes the job summary — the single place - # downstream jobs (post-result-comment) read outcome/report-url from. website-e2e-report: needs: [changes, website-e2e-shard] if: ${{ always() && (needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true') }} @@ -246,8 +227,6 @@ jobs: ] } - // True: Visual - // False: Other const failed = specsOf(report) .flatMap(spec => (spec.tests || []) .filter(isFailed) From 3db2b6b17374558215d5c5855d2be2414ff8f508 Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 13 Sep 2026 07:12:15 +0000 Subject: [PATCH 07/10] fix: harden website e2e shard reporting Amp-Thread-ID: https://ampcode.com/threads/T-01a097bd-51a4-74ed-8f02-4f7db2fe8351 --- .github/workflows/ci-website-e2e.yaml | 83 +---------- scripts/cicd/check-website-e2e-result.sh | 14 ++ scripts/cicd/prepare-website-e2e-report.sh | 31 ++++ scripts/cicd/website-e2e-workflow.test.ts | 166 +++++++++++++++++++++ 4 files changed, 215 insertions(+), 79 deletions(-) create mode 100755 scripts/cicd/check-website-e2e-result.sh create mode 100755 scripts/cicd/prepare-website-e2e-report.sh create mode 100644 scripts/cicd/website-e2e-workflow.test.ts diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index c4bfc71aef8..850ed36f9aa 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -70,24 +70,6 @@ jobs: with: name: website-blob-report-${{ matrix.shardIndex }} path: apps/website/blob-report/ - retention-days: 1 - - - name: Record shard status - if: ${{ always() }} - run: | - mkdir -p shard-status - if [ -d apps/website/blob-report ] && [ -n "$(ls -A apps/website/blob-report 2>/dev/null)" ]; then - echo reported > "shard-status/${{ matrix.shardIndex }}" - else - echo missing > "shard-status/${{ matrix.shardIndex }}" - fi - - - name: Upload shard status - if: ${{ always() }} - uses: actions/upload-artifact@v6 - with: - name: website-shard-status-${{ matrix.shardIndex }} - path: shard-status/ if-no-files-found: error retention-days: 1 @@ -103,7 +85,6 @@ jobs: report-url: ${{ steps.deploy.outputs.url }} screenshot-failures: ${{ steps.failures.outputs.screenshot }} other-failures: ${{ steps.failures.outputs.other }} - # Evaluated at job level (not from a step) — static expression. is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }} steps: - uses: actions/checkout@v7 @@ -111,18 +92,7 @@ jobs: - name: Install pnpm run: corepack enable && corepack prepare - - name: Download shard statuses - if: ${{ needs.website-e2e-shard.result != 'cancelled' }} - id: shard-statuses - continue-on-error: true - uses: actions/download-artifact@v8 - with: - path: apps/website/shard-status - pattern: website-shard-status-* - merge-multiple: true - - name: Download blob reports - if: ${{ needs.website-e2e-shard.result != 'cancelled' }} id: blob-reports continue-on-error: true uses: actions/download-artifact@v8 @@ -131,41 +101,13 @@ jobs: pattern: website-blob-report-* merge-multiple: true - # Three outcomes, not two. `infrastructure` means at least one shard - # never produced a blob report, so any merged report is partial and - # its pass/fail counts describe a subset of the suite. The gate job - # fails on all three of cancelled/infrastructure/failure; the split - # exists so the summary and PR comment stop rendering a partial run - # as an aggregate test result (previously a phantom `0/0`). - - name: Determine shard outcome + - name: Validate and merge blob reports id: outcome + working-directory: apps/website env: SHARDS_RESULT: ${{ needs.website-e2e-shard.result }} - SHARD_STATUSES_OUTCOME: ${{ steps.shard-statuses.outcome }} BLOB_REPORTS_OUTCOME: ${{ steps.blob-reports.outcome }} - run: | - statuses=$(find apps/website/shard-status -maxdepth 1 -type f 2>/dev/null | wc -l) - reports=$(find apps/website/all-blob-reports -type f -name '*.zip' 2>/dev/null | wc -l) - missing=$(grep -rlx missing apps/website/shard-status 2>/dev/null | wc -l) - if [ "$SHARD_STATUSES_OUTCOME" != "success" ] || \ - [ "$BLOB_REPORTS_OUTCOME" != "success" ] || \ - [ "$statuses" -ne 4 ] || [ "$reports" -ne 4 ] || \ - [ "$missing" -gt 0 ]; then - echo "Incomplete shard artifacts (statuses: $statuses/4, reports: $reports/4, missing: $missing)" >&2 - echo "result=infrastructure" >> "$GITHUB_OUTPUT" - elif [ "$SHARDS_RESULT" = "success" ]; then - echo "result=success" >> "$GITHUB_OUTPUT" - else - echo "result=failure" >> "$GITHUB_OUTPUT" - fi - - - name: Merge into HTML and JSON reports - if: ${{ needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }} - working-directory: apps/website - run: | - pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports - PLAYWRIGHT_JSON_OUTPUT_NAME=results.json \ - pnpm dlx @playwright/test merge-reports --reporter=json ./all-blob-reports + run: ../../scripts/cicd/prepare-website-e2e-report.sh all-blob-reports 4 - name: Upload test report uses: actions/upload-artifact@v6 @@ -291,9 +233,6 @@ jobs: await core.summary.addRaw(lines.join('\n')).write() - # Required-check gate: branch ruleset pins the context name `website-e2e`, - # so this stays a single non-matrix job (unlike website-e2e-shard above) - # that passes/fails/skips as one status regardless of shard count. website-e2e: if: ${{ always() }} needs: [changes, website-e2e-shard, website-e2e-report] @@ -306,23 +245,9 @@ jobs: SHARD_RESULT: ${{ needs.website-e2e-shard.result }} REPORT_RESULT: ${{ needs.website-e2e-report.result }} TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }} - run: | - if [ "$SHOULD_RUN" != "true" ]; then - echo "Website E2E skipped (no relevant changes)" - exit 0 - fi - # TEST_OUTCOME is checked separately from the job results: a shard - # can succeed and still emit no blob report, which leaves both job - # results green while the suite never actually reported. - if [ "$SHARD_RESULT" != "success" ] || [ "$REPORT_RESULT" != "success" ] || [ "$TEST_OUTCOME" != "success" ]; then - echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT, outcome: $TEST_OUTCOME)" - exit 1 - fi - echo "Website E2E passed" + run: scripts/cicd/check-website-e2e-result.sh post-starting-comment: - # Safe to comment from pull_request trigger: fork PRs are excluded by the guard below. - # This avoids a ci-*/pr-* workflow_run split for a comment that must appear immediately. needs: changes if: | github.event_name == 'pull_request' diff --git a/scripts/cicd/check-website-e2e-result.sh b/scripts/cicd/check-website-e2e-result.sh new file mode 100755 index 00000000000..3cd1f10e5c6 --- /dev/null +++ b/scripts/cicd/check-website-e2e-result.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$SHOULD_RUN" != true ]]; then + echo "Website E2E skipped (no relevant changes)" + exit 0 +fi + +if [[ "$SHARD_RESULT" != success || "$REPORT_RESULT" != success || "$TEST_OUTCOME" != success ]]; then + echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT, outcome: $TEST_OUTCOME)" + exit 1 +fi + +echo "Website E2E passed" diff --git a/scripts/cicd/prepare-website-e2e-report.sh b/scripts/cicd/prepare-website-e2e-report.sh new file mode 100755 index 00000000000..3d16328307c --- /dev/null +++ b/scripts/cicd/prepare-website-e2e-report.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +reports_dir=$1 +expected_reports=$2 +report_count=0 +if [[ -d "$reports_dir" ]]; then + report_count=$(find "$reports_dir" -type f -name '*.zip' | wc -l) +fi + +infrastructure_failure() { + echo "$1" >&2 + echo "result=infrastructure" >> "$GITHUB_OUTPUT" +} + +if [[ "$BLOB_REPORTS_OUTCOME" != success || "$report_count" -ne "$expected_reports" ]]; then + infrastructure_failure "Incomplete blob reports ($report_count/$expected_reports)" + exit 0 +fi + +if ! pnpm dlx @playwright/test@1.61.1 merge-reports --reporter=html "$reports_dir" || + ! PLAYWRIGHT_JSON_OUTPUT_NAME=results.json pnpm dlx @playwright/test@1.61.1 merge-reports --reporter=json "$reports_dir"; then + infrastructure_failure "Failed to merge blob reports" + exit 0 +fi + +if [[ "$SHARDS_RESULT" == success ]]; then + echo "result=success" >> "$GITHUB_OUTPUT" +else + echo "result=failure" >> "$GITHUB_OUTPUT" +fi diff --git a/scripts/cicd/website-e2e-workflow.test.ts b/scripts/cicd/website-e2e-workflow.test.ts new file mode 100644 index 00000000000..1b9fd53bc74 --- /dev/null +++ b/scripts/cicd/website-e2e-workflow.test.ts @@ -0,0 +1,166 @@ +import { spawnSync } from 'node:child_process' +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const ROOT = join(import.meta.dirname, '../..') +const PREPARE_REPORT = join( + import.meta.dirname, + 'prepare-website-e2e-report.sh' +) +const CHECK_RESULT = join(import.meta.dirname, 'check-website-e2e-result.sh') + +function reportFixture() { + const root = mkdtempSync(join(tmpdir(), 'website-e2e-report-')) + const reports = join(root, 'reports') + const bin = join(root, 'bin') + const output = join(root, 'github-output') + mkdirSync(reports) + mkdirSync(bin) + writeFileSync( + join(bin, 'pnpm'), + `#!/usr/bin/env bash +set -euo pipefail +echo "$*" >> "$CALLS" +[[ "\${MERGE_REPORTS_OK:-true}" == true ]] +`, + { mode: 0o755 } + ) + + return { + root, + reports, + output, + addReports(count: number) { + for (let index = 0; index < count; index++) + writeFileSync(join(reports, `${index}.zip`), '') + }, + run( + options: { + download?: string + mergeReports?: string + shards?: string + } = {} + ) { + const calls = join(root, 'calls') + writeFileSync(calls, '') + const result = spawnSync('bash', [PREPARE_REPORT, reports, '4'], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + BLOB_REPORTS_OUTCOME: options.download ?? 'success', + SHARDS_RESULT: options.shards ?? 'success', + MERGE_REPORTS_OK: options.mergeReports ?? 'true', + GITHUB_OUTPUT: output, + CALLS: calls + } + }) + return { + status: result.status, + output: readFileSync(output, 'utf8'), + calls: readFileSync(calls, 'utf8') + } + }, + [Symbol.dispose]() { + rmSync(root, { recursive: true, force: true }) + } + } +} + +function checkResult(env: Record) { + return spawnSync('bash', [CHECK_RESULT], { + encoding: 'utf8', + env: { ...process.env, ...env } + }) +} + +describe('website E2E workflow', () => { + it('merges every blob before reporting successful tests', () => { + using fixture = reportFixture() + fixture.addReports(4) + + const result = fixture.run() + + expect(result.status).toBe(0) + expect(result.output).toBe('result=success\n') + expect(result.calls).toContain('@playwright/test@1.61.1') + expect(result.calls).toContain('--reporter=html') + expect(result.calls).toContain('--reporter=json') + }) + + it('preserves a complete failing test run', () => { + using fixture = reportFixture() + fixture.addReports(4) + + expect(fixture.run({ shards: 'failure' }).output).toBe('result=failure\n') + }) + + it('reports infrastructure failure for incomplete artifacts', () => { + using failedDownload = reportFixture() + failedDownload.addReports(4) + using missingReport = reportFixture() + missingReport.addReports(3) + + expect(failedDownload.run({ download: 'failure' }).output).toBe( + 'result=infrastructure\n' + ) + expect(missingReport.run().output).toBe('result=infrastructure\n') + }) + + it('reports infrastructure failure when report generation fails', () => { + using fixture = reportFixture() + fixture.addReports(4) + + expect(fixture.run({ mergeReports: 'false' }).output).toBe( + 'result=infrastructure\n' + ) + }) + + it('keeps the matrix and completeness check synchronized', () => { + const workflow = readFileSync( + join(ROOT, '.github/workflows/ci-website-e2e.yaml'), + 'utf8' + ) + + expect(workflow).toContain('shardTotal: [4]') + expect(workflow).toContain( + 'prepare-website-e2e-report.sh all-blob-reports 4' + ) + }) + + it('passes only skipped or fully successful runs', () => { + expect( + checkResult({ + SHOULD_RUN: 'false', + SHARD_RESULT: 'skipped', + REPORT_RESULT: 'skipped', + TEST_OUTCOME: '' + }).status + ).toBe(0) + expect( + checkResult({ + SHOULD_RUN: 'true', + SHARD_RESULT: 'success', + REPORT_RESULT: 'success', + TEST_OUTCOME: 'success' + }).status + ).toBe(0) + expect( + checkResult({ + SHOULD_RUN: 'true', + SHARD_RESULT: 'success', + REPORT_RESULT: 'success', + TEST_OUTCOME: 'infrastructure' + }).status + ).toBe(1) + }) +}) From 8e3d4ef5d11e3260f503e51a08520b64a2d2a0c9 Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 13 Sep 2026 07:17:41 +0000 Subject: [PATCH 08/10] fix: checkout website e2e gate script Amp-Thread-ID: https://ampcode.com/threads/T-01a097bd-51a4-74ed-8f02-4f7db2fe8351 --- .github/workflows/ci-website-e2e.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 850ed36f9aa..90164a05b36 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -238,7 +238,11 @@ jobs: needs: [changes, website-e2e-shard, website-e2e-report] runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read steps: + - uses: actions/checkout@v7 + - name: Check website E2E result env: SHOULD_RUN: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }} From cd6598e014e358c99539364e831e109e332bff29 Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 13 Sep 2026 07:25:33 +0000 Subject: [PATCH 09/10] fix: trigger website e2e for helper changes Amp-Thread-ID: https://ampcode.com/threads/T-01a097bd-51a4-74ed-8f02-4f7db2fe8351 --- .github/actions/changes-filter/action.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/changes-filter/action.yaml b/.github/actions/changes-filter/action.yaml index 6dd8bc95ed9..b7f5925c6a1 100644 --- a/.github/actions/changes-filter/action.yaml +++ b/.github/actions/changes-filter/action.yaml @@ -41,7 +41,7 @@ outputs: candidate's synthetic head. value: ${{ (github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'merge_group' && inputs.evaluate-candidate != 'true') || steps.relevant.outputs.relevant == 'true' }} app-website-changes: - description: 'Shared deps, `apps/website/**`, or the website e2e workflow changed.' + description: 'Shared deps, `apps/website/**`, or website e2e workflow support changed.' value: ${{ github.event_name != 'pull_request' || steps.filter.outputs.deps == 'true' || steps.filter.outputs.app_website == 'true' }} app-frontend-changes: description: 'Shared deps or `src/**` changed.' @@ -71,6 +71,8 @@ runs: app_website: - 'apps/website/**' - '.github/workflows/ci-website-e2e.yaml' + - 'scripts/cicd/prepare-website-e2e-report.sh' + - 'scripts/cicd/check-website-e2e-result.sh' app_frontend: - 'src/**' packages: From 27e381bfc476f3d3b80a70359efbe7b97931380e Mon Sep 17 00:00:00 2001 From: Amp Date: Mon, 14 Sep 2026 18:24:41 +0000 Subject: [PATCH 10/10] fix: derive website Playwright report version Amp-Thread-ID: https://ampcode.com/threads/T-01a097bd-51a4-74ed-8f02-4f7db2fe8351 --- .github/workflows/ci-website-e2e.yaml | 4 +++- scripts/cicd/prepare-website-e2e-report.sh | 5 ++-- scripts/cicd/website-e2e-workflow.test.ts | 28 ++++++++++++---------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-website-e2e.yaml b/.github/workflows/ci-website-e2e.yaml index 90164a05b36..68f728e12d8 100644 --- a/.github/workflows/ci-website-e2e.yaml +++ b/.github/workflows/ci-website-e2e.yaml @@ -107,7 +107,9 @@ jobs: env: SHARDS_RESULT: ${{ needs.website-e2e-shard.result }} BLOB_REPORTS_OUTCOME: ${{ steps.blob-reports.outcome }} - run: ../../scripts/cicd/prepare-website-e2e-report.sh all-blob-reports 4 + run: | + PLAYWRIGHT_VERSION=$(pnpm list @playwright/test --depth 0 --json --lockfile-only | jq -r '.[0].devDependencies["@playwright/test"].version') + ../../scripts/cicd/prepare-website-e2e-report.sh all-blob-reports 4 "$PLAYWRIGHT_VERSION" - name: Upload test report uses: actions/upload-artifact@v6 diff --git a/scripts/cicd/prepare-website-e2e-report.sh b/scripts/cicd/prepare-website-e2e-report.sh index 3d16328307c..df4c5d4ead2 100755 --- a/scripts/cicd/prepare-website-e2e-report.sh +++ b/scripts/cicd/prepare-website-e2e-report.sh @@ -3,6 +3,7 @@ set -euo pipefail reports_dir=$1 expected_reports=$2 +playwright_version=$3 report_count=0 if [[ -d "$reports_dir" ]]; then report_count=$(find "$reports_dir" -type f -name '*.zip' | wc -l) @@ -18,8 +19,8 @@ if [[ "$BLOB_REPORTS_OUTCOME" != success || "$report_count" -ne "$expected_repor exit 0 fi -if ! pnpm dlx @playwright/test@1.61.1 merge-reports --reporter=html "$reports_dir" || - ! PLAYWRIGHT_JSON_OUTPUT_NAME=results.json pnpm dlx @playwright/test@1.61.1 merge-reports --reporter=json "$reports_dir"; then +if ! pnpm dlx "@playwright/test@$playwright_version" merge-reports --reporter=html "$reports_dir" || + ! PLAYWRIGHT_JSON_OUTPUT_NAME=results.json pnpm dlx "@playwright/test@$playwright_version" merge-reports --reporter=json "$reports_dir"; then infrastructure_failure "Failed to merge blob reports" exit 0 fi diff --git a/scripts/cicd/website-e2e-workflow.test.ts b/scripts/cicd/website-e2e-workflow.test.ts index 1b9fd53bc74..09dd7cfdb0c 100644 --- a/scripts/cicd/website-e2e-workflow.test.ts +++ b/scripts/cicd/website-e2e-workflow.test.ts @@ -52,18 +52,22 @@ echo "$*" >> "$CALLS" ) { const calls = join(root, 'calls') writeFileSync(calls, '') - const result = spawnSync('bash', [PREPARE_REPORT, reports, '4'], { - encoding: 'utf8', - env: { - ...process.env, - PATH: `${bin}:${process.env.PATH ?? ''}`, - BLOB_REPORTS_OUTCOME: options.download ?? 'success', - SHARDS_RESULT: options.shards ?? 'success', - MERGE_REPORTS_OK: options.mergeReports ?? 'true', - GITHUB_OUTPUT: output, - CALLS: calls + const result = spawnSync( + 'bash', + [PREPARE_REPORT, reports, '4', '1.61.1'], + { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + BLOB_REPORTS_OUTCOME: options.download ?? 'success', + SHARDS_RESULT: options.shards ?? 'success', + MERGE_REPORTS_OK: options.mergeReports ?? 'true', + GITHUB_OUTPUT: output, + CALLS: calls + } } - }) + ) return { status: result.status, output: readFileSync(output, 'utf8'), @@ -133,7 +137,7 @@ describe('website E2E workflow', () => { expect(workflow).toContain('shardTotal: [4]') expect(workflow).toContain( - 'prepare-website-e2e-report.sh all-blob-reports 4' + 'prepare-website-e2e-report.sh all-blob-reports 4 "$PLAYWRIGHT_VERSION"' ) })