Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/actions/changes-filter/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ outputs:
candidate's synthetic head.
value: ${{ (github.event_name != 'pull_request' && github.event_name != 'merge_group') || (github.event_name == 'merge_group' && inputs.evaluate-candidate != 'true') || steps.relevant.outputs.relevant == 'true' }}
app-website-changes:
description: 'Shared deps or `apps/website/**` changed.'
description: 'Shared deps, `apps/website/**`, or website e2e workflow support changed.'
value: ${{ github.event_name != 'pull_request' || steps.filter.outputs.deps == 'true' || steps.filter.outputs.app_website == 'true' }}
app-frontend-changes:
description: 'Shared deps or `src/**` changed.'
Expand Down Expand Up @@ -70,6 +70,9 @@ runs:
filters: |
app_website:
- 'apps/website/**'
- '.github/workflows/ci-website-e2e.yaml'
Comment thread
DrJKL marked this conversation as resolved.
Resolved
- 'scripts/cicd/prepare-website-e2e-report.sh'
- 'scripts/cicd/check-website-e2e-result.sh'
app_frontend:
- 'src/**'
packages:
Expand Down
142 changes: 109 additions & 33 deletions .github/workflows/ci-website-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
- id: changes
uses: ./.github/actions/changes-filter

website-e2e:
website-e2e-shard:
needs: changes
if: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }}
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
runs-on: ubuntu-latest
Expand All @@ -33,13 +33,11 @@ jobs:
timeout-minutes: 30
permissions:
contents: read
outputs:
test-outcome: ${{ steps.tests.outcome }}
report-url: ${{ steps.deploy.outputs.url }}
screenshot-failures: ${{ steps.failures.outputs.screenshot }}
other-failures: ${{ steps.failures.outputs.other }}
# Evaluated at job level (not from a step) — static expression.
is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }}
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2, 3, 4]
shardTotal: [4]
steps:
- uses: actions/checkout@v7

Expand All @@ -60,13 +58,62 @@ jobs:
PUBLIC_CUSTOMERIO_WRITE_KEY: test-e2e-write-key
run: pnpm --filter @comfyorg/website build

- name: Run Playwright tests
- name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
id: tests
run: pnpm --filter @comfyorg/website test:e2e
run: pnpm --filter @comfyorg/website test:e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
env:
PLAYWRIGHT_BLOB_OUTPUT_DIR: ${{ github.workspace }}/apps/website/blob-report
Comment thread
christian-byrne marked this conversation as resolved.
Resolved

- name: Upload test report
- name: Upload blob report
uses: actions/upload-artifact@v6
if: ${{ !cancelled() }}
with:
name: website-blob-report-${{ matrix.shardIndex }}
path: apps/website/blob-report/
if-no-files-found: error
retention-days: 1

website-e2e-report:
needs: [changes, website-e2e-shard]
if: ${{ always() && (needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 15

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we just eliminate the explicit timeout?

permissions:
contents: read
outputs:
test-outcome: ${{ steps.outcome.outputs.result }}
report-url: ${{ steps.deploy.outputs.url }}
screenshot-failures: ${{ steps.failures.outputs.screenshot }}
other-failures: ${{ steps.failures.outputs.other }}
is-pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }}
steps:
- uses: actions/checkout@v7

- name: Install pnpm
run: corepack enable && corepack prepare

- name: Download blob reports
id: blob-reports
continue-on-error: true
uses: actions/download-artifact@v8
with:
path: apps/website/all-blob-reports
pattern: website-blob-report-*
merge-multiple: true

- name: Validate and merge blob reports
id: outcome
working-directory: apps/website
env:
SHARDS_RESULT: ${{ needs.website-e2e-shard.result }}
BLOB_REPORTS_OUTCOME: ${{ steps.blob-reports.outcome }}
run: |
PLAYWRIGHT_VERSION=$(pnpm list @playwright/test --depth 0 --json --lockfile-only | jq -r '.[0].devDependencies["@playwright/test"].version')
../../scripts/cicd/prepare-website-e2e-report.sh all-blob-reports 4 "$PLAYWRIGHT_VERSION"

- name: Upload test report
uses: actions/upload-artifact@v6
if: ${{ !cancelled() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result != 'infrastructure' }}
with:
name: website-playwright-report
path: |
Expand All @@ -80,7 +127,8 @@ jobs:
if: >-
${{
always() &&
!cancelled() &&
needs.website-e2e-shard.result != 'cancelled' &&
steps.outcome.outputs.result != 'infrastructure' &&
(
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.fork == false
Expand All @@ -98,19 +146,19 @@ jobs:
OUTPUT=$(pnpx wrangler@^4.0.0 pages deploy apps/website/playwright-report \
--project-name=comfyui-website-e2e \
--branch="$BRANCH" 2>&1) && { DEPLOY_OK=true; break; } || echo "$OUTPUT"
[ $i -lt 3 ] && sleep 10
[ "$i" -lt 3 ] && sleep 10
done
echo "$OUTPUT"
if [ "$DEPLOY_OK" != "true" ]; then
echo "::error::All 3 deployment attempts failed"
exit 1
fi
URL=$(echo "$OUTPUT" | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1)
echo "url=${URL}" >> $GITHUB_OUTPUT
echo "url=${URL}" >> "$GITHUB_OUTPUT"

- name: Categorize failures
id: failures
if: always() && !cancelled() && steps.tests.outcome != 'success'
if: always() && needs.website-e2e-shard.result != 'cancelled' && steps.outcome.outputs.result == 'failure'
uses: actions/github-script@v9
with:
script: |
Expand All @@ -129,8 +177,6 @@ jobs:
]
}

// True: Visual
// False: Other
const failed = specsOf(report)
.flatMap(spec => (spec.tests || [])
.filter(isFailed)
Expand All @@ -141,16 +187,18 @@ jobs:
core.setOutput('other', failed.length - screenshotFailures)

- name: Write job summary
if: always() && !cancelled()
if: always() && needs.website-e2e-shard.result != 'cancelled'
uses: actions/github-script@v9
env:
TEST_OUTCOME: ${{ steps.tests.outcome }}
TEST_OUTCOME: ${{ steps.outcome.outputs.result }}
REPORT_URL: ${{ steps.deploy.outputs.url }}
SCREENSHOT_FAILURES: ${{ steps.failures.outputs.screenshot }}
OTHER_FAILURES: ${{ steps.failures.outputs.other }}
with:
script: |
const passed = process.env.TEST_OUTCOME === 'success'
const outcome = process.env.TEST_OUTCOME
const passed = outcome === 'success'
const infrastructure = outcome === 'infrastructure'
const reportUrl = process.env.REPORT_URL
const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0
const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0
Expand All @@ -159,15 +207,20 @@ jobs:

if (passed) {
lines.push('> [!TIP]', '> All tests passed.')
} else if (infrastructure) {
lines.push(
'> [!CAUTION]',
'> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.'
)
} else {
lines.push('> [!CAUTION]', '> Some tests failed.')
}

const rows = [
['Status', passed ? '✅ Passed' : '❌ Failed'],
['Status', passed ? '✅ Passed' : infrastructure ? '💥 Infrastructure failure' : '❌ Failed'],
['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_']
]
if (!passed) {
if (!passed && !infrastructure) {
rows.push(
['Screenshot diffs', String(screenshotFailures)],
['Other failures', String(otherFailures)]
Expand All @@ -182,9 +235,25 @@ jobs:

await core.summary.addRaw(lines.join('\n')).write()

website-e2e:
if: ${{ always() }}
needs: [changes, website-e2e-shard, website-e2e-report]
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@v7

- name: Check website E2E result
env:
SHOULD_RUN: ${{ needs.changes.outputs.app-website-changes == 'true' || needs.changes.outputs.packages-changes == 'true' }}
SHARD_RESULT: ${{ needs.website-e2e-shard.result }}
REPORT_RESULT: ${{ needs.website-e2e-report.result }}
TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }}
run: scripts/cicd/check-website-e2e-result.sh

post-starting-comment:
# Safe to comment from pull_request trigger: fork PRs are excluded by the guard below.
# This avoids a ci-*/pr-* workflow_run split for a comment that must appear immediately.
needs: changes
if: |
github.event_name == 'pull_request'
Expand Down Expand Up @@ -213,8 +282,8 @@ jobs:
> Tests are running… [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

post-result-comment:
needs: website-e2e
if: always() && !cancelled() && needs.website-e2e.outputs.is-pr == 'true'
needs: website-e2e-report
if: always() && !cancelled() && needs.website-e2e-report.outputs.is-pr == 'true'
runs-on: ubuntu-latest
permissions:
pull-requests: write
Expand All @@ -229,13 +298,15 @@ jobs:
id: content
uses: actions/github-script@v9
env:
TEST_OUTCOME: ${{ needs.website-e2e.outputs.test-outcome }}
REPORT_URL: ${{ needs.website-e2e.outputs.report-url }}
SCREENSHOT_FAILURES: ${{ needs.website-e2e.outputs.screenshot-failures }}
OTHER_FAILURES: ${{ needs.website-e2e.outputs.other-failures }}
TEST_OUTCOME: ${{ needs.website-e2e-report.outputs.test-outcome }}
REPORT_URL: ${{ needs.website-e2e-report.outputs.report-url }}
SCREENSHOT_FAILURES: ${{ needs.website-e2e-report.outputs.screenshot-failures }}
OTHER_FAILURES: ${{ needs.website-e2e-report.outputs.other-failures }}
with:
script: |
const passed = process.env.TEST_OUTCOME === 'success'
const outcome = process.env.TEST_OUTCOME
const passed = outcome === 'success'
const infrastructure = outcome === 'infrastructure'
const reportUrl = process.env.REPORT_URL
const screenshotFailures = parseInt(process.env.SCREENSHOT_FAILURES) || 0
const otherFailures = parseInt(process.env.OTHER_FAILURES) || 0
Expand All @@ -244,15 +315,20 @@ jobs:

if (passed) {
lines.push('> [!TIP]', '> All tests passed.')
} else if (infrastructure) {
lines.push(
'> [!CAUTION]',
'> A shard failed before producing a report, so the suite did not fully run. No pass/fail counts are available.'
)
} else {
lines.push('> [!CAUTION]', '> Some tests failed.')
}

const rows = [
['Status', passed ? '✅ Passed' : '❌ Failed'],
['Status', passed ? '✅ Passed' : infrastructure ? '💥 Infrastructure failure' : '❌ Failed'],
['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_']
]
if (!passed) {
if (!passed && !infrastructure) {
rows.push(
['Screenshot diffs', String(screenshotFailures)],
['Other failures', String(otherFailures)]
Expand Down
8 changes: 5 additions & 3 deletions apps/website/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ export default defineConfig({
fullyParallel: true,
forbidOnly: !!process.env.CI,
globalTimeout: process.env.CI ? 20 * 60_000 : 0,
reporter: process.env.CI
? [['list'], ['html'], ['json', { outputFile: 'results.json' }]]
: 'html',
reporter: process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR
? 'blob'
: process.env.CI
? [['list'], ['html'], ['json', { outputFile: 'results.json' }]]
: 'html',
expect: {
toHaveScreenshot: { maxDiffPixels: 100 }
},
Expand Down
14 changes: 14 additions & 0 deletions scripts/cicd/check-website-e2e-result.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail

if [[ "$SHOULD_RUN" != true ]]; then
echo "Website E2E skipped (no relevant changes)"
exit 0
fi

if [[ "$SHARD_RESULT" != success || "$REPORT_RESULT" != success || "$TEST_OUTCOME" != success ]]; then
echo "Website E2E failed (shards: $SHARD_RESULT, report: $REPORT_RESULT, outcome: $TEST_OUTCOME)"
exit 1
fi

echo "Website E2E passed"
32 changes: 32 additions & 0 deletions scripts/cicd/prepare-website-e2e-report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail

reports_dir=$1
expected_reports=$2
playwright_version=$3
report_count=0
if [[ -d "$reports_dir" ]]; then
report_count=$(find "$reports_dir" -type f -name '*.zip' | wc -l)
fi

infrastructure_failure() {
echo "$1" >&2
echo "result=infrastructure" >> "$GITHUB_OUTPUT"
}

if [[ "$BLOB_REPORTS_OUTCOME" != success || "$report_count" -ne "$expected_reports" ]]; then
infrastructure_failure "Incomplete blob reports ($report_count/$expected_reports)"
exit 0
fi

if ! pnpm dlx "@playwright/test@$playwright_version" merge-reports --reporter=html "$reports_dir" ||
! PLAYWRIGHT_JSON_OUTPUT_NAME=results.json pnpm dlx "@playwright/test@$playwright_version" merge-reports --reporter=json "$reports_dir"; then
infrastructure_failure "Failed to merge blob reports"
exit 0
fi

if [[ "$SHARDS_RESULT" == success ]]; then
echo "result=success" >> "$GITHUB_OUTPUT"
else
echo "result=failure" >> "$GITHUB_OUTPUT"
fi
Loading
Loading