Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/actions/find-workflow-run/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ inputs:
description: Status to output when no run exists
required: false
default: pending
artifact-name:
description: When set, a completed run is ready only when this artifact exists
required: false
token:
description: GitHub token for API access
required: true
Expand All @@ -34,6 +37,7 @@ runs:
WORKFLOW_ID: ${{ inputs.workflow-id }}
HEAD_SHA: ${{ inputs.head-sha }}
NOT_FOUND_STATUS: ${{ inputs.not-found-status }}
ARTIFACT_NAME: ${{ inputs.artifact-name }}
with:
github-token: ${{ inputs.token }}
script: |
Expand All @@ -56,6 +60,35 @@ runs:
return;
}

if (process.env.ARTIFACT_NAME) {
const artifacts = await github.paginate(
github.rest.actions.listWorkflowRunArtifacts,
{
owner: context.repo.owner,
repo: context.repo.repo,
run_id: run.id,
per_page: 100,
}
);
const artifact = artifacts.find(
candidate =>
candidate.name === process.env.ARTIFACT_NAME &&
!candidate.expired
);
if (!artifact) {
core.setOutput(
'status',
run.conclusion === 'success'
? process.env.NOT_FOUND_STATUS
: 'failed'
);
return;
}
core.setOutput('status', 'ready');
core.setOutput('run-id', String(run.id));
return;
Comment thread
DrJKL marked this conversation as resolved.
Resolved
}

if (run.conclusion !== 'success') {
core.setOutput('status', 'failed');
return;
Expand Down
61 changes: 61 additions & 0 deletions .github/workflows/ci-tests-e2e-coverage-package.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
name: 'CI: E2E Coverage Package'

on:
workflow_call:
inputs:
source_sha:
required: true
type: string

permissions:
contents: read

jobs:
package:
if: github.repository == 'Comfy-Org/ComfyUI_frontend'
runs-on: ubuntu-latest
timeout-minutes: 10

steps:
- name: Checkout sources for coverage packaging
uses: actions/checkout@v7
with:
ref: ${{ inputs.source_sha }}
sparse-checkout: |
src
packages
scripts/cicd/package-e2e-coverage.sh

- name: Download all shard coverage data
uses: actions/download-artifact@v8
with:
pattern: e2e-coverage-shard-*
path: temp/coverage-shards

- name: Install lcov
run: sudo apt-get install -y -qq lcov

- name: Package shard coverage
id: coverage
run: |
scripts/cicd/package-e2e-coverage.sh \
temp/coverage-shards \
coverage/playwright \
coverage/html

- name: Upload merged coverage data
if: steps.coverage.outputs.has-coverage == 'true'
uses: actions/upload-artifact@v6
with:
name: e2e-coverage
path: coverage/playwright/
retention-days: 30
if-no-files-found: warn

- name: Upload HTML report artifact
if: steps.coverage.outputs.has-coverage == 'true'
uses: actions/upload-artifact@v6
with:
name: e2e-coverage-html
path: coverage/html/
retention-days: 30
155 changes: 42 additions & 113 deletions .github/workflows/ci-tests-e2e-coverage.yaml
Original file line number Diff line number Diff line change
@@ -1,136 +1,63 @@
name: 'CI: E2E Coverage'

on:
workflow_call:
inputs:
source_branch:
required: true
type: string
source_pr:
required: false
type: string
source_run_id:
required: true
type: string
source_sha:
required: true
type: string
secrets:
CODECOV_TOKEN:
required: true
workflow_run:
workflows: ['CI: Tests E2E']
types:
- completed

concurrency:
group: e2e-coverage-${{ inputs.source_sha || github.event.workflow_run.head_sha }}
group: e2e-coverage-${{ github.event.workflow_run.head_sha }}
cancel-in-progress: true

permissions:
actions: read
contents: read
pull-requests: read

jobs:
merge:
# Not gated on full success: fail-fast:false means one flaky shard flips the whole run to failure, which would skip merging every other shard's real coverage.
publish:
if: >
github.repository == 'Comfy-Org/ComfyUI_frontend' &&
(github.event_name == 'workflow_call' ||
github.event.workflow_run.event != 'pull_request' ||
github.event.workflow_run.head_repository.full_name != github.repository ||
github.event.workflow_run.actor.login == 'dependabot[bot]') &&
github.event.workflow_run.conclusion != 'cancelled'
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
has-coverage: ${{ steps.coverage-shards.outputs.has-coverage }}
has-coverage: ${{ steps.coverage.outputs.has-coverage }}

steps:
- name: Checkout sources for genhtml
- name: Checkout trusted workflow actions
uses: actions/checkout@v7
with:
ref: ${{ inputs.source_sha || github.event.workflow_run.head_sha }}
sparse-checkout: |
src
packages
ref: ${{ github.event.repository.default_branch }}
sparse-checkout: .github/actions/resolve-pr-from-workflow-run
persist-credentials: false

- name: Download all shard coverage data
- name: Resolve PR from workflow_run context
id: pr-meta
if: github.event.workflow_run.event == 'pull_request'
uses: ./.github/actions/resolve-pr-from-workflow-run

- name: Download packaged coverage data
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
run_id: ${{ inputs.source_run_id || github.event.workflow_run.id }}
name: e2e-coverage-shard-.*
name_is_regexp: true
path: temp/coverage-shards
run_id: ${{ github.event.workflow_run.id }}
name: e2e-coverage
path: coverage/playwright
if_no_artifact_found: warn

- name: Detect shard coverage data
id: coverage-shards
- name: Detect packaged coverage data
id: coverage
run: |
if [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; then
if [ -f coverage/playwright/coverage.lcov ]; then
echo "has-coverage=true" >> "$GITHUB_OUTPUT"
else
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
echo "No E2E coverage shard artifacts found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY"
fi

- name: Install lcov
if: steps.coverage-shards.outputs.has-coverage == 'true'
run: sudo apt-get install -y -qq lcov

- name: Merge shard coverage into single LCOV
if: steps.coverage-shards.outputs.has-coverage == 'true'
run: |
mkdir -p coverage/playwright
LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f)
ADD_ARGS=""
for f in $LCOV_FILES; do ADD_ARGS="$ADD_ARGS -a $f"; done
lcov $ADD_ARGS -o coverage/playwright/coverage.lcov
wc -l coverage/playwright/coverage.lcov

- name: Validate merged coverage
if: steps.coverage-shards.outputs.has-coverage == 'true'
run: |
MERGED_SF=$(grep -c '^SF:' coverage/playwright/coverage.lcov || echo 0)
MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov)
MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov)
echo "### Merged coverage" >> "$GITHUB_STEP_SUMMARY"
echo "- **$MERGED_SF** source files" >> "$GITHUB_STEP_SUMMARY"
echo "- **$MERGED_LH / $MERGED_LF** lines hit" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "| Shard | Files | Lines Hit |" >> "$GITHUB_STEP_SUMMARY"
echo "|-------|-------|-----------|" >> "$GITHUB_STEP_SUMMARY"
for f in $(find temp/coverage-shards -name 'coverage.lcov' -type f | sort); do
SHARD=$(basename "$(dirname "$f")")
SHARD_SF=$(grep -c '^SF:' "$f" || echo 0)
SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$f")
echo "| $SHARD | $SHARD_SF | $SHARD_LH |" >> "$GITHUB_STEP_SUMMARY"
if [ "$MERGED_LH" -lt "$SHARD_LH" ]; then
echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss"
fi
done

- name: Assert coverage was mapped back to source
if: steps.coverage-shards.outputs.has-coverage == 'true'
run: |
MAPPED_SF=$(grep -cE '^SF:(src|packages)/' coverage/playwright/coverage.lcov || true)
echo "Source-mapped files: $MAPPED_SF" >> "$GITHUB_STEP_SUMMARY"
if [ "${MAPPED_SF:-0}" -lt 100 ]; then
echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $(grep -m 5 '^SF:' coverage/playwright/coverage.lcov | tr '\n' ' '). Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)."
exit 1
echo "No packaged E2E coverage artifact found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY"
fi

- name: Strip non-source entries from coverage
if: steps.coverage-shards.outputs.has-coverage == 'true'
run: |
lcov --remove coverage/playwright/coverage.lcov \
'*localhost-8188*' \
'assets/images/*' \
-o coverage/playwright/coverage.lcov \
--ignore-errors unused
wc -l coverage/playwright/coverage.lcov

- name: Upload merged coverage data

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optional simplification: make CI: Tests E2E the sole owner of the packaged LCOV artifact. Delete this re-upload, then point .github/workflows/coverage-slack-notify.yaml at ci-tests-e2e.yaml with workflow_conclusion: completed and search_artifacts: true. PR Report already consumes the producer artifact directly. This preserves partial coverage from failed E2E runs while removing a duplicate 30-day artifact and upload on every run.

if: steps.coverage-shards.outputs.has-coverage == 'true'
if: steps.coverage.outputs.has-coverage == 'true'
uses: actions/upload-artifact@v6
with:
name: e2e-coverage
Expand All @@ -139,41 +66,43 @@ jobs:
if-no-files-found: warn

- name: Upload E2E coverage to Codecov
if: steps.coverage-shards.outputs.has-coverage == 'true'
if: >
steps.coverage.outputs.has-coverage == 'true' &&
(github.event.workflow_run.event != 'pull_request' ||
steps.pr-meta.outputs.skip != 'true')
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
files: coverage/playwright/coverage.lcov
flags: e2e
override_branch: ${{ inputs.source_branch || github.event.workflow_run.head_branch }}
override_commit: ${{ inputs.source_sha || github.event.workflow_run.head_sha }}
override_pr: ${{ inputs.source_pr || github.event.workflow_run.pull_requests[0].number }}
override_branch: ${{ github.event.workflow_run.head_branch }}
override_commit: ${{ steps.pr-meta.outputs.head-sha || github.event.workflow_run.head_sha }}
override_pr: ${{ steps.pr-meta.outputs.number }}
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false

- name: Generate HTML coverage report
if: steps.coverage-shards.outputs.has-coverage == 'true'
run: |
genhtml coverage/playwright/coverage.lcov \
-o coverage/html \
--title "ComfyUI E2E Coverage" \
--no-function-coverage \
--precision 1 \
--ignore-errors source,unmapped \
--synthesize-missing
- name: Download packaged HTML report

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optional simplification: only the main-push deploy job consumes this HTML. Move the exact cross-run download into deploy using github.event.workflow_run.id, add actions: read to that job, and delete this download plus the re-upload below. PRs then retain the producer artifact without paying for another download, roughly 7.4 MB duplicate upload, and second retained copy.

if: steps.coverage.outputs.has-coverage == 'true'
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
run_id: ${{ github.event.workflow_run.id }}
name: e2e-coverage-html
path: coverage/html
if_no_artifact_found: warn

- name: Upload HTML report artifact
if: steps.coverage-shards.outputs.has-coverage == 'true'
if: steps.coverage.outputs.has-coverage == 'true'
uses: actions/upload-artifact@v6
with:
name: e2e-coverage-html
path: coverage/html/
retention-days: 30
if-no-files-found: warn

deploy:
needs: merge
needs: publish
if: >
github.event.workflow_run.head_branch == 'main' &&
needs.merge.outputs.has-coverage == 'true' &&
needs.publish.outputs.has-coverage == 'true' &&
github.event.workflow_run.event == 'push'
runs-on: ubuntu-latest
permissions:
Expand Down
9 changes: 1 addition & 8 deletions .github/workflows/ci-tests-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -229,18 +229,11 @@ jobs:
${{
always() &&
needs.changes.outputs.should-run == 'true' &&
github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.fork == false &&
github.event.pull_request.user.login != 'dependabot[bot]' &&
needs.playwright-tests-chromium-sharded.result != 'cancelled'
}}
uses: ./.github/workflows/ci-tests-e2e-coverage.yaml
uses: ./.github/workflows/ci-tests-e2e-coverage-package.yaml
with:
source_branch: ${{ github.head_ref || github.ref_name }}
source_pr: ${{ format('{0}', github.event.pull_request.number) }}
source_run_id: ${{ format('{0}', github.run_id) }}
source_sha: ${{ github.event.pull_request.head.sha || github.sha }}
secrets: inherit

# Records video only for spec files newly added in this PR, to keep cost bounded.
playwright-video-new-tests:
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/pr-report.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: 'PR: Unified Report'

on:
workflow_run:
workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: E2E Coverage']
workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: Tests E2E']
types:
- completed
branches-ignore:
Expand Down Expand Up @@ -77,8 +77,9 @@ jobs:
id: find-coverage
uses: ./.github/actions/find-workflow-run
with:
workflow-id: ci-tests-e2e-coverage.yaml
workflow-id: ci-tests-e2e.yaml
head-sha: ${{ steps.pr-meta.outputs.head-sha }}
artifact-name: e2e-coverage
not-found-status: skip
token: ${{ secrets.GITHUB_TOKEN }}

Expand Down
Loading
Loading