From bf2ca1b8a659f9d4c34d7125dff04aa1970325e1 Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 10:49:19 -0700 Subject: [PATCH 1/7] test: cover trusted E2E coverage handoff --- scripts/cicd/e2e-coverage-workflow.test.ts | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 scripts/cicd/e2e-coverage-workflow.test.ts diff --git a/scripts/cicd/e2e-coverage-workflow.test.ts b/scripts/cicd/e2e-coverage-workflow.test.ts new file mode 100644 index 00000000000..b74ad90bebe --- /dev/null +++ b/scripts/cicd/e2e-coverage-workflow.test.ts @@ -0,0 +1,62 @@ +import { readFileSync } from 'node:fs' + +import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' + +type WorkflowStep = { + if?: string + uses?: string + with?: Record +} + +type Workflow = { + jobs: Record< + string, + { + if?: string + steps?: WorkflowStep[] + with?: Record + } + > +} + +function readWorkflow(path: string): Workflow { + return parse(readFileSync(path, 'utf8')) as Workflow +} + +describe('E2E coverage workflows', () => { + it('packages fork coverage before entering the trusted workflow_run', () => { + const coverageWorkflow = readWorkflow( + '.github/workflows/ci-tests-e2e-coverage.yaml' + ) + const e2eWorkflow = readWorkflow('.github/workflows/ci-tests-e2e.yaml') + const coverageSteps = coverageWorkflow.jobs.merge.steps ?? [] + const checkout = coverageSteps.find((step) => + step.uses?.startsWith('actions/checkout@') + ) + const download = coverageSteps.find( + (step) => step.with?.name === 'e2e-coverage' + ) + const codecov = coverageSteps.find((step) => + step.uses?.startsWith('codecov/codecov-action@') + ) + const packageJob = e2eWorkflow.jobs['upload-e2e-coverage'] + + expect(checkout?.if).toContain("github.event_name == 'workflow_call'") + expect(download?.if).toContain("github.event_name == 'workflow_run'") + expect(codecov?.if).toContain("github.event_name == 'workflow_run'") + expect(codecov?.with).toMatchObject({ + override_branch: + '${{ inputs.source_branch || github.event.workflow_run.head_branch }}', + override_commit: + '${{ inputs.source_sha || github.event.workflow_run.head_sha }}', + override_pr: + '${{ inputs.source_pr || github.event.workflow_run.pull_requests[0].number }}' + }) + expect(packageJob.if).not.toContain('fork == false') + expect(packageJob.with).toMatchObject({ + source_branch: '${{ github.head_ref || github.ref_name }}', + source_sha: '${{ github.event.pull_request.head.sha || github.sha }}' + }) + }) +}) From 1efc228c4aeb39d90f9a0769447025d7756ed94d Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 10:53:29 -0700 Subject: [PATCH 2/7] fix: package fork E2E coverage before trusted upload --- .github/workflows/ci-tests-e2e-coverage.yaml | 50 +++++++++++++------- .github/workflows/ci-tests-e2e.yaml | 4 -- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-tests-e2e-coverage.yaml b/.github/workflows/ci-tests-e2e-coverage.yaml index bf9e77cbb81..bc41ee76548 100644 --- a/.github/workflows/ci-tests-e2e-coverage.yaml +++ b/.github/workflows/ci-tests-e2e-coverage.yaml @@ -17,7 +17,7 @@ on: type: string secrets: CODECOV_TOKEN: - required: true + required: false workflow_run: workflows: ['CI: Tests E2E'] types: @@ -32,41 +32,48 @@ permissions: jobs: merge: - # Not gated on full success: fail-fast:false means one flaky shard flips the whole run to failure, which would skip merging every other shard's real coverage. if: > github.repository == 'Comfy-Org/ComfyUI_frontend' && (github.event_name == 'workflow_call' || - github.event.workflow_run.event != 'pull_request' || - github.event.workflow_run.head_repository.full_name != github.repository || - github.event.workflow_run.actor.login == 'dependabot[bot]') && - github.event.workflow_run.conclusion != 'cancelled' + github.event.workflow_run.conclusion != 'cancelled') runs-on: ubuntu-latest timeout-minutes: 10 outputs: has-coverage: ${{ steps.coverage-shards.outputs.has-coverage }} steps: - - name: Checkout sources for genhtml + - name: Checkout sources for coverage packaging + if: github.event_name == 'workflow_call' uses: actions/checkout@v7 with: - ref: ${{ inputs.source_sha || github.event.workflow_run.head_sha }} + ref: ${{ inputs.source_sha }} sparse-checkout: | src packages - name: Download all shard coverage data + if: github.event_name == 'workflow_call' uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 with: - run_id: ${{ inputs.source_run_id || github.event.workflow_run.id }} + run_id: ${{ inputs.source_run_id }} name: e2e-coverage-shard-.* name_is_regexp: true path: temp/coverage-shards if_no_artifact_found: warn + - name: Download packaged coverage data + if: github.event_name == 'workflow_run' + uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 + with: + run_id: ${{ github.event.workflow_run.id }} + name: e2e-coverage + path: coverage/playwright + if_no_artifact_found: warn + - name: Detect shard coverage data id: coverage-shards run: | - if [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; then + if [ -f coverage/playwright/coverage.lcov ] || { [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; }; then echo "has-coverage=true" >> "$GITHUB_OUTPUT" else echo "has-coverage=false" >> "$GITHUB_OUTPUT" @@ -74,11 +81,11 @@ jobs: fi - name: Install lcov - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' run: sudo apt-get install -y -qq lcov - name: Merge shard coverage into single LCOV - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' run: | mkdir -p coverage/playwright LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f) @@ -88,7 +95,7 @@ jobs: wc -l coverage/playwright/coverage.lcov - name: Validate merged coverage - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' run: | MERGED_SF=$(grep -c '^SF:' coverage/playwright/coverage.lcov || echo 0) MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) @@ -110,7 +117,7 @@ jobs: done - name: Assert coverage was mapped back to source - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' run: | MAPPED_SF=$(grep -cE '^SF:(src|packages)/' coverage/playwright/coverage.lcov || true) echo "Source-mapped files: $MAPPED_SF" >> "$GITHUB_STEP_SUMMARY" @@ -120,7 +127,7 @@ jobs: fi - name: Strip non-source entries from coverage - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' run: | lcov --remove coverage/playwright/coverage.lcov \ '*localhost-8188*' \ @@ -139,7 +146,7 @@ jobs: if-no-files-found: warn - name: Upload E2E coverage to Codecov - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_run' && steps.coverage-shards.outputs.has-coverage == 'true' uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: files: coverage/playwright/coverage.lcov @@ -151,7 +158,7 @@ jobs: fail_ci_if_error: false - name: Generate HTML coverage report - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' run: | genhtml coverage/playwright/coverage.lcov \ -o coverage/html \ @@ -161,6 +168,15 @@ jobs: --ignore-errors source,unmapped \ --synthesize-missing + - name: Download packaged HTML report + if: github.event_name == 'workflow_run' && steps.coverage-shards.outputs.has-coverage == 'true' + uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 + with: + run_id: ${{ github.event.workflow_run.id }} + name: e2e-coverage-html + path: coverage/html + if_no_artifact_found: warn + - name: Upload HTML report artifact if: steps.coverage-shards.outputs.has-coverage == 'true' uses: actions/upload-artifact@v6 diff --git a/.github/workflows/ci-tests-e2e.yaml b/.github/workflows/ci-tests-e2e.yaml index 14839851cda..17a1412ace3 100644 --- a/.github/workflows/ci-tests-e2e.yaml +++ b/.github/workflows/ci-tests-e2e.yaml @@ -229,9 +229,6 @@ jobs: ${{ always() && needs.changes.outputs.should-run == 'true' && - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.fork == false && - github.event.pull_request.user.login != 'dependabot[bot]' && needs.playwright-tests-chromium-sharded.result != 'cancelled' }} uses: ./.github/workflows/ci-tests-e2e-coverage.yaml @@ -240,7 +237,6 @@ jobs: source_pr: ${{ format('{0}', github.event.pull_request.number) }} source_run_id: ${{ format('{0}', github.run_id) }} source_sha: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: inherit # Records video only for spec files newly added in this PR, to keep cost bounded. playwright-video-new-tests: From 5a84b49220565633aef21556ebedb072f84bdf17 Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 13:31:03 -0700 Subject: [PATCH 3/7] fix: separate E2E coverage producer and publisher --- .github/actions/find-workflow-run/action.yaml | 28 +++ .../ci-tests-e2e-coverage-package.yaml | 133 +++++++++++++ .github/workflows/ci-tests-e2e-coverage.yaml | 157 ++++------------ .github/workflows/ci-tests-e2e.yaml | 4 +- .github/workflows/pr-report.yaml | 5 +- scripts/cicd/e2e-coverage-workflow.test.ts | 174 ++++++++++++++---- 6 files changed, 343 insertions(+), 158 deletions(-) create mode 100644 .github/workflows/ci-tests-e2e-coverage-package.yaml diff --git a/.github/actions/find-workflow-run/action.yaml b/.github/actions/find-workflow-run/action.yaml index 620394eb729..e8dfd3a3446 100644 --- a/.github/actions/find-workflow-run/action.yaml +++ b/.github/actions/find-workflow-run/action.yaml @@ -12,6 +12,9 @@ inputs: description: Status to output when no run exists required: false default: pending + artifact-name: + description: When set, a completed run is ready only when this artifact exists + required: false token: description: GitHub token for API access required: true @@ -34,6 +37,7 @@ runs: WORKFLOW_ID: ${{ inputs.workflow-id }} HEAD_SHA: ${{ inputs.head-sha }} NOT_FOUND_STATUS: ${{ inputs.not-found-status }} + ARTIFACT_NAME: ${{ inputs.artifact-name }} with: github-token: ${{ inputs.token }} script: | @@ -56,6 +60,30 @@ runs: return; } + if (process.env.ARTIFACT_NAME) { + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + } + ); + const artifact = artifacts.find( + candidate => + candidate.name === process.env.ARTIFACT_NAME && + !candidate.expired + ); + if (!artifact) { + core.setOutput('status', process.env.NOT_FOUND_STATUS); + return; + } + core.setOutput('status', 'ready'); + core.setOutput('run-id', String(run.id)); + return; + } + if (run.conclusion !== 'success') { core.setOutput('status', 'failed'); return; diff --git a/.github/workflows/ci-tests-e2e-coverage-package.yaml b/.github/workflows/ci-tests-e2e-coverage-package.yaml new file mode 100644 index 00000000000..755f0e89e24 --- /dev/null +++ b/.github/workflows/ci-tests-e2e-coverage-package.yaml @@ -0,0 +1,133 @@ +name: 'CI: E2E Coverage Package' + +on: + workflow_call: + inputs: + source_run_id: + required: true + type: string + source_sha: + required: true + type: string + +permissions: + actions: read + contents: read + +jobs: + package: + if: github.repository == 'Comfy-Org/ComfyUI_frontend' + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout sources for coverage packaging + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.source_sha }} + sparse-checkout: | + src + packages + + - name: Download all shard coverage data + uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 + with: + run_id: ${{ inputs.source_run_id }} + name: e2e-coverage-shard-.* + name_is_regexp: true + path: temp/coverage-shards + if_no_artifact_found: warn + + - name: Detect shard coverage data + id: coverage + run: | + if [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; then + echo "has-coverage=true" >> "$GITHUB_OUTPUT" + else + echo "has-coverage=false" >> "$GITHUB_OUTPUT" + echo "No E2E coverage shard artifacts found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Install lcov + if: steps.coverage.outputs.has-coverage == 'true' + run: sudo apt-get install -y -qq lcov + + - name: Merge shard coverage into single LCOV + if: steps.coverage.outputs.has-coverage == 'true' + run: | + mkdir -p coverage/playwright + LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f) + ADD_ARGS="" + for f in $LCOV_FILES; do ADD_ARGS="$ADD_ARGS -a $f"; done + lcov $ADD_ARGS -o coverage/playwright/coverage.lcov + wc -l coverage/playwright/coverage.lcov + + - name: Validate merged coverage + if: steps.coverage.outputs.has-coverage == 'true' + run: | + MERGED_SF=$(grep -c '^SF:' coverage/playwright/coverage.lcov || echo 0) + MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) + MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) + echo "### Merged coverage" >> "$GITHUB_STEP_SUMMARY" + echo "- **$MERGED_SF** source files" >> "$GITHUB_STEP_SUMMARY" + echo "- **$MERGED_LH / $MERGED_LF** lines hit" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Shard | Files | Lines Hit |" >> "$GITHUB_STEP_SUMMARY" + echo "|-------|-------|-----------|" >> "$GITHUB_STEP_SUMMARY" + for f in $(find temp/coverage-shards -name 'coverage.lcov' -type f | sort); do + SHARD=$(basename "$(dirname "$f")") + SHARD_SF=$(grep -c '^SF:' "$f" || echo 0) + SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$f") + echo "| $SHARD | $SHARD_SF | $SHARD_LH |" >> "$GITHUB_STEP_SUMMARY" + if [ "$MERGED_LH" -lt "$SHARD_LH" ]; then + echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" + fi + done + + - name: Assert coverage was mapped back to source + if: steps.coverage.outputs.has-coverage == 'true' + run: | + MAPPED_SF=$(grep -cE '^SF:(src|packages)/' coverage/playwright/coverage.lcov || true) + echo "Source-mapped files: $MAPPED_SF" >> "$GITHUB_STEP_SUMMARY" + if [ "${MAPPED_SF:-0}" -lt 100 ]; then + echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $(grep -m 5 '^SF:' coverage/playwright/coverage.lcov | tr '\n' ' '). Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)." + exit 1 + fi + + - name: Strip non-source entries from coverage + if: steps.coverage.outputs.has-coverage == 'true' + run: | + lcov --remove coverage/playwright/coverage.lcov \ + '*localhost-8188*' \ + 'assets/images/*' \ + -o coverage/playwright/coverage.lcov \ + --ignore-errors unused + wc -l coverage/playwright/coverage.lcov + + - name: Upload merged coverage data + if: steps.coverage.outputs.has-coverage == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: e2e-coverage + path: coverage/playwright/ + retention-days: 30 + if-no-files-found: warn + + - name: Generate HTML coverage report + if: steps.coverage.outputs.has-coverage == 'true' + run: | + genhtml coverage/playwright/coverage.lcov \ + -o coverage/html \ + --title "ComfyUI E2E Coverage" \ + --no-function-coverage \ + --precision 1 \ + --ignore-errors source,unmapped \ + --synthesize-missing + + - name: Upload HTML report artifact + if: steps.coverage.outputs.has-coverage == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: e2e-coverage-html + path: coverage/html/ + retention-days: 30 diff --git a/.github/workflows/ci-tests-e2e-coverage.yaml b/.github/workflows/ci-tests-e2e-coverage.yaml index bc41ee76548..e2810157424 100644 --- a/.github/workflows/ci-tests-e2e-coverage.yaml +++ b/.github/workflows/ci-tests-e2e-coverage.yaml @@ -1,68 +1,44 @@ name: 'CI: E2E Coverage' on: - workflow_call: - inputs: - source_branch: - required: true - type: string - source_pr: - required: false - type: string - source_run_id: - required: true - type: string - source_sha: - required: true - type: string - secrets: - CODECOV_TOKEN: - required: false workflow_run: workflows: ['CI: Tests E2E'] types: - completed concurrency: - group: e2e-coverage-${{ inputs.source_sha || github.event.workflow_run.head_sha }} + group: e2e-coverage-${{ github.event.workflow_run.head_sha }} cancel-in-progress: true permissions: + actions: read contents: read + pull-requests: read jobs: - merge: + publish: if: > github.repository == 'Comfy-Org/ComfyUI_frontend' && - (github.event_name == 'workflow_call' || - github.event.workflow_run.conclusion != 'cancelled') + github.event.workflow_run.conclusion != 'cancelled' runs-on: ubuntu-latest timeout-minutes: 10 outputs: - has-coverage: ${{ steps.coverage-shards.outputs.has-coverage }} + has-coverage: ${{ steps.coverage.outputs.has-coverage }} steps: - - name: Checkout sources for coverage packaging - if: github.event_name == 'workflow_call' - uses: actions/checkout@v7 + - name: Checkout trusted workflow actions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.source_sha }} - sparse-checkout: | - src - packages + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: .github/actions/resolve-pr-from-workflow-run + persist-credentials: false - - name: Download all shard coverage data - if: github.event_name == 'workflow_call' - uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 - with: - run_id: ${{ inputs.source_run_id }} - name: e2e-coverage-shard-.* - name_is_regexp: true - path: temp/coverage-shards - if_no_artifact_found: warn + - name: Resolve PR from workflow_run context + id: pr-meta + if: github.event.workflow_run.event == 'pull_request' + uses: ./.github/actions/resolve-pr-from-workflow-run - name: Download packaged coverage data - if: github.event_name == 'workflow_run' uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 with: run_id: ${{ github.event.workflow_run.id }} @@ -70,75 +46,19 @@ jobs: path: coverage/playwright if_no_artifact_found: warn - - name: Detect shard coverage data - id: coverage-shards + - name: Detect packaged coverage data + id: coverage run: | - if [ -f coverage/playwright/coverage.lcov ] || { [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; }; then + if [ -f coverage/playwright/coverage.lcov ]; then echo "has-coverage=true" >> "$GITHUB_OUTPUT" else echo "has-coverage=false" >> "$GITHUB_OUTPUT" - echo "No E2E coverage shard artifacts found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Install lcov - if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' - run: sudo apt-get install -y -qq lcov - - - name: Merge shard coverage into single LCOV - if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' - run: | - mkdir -p coverage/playwright - LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f) - ADD_ARGS="" - for f in $LCOV_FILES; do ADD_ARGS="$ADD_ARGS -a $f"; done - lcov $ADD_ARGS -o coverage/playwright/coverage.lcov - wc -l coverage/playwright/coverage.lcov - - - name: Validate merged coverage - if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' - run: | - MERGED_SF=$(grep -c '^SF:' coverage/playwright/coverage.lcov || echo 0) - MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) - MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) - echo "### Merged coverage" >> "$GITHUB_STEP_SUMMARY" - echo "- **$MERGED_SF** source files" >> "$GITHUB_STEP_SUMMARY" - echo "- **$MERGED_LH / $MERGED_LF** lines hit" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "| Shard | Files | Lines Hit |" >> "$GITHUB_STEP_SUMMARY" - echo "|-------|-------|-----------|" >> "$GITHUB_STEP_SUMMARY" - for f in $(find temp/coverage-shards -name 'coverage.lcov' -type f | sort); do - SHARD=$(basename "$(dirname "$f")") - SHARD_SF=$(grep -c '^SF:' "$f" || echo 0) - SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$f") - echo "| $SHARD | $SHARD_SF | $SHARD_LH |" >> "$GITHUB_STEP_SUMMARY" - if [ "$MERGED_LH" -lt "$SHARD_LH" ]; then - echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" - fi - done - - - name: Assert coverage was mapped back to source - if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' - run: | - MAPPED_SF=$(grep -cE '^SF:(src|packages)/' coverage/playwright/coverage.lcov || true) - echo "Source-mapped files: $MAPPED_SF" >> "$GITHUB_STEP_SUMMARY" - if [ "${MAPPED_SF:-0}" -lt 100 ]; then - echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $(grep -m 5 '^SF:' coverage/playwright/coverage.lcov | tr '\n' ' '). Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)." - exit 1 + echo "No packaged E2E coverage artifact found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY" fi - - name: Strip non-source entries from coverage - if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' - run: | - lcov --remove coverage/playwright/coverage.lcov \ - '*localhost-8188*' \ - 'assets/images/*' \ - -o coverage/playwright/coverage.lcov \ - --ignore-errors unused - wc -l coverage/playwright/coverage.lcov - - name: Upload merged coverage data - if: steps.coverage-shards.outputs.has-coverage == 'true' - uses: actions/upload-artifact@v6 + if: steps.coverage.outputs.has-coverage == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: e2e-coverage path: coverage/playwright/ @@ -146,30 +66,22 @@ jobs: if-no-files-found: warn - name: Upload E2E coverage to Codecov - if: github.event_name == 'workflow_run' && steps.coverage-shards.outputs.has-coverage == 'true' + if: > + steps.coverage.outputs.has-coverage == 'true' && + (github.event.workflow_run.event != 'pull_request' || + steps.pr-meta.outputs.skip != 'true') uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: files: coverage/playwright/coverage.lcov flags: e2e - override_branch: ${{ inputs.source_branch || github.event.workflow_run.head_branch }} - override_commit: ${{ inputs.source_sha || github.event.workflow_run.head_sha }} - override_pr: ${{ inputs.source_pr || github.event.workflow_run.pull_requests[0].number }} + override_branch: ${{ github.event.workflow_run.head_branch }} + override_commit: ${{ steps.pr-meta.outputs.head-sha || github.event.workflow_run.head_sha }} + override_pr: ${{ steps.pr-meta.outputs.number }} token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false - - name: Generate HTML coverage report - if: github.event_name == 'workflow_call' && steps.coverage-shards.outputs.has-coverage == 'true' - run: | - genhtml coverage/playwright/coverage.lcov \ - -o coverage/html \ - --title "ComfyUI E2E Coverage" \ - --no-function-coverage \ - --precision 1 \ - --ignore-errors source,unmapped \ - --synthesize-missing - - name: Download packaged HTML report - if: github.event_name == 'workflow_run' && steps.coverage-shards.outputs.has-coverage == 'true' + if: steps.coverage.outputs.has-coverage == 'true' uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 with: run_id: ${{ github.event.workflow_run.id }} @@ -178,18 +90,19 @@ jobs: if_no_artifact_found: warn - name: Upload HTML report artifact - if: steps.coverage-shards.outputs.has-coverage == 'true' - uses: actions/upload-artifact@v6 + if: steps.coverage.outputs.has-coverage == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: e2e-coverage-html path: coverage/html/ retention-days: 30 + if-no-files-found: warn deploy: - needs: merge + needs: publish if: > github.event.workflow_run.head_branch == 'main' && - needs.merge.outputs.has-coverage == 'true' && + needs.publish.outputs.has-coverage == 'true' && github.event.workflow_run.event == 'push' runs-on: ubuntu-latest permissions: @@ -200,7 +113,7 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - name: Download HTML report - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-coverage-html path: coverage/html diff --git a/.github/workflows/ci-tests-e2e.yaml b/.github/workflows/ci-tests-e2e.yaml index 17a1412ace3..c15509c1bc9 100644 --- a/.github/workflows/ci-tests-e2e.yaml +++ b/.github/workflows/ci-tests-e2e.yaml @@ -231,10 +231,8 @@ jobs: needs.changes.outputs.should-run == 'true' && needs.playwright-tests-chromium-sharded.result != 'cancelled' }} - uses: ./.github/workflows/ci-tests-e2e-coverage.yaml + uses: ./.github/workflows/ci-tests-e2e-coverage-package.yaml with: - source_branch: ${{ github.head_ref || github.ref_name }} - source_pr: ${{ format('{0}', github.event.pull_request.number) }} source_run_id: ${{ format('{0}', github.run_id) }} source_sha: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/pr-report.yaml b/.github/workflows/pr-report.yaml index ffd3821aa99..90cd77ceb25 100644 --- a/.github/workflows/pr-report.yaml +++ b/.github/workflows/pr-report.yaml @@ -2,7 +2,7 @@ name: 'PR: Unified Report' on: workflow_run: - workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: E2E Coverage'] + workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: Tests E2E'] types: - completed branches-ignore: @@ -77,8 +77,9 @@ jobs: id: find-coverage uses: ./.github/actions/find-workflow-run with: - workflow-id: ci-tests-e2e-coverage.yaml + workflow-id: ci-tests-e2e.yaml head-sha: ${{ steps.pr-meta.outputs.head-sha }} + artifact-name: e2e-coverage not-found-status: skip token: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/cicd/e2e-coverage-workflow.test.ts b/scripts/cicd/e2e-coverage-workflow.test.ts index b74ad90bebe..fe20f0594f7 100644 --- a/scripts/cicd/e2e-coverage-workflow.test.ts +++ b/scripts/cicd/e2e-coverage-workflow.test.ts @@ -4,59 +4,171 @@ import { describe, expect, it } from 'vitest' import { parse } from 'yaml' type WorkflowStep = { + name?: string if?: string uses?: string - with?: Record + run?: string + with?: Record +} + +type WorkflowJob = { + if?: string + uses?: string + secrets?: unknown + steps?: WorkflowStep[] + with?: Record } type Workflow = { - jobs: Record< - string, - { - if?: string - steps?: WorkflowStep[] - with?: Record - } - > + on?: Record + jobs: Record +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function hasOptionalString( + value: Record, + key: string +): boolean { + return value[key] === undefined || typeof value[key] === 'string' +} + +function isWorkflowStep(value: unknown): value is WorkflowStep { + return ( + isRecord(value) && + hasOptionalString(value, 'name') && + hasOptionalString(value, 'if') && + hasOptionalString(value, 'uses') && + hasOptionalString(value, 'run') && + (value.with === undefined || isRecord(value.with)) + ) +} + +function isWorkflowJob(value: unknown): value is WorkflowJob { + return ( + isRecord(value) && + hasOptionalString(value, 'if') && + hasOptionalString(value, 'uses') && + (value.steps === undefined || + (Array.isArray(value.steps) && value.steps.every(isWorkflowStep))) && + (value.with === undefined || isRecord(value.with)) + ) +} + +function isWorkflow(value: unknown): value is Workflow { + return ( + isRecord(value) && + isRecord(value.jobs) && + Object.values(value.jobs).every(isWorkflowJob) && + (value.on === undefined || isRecord(value.on)) + ) } function readWorkflow(path: string): Workflow { - return parse(readFileSync(path, 'utf8')) as Workflow + const value: unknown = parse(readFileSync(path, 'utf8')) + if (!isWorkflow(value)) { + throw new Error(`Invalid workflow structure in ${path}`) + } + return value } describe('E2E coverage workflows', () => { - it('packages fork coverage before entering the trusted workflow_run', () => { - const coverageWorkflow = readWorkflow( - '.github/workflows/ci-tests-e2e-coverage.yaml' + it('packages coverage in the unprivileged E2E run', () => { + const packageWorkflow = readWorkflow( + '.github/workflows/ci-tests-e2e-coverage-package.yaml' ) const e2eWorkflow = readWorkflow('.github/workflows/ci-tests-e2e.yaml') - const coverageSteps = coverageWorkflow.jobs.merge.steps ?? [] - const checkout = coverageSteps.find((step) => + const packageSteps = packageWorkflow.jobs.package.steps ?? [] + const checkout = packageSteps.find((step) => + step.uses?.startsWith('actions/checkout@') + ) + const packageJob = e2eWorkflow.jobs['upload-e2e-coverage'] + + expect(packageWorkflow.on).toHaveProperty('workflow_call') + expect(checkout?.with?.ref).toBe('${{ inputs.source_sha }}') + expect(packageSteps.some((step) => step.run?.includes('lcov '))).toBe(true) + expect(packageSteps.some((step) => step.run?.includes('genhtml '))).toBe( + true + ) + expect(packageJob.uses).toBe( + './.github/workflows/ci-tests-e2e-coverage-package.yaml' + ) + expect(packageJob.secrets).toBeUndefined() + expect(packageJob.if).not.toContain('fork == false') + expect(packageJob.with).toMatchObject({ + source_run_id: "${{ format('{0}', github.run_id) }}", + source_sha: '${{ github.event.pull_request.head.sha || github.sha }}' + }) + }) + + it('publishes only fixed artifacts from the exact triggering run', () => { + const publishWorkflow = readWorkflow( + '.github/workflows/ci-tests-e2e-coverage.yaml' + ) + const publishSteps = publishWorkflow.jobs.publish.steps ?? [] + const checkouts = publishSteps.filter((step) => step.uses?.startsWith('actions/checkout@') ) - const download = coverageSteps.find( - (step) => step.with?.name === 'e2e-coverage' + const coverageDownloads = publishSteps.filter( + (step) => + step.uses?.startsWith('dawidd6/action-download-artifact@') && + step.with?.name === 'e2e-coverage' ) - const codecov = coverageSteps.find((step) => + const resolver = publishSteps.find( + (step) => step.uses === './.github/actions/resolve-pr-from-workflow-run' + ) + const codecov = publishSteps.find((step) => step.uses?.startsWith('codecov/codecov-action@') ) - const packageJob = e2eWorkflow.jobs['upload-e2e-coverage'] - expect(checkout?.if).toContain("github.event_name == 'workflow_call'") - expect(download?.if).toContain("github.event_name == 'workflow_run'") - expect(codecov?.if).toContain("github.event_name == 'workflow_run'") + expect(publishWorkflow.on).toHaveProperty('workflow_run') + expect(checkouts).not.toHaveLength(0) + for (const checkout of checkouts) { + expect(checkout.with).toMatchObject({ + ref: '${{ github.event.repository.default_branch }}', + 'persist-credentials': false + }) + } + expect(coverageDownloads).toHaveLength(1) + expect(coverageDownloads[0]?.with).toMatchObject({ + run_id: '${{ github.event.workflow_run.id }}', + name: 'e2e-coverage', + path: 'coverage/playwright' + }) + expect(resolver?.if).toContain("workflow_run.event == 'pull_request'") expect(codecov?.with).toMatchObject({ - override_branch: - '${{ inputs.source_branch || github.event.workflow_run.head_branch }}', + files: 'coverage/playwright/coverage.lcov', + override_branch: '${{ github.event.workflow_run.head_branch }}', override_commit: - '${{ inputs.source_sha || github.event.workflow_run.head_sha }}', - override_pr: - '${{ inputs.source_pr || github.event.workflow_run.pull_requests[0].number }}' + '${{ steps.pr-meta.outputs.head-sha || github.event.workflow_run.head_sha }}', + override_pr: '${{ steps.pr-meta.outputs.number }}' }) - expect(packageJob.if).not.toContain('fork == false') - expect(packageJob.with).toMatchObject({ - source_branch: '${{ github.head_ref || github.ref_name }}', - source_sha: '${{ github.event.pull_request.head.sha || github.sha }}' + expect( + publishSteps.some( + (step) => + step.run !== undefined && /(^|\n)\s*(lcov|genhtml)\s/.test(step.run) + ) + ).toBe(false) + }) + + it('makes packaged PR coverage discoverable from the E2E run', () => { + const reportWorkflow = readWorkflow('.github/workflows/pr-report.yaml') + const reportSteps = reportWorkflow.jobs.comment.steps ?? [] + const workflowRun = reportWorkflow.on?.workflow_run + const coverageLookup = reportSteps.find( + (step) => step.name === 'Find coverage workflow run' + ) + + expect(workflowRun).toMatchObject({ + workflows: expect.arrayContaining(['CI: Tests E2E']) + }) + expect(coverageLookup?.with).toMatchObject({ + 'workflow-id': 'ci-tests-e2e.yaml', + 'head-sha': '${{ steps.pr-meta.outputs.head-sha }}', + 'artifact-name': 'e2e-coverage', + 'not-found-status': 'skip' }) }) }) From a9d6f201f1e186faa699e8fd22708782d903e627 Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 13:38:58 -0700 Subject: [PATCH 4/7] fix: harden E2E coverage packaging --- .../ci-tests-e2e-coverage-package.yaml | 20 ++++++++++--------- .github/workflows/ci-tests-e2e-coverage.yaml | 8 ++++---- scripts/cicd/e2e-coverage-workflow.test.ts | 15 +++++++++++++- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-tests-e2e-coverage-package.yaml b/.github/workflows/ci-tests-e2e-coverage-package.yaml index 755f0e89e24..441adae95f6 100644 --- a/.github/workflows/ci-tests-e2e-coverage-package.yaml +++ b/.github/workflows/ci-tests-e2e-coverage-package.yaml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout sources for coverage packaging - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@v7 with: ref: ${{ inputs.source_sha }} sparse-checkout: | @@ -56,10 +56,11 @@ jobs: if: steps.coverage.outputs.has-coverage == 'true' run: | mkdir -p coverage/playwright - LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f) - ADD_ARGS="" - for f in $LCOV_FILES; do ADD_ARGS="$ADD_ARGS -a $f"; done - lcov $ADD_ARGS -o coverage/playwright/coverage.lcov + ADD_ARGS=() + while IFS= read -r -d '' f; do + ADD_ARGS+=(-a "$f") + done < <(find temp/coverage-shards -name 'coverage.lcov' -type f -print0) + lcov "${ADD_ARGS[@]}" -o coverage/playwright/coverage.lcov wc -l coverage/playwright/coverage.lcov - name: Validate merged coverage @@ -74,15 +75,16 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "| Shard | Files | Lines Hit |" >> "$GITHUB_STEP_SUMMARY" echo "|-------|-------|-----------|" >> "$GITHUB_STEP_SUMMARY" - for f in $(find temp/coverage-shards -name 'coverage.lcov' -type f | sort); do + while IFS= read -r -d '' f; do SHARD=$(basename "$(dirname "$f")") SHARD_SF=$(grep -c '^SF:' "$f" || echo 0) SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$f") echo "| $SHARD | $SHARD_SF | $SHARD_LH |" >> "$GITHUB_STEP_SUMMARY" if [ "$MERGED_LH" -lt "$SHARD_LH" ]; then echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" + exit 1 fi - done + done < <(find temp/coverage-shards -name 'coverage.lcov' -type f -print0 | sort -z) - name: Assert coverage was mapped back to source if: steps.coverage.outputs.has-coverage == 'true' @@ -106,7 +108,7 @@ jobs: - name: Upload merged coverage data if: steps.coverage.outputs.has-coverage == 'true' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@v6 with: name: e2e-coverage path: coverage/playwright/ @@ -126,7 +128,7 @@ jobs: - name: Upload HTML report artifact if: steps.coverage.outputs.has-coverage == 'true' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@v6 with: name: e2e-coverage-html path: coverage/html/ diff --git a/.github/workflows/ci-tests-e2e-coverage.yaml b/.github/workflows/ci-tests-e2e-coverage.yaml index e2810157424..4e1ffd22f18 100644 --- a/.github/workflows/ci-tests-e2e-coverage.yaml +++ b/.github/workflows/ci-tests-e2e-coverage.yaml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout trusted workflow actions - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@v7 with: ref: ${{ github.event.repository.default_branch }} sparse-checkout: .github/actions/resolve-pr-from-workflow-run @@ -58,7 +58,7 @@ jobs: - name: Upload merged coverage data if: steps.coverage.outputs.has-coverage == 'true' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@v6 with: name: e2e-coverage path: coverage/playwright/ @@ -91,7 +91,7 @@ jobs: - name: Upload HTML report artifact if: steps.coverage.outputs.has-coverage == 'true' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@v6 with: name: e2e-coverage-html path: coverage/html/ @@ -113,7 +113,7 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - name: Download HTML report - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@v8 with: name: e2e-coverage-html path: coverage/html diff --git a/scripts/cicd/e2e-coverage-workflow.test.ts b/scripts/cicd/e2e-coverage-workflow.test.ts index fe20f0594f7..f34e7a8b310 100644 --- a/scripts/cicd/e2e-coverage-workflow.test.ts +++ b/scripts/cicd/e2e-coverage-workflow.test.ts @@ -84,11 +84,24 @@ describe('E2E coverage workflows', () => { const checkout = packageSteps.find((step) => step.uses?.startsWith('actions/checkout@') ) + const merge = packageSteps.find( + (step) => step.name === 'Merge shard coverage into single LCOV' + ) + const validation = packageSteps.find( + (step) => step.name === 'Validate merged coverage' + ) const packageJob = e2eWorkflow.jobs['upload-e2e-coverage'] expect(packageWorkflow.on).toHaveProperty('workflow_call') + expect(checkout?.uses).toBe('actions/checkout@v7') expect(checkout?.with?.ref).toBe('${{ inputs.source_sha }}') - expect(packageSteps.some((step) => step.run?.includes('lcov '))).toBe(true) + expect(merge?.run).toContain( + "find temp/coverage-shards -name 'coverage.lcov' -type f -print0" + ) + expect(merge?.run).toContain("while IFS= read -r -d '' f") + expect(merge?.run).toContain('lcov "${ADD_ARGS[@]}"') + expect(validation?.run).toContain("while IFS= read -r -d '' f") + expect(validation?.run).toMatch(/possible data loss"\n\s+exit 1/) expect(packageSteps.some((step) => step.run?.includes('genhtml '))).toBe( true ) From 0be708729008179a33b5f322bc4017085dcb1857 Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 14:13:48 -0700 Subject: [PATCH 5/7] test: exercise E2E coverage packaging behavior --- .../ci-tests-e2e-coverage-package.yaml | 81 +------- scripts/cicd/e2e-coverage-workflow.test.ts | 19 +- scripts/cicd/package-e2e-coverage.sh | 81 ++++++++ scripts/cicd/package-e2e-coverage.test.ts | 173 ++++++++++++++++++ 4 files changed, 267 insertions(+), 87 deletions(-) create mode 100755 scripts/cicd/package-e2e-coverage.sh create mode 100644 scripts/cicd/package-e2e-coverage.test.ts diff --git a/.github/workflows/ci-tests-e2e-coverage-package.yaml b/.github/workflows/ci-tests-e2e-coverage-package.yaml index 441adae95f6..6218606a14f 100644 --- a/.github/workflows/ci-tests-e2e-coverage-package.yaml +++ b/.github/workflows/ci-tests-e2e-coverage-package.yaml @@ -28,6 +28,7 @@ jobs: sparse-checkout: | src packages + scripts/cicd/package-e2e-coverage.sh - name: Download all shard coverage data uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 @@ -38,73 +39,16 @@ jobs: path: temp/coverage-shards if_no_artifact_found: warn - - name: Detect shard coverage data - id: coverage - run: | - if [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; then - echo "has-coverage=true" >> "$GITHUB_OUTPUT" - else - echo "has-coverage=false" >> "$GITHUB_OUTPUT" - echo "No E2E coverage shard artifacts found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY" - fi - - name: Install lcov - if: steps.coverage.outputs.has-coverage == 'true' run: sudo apt-get install -y -qq lcov - - name: Merge shard coverage into single LCOV - if: steps.coverage.outputs.has-coverage == 'true' - run: | - mkdir -p coverage/playwright - ADD_ARGS=() - while IFS= read -r -d '' f; do - ADD_ARGS+=(-a "$f") - done < <(find temp/coverage-shards -name 'coverage.lcov' -type f -print0) - lcov "${ADD_ARGS[@]}" -o coverage/playwright/coverage.lcov - wc -l coverage/playwright/coverage.lcov - - - name: Validate merged coverage - if: steps.coverage.outputs.has-coverage == 'true' - run: | - MERGED_SF=$(grep -c '^SF:' coverage/playwright/coverage.lcov || echo 0) - MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) - MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) - echo "### Merged coverage" >> "$GITHUB_STEP_SUMMARY" - echo "- **$MERGED_SF** source files" >> "$GITHUB_STEP_SUMMARY" - echo "- **$MERGED_LH / $MERGED_LF** lines hit" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "| Shard | Files | Lines Hit |" >> "$GITHUB_STEP_SUMMARY" - echo "|-------|-------|-----------|" >> "$GITHUB_STEP_SUMMARY" - while IFS= read -r -d '' f; do - SHARD=$(basename "$(dirname "$f")") - SHARD_SF=$(grep -c '^SF:' "$f" || echo 0) - SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$f") - echo "| $SHARD | $SHARD_SF | $SHARD_LH |" >> "$GITHUB_STEP_SUMMARY" - if [ "$MERGED_LH" -lt "$SHARD_LH" ]; then - echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" - exit 1 - fi - done < <(find temp/coverage-shards -name 'coverage.lcov' -type f -print0 | sort -z) - - - name: Assert coverage was mapped back to source - if: steps.coverage.outputs.has-coverage == 'true' - run: | - MAPPED_SF=$(grep -cE '^SF:(src|packages)/' coverage/playwright/coverage.lcov || true) - echo "Source-mapped files: $MAPPED_SF" >> "$GITHUB_STEP_SUMMARY" - if [ "${MAPPED_SF:-0}" -lt 100 ]; then - echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $(grep -m 5 '^SF:' coverage/playwright/coverage.lcov | tr '\n' ' '). Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)." - exit 1 - fi - - - name: Strip non-source entries from coverage - if: steps.coverage.outputs.has-coverage == 'true' + - name: Package shard coverage + id: coverage run: | - lcov --remove coverage/playwright/coverage.lcov \ - '*localhost-8188*' \ - 'assets/images/*' \ - -o coverage/playwright/coverage.lcov \ - --ignore-errors unused - wc -l coverage/playwright/coverage.lcov + scripts/cicd/package-e2e-coverage.sh \ + temp/coverage-shards \ + coverage/playwright \ + coverage/html - name: Upload merged coverage data if: steps.coverage.outputs.has-coverage == 'true' @@ -115,17 +59,6 @@ jobs: retention-days: 30 if-no-files-found: warn - - name: Generate HTML coverage report - if: steps.coverage.outputs.has-coverage == 'true' - run: | - genhtml coverage/playwright/coverage.lcov \ - -o coverage/html \ - --title "ComfyUI E2E Coverage" \ - --no-function-coverage \ - --precision 1 \ - --ignore-errors source,unmapped \ - --synthesize-missing - - name: Upload HTML report artifact if: steps.coverage.outputs.has-coverage == 'true' uses: actions/upload-artifact@v6 diff --git a/scripts/cicd/e2e-coverage-workflow.test.ts b/scripts/cicd/e2e-coverage-workflow.test.ts index f34e7a8b310..7881976617d 100644 --- a/scripts/cicd/e2e-coverage-workflow.test.ts +++ b/scripts/cicd/e2e-coverage-workflow.test.ts @@ -84,26 +84,19 @@ describe('E2E coverage workflows', () => { const checkout = packageSteps.find((step) => step.uses?.startsWith('actions/checkout@') ) - const merge = packageSteps.find( - (step) => step.name === 'Merge shard coverage into single LCOV' - ) - const validation = packageSteps.find( - (step) => step.name === 'Validate merged coverage' + const packageCoverage = packageSteps.find( + (step) => step.name === 'Package shard coverage' ) const packageJob = e2eWorkflow.jobs['upload-e2e-coverage'] expect(packageWorkflow.on).toHaveProperty('workflow_call') expect(checkout?.uses).toBe('actions/checkout@v7') expect(checkout?.with?.ref).toBe('${{ inputs.source_sha }}') - expect(merge?.run).toContain( - "find temp/coverage-shards -name 'coverage.lcov' -type f -print0" + expect(checkout?.with?.['sparse-checkout']).toContain( + 'scripts/cicd/package-e2e-coverage.sh' ) - expect(merge?.run).toContain("while IFS= read -r -d '' f") - expect(merge?.run).toContain('lcov "${ADD_ARGS[@]}"') - expect(validation?.run).toContain("while IFS= read -r -d '' f") - expect(validation?.run).toMatch(/possible data loss"\n\s+exit 1/) - expect(packageSteps.some((step) => step.run?.includes('genhtml '))).toBe( - true + expect(packageCoverage?.run).toContain( + 'scripts/cicd/package-e2e-coverage.sh' ) expect(packageJob.uses).toBe( './.github/workflows/ci-tests-e2e-coverage-package.yaml' diff --git a/scripts/cicd/package-e2e-coverage.sh b/scripts/cicd/package-e2e-coverage.sh new file mode 100755 index 00000000000..14bdebed473 --- /dev/null +++ b/scripts/cicd/package-e2e-coverage.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +SHARDS_DIR="${1:?Usage: package-e2e-coverage.sh }" +COVERAGE_DIR="${2:?Usage: package-e2e-coverage.sh }" +HTML_DIR="${3:?Usage: package-e2e-coverage.sh }" + +append_summary() { + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + printf '%s\n' "$1" >> "$GITHUB_STEP_SUMMARY" + fi +} + +COVERAGE_FILES=() +if [[ -d "$SHARDS_DIR" ]]; then + while IFS= read -r -d '' file; do + COVERAGE_FILES+=("$file") + done < <(find "$SHARDS_DIR" -name 'coverage.lcov' -type f -print0 | sort -z) +fi + +if [[ ${#COVERAGE_FILES[@]} -eq 0 ]]; then + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo 'has-coverage=false' >> "$GITHUB_OUTPUT" + fi + append_summary 'No E2E coverage shard artifacts found; treating this run as skipped.' + exit 0 +fi + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo 'has-coverage=true' >> "$GITHUB_OUTPUT" +fi + +mkdir -p "$COVERAGE_DIR" +ADD_ARGS=() +for file in "${COVERAGE_FILES[@]}"; do + ADD_ARGS+=(-a "$file") +done +lcov "${ADD_ARGS[@]}" -o "$COVERAGE_DIR/coverage.lcov" + +MERGED_SF=$(grep -c '^SF:' "$COVERAGE_DIR/coverage.lcov" || true) +MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$COVERAGE_DIR/coverage.lcov") +MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' "$COVERAGE_DIR/coverage.lcov") +append_summary '### Merged coverage' +append_summary "- **$MERGED_SF** source files" +append_summary "- **$MERGED_LH / $MERGED_LF** lines hit" +append_summary '' +append_summary '| Shard | Files | Lines Hit |' +append_summary '|-------|-------|-----------|' + +for file in "${COVERAGE_FILES[@]}"; do + SHARD=$(basename "$(dirname "$file")") + SHARD_SF=$(grep -c '^SF:' "$file" || true) + SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$file") + append_summary "| $SHARD | $SHARD_SF | $SHARD_LH |" + if [[ "$MERGED_LH" -lt "$SHARD_LH" ]]; then + echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" + exit 1 + fi +done + +MAPPED_SF=$(grep -cE '^SF:(src|packages)/' "$COVERAGE_DIR/coverage.lcov" || true) +append_summary "Source-mapped files: $MAPPED_SF" +if [[ "${MAPPED_SF:-0}" -lt 100 ]]; then + OBSERVED_PATHS=$(grep -m 5 '^SF:' "$COVERAGE_DIR/coverage.lcov" | tr '\n' ' ' || true) + echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $OBSERVED_PATHS. Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)." + exit 1 +fi + +lcov --remove "$COVERAGE_DIR/coverage.lcov" \ + '*localhost-8188*' \ + 'assets/images/*' \ + -o "$COVERAGE_DIR/coverage.lcov" \ + --ignore-errors unused + +genhtml "$COVERAGE_DIR/coverage.lcov" \ + -o "$HTML_DIR" \ + --title 'ComfyUI E2E Coverage' \ + --no-function-coverage \ + --precision 1 \ + --ignore-errors source,unmapped \ + --synthesize-missing diff --git a/scripts/cicd/package-e2e-coverage.test.ts b/scripts/cicd/package-e2e-coverage.test.ts new file mode 100644 index 00000000000..b8d1b4ad736 --- /dev/null +++ b/scripts/cicd/package-e2e-coverage.test.ts @@ -0,0 +1,173 @@ +import { spawnSync } from 'node:child_process' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const SCRIPT = join(import.meta.dirname, 'package-e2e-coverage.sh') + +function coverage(sourcePrefix: string) { + return Array.from( + { length: 100 }, + (_, index) => + `SF:${sourcePrefix}/file ${index}.ts\nDA:1,1\nLF:1\nLH:1\nend_of_record\n` + ).join('') +} + +function coverageFixture() { + const root = mkdtempSync(join(tmpdir(), 'e2e-coverage-')) + const shards = join(root, 'shards') + const output = join(root, 'coverage') + const html = join(root, 'html') + const bin = join(root, 'bin') + const githubOutput = join(root, 'github-output') + const summary = join(root, 'summary') + mkdirSync(shards) + mkdirSync(bin) + + writeFileSync( + join(bin, 'lcov'), + `#!/usr/bin/env bash +set -euo pipefail +if [[ "$1" == "--remove" ]]; then + exit 0 +fi +inputs=() +output='' +while [[ $# -gt 0 ]]; do + case "$1" in + -a) + inputs+=("$2") + shift 2 + ;; + -o) + output="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +if [[ "\${STUB_LCOV_DROP:-false}" == "true" ]]; then + printf 'SF:src/dropped.ts\\nDA:1,0\\nLF:1\\nLH:0\\nend_of_record\\n' > "$output" +else + : > "$output" + for input in "\${inputs[@]}"; do + cat "$input" >> "$output" + done +fi +`, + { mode: 0o755 } + ) + writeFileSync( + join(bin, 'genhtml'), + `#!/usr/bin/env bash +set -euo pipefail +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-o" ]]; then + mkdir -p "$2" + touch "$2/index.html" + exit 0 + fi + shift +done +exit 1 +`, + { mode: 0o755 } + ) + + return { + root, + shards, + output, + html, + githubOutput, + summary, + writeShard(name: string, contents: string) { + const shard = join(shards, name) + mkdirSync(shard, { recursive: true }) + writeFileSync(join(shard, 'coverage.lcov'), contents) + }, + run(extraEnv: NodeJS.ProcessEnv = {}) { + const result = spawnSync('bash', [SCRIPT, shards, output, html], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + GITHUB_OUTPUT: githubOutput, + GITHUB_STEP_SUMMARY: summary, + ...extraEnv + } + }) + return { + status: result.status, + output: `${result.stdout}${result.stderr}` + } + }, + [Symbol.dispose]() { + rmSync(root, { recursive: true, force: true }) + } + } +} + +describe('package-e2e-coverage.sh', () => { + it('packages available coverage from a partial run with a whitespace path', () => { + using fixture = coverageFixture() + fixture.writeShard('failed shard 1', coverage('src')) + + const result = fixture.run() + + expect(result.status).toBe(0) + expect(readFileSync(fixture.githubOutput, 'utf8')).toBe( + 'has-coverage=true\n' + ) + expect( + readFileSync(join(fixture.output, 'coverage.lcov'), 'utf8') + ).toContain('SF:src/file 0.ts') + expect(readFileSync(fixture.summary, 'utf8')).toContain('failed shard 1') + expect(existsSync(join(fixture.html, 'index.html'))).toBe(true) + }) + + it('skips successfully when no shard artifacts exist', () => { + using fixture = coverageFixture() + + const result = fixture.run() + + expect(result.status).toBe(0) + expect(readFileSync(fixture.githubOutput, 'utf8')).toBe( + 'has-coverage=false\n' + ) + expect(existsSync(join(fixture.output, 'coverage.lcov'))).toBe(false) + }) + + it('fails when merging loses shard coverage', () => { + using fixture = coverageFixture() + fixture.writeShard('partial-shard', coverage('src')) + + const result = fixture.run({ STUB_LCOV_DROP: 'true' }) + + expect(result.status).toBe(1) + expect(result.output).toContain('possible data loss') + expect(existsSync(join(fixture.html, 'index.html'))).toBe(false) + }) + + it('fails when coverage is not mapped to repository sources', () => { + using fixture = coverageFixture() + fixture.writeShard('served-bundle', coverage('assets')) + + const result = fixture.run() + + expect(result.status).toBe(1) + expect(result.output).toContain('Only 0 files under src/ or packages/') + expect(existsSync(join(fixture.html, 'index.html'))).toBe(false) + }) +}) From 007a524b69f3975bec7164e79c8529f9a2417a28 Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 15:21:09 -0700 Subject: [PATCH 6/7] fix: validate merged E2E coverage per line --- scripts/cicd/package-e2e-coverage.sh | 29 +++++++++++++++++++++-- scripts/cicd/package-e2e-coverage.test.ts | 15 +++++++----- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/scripts/cicd/package-e2e-coverage.sh b/scripts/cicd/package-e2e-coverage.sh index 14bdebed473..9dc27f15220 100755 --- a/scripts/cicd/package-e2e-coverage.sh +++ b/scripts/cicd/package-e2e-coverage.sh @@ -52,8 +52,33 @@ for file in "${COVERAGE_FILES[@]}"; do SHARD_SF=$(grep -c '^SF:' "$file" || true) SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$file") append_summary "| $SHARD | $SHARD_SF | $SHARD_LH |" - if [[ "$MERGED_LH" -lt "$SHARD_LH" ]]; then - echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" + if ! awk ' + FNR == NR { + if ($0 ~ /^SF:/) { + source = substr($0, 4) + } else if ($0 ~ /^DA:/) { + split(substr($0, 4), line_data, ",") + if (line_data[2] > 0) { + merged_hits[source SUBSEP line_data[1]] = 1 + } + } + next + } + $0 ~ /^SF:/ { + source = substr($0, 4) + } + $0 ~ /^DA:/ { + split(substr($0, 4), line_data, ",") + if (line_data[2] > 0 && !(source SUBSEP line_data[1] in merged_hits)) { + printf "Missing merged hit: %s:%s\n", source, line_data[1] > "/dev/stderr" + missing = 1 + } + } + END { + exit missing + } + ' "$COVERAGE_DIR/coverage.lcov" "$file"; then + echo "::error::Merged coverage omitted hit lines from $SHARD" exit 1 fi done diff --git a/scripts/cicd/package-e2e-coverage.test.ts b/scripts/cicd/package-e2e-coverage.test.ts index b8d1b4ad736..da8cbb7601b 100644 --- a/scripts/cicd/package-e2e-coverage.test.ts +++ b/scripts/cicd/package-e2e-coverage.test.ts @@ -57,8 +57,8 @@ while [[ $# -gt 0 ]]; do ;; esac done -if [[ "\${STUB_LCOV_DROP:-false}" == "true" ]]; then - printf 'SF:src/dropped.ts\\nDA:1,0\\nLF:1\\nLH:0\\nend_of_record\\n' > "$output" +if [[ "\${STUB_LCOV_KEEP_FIRST:-false}" == "true" ]]; then + cat "\${inputs[0]}" > "$output" else : > "$output" for input in "\${inputs[@]}"; do @@ -149,14 +149,17 @@ describe('package-e2e-coverage.sh', () => { expect(existsSync(join(fixture.output, 'coverage.lcov'))).toBe(false) }) - it('fails when merging loses shard coverage', () => { + it('fails when merging omits a shard with an equal hit count', () => { using fixture = coverageFixture() - fixture.writeShard('partial-shard', coverage('src')) + fixture.writeShard('shard-a', coverage('src/shard-a')) + fixture.writeShard('shard-b', coverage('packages/shard-b')) - const result = fixture.run({ STUB_LCOV_DROP: 'true' }) + const result = fixture.run({ STUB_LCOV_KEEP_FIRST: 'true' }) expect(result.status).toBe(1) - expect(result.output).toContain('possible data loss') + expect(result.output).toContain( + 'Merged coverage omitted hit lines from shard-b' + ) expect(existsSync(join(fixture.html, 'index.html'))).toBe(false) }) From 4131a21700e2ba9ee1295b0ecc48148abc75241f Mon Sep 17 00:00:00 2001 From: benjcooley Date: Mon, 31 Aug 2026 16:07:27 -0700 Subject: [PATCH 7/7] fix: simplify E2E coverage handoff --- .github/actions/find-workflow-run/action.yaml | 7 +- .../ci-tests-e2e-coverage-package.yaml | 11 +- .github/workflows/ci-tests-e2e.yaml | 1 - scripts/cicd/e2e-coverage-workflow.test.ts | 180 ------------------ scripts/cicd/package-e2e-coverage.sh | 29 --- scripts/cicd/package-e2e-coverage.test.ts | 31 +-- 6 files changed, 14 insertions(+), 245 deletions(-) delete mode 100644 scripts/cicd/e2e-coverage-workflow.test.ts diff --git a/.github/actions/find-workflow-run/action.yaml b/.github/actions/find-workflow-run/action.yaml index e8dfd3a3446..da582190276 100644 --- a/.github/actions/find-workflow-run/action.yaml +++ b/.github/actions/find-workflow-run/action.yaml @@ -76,7 +76,12 @@ runs: !candidate.expired ); if (!artifact) { - core.setOutput('status', process.env.NOT_FOUND_STATUS); + core.setOutput( + 'status', + run.conclusion === 'success' + ? process.env.NOT_FOUND_STATUS + : 'failed' + ); return; } core.setOutput('status', 'ready'); diff --git a/.github/workflows/ci-tests-e2e-coverage-package.yaml b/.github/workflows/ci-tests-e2e-coverage-package.yaml index 6218606a14f..1c0cb0d4487 100644 --- a/.github/workflows/ci-tests-e2e-coverage-package.yaml +++ b/.github/workflows/ci-tests-e2e-coverage-package.yaml @@ -3,15 +3,11 @@ name: 'CI: E2E Coverage Package' on: workflow_call: inputs: - source_run_id: - required: true - type: string source_sha: required: true type: string permissions: - actions: read contents: read jobs: @@ -31,13 +27,10 @@ jobs: scripts/cicd/package-e2e-coverage.sh - name: Download all shard coverage data - uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 + uses: actions/download-artifact@v8 with: - run_id: ${{ inputs.source_run_id }} - name: e2e-coverage-shard-.* - name_is_regexp: true + pattern: e2e-coverage-shard-* path: temp/coverage-shards - if_no_artifact_found: warn - name: Install lcov run: sudo apt-get install -y -qq lcov diff --git a/.github/workflows/ci-tests-e2e.yaml b/.github/workflows/ci-tests-e2e.yaml index c15509c1bc9..aeb98600eff 100644 --- a/.github/workflows/ci-tests-e2e.yaml +++ b/.github/workflows/ci-tests-e2e.yaml @@ -233,7 +233,6 @@ jobs: }} uses: ./.github/workflows/ci-tests-e2e-coverage-package.yaml with: - source_run_id: ${{ format('{0}', github.run_id) }} source_sha: ${{ github.event.pull_request.head.sha || github.sha }} # Records video only for spec files newly added in this PR, to keep cost bounded. diff --git a/scripts/cicd/e2e-coverage-workflow.test.ts b/scripts/cicd/e2e-coverage-workflow.test.ts deleted file mode 100644 index 7881976617d..00000000000 --- a/scripts/cicd/e2e-coverage-workflow.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { readFileSync } from 'node:fs' - -import { describe, expect, it } from 'vitest' -import { parse } from 'yaml' - -type WorkflowStep = { - name?: string - if?: string - uses?: string - run?: string - with?: Record -} - -type WorkflowJob = { - if?: string - uses?: string - secrets?: unknown - steps?: WorkflowStep[] - with?: Record -} - -type Workflow = { - on?: Record - jobs: Record -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -function hasOptionalString( - value: Record, - key: string -): boolean { - return value[key] === undefined || typeof value[key] === 'string' -} - -function isWorkflowStep(value: unknown): value is WorkflowStep { - return ( - isRecord(value) && - hasOptionalString(value, 'name') && - hasOptionalString(value, 'if') && - hasOptionalString(value, 'uses') && - hasOptionalString(value, 'run') && - (value.with === undefined || isRecord(value.with)) - ) -} - -function isWorkflowJob(value: unknown): value is WorkflowJob { - return ( - isRecord(value) && - hasOptionalString(value, 'if') && - hasOptionalString(value, 'uses') && - (value.steps === undefined || - (Array.isArray(value.steps) && value.steps.every(isWorkflowStep))) && - (value.with === undefined || isRecord(value.with)) - ) -} - -function isWorkflow(value: unknown): value is Workflow { - return ( - isRecord(value) && - isRecord(value.jobs) && - Object.values(value.jobs).every(isWorkflowJob) && - (value.on === undefined || isRecord(value.on)) - ) -} - -function readWorkflow(path: string): Workflow { - const value: unknown = parse(readFileSync(path, 'utf8')) - if (!isWorkflow(value)) { - throw new Error(`Invalid workflow structure in ${path}`) - } - return value -} - -describe('E2E coverage workflows', () => { - it('packages coverage in the unprivileged E2E run', () => { - const packageWorkflow = readWorkflow( - '.github/workflows/ci-tests-e2e-coverage-package.yaml' - ) - const e2eWorkflow = readWorkflow('.github/workflows/ci-tests-e2e.yaml') - const packageSteps = packageWorkflow.jobs.package.steps ?? [] - const checkout = packageSteps.find((step) => - step.uses?.startsWith('actions/checkout@') - ) - const packageCoverage = packageSteps.find( - (step) => step.name === 'Package shard coverage' - ) - const packageJob = e2eWorkflow.jobs['upload-e2e-coverage'] - - expect(packageWorkflow.on).toHaveProperty('workflow_call') - expect(checkout?.uses).toBe('actions/checkout@v7') - expect(checkout?.with?.ref).toBe('${{ inputs.source_sha }}') - expect(checkout?.with?.['sparse-checkout']).toContain( - 'scripts/cicd/package-e2e-coverage.sh' - ) - expect(packageCoverage?.run).toContain( - 'scripts/cicd/package-e2e-coverage.sh' - ) - expect(packageJob.uses).toBe( - './.github/workflows/ci-tests-e2e-coverage-package.yaml' - ) - expect(packageJob.secrets).toBeUndefined() - expect(packageJob.if).not.toContain('fork == false') - expect(packageJob.with).toMatchObject({ - source_run_id: "${{ format('{0}', github.run_id) }}", - source_sha: '${{ github.event.pull_request.head.sha || github.sha }}' - }) - }) - - it('publishes only fixed artifacts from the exact triggering run', () => { - const publishWorkflow = readWorkflow( - '.github/workflows/ci-tests-e2e-coverage.yaml' - ) - const publishSteps = publishWorkflow.jobs.publish.steps ?? [] - const checkouts = publishSteps.filter((step) => - step.uses?.startsWith('actions/checkout@') - ) - const coverageDownloads = publishSteps.filter( - (step) => - step.uses?.startsWith('dawidd6/action-download-artifact@') && - step.with?.name === 'e2e-coverage' - ) - const resolver = publishSteps.find( - (step) => step.uses === './.github/actions/resolve-pr-from-workflow-run' - ) - const codecov = publishSteps.find((step) => - step.uses?.startsWith('codecov/codecov-action@') - ) - - expect(publishWorkflow.on).toHaveProperty('workflow_run') - expect(checkouts).not.toHaveLength(0) - for (const checkout of checkouts) { - expect(checkout.with).toMatchObject({ - ref: '${{ github.event.repository.default_branch }}', - 'persist-credentials': false - }) - } - expect(coverageDownloads).toHaveLength(1) - expect(coverageDownloads[0]?.with).toMatchObject({ - run_id: '${{ github.event.workflow_run.id }}', - name: 'e2e-coverage', - path: 'coverage/playwright' - }) - expect(resolver?.if).toContain("workflow_run.event == 'pull_request'") - expect(codecov?.with).toMatchObject({ - files: 'coverage/playwright/coverage.lcov', - override_branch: '${{ github.event.workflow_run.head_branch }}', - override_commit: - '${{ steps.pr-meta.outputs.head-sha || github.event.workflow_run.head_sha }}', - override_pr: '${{ steps.pr-meta.outputs.number }}' - }) - expect( - publishSteps.some( - (step) => - step.run !== undefined && /(^|\n)\s*(lcov|genhtml)\s/.test(step.run) - ) - ).toBe(false) - }) - - it('makes packaged PR coverage discoverable from the E2E run', () => { - const reportWorkflow = readWorkflow('.github/workflows/pr-report.yaml') - const reportSteps = reportWorkflow.jobs.comment.steps ?? [] - const workflowRun = reportWorkflow.on?.workflow_run - const coverageLookup = reportSteps.find( - (step) => step.name === 'Find coverage workflow run' - ) - - expect(workflowRun).toMatchObject({ - workflows: expect.arrayContaining(['CI: Tests E2E']) - }) - expect(coverageLookup?.with).toMatchObject({ - 'workflow-id': 'ci-tests-e2e.yaml', - 'head-sha': '${{ steps.pr-meta.outputs.head-sha }}', - 'artifact-name': 'e2e-coverage', - 'not-found-status': 'skip' - }) - }) -}) diff --git a/scripts/cicd/package-e2e-coverage.sh b/scripts/cicd/package-e2e-coverage.sh index 9dc27f15220..1b09e83c144 100755 --- a/scripts/cicd/package-e2e-coverage.sh +++ b/scripts/cicd/package-e2e-coverage.sh @@ -52,35 +52,6 @@ for file in "${COVERAGE_FILES[@]}"; do SHARD_SF=$(grep -c '^SF:' "$file" || true) SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$file") append_summary "| $SHARD | $SHARD_SF | $SHARD_LH |" - if ! awk ' - FNR == NR { - if ($0 ~ /^SF:/) { - source = substr($0, 4) - } else if ($0 ~ /^DA:/) { - split(substr($0, 4), line_data, ",") - if (line_data[2] > 0) { - merged_hits[source SUBSEP line_data[1]] = 1 - } - } - next - } - $0 ~ /^SF:/ { - source = substr($0, 4) - } - $0 ~ /^DA:/ { - split(substr($0, 4), line_data, ",") - if (line_data[2] > 0 && !(source SUBSEP line_data[1] in merged_hits)) { - printf "Missing merged hit: %s:%s\n", source, line_data[1] > "/dev/stderr" - missing = 1 - } - } - END { - exit missing - } - ' "$COVERAGE_DIR/coverage.lcov" "$file"; then - echo "::error::Merged coverage omitted hit lines from $SHARD" - exit 1 - fi done MAPPED_SF=$(grep -cE '^SF:(src|packages)/' "$COVERAGE_DIR/coverage.lcov" || true) diff --git a/scripts/cicd/package-e2e-coverage.test.ts b/scripts/cicd/package-e2e-coverage.test.ts index da8cbb7601b..8da80b805ce 100644 --- a/scripts/cicd/package-e2e-coverage.test.ts +++ b/scripts/cicd/package-e2e-coverage.test.ts @@ -57,14 +57,10 @@ while [[ $# -gt 0 ]]; do ;; esac done -if [[ "\${STUB_LCOV_KEEP_FIRST:-false}" == "true" ]]; then - cat "\${inputs[0]}" > "$output" -else - : > "$output" - for input in "\${inputs[@]}"; do - cat "$input" >> "$output" - done -fi +: > "$output" +for input in "\${inputs[@]}"; do + cat "$input" >> "$output" +done `, { mode: 0o755 } ) @@ -97,15 +93,14 @@ exit 1 mkdirSync(shard, { recursive: true }) writeFileSync(join(shard, 'coverage.lcov'), contents) }, - run(extraEnv: NodeJS.ProcessEnv = {}) { + run() { const result = spawnSync('bash', [SCRIPT, shards, output, html], { encoding: 'utf8', env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ''}`, GITHUB_OUTPUT: githubOutput, - GITHUB_STEP_SUMMARY: summary, - ...extraEnv + GITHUB_STEP_SUMMARY: summary } }) return { @@ -149,20 +144,6 @@ describe('package-e2e-coverage.sh', () => { expect(existsSync(join(fixture.output, 'coverage.lcov'))).toBe(false) }) - it('fails when merging omits a shard with an equal hit count', () => { - using fixture = coverageFixture() - fixture.writeShard('shard-a', coverage('src/shard-a')) - fixture.writeShard('shard-b', coverage('packages/shard-b')) - - const result = fixture.run({ STUB_LCOV_KEEP_FIRST: 'true' }) - - expect(result.status).toBe(1) - expect(result.output).toContain( - 'Merged coverage omitted hit lines from shard-b' - ) - expect(existsSync(join(fixture.html, 'index.html'))).toBe(false) - }) - it('fails when coverage is not mapped to repository sources', () => { using fixture = coverageFixture() fixture.writeShard('served-bundle', coverage('assets'))