diff --git a/.github/actions/find-workflow-run/action.yaml b/.github/actions/find-workflow-run/action.yaml index 620394eb729..da582190276 100644 --- a/.github/actions/find-workflow-run/action.yaml +++ b/.github/actions/find-workflow-run/action.yaml @@ -12,6 +12,9 @@ inputs: description: Status to output when no run exists required: false default: pending + artifact-name: + description: When set, a completed run is ready only when this artifact exists + required: false token: description: GitHub token for API access required: true @@ -34,6 +37,7 @@ runs: WORKFLOW_ID: ${{ inputs.workflow-id }} HEAD_SHA: ${{ inputs.head-sha }} NOT_FOUND_STATUS: ${{ inputs.not-found-status }} + ARTIFACT_NAME: ${{ inputs.artifact-name }} with: github-token: ${{ inputs.token }} script: | @@ -56,6 +60,35 @@ runs: return; } + if (process.env.ARTIFACT_NAME) { + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + } + ); + const artifact = artifacts.find( + candidate => + candidate.name === process.env.ARTIFACT_NAME && + !candidate.expired + ); + if (!artifact) { + core.setOutput( + 'status', + run.conclusion === 'success' + ? process.env.NOT_FOUND_STATUS + : 'failed' + ); + return; + } + core.setOutput('status', 'ready'); + core.setOutput('run-id', String(run.id)); + return; + } + if (run.conclusion !== 'success') { core.setOutput('status', 'failed'); return; diff --git a/.github/workflows/ci-tests-e2e-coverage-package.yaml b/.github/workflows/ci-tests-e2e-coverage-package.yaml new file mode 100644 index 00000000000..1c0cb0d4487 --- /dev/null +++ b/.github/workflows/ci-tests-e2e-coverage-package.yaml @@ -0,0 +1,61 @@ +name: 'CI: E2E Coverage Package' + +on: + workflow_call: + inputs: + source_sha: + required: true + type: string + +permissions: + contents: read + +jobs: + package: + if: github.repository == 'Comfy-Org/ComfyUI_frontend' + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout sources for coverage packaging + uses: actions/checkout@v7 + with: + ref: ${{ inputs.source_sha }} + sparse-checkout: | + src + packages + scripts/cicd/package-e2e-coverage.sh + + - name: Download all shard coverage data + uses: actions/download-artifact@v8 + with: + pattern: e2e-coverage-shard-* + path: temp/coverage-shards + + - name: Install lcov + run: sudo apt-get install -y -qq lcov + + - name: Package shard coverage + id: coverage + run: | + scripts/cicd/package-e2e-coverage.sh \ + temp/coverage-shards \ + coverage/playwright \ + coverage/html + + - name: Upload merged coverage data + if: steps.coverage.outputs.has-coverage == 'true' + uses: actions/upload-artifact@v6 + with: + name: e2e-coverage + path: coverage/playwright/ + retention-days: 30 + if-no-files-found: warn + + - name: Upload HTML report artifact + if: steps.coverage.outputs.has-coverage == 'true' + uses: actions/upload-artifact@v6 + with: + name: e2e-coverage-html + path: coverage/html/ + retention-days: 30 diff --git a/.github/workflows/ci-tests-e2e-coverage.yaml b/.github/workflows/ci-tests-e2e-coverage.yaml index bf9e77cbb81..4e1ffd22f18 100644 --- a/.github/workflows/ci-tests-e2e-coverage.yaml +++ b/.github/workflows/ci-tests-e2e-coverage.yaml @@ -1,136 +1,63 @@ name: 'CI: E2E Coverage' on: - workflow_call: - inputs: - source_branch: - required: true - type: string - source_pr: - required: false - type: string - source_run_id: - required: true - type: string - source_sha: - required: true - type: string - secrets: - CODECOV_TOKEN: - required: true workflow_run: workflows: ['CI: Tests E2E'] types: - completed concurrency: - group: e2e-coverage-${{ inputs.source_sha || github.event.workflow_run.head_sha }} + group: e2e-coverage-${{ github.event.workflow_run.head_sha }} cancel-in-progress: true permissions: + actions: read contents: read + pull-requests: read jobs: - merge: - # Not gated on full success: fail-fast:false means one flaky shard flips the whole run to failure, which would skip merging every other shard's real coverage. + publish: if: > github.repository == 'Comfy-Org/ComfyUI_frontend' && - (github.event_name == 'workflow_call' || - github.event.workflow_run.event != 'pull_request' || - github.event.workflow_run.head_repository.full_name != github.repository || - github.event.workflow_run.actor.login == 'dependabot[bot]') && github.event.workflow_run.conclusion != 'cancelled' runs-on: ubuntu-latest timeout-minutes: 10 outputs: - has-coverage: ${{ steps.coverage-shards.outputs.has-coverage }} + has-coverage: ${{ steps.coverage.outputs.has-coverage }} steps: - - name: Checkout sources for genhtml + - name: Checkout trusted workflow actions uses: actions/checkout@v7 with: - ref: ${{ inputs.source_sha || github.event.workflow_run.head_sha }} - sparse-checkout: | - src - packages + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: .github/actions/resolve-pr-from-workflow-run + persist-credentials: false - - name: Download all shard coverage data + - name: Resolve PR from workflow_run context + id: pr-meta + if: github.event.workflow_run.event == 'pull_request' + uses: ./.github/actions/resolve-pr-from-workflow-run + + - name: Download packaged coverage data uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 with: - run_id: ${{ inputs.source_run_id || github.event.workflow_run.id }} - name: e2e-coverage-shard-.* - name_is_regexp: true - path: temp/coverage-shards + run_id: ${{ github.event.workflow_run.id }} + name: e2e-coverage + path: coverage/playwright if_no_artifact_found: warn - - name: Detect shard coverage data - id: coverage-shards + - name: Detect packaged coverage data + id: coverage run: | - if [ -d temp/coverage-shards ] && find temp/coverage-shards -name 'coverage.lcov' -type f | grep -q .; then + if [ -f coverage/playwright/coverage.lcov ]; then echo "has-coverage=true" >> "$GITHUB_OUTPUT" else echo "has-coverage=false" >> "$GITHUB_OUTPUT" - echo "No E2E coverage shard artifacts found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Install lcov - if: steps.coverage-shards.outputs.has-coverage == 'true' - run: sudo apt-get install -y -qq lcov - - - name: Merge shard coverage into single LCOV - if: steps.coverage-shards.outputs.has-coverage == 'true' - run: | - mkdir -p coverage/playwright - LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f) - ADD_ARGS="" - for f in $LCOV_FILES; do ADD_ARGS="$ADD_ARGS -a $f"; done - lcov $ADD_ARGS -o coverage/playwright/coverage.lcov - wc -l coverage/playwright/coverage.lcov - - - name: Validate merged coverage - if: steps.coverage-shards.outputs.has-coverage == 'true' - run: | - MERGED_SF=$(grep -c '^SF:' coverage/playwright/coverage.lcov || echo 0) - MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) - MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' coverage/playwright/coverage.lcov) - echo "### Merged coverage" >> "$GITHUB_STEP_SUMMARY" - echo "- **$MERGED_SF** source files" >> "$GITHUB_STEP_SUMMARY" - echo "- **$MERGED_LH / $MERGED_LF** lines hit" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "| Shard | Files | Lines Hit |" >> "$GITHUB_STEP_SUMMARY" - echo "|-------|-------|-----------|" >> "$GITHUB_STEP_SUMMARY" - for f in $(find temp/coverage-shards -name 'coverage.lcov' -type f | sort); do - SHARD=$(basename "$(dirname "$f")") - SHARD_SF=$(grep -c '^SF:' "$f" || echo 0) - SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$f") - echo "| $SHARD | $SHARD_SF | $SHARD_LH |" >> "$GITHUB_STEP_SUMMARY" - if [ "$MERGED_LH" -lt "$SHARD_LH" ]; then - echo "::error::Merged LH ($MERGED_LH) < shard LH ($SHARD_LH) in $SHARD — possible data loss" - fi - done - - - name: Assert coverage was mapped back to source - if: steps.coverage-shards.outputs.has-coverage == 'true' - run: | - MAPPED_SF=$(grep -cE '^SF:(src|packages)/' coverage/playwright/coverage.lcov || true) - echo "Source-mapped files: $MAPPED_SF" >> "$GITHUB_STEP_SUMMARY" - if [ "${MAPPED_SF:-0}" -lt 100 ]; then - echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $(grep -m 5 '^SF:' coverage/playwright/coverage.lcov | tr '\n' ' '). Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)." - exit 1 + echo "No packaged E2E coverage artifact found; treating this run as skipped." >> "$GITHUB_STEP_SUMMARY" fi - - name: Strip non-source entries from coverage - if: steps.coverage-shards.outputs.has-coverage == 'true' - run: | - lcov --remove coverage/playwright/coverage.lcov \ - '*localhost-8188*' \ - 'assets/images/*' \ - -o coverage/playwright/coverage.lcov \ - --ignore-errors unused - wc -l coverage/playwright/coverage.lcov - - name: Upload merged coverage data - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: steps.coverage.outputs.has-coverage == 'true' uses: actions/upload-artifact@v6 with: name: e2e-coverage @@ -139,41 +66,43 @@ jobs: if-no-files-found: warn - name: Upload E2E coverage to Codecov - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: > + steps.coverage.outputs.has-coverage == 'true' && + (github.event.workflow_run.event != 'pull_request' || + steps.pr-meta.outputs.skip != 'true') uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: files: coverage/playwright/coverage.lcov flags: e2e - override_branch: ${{ inputs.source_branch || github.event.workflow_run.head_branch }} - override_commit: ${{ inputs.source_sha || github.event.workflow_run.head_sha }} - override_pr: ${{ inputs.source_pr || github.event.workflow_run.pull_requests[0].number }} + override_branch: ${{ github.event.workflow_run.head_branch }} + override_commit: ${{ steps.pr-meta.outputs.head-sha || github.event.workflow_run.head_sha }} + override_pr: ${{ steps.pr-meta.outputs.number }} token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false - - name: Generate HTML coverage report - if: steps.coverage-shards.outputs.has-coverage == 'true' - run: | - genhtml coverage/playwright/coverage.lcov \ - -o coverage/html \ - --title "ComfyUI E2E Coverage" \ - --no-function-coverage \ - --precision 1 \ - --ignore-errors source,unmapped \ - --synthesize-missing + - name: Download packaged HTML report + if: steps.coverage.outputs.has-coverage == 'true' + uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 + with: + run_id: ${{ github.event.workflow_run.id }} + name: e2e-coverage-html + path: coverage/html + if_no_artifact_found: warn - name: Upload HTML report artifact - if: steps.coverage-shards.outputs.has-coverage == 'true' + if: steps.coverage.outputs.has-coverage == 'true' uses: actions/upload-artifact@v6 with: name: e2e-coverage-html path: coverage/html/ retention-days: 30 + if-no-files-found: warn deploy: - needs: merge + needs: publish if: > github.event.workflow_run.head_branch == 'main' && - needs.merge.outputs.has-coverage == 'true' && + needs.publish.outputs.has-coverage == 'true' && github.event.workflow_run.event == 'push' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/ci-tests-e2e.yaml b/.github/workflows/ci-tests-e2e.yaml index 14839851cda..aeb98600eff 100644 --- a/.github/workflows/ci-tests-e2e.yaml +++ b/.github/workflows/ci-tests-e2e.yaml @@ -229,18 +229,11 @@ jobs: ${{ always() && needs.changes.outputs.should-run == 'true' && - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.fork == false && - github.event.pull_request.user.login != 'dependabot[bot]' && needs.playwright-tests-chromium-sharded.result != 'cancelled' }} - uses: ./.github/workflows/ci-tests-e2e-coverage.yaml + uses: ./.github/workflows/ci-tests-e2e-coverage-package.yaml with: - source_branch: ${{ github.head_ref || github.ref_name }} - source_pr: ${{ format('{0}', github.event.pull_request.number) }} - source_run_id: ${{ format('{0}', github.run_id) }} source_sha: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: inherit # Records video only for spec files newly added in this PR, to keep cost bounded. playwright-video-new-tests: diff --git a/.github/workflows/pr-report.yaml b/.github/workflows/pr-report.yaml index ffd3821aa99..90cd77ceb25 100644 --- a/.github/workflows/pr-report.yaml +++ b/.github/workflows/pr-report.yaml @@ -2,7 +2,7 @@ name: 'PR: Unified Report' on: workflow_run: - workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: E2E Coverage'] + workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: Tests E2E'] types: - completed branches-ignore: @@ -77,8 +77,9 @@ jobs: id: find-coverage uses: ./.github/actions/find-workflow-run with: - workflow-id: ci-tests-e2e-coverage.yaml + workflow-id: ci-tests-e2e.yaml head-sha: ${{ steps.pr-meta.outputs.head-sha }} + artifact-name: e2e-coverage not-found-status: skip token: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/cicd/package-e2e-coverage.sh b/scripts/cicd/package-e2e-coverage.sh new file mode 100755 index 00000000000..1b09e83c144 --- /dev/null +++ b/scripts/cicd/package-e2e-coverage.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +SHARDS_DIR="${1:?Usage: package-e2e-coverage.sh }" +COVERAGE_DIR="${2:?Usage: package-e2e-coverage.sh }" +HTML_DIR="${3:?Usage: package-e2e-coverage.sh }" + +append_summary() { + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + printf '%s\n' "$1" >> "$GITHUB_STEP_SUMMARY" + fi +} + +COVERAGE_FILES=() +if [[ -d "$SHARDS_DIR" ]]; then + while IFS= read -r -d '' file; do + COVERAGE_FILES+=("$file") + done < <(find "$SHARDS_DIR" -name 'coverage.lcov' -type f -print0 | sort -z) +fi + +if [[ ${#COVERAGE_FILES[@]} -eq 0 ]]; then + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo 'has-coverage=false' >> "$GITHUB_OUTPUT" + fi + append_summary 'No E2E coverage shard artifacts found; treating this run as skipped.' + exit 0 +fi + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo 'has-coverage=true' >> "$GITHUB_OUTPUT" +fi + +mkdir -p "$COVERAGE_DIR" +ADD_ARGS=() +for file in "${COVERAGE_FILES[@]}"; do + ADD_ARGS+=(-a "$file") +done +lcov "${ADD_ARGS[@]}" -o "$COVERAGE_DIR/coverage.lcov" + +MERGED_SF=$(grep -c '^SF:' "$COVERAGE_DIR/coverage.lcov" || true) +MERGED_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$COVERAGE_DIR/coverage.lcov") +MERGED_LF=$(awk -F: '/^LF:/{s+=$2}END{print s+0}' "$COVERAGE_DIR/coverage.lcov") +append_summary '### Merged coverage' +append_summary "- **$MERGED_SF** source files" +append_summary "- **$MERGED_LH / $MERGED_LF** lines hit" +append_summary '' +append_summary '| Shard | Files | Lines Hit |' +append_summary '|-------|-------|-----------|' + +for file in "${COVERAGE_FILES[@]}"; do + SHARD=$(basename "$(dirname "$file")") + SHARD_SF=$(grep -c '^SF:' "$file" || true) + SHARD_LH=$(awk -F: '/^LH:/{s+=$2}END{print s+0}' "$file") + append_summary "| $SHARD | $SHARD_SF | $SHARD_LH |" +done + +MAPPED_SF=$(grep -cE '^SF:(src|packages)/' "$COVERAGE_DIR/coverage.lcov" || true) +append_summary "Source-mapped files: $MAPPED_SF" +if [[ "${MAPPED_SF:-0}" -lt 100 ]]; then + OBSERVED_PATHS=$(grep -m 5 '^SF:' "$COVERAGE_DIR/coverage.lcov" | tr '\n' ' ' || true) + echo "::error::Only $MAPPED_SF files under src/ or packages/ in the merged tracefile. Observed paths: $OBSERVED_PATHS. Served bundle paths mean the E2E build dropped its '//# sourceMappingURL=' comment — check it ran with COLLECT_COVERAGE=true (vite.config.mts build.sourcemap)." + exit 1 +fi + +lcov --remove "$COVERAGE_DIR/coverage.lcov" \ + '*localhost-8188*' \ + 'assets/images/*' \ + -o "$COVERAGE_DIR/coverage.lcov" \ + --ignore-errors unused + +genhtml "$COVERAGE_DIR/coverage.lcov" \ + -o "$HTML_DIR" \ + --title 'ComfyUI E2E Coverage' \ + --no-function-coverage \ + --precision 1 \ + --ignore-errors source,unmapped \ + --synthesize-missing diff --git a/scripts/cicd/package-e2e-coverage.test.ts b/scripts/cicd/package-e2e-coverage.test.ts new file mode 100644 index 00000000000..8da80b805ce --- /dev/null +++ b/scripts/cicd/package-e2e-coverage.test.ts @@ -0,0 +1,157 @@ +import { spawnSync } from 'node:child_process' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const SCRIPT = join(import.meta.dirname, 'package-e2e-coverage.sh') + +function coverage(sourcePrefix: string) { + return Array.from( + { length: 100 }, + (_, index) => + `SF:${sourcePrefix}/file ${index}.ts\nDA:1,1\nLF:1\nLH:1\nend_of_record\n` + ).join('') +} + +function coverageFixture() { + const root = mkdtempSync(join(tmpdir(), 'e2e-coverage-')) + const shards = join(root, 'shards') + const output = join(root, 'coverage') + const html = join(root, 'html') + const bin = join(root, 'bin') + const githubOutput = join(root, 'github-output') + const summary = join(root, 'summary') + mkdirSync(shards) + mkdirSync(bin) + + writeFileSync( + join(bin, 'lcov'), + `#!/usr/bin/env bash +set -euo pipefail +if [[ "$1" == "--remove" ]]; then + exit 0 +fi +inputs=() +output='' +while [[ $# -gt 0 ]]; do + case "$1" in + -a) + inputs+=("$2") + shift 2 + ;; + -o) + output="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +: > "$output" +for input in "\${inputs[@]}"; do + cat "$input" >> "$output" +done +`, + { mode: 0o755 } + ) + writeFileSync( + join(bin, 'genhtml'), + `#!/usr/bin/env bash +set -euo pipefail +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-o" ]]; then + mkdir -p "$2" + touch "$2/index.html" + exit 0 + fi + shift +done +exit 1 +`, + { mode: 0o755 } + ) + + return { + root, + shards, + output, + html, + githubOutput, + summary, + writeShard(name: string, contents: string) { + const shard = join(shards, name) + mkdirSync(shard, { recursive: true }) + writeFileSync(join(shard, 'coverage.lcov'), contents) + }, + run() { + const result = spawnSync('bash', [SCRIPT, shards, output, html], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + GITHUB_OUTPUT: githubOutput, + GITHUB_STEP_SUMMARY: summary + } + }) + return { + status: result.status, + output: `${result.stdout}${result.stderr}` + } + }, + [Symbol.dispose]() { + rmSync(root, { recursive: true, force: true }) + } + } +} + +describe('package-e2e-coverage.sh', () => { + it('packages available coverage from a partial run with a whitespace path', () => { + using fixture = coverageFixture() + fixture.writeShard('failed shard 1', coverage('src')) + + const result = fixture.run() + + expect(result.status).toBe(0) + expect(readFileSync(fixture.githubOutput, 'utf8')).toBe( + 'has-coverage=true\n' + ) + expect( + readFileSync(join(fixture.output, 'coverage.lcov'), 'utf8') + ).toContain('SF:src/file 0.ts') + expect(readFileSync(fixture.summary, 'utf8')).toContain('failed shard 1') + expect(existsSync(join(fixture.html, 'index.html'))).toBe(true) + }) + + it('skips successfully when no shard artifacts exist', () => { + using fixture = coverageFixture() + + const result = fixture.run() + + expect(result.status).toBe(0) + expect(readFileSync(fixture.githubOutput, 'utf8')).toBe( + 'has-coverage=false\n' + ) + expect(existsSync(join(fixture.output, 'coverage.lcov'))).toBe(false) + }) + + it('fails when coverage is not mapped to repository sources', () => { + using fixture = coverageFixture() + fixture.writeShard('served-bundle', coverage('assets')) + + const result = fixture.run() + + expect(result.status).toBe(1) + expect(result.output).toContain('Only 0 files under src/ or packages/') + expect(existsSync(join(fixture.html, 'index.html'))).toBe(false) + }) +})