diff --git a/.github/actions/collect-coverage/action.yml b/.github/actions/collect-coverage/action.yml new file mode 100644 index 00000000000..2646fdf6279 --- /dev/null +++ b/.github/actions/collect-coverage/action.yml @@ -0,0 +1,36 @@ +name: Collect container coverage +description: Copy Go coverage data from running or stopped containers. + +inputs: + destination: + description: Host directory that receives per-container coverage directories. + required: true + image: + description: Optional container image filter, for example chainlink-cover:local. + required: false + default: "" + +runs: + using: composite + steps: + - name: Copy coverage data + shell: bash + env: + DESTINATION: ${{ inputs.destination }} + IMAGE_FILTER: ${{ inputs.image }} + run: | + set -euo pipefail + mkdir -p "${DESTINATION}" + for container in $(docker ps -aq); do + image="$(docker inspect -f '{{.Config.Image}}' "${container}" 2>/dev/null || true)" + if [[ -n "${IMAGE_FILTER}" && "${image}" != "${IMAGE_FILTER}"* ]]; then + continue + fi + if [[ "${image}" == *"-coverage"* ]]; then + docker kill --signal=SIGUSR1 "${container}" >/dev/null 2>&1 || true + sleep 0.2 + fi + if docker cp "${container}:/var/tmp/go-coverage/." "${DESTINATION}/${container}" 2>/dev/null; then + echo "Copied coverage from ${container} (${image})" + fi + done diff --git a/.github/actions/start-coverage-flusher/action.yml b/.github/actions/start-coverage-flusher/action.yml new file mode 100644 index 00000000000..e1c21eae1dc --- /dev/null +++ b/.github/actions/start-coverage-flusher/action.yml @@ -0,0 +1,40 @@ +name: Start coverage flusher +description: Periodically sends SIGUSR1 to coverage-instrumented containers so covcounters are written before teardown. + +inputs: + interval: + description: Seconds between flushes. + required: false + default: "15" + pidfile: + description: File used to record the background process ID. + required: false + default: /tmp/chainlink-coverage-flusher.pid + +runs: + using: composite + steps: + - name: Start flusher + shell: bash + env: + INTERVAL: ${{ inputs.interval }} + PIDFILE: ${{ inputs.pidfile }} + run: | + set -euo pipefail + cat > /tmp/chainlink-coverage-flusher.sh <<'EOF' + #!/usr/bin/env bash + set -u + interval="${INTERVAL:-15}" + while true; do + for container in $(docker ps -q 2>/dev/null || true); do + image="$(docker inspect -f '{{.Config.Image}}' "${container}" 2>/dev/null || true)" + if [[ "${image}" == *"-coverage"* ]]; then + docker kill --signal=SIGUSR1 "${container}" >/dev/null 2>&1 || true + fi + done + sleep "${interval}" + done + EOF + chmod +x /tmp/chainlink-coverage-flusher.sh + nohup env INTERVAL="${INTERVAL}" /tmp/chainlink-coverage-flusher.sh >/tmp/chainlink-coverage-flusher.log 2>&1 & + echo $! > "${PIDFILE}" diff --git a/.github/workflows/ccip-system-tests.yaml b/.github/workflows/ccip-system-tests.yaml index 191ba84227c..460486b7f56 100644 --- a/.github/workflows/ccip-system-tests.yaml +++ b/.github/workflows/ccip-system-tests.yaml @@ -296,6 +296,9 @@ jobs: echo "Baseline image: ${image}:${version}" fi + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CCIP smoke tests id: run-tests shell: bash @@ -364,6 +367,30 @@ jobs: # unique name per matrix job to avoid 409 conflict when multiple jobs upload in the same workflow run artifact-name: ${{ matrix.tests.test_id }}_test_logs + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index e0190836946..988262e07cb 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -16,6 +16,12 @@ on: schedule: - cron: "0 0,1,2,3,4,5,6,20,21,22,23 * * *" # Run regularly at night to get better flaky test data workflow_dispatch: + inputs: + race_coverage: + description: "Run the nightly race-coverage pass on this manual run (test hook)." + required: false + default: false + type: boolean jobs: filter: @@ -323,6 +329,7 @@ jobs: setup-sui: "false" install-loopps: "false" build-cache-write: always + coverage: "true" - cmd: go_core_tests_integration os: @@ -341,6 +348,7 @@ jobs: setup-aptos: "false" setup-sui: "false" build-cache-write: always + coverage: "true" - cmd: go_core_fuzz os: @@ -379,12 +387,17 @@ jobs: install-loopps: "false" build-cache-version: go_core_race_tests build-cache-write: always + # Nightly race runs also produce a coverage artifact so a race- + # instrumented run can be inspected later. It is deliberately not + # merged into the project coverage calculation; see collect.sh. + coverage_on_schedule: "true" name: Core Tests (${{ matrix.type.cmd }}) # Be careful modifying the job name, as it is used to fetch the job URL # We don't directly merge dependabot PRs, so let's not waste the resources if: ${{ github.actor != 'dependabot[bot]' }} needs: [filter] - timeout-minutes: 60 + # Scheduled race runs with coverage are much slower than PR runs. + timeout-minutes: 120 # Use ubuntu-latest for jobs that will be skipped runs-on: ${{ matrix.type.should-run == 'true' && matrix.type.os || 'ubuntu-latest' }} @@ -487,12 +500,14 @@ jobs: - name: Run tests if: ${{ matrix.type.should-run == 'true' }} - timeout-minutes: 50 + timeout-minutes: 90 continue-on-error: ${{ matrix.type.trunk-auto-quarantine == 'true' }} id: run-tests shell: bash env: CL_DATABASE_URL: ${{ env.DB_URL }} + COVERAGE: ${{ matrix.type.coverage || (matrix.type.coverage_on_schedule == 'true' && (github.event_name == 'schedule' || inputs.race_coverage)) || 'false' }} + GOCOVERDIR: ${{ github.workspace }}/.covdata/${{ matrix.type.cmd }} OUTPUT_FILE: ./output.txt PRODUCE_JUNIT_XML: ${{ matrix.type.trunk-auto-quarantine }} TRUNK_AUTO_QUARANTINE: ${{ matrix.type.trunk-auto-quarantine }} @@ -564,6 +579,27 @@ jobs: ./junit.xml retention-days: 7 + - name: Store coverage artifacts + if: + ${{ always() && matrix.type.should-run == 'true' && (matrix.type.coverage + == 'true' || (matrix.type.coverage_on_schedule == 'true' && + (github.event_name == 'schedule' || inputs.race_coverage))) }} + uses: actions/upload-artifact@v7 + with: + name: coverage-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/${{ matrix.type.cmd }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() && matrix.type.should-run == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: test-results-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }} + path: junit.xml + retention-days: 7 + if-no-files-found: ignore + - name: Notify Slack on Race Test Failure if: | failure() && @@ -599,6 +635,8 @@ jobs: - name: Run Tests env: OUTPUT_FILE: ./output.txt + COVERAGE: "true" + GOCOVERDIR: ${{ github.workspace }}/.covdata/scripts run: ./tools/bin/go_core_scripts_tests ./... - name: Store test report artifacts @@ -611,6 +649,15 @@ jobs: ./coverage.txt retention-days: 7 + - name: Store coverage artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-core-scripts-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/scripts + retention-days: 7 + if-no-files-found: warn + clean: name: Clean Go Tidy & Generate if: ${{ github.actor != 'dependabot[bot]' }} diff --git a/.github/workflows/ci-deployments.yml b/.github/workflows/ci-deployments.yml index 3d33e0a9cab..848a76b9060 100644 --- a/.github/workflows/ci-deployments.yml +++ b/.github/workflows/ci-deployments.yml @@ -199,6 +199,8 @@ jobs: shell: bash env: CL_DATABASE_URL: ${{ env.DB_URL }} + COVERAGE: "true" + GOCOVERDIR: ${{ github.workspace }}/.covdata/deployment-shard-${{ matrix.shard-id }} OUTPUT_FILE: ./output.txt PRODUCE_JUNIT_XML: "true" TRUNK_AUTO_QUARANTINE: "true" @@ -238,6 +240,24 @@ jobs: ./deployment/output-short.txt retention-days: 7 + - name: Store coverage artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-deployment-shard-${{ matrix.shard-id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/deployment-shard-${{ matrix.shard-id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-deployment-shard-${{ matrix.shard-id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: junit.xml + retention-days: 7 + if-no-files-found: ignore + deployment-tests-result: # Do not modify this job name, it is used by branch protection rules name: Core Tests (go_core_deployment_tests) diff --git a/.github/workflows/ci-tools.yml b/.github/workflows/ci-tools.yml index e05f54b96bd..99613ba12a7 100644 --- a/.github/workflows/ci-tools.yml +++ b/.github/workflows/ci-tools.yml @@ -72,4 +72,19 @@ jobs: - name: Run tests working-directory: ${{ matrix.target.dir }} - run: gotestsum --format github-actions -- ${{ matrix.target.packages }} + env: + GOCOVERDIR: ${{ github.workspace }}/.covdata/tools-${{ strategy.job-index }} + run: | + mkdir -p "$GOCOVERDIR" + gotestsum --format github-actions -- ${{ matrix.target.packages }} \ + -cover -covermode=atomic -coverpkg=./... \ + -args -test.gocoverdir="$GOCOVERDIR" + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-tools-${{ strategy.job-index }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/tools-${{ strategy.job-index }} + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml new file mode 100644 index 00000000000..57e1a23e0aa --- /dev/null +++ b/.github/workflows/coverage-collector.yml @@ -0,0 +1,177 @@ +name: Coverage Collector + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + target_sha: + description: Commit SHA whose coverage artifacts should be collected + required: true + type: string + pr_number: + description: PR number to comment on (optional) + required: false + type: string + wait_for_runs: + description: Wait for the other watched workflows for this SHA + required: false + default: true + type: boolean + update_badge: + description: Update the coverage badge branch from this manual run (develop only) + required: false + default: false + type: boolean + workflow_run: + workflows: + - CI Core + - Deployment Tests + - Integration In-Memory Tests + - Integration Tests + - CI Tools + - CRE System Tests + - CRE Regression System Tests + - CRE Mixed-Env Tests + - CRE Local Environment Tests + - CCIP System Tests + - Legacy System Tests + - Legacy Non Functional Tests + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: write + +concurrency: + group: coverage-${{ inputs.target_sha || github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} + cancel-in-progress: true + +jobs: + collect: + name: Collect coverage + if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' || github.event.workflow_run.conclusion != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Wait for PR workflows to start + if: ${{ github.event_name == 'pull_request' }} + run: sleep 60 + + - name: Collect and publish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET_SHA: ${{ inputs.target_sha || github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number || (github.event.workflow_run.pull_requests[0].number || '') }} + PATCH_BASE: "" + PATCH_HEAD: "" + WAIT_FOR_RUNS: ${{ github.event_name != 'workflow_dispatch' && 'true' || (inputs.wait_for_runs == true && 'true' || 'false') }} + WAIT_TIMEOUT: "2700" + run: ./tools/coverage/collect.sh + + - name: Upload merged report + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-report-${{ inputs.target_sha || github.event.pull_request.head.sha || github.event.workflow_run.head_sha }}-${{ github.run_id }} + path: coverage-report + retention-days: 30 + if-no-files-found: warn + badge: + name: Update coverage badge + needs: [collect] + if: | + (github.event_name == 'workflow_run' && + github.event.workflow_run.name == 'Integration Tests' && + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.head_branch == 'develop' && + github.event.workflow_run.conclusion == 'success') || + (github.event_name == 'workflow_dispatch' && + inputs.update_badge && + github.ref == 'refs/heads/develop') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + # Only this trusted, scheduled badge job may write the coverage branch. + actions: read + contents: write + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Download coverage report + uses: actions/download-artifact@v7 + with: + name: coverage-report-${{ github.event.workflow_run.head_sha || inputs.target_sha }}-${{ github.run_id }} + path: ${{ runner.temp }}/coverage-report + + - name: Require a full report + id: full + run: | + set -euo pipefail + for file in unit.txt integration.txt e2e.txt any.json; do + if [[ ! -f "$RUNNER_TEMP/coverage-report/$file" ]]; then + echo "Missing $file; not updating the badge from a partial report." + echo "full=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + done + echo "full=true" >> "$GITHUB_OUTPUT" + + - name: Generate badge and state + if: steps.full.outputs.full == 'true' + env: + SHA: ${{ github.event.workflow_run.head_sha || inputs.target_sha }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ARTIFACT: coverage-report-${{ github.event.workflow_run.head_sha || inputs.target_sha }}-${{ github.run_id }} + run: | + python3 tools/coverage/badge.py \ + "$RUNNER_TEMP/coverage-report/any.json" \ + "$RUNNER_TEMP/coverage-badge.svg" \ + "$RUNNER_TEMP/coverage-state.json" \ + "$SHA" "$ARTIFACT" "$REPO" "$RUN_URL" + + - name: Publish coverage branch + if: steps.full.outputs.full == 'true' + env: + SHA: ${{ github.event.workflow_run.head_sha || inputs.target_sha }} + run: | + set -euo pipefail + worktree="$RUNNER_TEMP/coverage-branch" + rm -rf "$worktree" + if git ls-remote --exit-code --heads origin coverage >/dev/null 2>&1; then + git fetch --no-tags origin coverage:refs/remotes/origin/coverage + git worktree add --detach "$worktree" refs/remotes/origin/coverage + else + git worktree add --detach "$worktree" + cd "$worktree" + git checkout --orphan coverage + git rm -rf . >/dev/null 2>&1 || true + fi + cd "$worktree" + mkdir -p latest + cp "$RUNNER_TEMP/coverage-badge.svg" latest/badge.svg + cp "$RUNNER_TEMP/coverage-state.json" latest/report.json + git add latest + if git diff --cached --quiet; then + echo "Coverage badge unchanged; not committing." + exit 0 + fi + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -m "coverage: update badge for ${SHA}" + git push origin HEAD:refs/heads/coverage diff --git a/.github/workflows/coverage-patch.yml b/.github/workflows/coverage-patch.yml new file mode 100644 index 00000000000..245df854f31 --- /dev/null +++ b/.github/workflows/coverage-patch.yml @@ -0,0 +1,70 @@ +name: Coverage Patch + +# Fast, non-blocking patch coverage for the packages touched by a PR. +# This is intentionally not the full coverage suite: it runs affected packages +# with -short, reuses the normal Go build/module caches, and posts a sticky PR +# comment. The nightly full report remains the authoritative number. + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "**/*.go" + - "**/go.mod" + - "**/go.sum" + +permissions: + contents: read + pull-requests: write + +concurrency: + group: coverage-patch-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + patch: + name: Patch coverage + if: ${{ !github.event.pull_request.draft && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version-file: go.mod + go-module-file: go.sum + build-cache-version: coverage-patch + # Upsert the branch-specific build cache so later pushes on this PR + # reuse the coverage build outputs. The cache version is unique to + # this job, so concurrent PRs cannot race on one key. + build-cache-write: always + + - name: Test coverage tooling + run: python3 -m unittest discover -s tools/coverage -p 'test_*.py' -v + + - name: Run affected-package tests and publish patch coverage + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_REPOSITORY: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + OUT: coverage-patch + SHORT: "true" + TEST_TIMEOUT: "10m" + run: ./tools/coverage/patch.sh + + - name: Upload patch report + if: always() + uses: actions/upload-artifact@v7 + with: + name: coverage-patch-${{ github.event.pull_request.head.sha }} + path: coverage-patch + retention-days: 1 + if-no-files-found: warn diff --git a/.github/workflows/cre-local-env-tests.yaml b/.github/workflows/cre-local-env-tests.yaml index 5a5fb7c280a..5f16d439cd8 100644 --- a/.github/workflows/cre-local-env-tests.yaml +++ b/.github/workflows/cre-local-env-tests.yaml @@ -196,6 +196,9 @@ jobs: total_seconds=$(echo "$minutes * 60 + $seconds" | bc) echo "execution_time=$total_seconds" >> "${GITHUB_OUTPUT}" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Deploy Cron workflow (Go) shell: bash working-directory: core/scripts/cre/environment @@ -324,6 +327,21 @@ jobs: docker logs "$c" > "logs/${c}.log" 2>&1 done + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-local-cre + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-local-cre-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-local-cre + retention-days: 7 + if-no-files-found: warn + - name: Upload all artifacts as single package if: failure() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/cre-mixed-env-tests.yaml b/.github/workflows/cre-mixed-env-tests.yaml index 2d5cd6441c6..8ab4c92fbd6 100644 --- a/.github/workflows/cre-mixed-env-tests.yaml +++ b/.github/workflows/cre-mixed-env-tests.yaml @@ -373,6 +373,9 @@ jobs: aws-role-duration-seconds: "1800" set-git-config: "true" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CRE Smoke system tests (mixed-env) id: run-tests if: ${{ steps.baseline.outputs.skip != 'true' }} @@ -454,6 +457,31 @@ jobs: trunk-upload-only: ${{ env.ENABLE_AUTO_QUARANTINE != 'true' }} artifact-name: mixed-env-${{ matrix.tests.test_id }}_test_logs + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + image: ${{ env.CHAINLINK_IMAGE_FULL }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit-report.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/cre-regression-system-tests.yaml b/.github/workflows/cre-regression-system-tests.yaml index 42bbb525b77..5f6daa42c4d 100644 --- a/.github/workflows/cre-regression-system-tests.yaml +++ b/.github/workflows/cre-regression-system-tests.yaml @@ -212,6 +212,9 @@ jobs: cleanup-on-error: "false" working-directory: core/scripts/cre/environment + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CRE Regression system tests id: run-regression-tests shell: bash @@ -271,6 +274,31 @@ jobs: artifact-name: ${{ matrix.tests.test_id }}_test_logs trunk-test-collection-id: qt0j1cVm # chainlink-e2e + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + image: ${{ env.CHAINLINK_IMAGE_FULL }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit-report-regression.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 056f9866053..0aafeb101d2 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -351,6 +351,9 @@ jobs: aws-role-duration-seconds: "1800" set-git-config: "true" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CRE Smoke system tests id: run-tests shell: bash @@ -411,6 +414,31 @@ jobs: artifact-name: ${{ matrix.tests.test_id }}_test_logs trunk-test-collection-id: qt0j1cVm # chainlink-e2e + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + image: ${{ env.CHAINLINK_IMAGE_FULL }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit-report.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/integration-in-memory-tests.yml b/.github/workflows/integration-in-memory-tests.yml index f1e007a065d..ffbe9572c96 100644 --- a/.github/workflows/integration-in-memory-tests.yml +++ b/.github/workflows/integration-in-memory-tests.yml @@ -233,7 +233,9 @@ jobs: run: | set -euo pipefail mkdir -p integration-tests/bin - (cd integration-tests && go test -c -ldflags="-s -w" -o ./bin/ccip-inmemory.test ./smoke/ccip) + (cd integration-tests && go test -c -cover -covermode=atomic \ + -coverpkg=github.com/smartcontractkit/chainlink/v2/...,github.com/smartcontractkit/chainlink/deployment/... \ + -ldflags="-s -w" -o ./bin/ccip-inmemory.test ./smoke/ccip) # Pre-build preparetest so matrix jobs skip ~1min of module downloads + compile each go build -o integration-tests/bin/preparetest ./core/store/cmd/preparetest GOBIN="$PWD/integration-tests/bin" go install gotest.tools/gotestsum@v1.13.0 @@ -344,12 +346,15 @@ jobs: TEST_RUN: ${{ matrix.tests.test }} TEST_TIMEOUT: ${{ matrix.tests.timeout }} PARALLEL: ${{ matrix.tests.parallel }} + GOCOVERDIR: ${{ github.workspace }}/.covdata/inmemory-${{ strategy.job-index }} run: | + mkdir -p "${GOCOVERDIR}" gotestsum --jsonfile=/tmp/gotest.log --junitfile=/tmp/junit.xml \ --format=github-actions --raw-command -- \ go tool test2json -t -p github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip \ ../../bin/ccip-inmemory.test -test.v -test.run "${TEST_RUN}" \ - -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL}" + -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL}" \ + -test.gocoverdir="${GOCOVERDIR}" - name: Ensure Junit XML file exists if: always() @@ -371,6 +376,24 @@ jobs: trunk-token: ${{ secrets.TRUNK_API_KEY }} artifact-name: ${{ matrix.tests.test_id }}_test_logs + - name: Store coverage artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-inmemory-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/inmemory-${{ strategy.job-index }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-inmemory-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit.xml + retention-days: 7 + if-no-files-found: ignore + - name: Collect Diagnostic Logs on Failure if: ${{ steps.run-tests.outcome != 'success' || cancelled() }} shell: bash diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 955230c1dad..385f3447348 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,6 +11,9 @@ on: - "*" branches: - develop + schedule: + # Nightly full E2E coverage for the full coverage review/badge. + - cron: "0 3 * * *" workflow_dispatch: inputs: cl_ref: @@ -30,6 +33,16 @@ on: required: false type: string default: "chainlink" + coverage: + description: "Build the Chainlink image with Go coverage instrumentation and use the -coverage tag." + required: false + type: boolean + default: false + debug: + description: "Build the Chainlink image with dev + coverage + race instrumentation and use the -debug tag." + required: false + type: boolean + default: false # Only run 1 of this workflow at a time per PR, but don't cancel runs on develop concurrency: @@ -41,6 +54,8 @@ env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name || 'chainlink-integration-tests' }} CHAINLINK_REF: ${{ inputs.cl_ref || github.sha }} TEST_SUITE: smoke + E2E_COVERAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage')) || github.event_name == 'schedule' }} + E2E_DEBUG: ${{ (github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug')) }} jobs: enforce-ctf-version: @@ -312,7 +327,7 @@ jobs: uses: smartcontractkit/.github/actions/ecr-image-exists@ecr-image-exists/0.2.0 with: repository: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} - tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }} + tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG == 'true' && '-debug' || env.E2E_COVERAGE == 'true' && '-coverage' || '') || '' }} aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Explain build/no-build decision @@ -358,7 +373,9 @@ jobs: # Rebuild the final-stage apt layer daily so `apt-get upgrade` isn't served stale from cache. docker-additional-build-args: | APT_CACHE_BUST=${{ steps.apt-cache-bust.outputs.key }} - image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }} + ${{ matrix.image.name == '' && format('GO_COVER_FLAG={0}', env.E2E_COVERAGE) || '' }} + ${{ matrix.image.name == '' && format('GO_DEBUG_FLAG={0}', env.E2E_DEBUG) || '' }} + image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG == 'true' && '-debug' || env.E2E_COVERAGE == 'true' && '-coverage' || '') || '' }} dockerfile: ${{ matrix.image.dockerfile }} docker-registry-url: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com docker-repository-name: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} @@ -501,7 +518,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} secrets: inherit run-core-cre-e2e-regression-tests: @@ -519,7 +536,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} secrets: inherit run-core-cre-mixed-env-tests: @@ -537,7 +554,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} # Base-branch commit this PR is built on top of, so the mixed-env baseline is the exact # branch the PR image derives from (develop or a release branch) and the 2-2 split # isolates only the PR's changes. Empty for non-PR events; the workflow then falls back @@ -845,7 +862,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} baseline_ref: ${{ github.event.pull_request.base.sha }} base_ref_name: ${{ github.event.pull_request.base.ref }} secrets: inherit diff --git a/.github/workflows/legacy-non-functional-tests.yml b/.github/workflows/legacy-non-functional-tests.yml index 7c6e647ca2c..7da7e5f082b 100644 --- a/.github/workflows/legacy-non-functional-tests.yml +++ b/.github/workflows/legacy-non-functional-tests.yml @@ -179,6 +179,9 @@ jobs: echo "Setting up environment with: $ENV_CMD" eval "$ENV_CMD" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run tests working-directory: devenv/tests/${{ matrix.tests_dir }} env: @@ -188,6 +191,22 @@ jobs: echo "Running tests for: $CHAINLINK_IMAGE, test: $TESTCMD" eval "$TESTCMD" + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.logs_archive_name }} + image: ${{ env.CHAINLINK_IMAGE }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.logs_archive_name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.logs_archive_name }} + retention-days: 7 + if-no-files-found: warn + - name: Upload Logs if: always() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/legacy-system-tests.yml b/.github/workflows/legacy-system-tests.yml index c7bbd997d74..583fdbc0ca5 100644 --- a/.github/workflows/legacy-system-tests.yml +++ b/.github/workflows/legacy-system-tests.yml @@ -248,6 +248,9 @@ jobs: echo "Setting up environment with: $ENV_CMD" eval "$ENV_CMD" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run tests working-directory: devenv/tests/${{ matrix.tests_dir }} env: @@ -257,6 +260,22 @@ jobs: echo "Running tests for: $CHAINLINK_IMAGE, test: $TESTCMD" eval "$TESTCMD" + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.logs_archive_name }} + image: ${{ env.CHAINLINK_IMAGE }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.logs_archive_name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.logs_archive_name }} + retention-days: 7 + if-no-files-found: warn + - name: Upload Logs if: always() uses: actions/upload-artifact@v7 diff --git a/GNUmakefile b/GNUmakefile index 1f178d09597..9eac2444615 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -84,7 +84,11 @@ install-chainlink-dev: operator-ui ## Install the chainlink binary. .PHONY: install-chainlink-cover install-chainlink-cover: operator-ui ## Install the chainlink binary with cover flag. - go install -cover $(GOFLAGS) . + go install -tags "dev coverage" -cover -covermode=atomic $(GOFLAGS) . + +.PHONY: install-chainlink-debug +install-chainlink-debug: operator-ui ## Install the chainlink binary with dev, coverage, and race instrumentation. + go install -tags "dev coverage" -cover -covermode=atomic -race $(GCFLAGS) $(GOFLAGS) . .PHONY: chainlink chainlink: ## Build the chainlink binary. diff --git a/README.md b/README.md index d003c1bd806..8cdf07dec46 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ [![GitHub contributors](https://img.shields.io/github/contributors-anon/smartcontractkit/chainlink?style=flat-square)](https://github.com/smartcontractkit/chainlink/graphs/contributors) [![GitHub commit activity](https://img.shields.io/github/commit-activity/y/smartcontractkit/chainlink?style=flat-square)](https://github.com/smartcontractkit/chainlink/commits/master) [![Official documentation](https://img.shields.io/static/v1?label=docs&message=latest&color=blue)](https://docs.chain.link/) +[![Coverage](https://raw.githubusercontent.com/smartcontractkit/chainlink/coverage/latest/badge.svg)](https://github.com/smartcontractkit/chainlink/actions/workflows/coverage-collector.yml) [Chainlink](https://chain.link/) expands the capabilities of smart contracts by enabling access to real-world data and off-chain computation while maintaining the security and reliability guarantees inherent to blockchain technology. diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index e7024c030ce..9bfaeb24768 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -97,11 +97,17 @@ FROM deps AS build-chainlink ARG COMMIT_SHA ARG VERSION_TAG ARG CL_IS_PROD_BUILD=true +ARG GO_COVER_FLAG=false +ARG GO_DEBUG_FLAG=false RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ mkdir -p /gobins && \ - if [ "$CL_IS_PROD_BUILD" = "false" ]; then \ + if [ "$GO_DEBUG_FLAG" = "true" ]; then \ + CGO_ENABLED=1 GOBIN=/gobins make install-chainlink-debug; \ + elif [ "$GO_COVER_FLAG" = "true" ]; then \ + GOBIN=/gobins make install-chainlink-cover; \ + elif [ "$CL_IS_PROD_BUILD" = "false" ]; then \ GOBIN=/gobins make install-chainlink-dev; \ else \ GOBIN=/gobins make install-chainlink; \ diff --git a/core/coverage_flush.go b/core/coverage_flush.go new file mode 100644 index 00000000000..656f01d34d9 --- /dev/null +++ b/core/coverage_flush.go @@ -0,0 +1,42 @@ +//go:build coverage + +package core + +import ( + "os" + "os/signal" + "runtime/coverage" + "sync" + "syscall" + "time" +) + +// init installs a SIGUSR1 handler and a periodic flush used by E2E coverage +// collection. Flushing while the process is running means coverage is not lost +// when a container is later stopped with SIGKILL. +func init() { + dir := os.Getenv("GOCOVERDIR") + if dir == "" { + return + } + var mu sync.Mutex + flush := func() { + mu.Lock() + defer mu.Unlock() + _ = coverage.WriteCountersDir(dir) + } + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGUSR1) + go func() { + for range ch { + flush() + } + }() + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + flush() + } + }() +} diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index 467dda4e92f..83ecf24ce67 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -3,10 +3,21 @@ set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} TIMEOUT="${TIMEOUT:-10s}" COUNT="${COUNT:-5}" +COVERAGE=${COVERAGE:-false} + +COVERAGE_FLAGS="" +COVERAGE_ARGS="" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + COVERAGE_FLAGS="-covermode=atomic -cover -coverpkg=${COVERPKG:-./...}" + COVERAGE_ARGS="-args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi echo "Test execution results: ---------------------" echo "" -GORACE="log_path=$PWD/race" go test -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE +GORACE="log_path=$PWD/race" go test -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $COVERAGE_FLAGS "$1" $COVERAGE_ARGS | cat > $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} diff --git a/tools/bin/go_core_scripts_tests b/tools/bin/go_core_scripts_tests index e4380264215..2adc76fd87a 100755 --- a/tools/bin/go_core_scripts_tests +++ b/tools/bin/go_core_scripts_tests @@ -5,14 +5,18 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} EXTRA_FLAGS="" +COVERAGE=${COVERAGE:-false} cd ./core/scripts || exit go mod download echo "Test execution results: ---------------------" echo "" -if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then - EXTRA_FLAGS="-covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + EXTRA_FLAGS="-covermode=atomic -cover -coverpkg=${COVERPKG:-./...} -args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" fi go test ./... $EXTRA_FLAGS | tee $OUTPUT_FILE | grep -Ev '\[no test files\]|\[no tests to run\]' EXITCODE=${PIPESTATUS[0]} diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index f2ad773f820..d4473a02cba 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -6,6 +6,7 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"} GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"15m"} +PACKAGES=${PACKAGES:-"./..."} echo "Installing gotestsum..." go install gotest.tools/gotestsum@v1.13.0 @@ -16,8 +17,6 @@ echo "Running go_core_tests for event: $GITHUB_EVENT_NAME" GO_TEST_FLAGS="-timeout=${GO_TEST_TIMEOUT}" if [[ "$GITHUB_EVENT_NAME" == 'push' || "$GITHUB_EVENT_NAME" == 'workflow_dispatch' || "$GITHUB_EVENT_NAME" == 'schedule' ]]; then GO_TEST_FLAGS="$GO_TEST_FLAGS -count=1" -elif [[ "$GITHUB_EVENT_NAME" == 'schedule' ]]; then - GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge_group' ]]; then RERUN_FLAGS="" if [[ "$TRUNK_AUTO_QUARANTINE" != "true" ]]; then @@ -28,6 +27,14 @@ elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge fi fi +COVERAGE=${COVERAGE:-false} +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -cover -coverpkg=${COVERPKG:-./...} -args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi + if [[ "$PRODUCE_JUNIT_XML" == "true" ]]; then JUNIT_FLAG="--junitfile=$JUNIT_FILE" fi @@ -45,7 +52,7 @@ gotestsum \ --format='standard-quiet' \ --hide-summary=skipped \ $RERUN_FLAGS \ - --packages='./...' \ + --packages="$PACKAGES" \ --jsonfile "$OUTPUT_FILE" \ --post-run-command "sh -c \"echo; echo 'Slowest tests'; echo '----------------------------------------------'; echo; gotestsum tool slowest --num 20 --jsonfile ${OUTPUT_FILE}\"" \ "$JUNIT_FLAG" \ diff --git a/tools/bin/go_core_tests_integration b/tools/bin/go_core_tests_integration index aaaa7fe7769..3850dc96199 100755 --- a/tools/bin/go_core_tests_integration +++ b/tools/bin/go_core_tests_integration @@ -19,11 +19,6 @@ echo "Running go_core_tests_integration for event: $GITHUB_EVENT_NAME" GO_TEST_FLAGS="-timeout=${GO_TEST_TIMEOUT}" if [[ "$GITHUB_EVENT_NAME" == 'push' ]]; then GO_TEST_FLAGS="$GO_TEST_FLAGS -count=1" -elif [[ "$GITHUB_EVENT_NAME" == 'schedule' ]]; then - # Experimental code to minimize size of this coverage report - # ALL_IMPORTS=$(go list -f '{{ join .Imports "\n" }}' $INTEGRATION_TEST_DIRS | sort -u) - # COVERPKG_DIRS=$(echo "$INTEGRATION_TEST_DIRS $ALL_IMPORTS" | grep "smartcontractkit/chainlink" | tr '\n' ',') - GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge_group' ]]; then RERUN_FLAGS="" if [[ "$TRUNK_AUTO_QUARANTINE" != "true" ]]; then @@ -34,6 +29,16 @@ elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge fi fi +COVERAGE=${COVERAGE:-false} +COVERAGE_ARGS="" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -cover -coverpkg=${COVERPKG:-./...}" + COVERAGE_ARGS="-args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi + if [[ "$PRODUCE_JUNIT_XML" == "true" ]]; then JUNIT_FLAG="--junitfile=$JUNIT_FILE" fi @@ -66,7 +71,7 @@ gotestsum \ --jsonfile "$OUTPUT_FILE" \ --post-run-command "sh -c \"echo; echo 'Slowest tests'; echo '----------------------------------------------'; echo; gotestsum tool slowest --num 20 --jsonfile ${OUTPUT_FILE}\"" \ "$JUNIT_FLAG" \ - -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED + -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED $COVERAGE_ARGS EXITCODE=${PIPESTATUS[0]} diff --git a/tools/bin/go_deployment_tests b/tools/bin/go_deployment_tests index b5c350c65d7..9dca828cc34 100755 --- a/tools/bin/go_deployment_tests +++ b/tools/bin/go_deployment_tests @@ -32,8 +32,6 @@ GO_TEST_FLAGS="-timeout=${GO_TEST_TIMEOUT}" GO_TEST_FLAGS="$GO_TEST_FLAGS -p=2 -parallel=1" if [[ "$GITHUB_EVENT_NAME" == 'push' || "$GITHUB_EVENT_NAME" == 'workflow_dispatch' || "$GITHUB_EVENT_NAME" == 'schedule' ]]; then GO_TEST_FLAGS="$GO_TEST_FLAGS -count=1" -elif [[ "$GITHUB_EVENT_NAME" == 'schedule' ]]; then - GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge_group' ]]; then RERUN_FLAGS="" if [[ "$TRUNK_AUTO_QUARANTINE" != "true" ]]; then @@ -44,6 +42,16 @@ elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge fi fi +COVERAGE=${COVERAGE:-false} +COVERAGE_ARGS="" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -cover -coverpkg=${COVERPKG:-./...}" + COVERAGE_ARGS="-args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi + if [[ "$PRODUCE_JUNIT_XML" == "true" ]]; then JUNIT_FLAG="--junitfile=$JUNIT_FILE" fi @@ -88,7 +96,7 @@ gotestsum \ --jsonfile "$OUTPUT_FILE" \ --post-run-command "sh -c \"echo; echo 'Slowest tests'; echo '----------------------------------------------'; echo; gotestsum tool slowest --num 20 --jsonfile ${OUTPUT_FILE}\"" \ "$JUNIT_FLAG" \ - -- $GO_TEST_FLAGS "${SHARD_PACKAGES[@]}" + -- $GO_TEST_FLAGS "${SHARD_PACKAGES[@]}" $COVERAGE_ARGS EXITCODE=${PIPESTATUS[0]} diff --git a/tools/coverage/badge.py b/tools/coverage/badge.py new file mode 100644 index 00000000000..5a7706e0c85 --- /dev/null +++ b/tools/coverage/badge.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Generate a small coverage badge and latest-report JSON from tools/coverage output. + +Usage: + badge.py any.json badge.svg report.json SHA artifact_name repo run_url + +The badge uses the source-only global percentage, which excludes tests, +generated code, and mocks. The report JSON is the only state kept in the +orphan `coverage` branch. +""" + +from __future__ import annotations + +import datetime as dt +import json +import sys +from pathlib import Path + + +def color_for(percent: float) -> str: + if percent >= 70: + return "#2e7d32" + if percent >= 55: + return "#f9a825" + if percent >= 45: + return "#ef6c00" + return "#c62828" + + +def main() -> int: + if len(sys.argv) != 8: + print(__doc__, file=sys.stderr) + return 2 + + report_path, svg_path, state_path, sha, artifact, repo, run_url = sys.argv[1:] + report = json.loads(Path(report_path).read_text(encoding="utf-8")) + source = report.get("GlobalSourceOnly") or report.get("Global") or {} + percent = float(source.get("Percent", 0.0)) + raw = float((report.get("Global") or {}).get("Percent", 0.0)) + patch = report.get("Patch") or {} + patch_percent = patch.get("Percent") + + value = f"{percent:.1f}%" + color = color_for(percent) + svg = ( + '' + '' + f'' + f'' + '' + 'coverage' + f'{value}' + "\n" + ) + Path(svg_path).write_text(svg, encoding="utf-8") + + state = { + "sha": sha, + "repo": repo, + "global_raw": raw, + "global_source_only": percent, + "patch": patch_percent, + "report_artifact": artifact, + "run_url": run_url, + "updated_at": dt.datetime.now(dt.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + } + Path(state_path).write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + print(f"coverage badge: {value} ({color})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh new file mode 100755 index 00000000000..4970a737daa --- /dev/null +++ b/tools/coverage/collect.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Collect coverage artifacts for a commit and publish a PR report. +# +# Required environment: +# GITHUB_REPOSITORY owner/repo +# GITHUB_TOKEN token with actions:read, pull-requests:write +# TARGET_SHA commit whose workflow runs should be collected +# +# Optional: +# PR_NUMBER PR number; inferred from TARGET_SHA when empty +# OUTPUT_DIR report output directory (default: coverage-report) +# ARTIFACTS_DIR raw artifact download directory (default: $RUNNER_TEMP/coverage-artifacts) +# WAIT_FOR_RUNS wait for other workflow runs for TARGET_SHA (true/false) +# WAIT_TIMEOUT seconds to wait for runs (default: 2700) +set -euo pipefail + +REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +TARGET_SHA="${TARGET_SHA:?TARGET_SHA is required}" +OUT="${OUTPUT_DIR:-coverage-report}" +ARTIFACTS="${ARTIFACTS_DIR:-${RUNNER_TEMP:-/tmp}/coverage-artifacts}" +rm -rf "$ARTIFACTS" +mkdir -p "$OUT" "$ARTIFACTS" + +if [[ -z "${PR_NUMBER:-}" ]]; then + PR_NUMBER="$(gh api "repos/${REPO}/commits/${TARGET_SHA}/pulls" \ + --jq '[.[] | select(.state == "open")][0].number // empty' 2>/dev/null || true)" +fi +BASE_SHA="" +if [[ -n "${PR_NUMBER:-}" ]]; then + BASE_SHA="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.base.sha' 2>/dev/null || true)" +fi +PATCH_BASE="${PATCH_BASE:-}" +PATCH_HEAD="${PATCH_HEAD:-$TARGET_SHA}" + +if [[ "${WAIT_FOR_RUNS:-false}" == "true" ]]; then + deadline=$((SECONDS + ${WAIT_TIMEOUT:-2700})) + sleep 30 + while (( SECONDS < deadline )); do + runs_json="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 500 \ + --json workflowName,status 2>/dev/null || echo '[]')" + watched='["CI Core","Deployment Tests","Integration In-Memory Tests","Integration Tests","CI Tools","CRE System Tests","CRE Regression System Tests","CRE Mixed-Env Tests","CRE Local Environment Tests","CCIP System Tests","Legacy System Tests","Legacy Non Functional Tests"]' + pending="$(printf '%s' "$runs_json" | jq -r --argjson watched "$watched" \ + '.[] | select(.workflowName as $n | $watched | index($n)) | select(.status == "queued" or .status == "in_progress") | .workflowName' \ + 2>/dev/null | sort -u || true)" + if [[ -z "$pending" ]]; then + break + fi + echo "Waiting for: ${pending//$'\n'/, }" + sleep 30 + done +fi + +echo "Collecting coverage for ${REPO}@${TARGET_SHA} (PR ${PR_NUMBER:-none}, base ${BASE_SHA:-none})" + +if [[ -n "$BASE_SHA" ]]; then + git fetch --no-tags origin "$BASE_SHA" "$TARGET_SHA" >/dev/null 2>&1 || true +fi +if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + git fetch --no-tags origin "$PATCH_BASE" "$PATCH_HEAD" >/dev/null 2>&1 || true +fi +if [[ -z "$PATCH_BASE" && -n "$BASE_SHA" ]]; then + PATCH_BASE="$(git merge-base "$BASE_SHA" "$TARGET_SHA" 2>/dev/null || echo "$BASE_SHA")" +fi + +runs="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 500 --json databaseId --jq '.[].databaseId' 2>/dev/null || true)" +for run in $runs; do + names="$(gh api "repos/${REPO}/actions/runs/${run}/artifacts?per_page=100" --jq '.artifacts[].name' 2>/dev/null \ + | grep -E '^(coverage-(core|deployment|inmemory|tools|e2e)-|test-results-)' \ + | grep -v '^coverage-core-go_core_race_tests-' || true)" + # Nightly race coverage is a separate 7-day artifact for race-instrumented + # inspection. It is intentionally excluded from the project and patch + # coverage calculation: -race changes runtime/timing and would make the + # normal coverage trend incomparable. + while IFS= read -r name; do + [[ -z "$name" ]] && continue + dest="${ARTIFACTS}/${run}/${name}" + mkdir -p "$dest" + echo "Downloading ${name} from run ${run}" + gh run download "$run" --repo "$REPO" -n "$name" -D "$dest" || \ + echo "::warning::Failed to download ${name} from ${run}" + done <<< "$names" +done + +unit_dirs="" +integration_dirs="" +e2e_dirs="" +while IFS= read -r dir; do + case "$dir" in + */coverage-core-go_core_tests-*|*/coverage-core-scripts-*|*/coverage-tools-*) + unit_dirs="${unit_dirs}${unit_dirs:+,}${dir}" ;; + */coverage-core-go_core_tests_integration-*|*/coverage-deployment-*|*/coverage-inmemory-*) + integration_dirs="${integration_dirs}${integration_dirs:+,}${dir}" ;; + */coverage-e2e-*) + e2e_dirs="${e2e_dirs}${e2e_dirs:+,}${dir}" ;; + esac +done < <(find "$ARTIFACTS" -type f -name 'covmeta.*' -exec dirname {} \; 2>/dev/null | sort -u) + +merge_and_report() { + local category="$1" dirs="$2" + [[ -z "$dirs" ]] && return 0 + echo "Merging ${category}" + if ! go run ./tools/coverage merge --covdata "$dirs" --out "${OUT}/${category}.txt"; then + echo "::warning::Failed to merge ${category} coverage" + return 0 + fi + local args=(--profile "${OUT}/${category}.txt" --category "$category" + --json "${OUT}/${category}.json" --markdown "${OUT}/${category}.md") + if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") + fi + if ! go run ./tools/coverage report "${args[@]}"; then + echo "::warning::Failed to compute ${category} patch coverage; retrying without patch" + args=(--profile "${OUT}/${category}.txt" --category "$category" + --json "${OUT}/${category}.json" --markdown "${OUT}/${category}.md") + go run ./tools/coverage report "${args[@]}" || echo "::warning::Failed to report ${category} coverage" + fi +} + +merge_and_report unit "$unit_dirs" +merge_and_report integration "$integration_dirs" +merge_and_report e2e "$e2e_dirs" + +profile_list="" +for category in unit integration e2e; do + if [[ -f "${OUT}/${category}.txt" ]]; then + profile_list="${profile_list}${profile_list:+,}${OUT}/${category}.txt" + fi +done +if [[ -n "$profile_list" ]]; then + if go run ./tools/coverage merge-profiles --profile "$profile_list" --out "${OUT}/any.txt"; then + args=(--profile "${OUT}/any.txt" --category any + --json "${OUT}/any.json" --markdown "${OUT}/any.md") + if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") + fi + if ! go run ./tools/coverage report "${args[@]}"; then + echo "::warning::Failed to compute any patch coverage; retrying without patch" + args=(--profile "${OUT}/any.txt" --category any + --json "${OUT}/any.json" --markdown "${OUT}/any.md") + go run ./tools/coverage report "${args[@]}" || echo "::warning::Failed to report any coverage" + fi + else + echo "::warning::Failed to merge any coverage" + fi +fi + +test_xmls="$(find "$ARTIFACTS" -type f -name '*.xml' -path '*test-results-*' 2>/dev/null | sort -u)" +if [[ -n "$test_xmls" ]]; then + python3 - "$OUT/test-results.json" $test_xmls <<'PY' || echo "::warning::Failed to parse test results" +import json, sys, xml.etree.ElementTree as ET +files = sys.argv[2:] +totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0, "files": len(files)} +for path in files: + try: + root = ET.parse(path).getroot() + except Exception: + continue + suites = root.findall("testsuite") if root.tag == "testsuites" else [root] + for suite in suites: + for key in ("tests", "failures", "errors", "skipped"): + try: + totals[key] += int(suite.get(key, "0")) + except ValueError: + pass +with open(sys.argv[1], "w", encoding="utf-8") as f: + json.dump(totals, f) +PY +fi + +if ! compgen -G "${OUT}/*.md" >/dev/null; then + echo "No coverage profiles were produced for ${TARGET_SHA}" + exit 0 +fi + +{ + echo "" + echo "# Coverage report for \`${TARGET_SHA:0:9}\`" + echo + echo "Commit: \`${TARGET_SHA}\`" + if [[ -n "${PR_NUMBER:-}" ]]; then echo "PR: #${PR_NUMBER}"; fi + if [[ -n "$BASE_SHA" ]]; then echo "Base: \`${BASE_SHA}\`"; fi + echo + echo "Targets: project coverage should not drop more than 1%; patch target is 80%. This report is informational." + echo + if [[ -f "${OUT}/test-results.json" ]]; then + echo "### Test results" + echo + echo '```json' + cat "${OUT}/test-results.json" + echo + echo '```' + echo + fi + for category in any unit integration e2e; do + if [[ -f "${OUT}/${category}.md" ]]; then + cat "${OUT}/${category}.md" + echo + fi + done +} > "${OUT}/summary.md" + +cat "${OUT}/summary.md" + +if [[ -n "${PR_NUMBER:-}" ]]; then + marker="" + comment_id="$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \ + | jq -r --arg marker "$marker" '.[] | select(.user.login == "github-actions[bot]") | select(.body | contains($marker)) | .id' 2>/dev/null \ + | head -1 || true)" + if [[ -n "$comment_id" ]]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body="@${OUT}/summary.md" || \ + echo "::warning::Failed to update coverage comment" + else + gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "${OUT}/summary.md" || \ + echo "::warning::Failed to create coverage comment" + fi +fi + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + cat "${OUT}/summary.md" >> "$GITHUB_STEP_SUMMARY" +fi diff --git a/tools/coverage/main.go b/tools/coverage/main.go new file mode 100644 index 00000000000..fd2a41e8e54 --- /dev/null +++ b/tools/coverage/main.go @@ -0,0 +1,607 @@ +// Command coverage computes Go coverage summaries and PR patch coverage. +// +// It is intentionally dependency-free so it can run in CI jobs that only have +// the Go toolchain and a coverage profile. +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" +) + +type block struct { + File string + StartLine int + StartCol int + EndLine int + EndCol int + NumStmt int + Count int64 +} + +type lineRange struct { + Start int + End int +} + +type filePatch struct { + Path string + TestFile bool + Added []lineRange + Statements int + Covered int + Missed int + MissedSpans []string +} + +type stats struct { + Statements int + Covered int + Missed int + Percent float64 +} + +type patchStats struct { + ChangedFiles int + ChangedLines int + Statements int + Covered int + Missed int + Percent float64 + Files []filePatch +} + +type report struct { + Category string + Profile []string + Base string + Head string + Global stats + GlobalSourceOnly stats + Patch *patchStats + PatchSourceOnly *patchStats +} + +var modulePrefixes = []string{ + "github.com/smartcontractkit/chainlink/v2/", + "github.com/smartcontractkit/chainlink/", +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + var err error + switch os.Args[1] { + case "report": + err = runReport(os.Args[2:]) + case "merge": + err = runMerge(os.Args[2:]) + case "merge-profiles": + err = runMergeProfiles(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + err = fmt.Errorf("unknown subcommand %q", os.Args[1]) + } + if err != nil { + fmt.Fprintln(os.Stderr, "coverage:", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, `usage: + coverage report --profile file[,file...] [--base SHA --head SHA] [--repo DIR] + [--category NAME] [--json FILE] [--markdown FILE] + [--min-global PCT] [--min-patch PCT] + coverage merge --covdata dir[,dir...] --out coverage.txt + coverage merge-profiles --profile file[,file...] --out coverage.txt`) +} + +func runMerge(args []string) error { + fs := flag.NewFlagSet("merge", flag.ExitOnError) + covdata := fs.String("covdata", "", "comma-separated covdata directories") + out := fs.String("out", "coverage.txt", "text profile output path") + if err := fs.Parse(args); err != nil { + return err + } + dirs := splitList(*covdata) + if len(dirs) == 0 { + return errors.New("--covdata is required") + } + for _, dir := range dirs { + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + return fmt.Errorf("covdata directory %q is not accessible", dir) + } + } + tmp, err := os.MkdirTemp("", "coverage-merge-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + merge := exec.CommandContext(context.Background(), "go", "tool", "covdata", "merge", "-i="+strings.Join(dirs, ","), "-o="+tmp) //nolint:gosec // paths are validated above + merge.Stdout = os.Stdout + merge.Stderr = os.Stderr + if err := merge.Run(); err != nil { + return fmt.Errorf("covdata merge: %w", err) + } + textfmt := exec.CommandContext(context.Background(), "go", "tool", "covdata", "textfmt", "-i="+tmp, "-o="+*out) //nolint:gosec // output path is user-provided CLI input + textfmt.Stdout = os.Stdout + textfmt.Stderr = os.Stderr + if err := textfmt.Run(); err != nil { + return fmt.Errorf("covdata textfmt: %w", err) + } + return nil +} + +func runMergeProfiles(args []string) error { + fs := flag.NewFlagSet("merge-profiles", flag.ExitOnError) + profiles := fs.String("profile", "", "comma-separated text coverage profiles") + out := fs.String("out", "coverage.txt", "text profile output path") + if err := fs.Parse(args); err != nil { + return err + } + paths := splitList(*profiles) + if len(paths) == 0 { + return errors.New("--profile is required") + } + blocks, err := loadProfiles(paths) + if err != nil { + return err + } + f, err := os.Create(*out) + if err != nil { + return err + } + defer f.Close() + if _, err := fmt.Fprintln(f, "mode: atomic"); err != nil { + return err + } + keys := make([]string, 0, len(blocks)) + for key := range blocks { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + a, b := blocks[keys[i]], blocks[keys[j]] + if a.File != b.File { + return a.File < b.File + } + if a.StartLine != b.StartLine { + return a.StartLine < b.StartLine + } + if a.StartCol != b.StartCol { + return a.StartCol < b.StartCol + } + if a.EndLine != b.EndLine { + return a.EndLine < b.EndLine + } + return a.EndCol < b.EndCol + }) + for _, key := range keys { + b := blocks[key] + if _, err := fmt.Fprintf(f, "%s:%d.%d,%d.%d %d %d\n", b.File, b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt, b.Count); err != nil { + return err + } + } + return nil +} + +func runReport(args []string) error { + fs := flag.NewFlagSet("report", flag.ExitOnError) + profiles := fs.String("profile", "", "comma-separated text coverage profiles") + base := fs.String("base", "", "base git revision for patch coverage") + head := fs.String("head", "", "head git revision for patch coverage") + repo := fs.String("repo", ".", "repository root") + category := fs.String("category", "all", "coverage category label") + jsonOut := fs.String("json", "", "JSON output path (default stdout when --markdown is empty)") + mdOut := fs.String("markdown", "", "Markdown output path") + minGlobal := fs.Float64("min-global", -1, "fail if global coverage is below this percentage") + minPatch := fs.Float64("min-patch", -1, "fail if patch coverage is below this percentage") + if err := fs.Parse(args); err != nil { + return err + } + profilePaths := splitList(*profiles) + if len(profilePaths) == 0 { + return errors.New("--profile is required") + } + + blocks, err := loadProfiles(profilePaths) + if err != nil { + return err + } + rep := report{ + Category: *category, + Profile: profilePaths, + Base: *base, + Head: *head, + } + rep.Global = summarize(blocks, func(string) bool { return true }) + rep.GlobalSourceOnly = summarize(blocks, func(path string) bool { return !excluded(path) }) + + if *base != "" || *head != "" { + if *base == "" || *head == "" { + return errors.New("--base and --head must be provided together") + } + diff, err := gitDiff(*repo, *base, *head) + if err != nil { + return err + } + rep.Patch = computePatch(blocks, diff, func(string) bool { return true }) + rep.PatchSourceOnly = computePatch(blocks, diff, func(path string) bool { + return !excluded(path) && !strings.HasSuffix(path, "_test.go") + }) + } + + md := markdown(rep) + if *mdOut != "" { + if err := os.WriteFile(*mdOut, []byte(md), 0o600); err != nil { + return err + } + } + if *jsonOut != "" { + data, err := json.MarshalIndent(rep, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(*jsonOut, append(data, '\n'), 0o600); err != nil { + return err + } + } + if *mdOut == "" && *jsonOut == "" { + fmt.Print(md) + } + + var failures []string + if *minGlobal >= 0 && rep.Global.Percent < *minGlobal { + failures = append(failures, fmt.Sprintf("global coverage %.2f%% is below %.2f%%", rep.Global.Percent, *minGlobal)) + } + if *minPatch >= 0 { + if rep.Patch == nil { + failures = append(failures, "patch coverage requested but --base/--head not provided") + } else if rep.Patch.Percent < *minPatch { + failures = append(failures, fmt.Sprintf("patch coverage %.2f%% is below %.2f%%", rep.Patch.Percent, *minPatch)) + } + } + if len(failures) > 0 { + return fmt.Errorf("%s", strings.Join(failures, "; ")) + } + return nil +} + +func loadProfiles(paths []string) (map[string]block, error) { + merged := make(map[string]block) + for _, path := range paths { + f, err := os.Open(path) + if err != nil { + return nil, err + } + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "mode:") { + continue + } + b, err := parseBlock(line) + if err != nil { + f.Close() + return nil, fmt.Errorf("%s: %w", path, err) + } + key := blockKey(b) + if old, ok := merged[key]; !ok || b.Count > old.Count { + merged[key] = b + } + } + if err := sc.Err(); err != nil { + f.Close() + return nil, err + } + f.Close() + } + return merged, nil +} + +func parseBlock(line string) (block, error) { + fields := strings.Fields(line) + if len(fields) != 3 { + return block{}, fmt.Errorf("malformed coverage line %q", line) + } + loc := fields[0] + filePart, span, ok := strings.CutLast(loc, ":") + if !ok { + return block{}, fmt.Errorf("malformed coverage location %q", loc) + } + file := normalizePath(filePart) + startPart, endPart, ok := strings.Cut(span, ",") + if !ok { + return block{}, fmt.Errorf("malformed coverage span %q", span) + } + start, err := parsePos(startPart) + if err != nil { + return block{}, err + } + end, err := parsePos(endPart) + if err != nil { + return block{}, err + } + numStmt, err := strconv.Atoi(fields[1]) + if err != nil { + return block{}, err + } + count, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return block{}, err + } + return block{ + File: file, + StartLine: start[0], + StartCol: start[1], + EndLine: end[0], + EndCol: end[1], + NumStmt: numStmt, + Count: count, + }, nil +} + +func parsePos(s string) ([2]int, error) { + var pos [2]int + linePart, colPart, ok := strings.Cut(s, ".") + if !ok { + return pos, fmt.Errorf("malformed position %q", s) + } + line, err := strconv.Atoi(linePart) + if err != nil { + return pos, err + } + col, err := strconv.Atoi(colPart) + if err != nil { + return pos, err + } + pos[0], pos[1] = line, col + return pos, nil +} + +func blockKey(b block) string { + return fmt.Sprintf("%s:%d.%d,%d.%d", b.File, b.StartLine, b.StartCol, b.EndLine, b.EndCol) +} + +func normalizePath(path string) string { + for _, prefix := range modulePrefixes { + if stripped, ok := strings.CutPrefix(path, prefix); ok { + path = stripped + break + } + } + path = strings.TrimPrefix(path, "./") + return path +} + +func excluded(path string) bool { + base := filepath.Base(path) + for part := range strings.SplitSeq(path, "/") { + switch part { + case "mocks", "testutils", "testhelpers", "cltest", "testdata", "fixtures": + return true + } + } + if strings.HasPrefix(base, "mock_") || strings.HasSuffix(base, "_mock.go") { + return true + } + for _, suffix := range []string{".pb.go", "_gen.go", "_codecgen.go"} { + if strings.HasSuffix(base, suffix) { + return true + } + } + return false +} + +func summarize(blocks map[string]block, include func(string) bool) stats { + var s stats + for _, b := range blocks { + if !include(b.File) { + continue + } + s.Statements += b.NumStmt + if b.Count > 0 { + s.Covered += b.NumStmt + } + } + s.Missed = s.Statements - s.Covered + s.Percent = percent(s.Covered, s.Statements) + return s +} + +func computePatch(blocks map[string]block, diff map[string][]lineRange, include func(string) bool) *patchStats { + ps := &patchStats{} + paths := make([]string, 0, len(diff)) + for path := range diff { + paths = append(paths, path) + } + sort.Strings(paths) + ps.ChangedFiles = len(paths) + for _, path := range paths { + ranges := diff[path] + for _, r := range ranges { + ps.ChangedLines += r.End - r.Start + 1 + } + fp := filePatch{ + Path: path, + TestFile: strings.HasSuffix(path, "_test.go"), + Added: ranges, + } + if include(path) && !fp.TestFile { + seen := make(map[string]bool) + missed := make(map[string]bool) + for key, b := range blocks { + if b.File != path || seen[key] || !intersects(b, ranges) { + continue + } + seen[key] = true + fp.Statements += b.NumStmt + if b.Count > 0 { + fp.Covered += b.NumStmt + } else { + fp.Missed += b.NumStmt + missed[fmt.Sprintf("%d-%d", b.StartLine, b.EndLine)] = true + } + } + for span := range missed { + fp.MissedSpans = append(fp.MissedSpans, span) + } + sort.Strings(fp.MissedSpans) + ps.Statements += fp.Statements + ps.Covered += fp.Covered + ps.Missed += fp.Missed + } + ps.Files = append(ps.Files, fp) + } + ps.Percent = percent(ps.Covered, ps.Statements) + return ps +} + +func intersects(b block, ranges []lineRange) bool { + for _, r := range ranges { + if b.EndLine >= r.Start && b.StartLine <= r.End { + return true + } + } + return false +} + +func percent(covered, total int) float64 { + if total == 0 { + return 0 + } + return float64(int(10000*float64(covered)/float64(total)+0.5)) / 100 +} + +func gitDiff(repo, base, head string) (map[string][]lineRange, error) { + cmd := exec.CommandContext(context.Background(), "git", "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff", base+".."+head, "--", "*.go") //nolint:gosec // revisions come from the caller + cmd.Dir = repo + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git diff: %w", err) + } + return parseDiff(string(out)), nil +} + +func parseDiff(diff string) map[string][]lineRange { + files := make(map[string][]lineRange) + var current string + sc := bufio.NewScanner(strings.NewReader(diff)) + sc.Buffer(make([]byte, 64*1024), 8*1024*1024) + for sc.Scan() { + line := sc.Text() + switch { + case strings.HasPrefix(line, "+++ "): + path := strings.TrimPrefix(line, "+++ ") + path = strings.TrimPrefix(path, "b/") + if path == "/dev/null" { + current = "" + } else { + current = path + } + case strings.HasPrefix(line, "@@ ") && current != "": + if r, ok := parseHunk(line); ok { + files[current] = append(files[current], r) + } + } + } + return files +} + +func parseHunk(line string) (lineRange, bool) { + plus := strings.Index(line, " +") + if plus < 0 { + return lineRange{}, false + } + rest := line[plus+1:] + end := strings.Index(rest, " @@") + if end < 0 { + return lineRange{}, false + } + rest = rest[:end] + startStr := rest + count := 1 + if comma := strings.Index(rest, ","); comma >= 0 { + startStr = rest[:comma] + n, err := strconv.Atoi(rest[comma+1:]) + if err != nil { + return lineRange{}, false + } + count = n + } + start, err := strconv.Atoi(startStr) + if err != nil || count <= 0 || start <= 0 { + return lineRange{}, false + } + return lineRange{Start: start, End: start + count - 1}, true +} + +func splitList(value string) []string { + var out []string + for item := range strings.SplitSeq(value, ",") { + if trimmed := strings.TrimSpace(item); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +func markdown(rep report) string { + var b strings.Builder + fmt.Fprintf(&b, "## Coverage — %s\n\n", rep.Category) + if rep.Base != "" || rep.Head != "" { + fmt.Fprintf(&b, "`%s..%s`\n\n", rep.Base, rep.Head) + } + b.WriteString("### Global\n\n") + b.WriteString("| Scope | Statements | Covered | Missed | Coverage |\n") + b.WriteString("|---|---:|---:|---:|---:|\n") + writeStatsRow(&b, "raw", rep.Global) + writeStatsRow(&b, "source-only", rep.GlobalSourceOnly) + if rep.Patch != nil { + b.WriteString("\n### Patch\n\n") + b.WriteString("| Scope | Changed files | Changed lines | Statements | Covered | Missed | Coverage |\n") + b.WriteString("|---|---:|---:|---:|---:|---:|---:|\n") + writePatchRow(&b, "raw", rep.Patch) + if rep.PatchSourceOnly != nil { + writePatchRow(&b, "source-only", rep.PatchSourceOnly) + } + if len(rep.Patch.Files) > 0 { + b.WriteString("\n### Changed files\n\n") + b.WriteString("| File | Statements | Covered | Missed | Coverage | Missed spans |\n") + b.WriteString("|---|---:|---:|---:|---:|---|\n") + for _, f := range rep.Patch.Files { + if f.TestFile || f.Statements == 0 { + continue + } + fmt.Fprintf(&b, "| `%s` | %d | %d | %d | %.2f%% | %s |\n", + f.Path, f.Statements, f.Covered, f.Missed, percent(f.Covered, f.Statements), strings.Join(f.MissedSpans, ", ")) + } + } + } + return b.String() +} + +func writeStatsRow(b *strings.Builder, name string, s stats) { + fmt.Fprintf(b, "| %s | %d | %d | %d | **%.2f%%** |\n", name, s.Statements, s.Covered, s.Missed, s.Percent) +} + +func writePatchRow(b *strings.Builder, name string, p *patchStats) { + fmt.Fprintf(b, "| %s | %d | %d | %d | %d | %d | **%.2f%%** |\n", + name, p.ChangedFiles, p.ChangedLines, p.Statements, p.Covered, p.Missed, p.Percent) +} diff --git a/tools/coverage/main_test.go b/tools/coverage/main_test.go new file mode 100644 index 00000000000..5e07ddbd411 --- /dev/null +++ b/tools/coverage/main_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseBlockAndNormalize(t *testing.T) { + t.Parallel() + b, err := parseBlock("github.com/smartcontractkit/chainlink/v2/core/foo.go:10.2,12.4 3 7") + if err != nil { + t.Fatal(err) + } + if b.File != "core/foo.go" || b.StartLine != 10 || b.EndLine != 12 || b.NumStmt != 3 || b.Count != 7 { + t.Fatalf("unexpected block: %+v", b) + } + if got := normalizePath("github.com/smartcontractkit/chainlink/deployment/foo.go"); got != "deployment/foo.go" { + t.Fatalf("unexpected deployment path: %q", got) + } +} + +func TestSummarizeAndExclude(t *testing.T) { + t.Parallel() + blocks := map[string]block{ + "a": {File: "core/a.go", NumStmt: 4, Count: 1}, + "b": {File: "core/a.go", NumStmt: 2, Count: 0}, + "c": {File: "core/mocks/m.go", NumStmt: 10, Count: 0}, + } + raw := summarize(blocks, func(string) bool { return true }) + if raw.Statements != 16 || raw.Covered != 4 || raw.Missed != 12 { + t.Fatalf("unexpected raw stats: %+v", raw) + } + source := summarize(blocks, func(path string) bool { return !excluded(path) }) + if source.Statements != 6 || source.Covered != 4 || source.Missed != 2 { + t.Fatalf("unexpected source stats: %+v", source) + } +} + +func TestParseDiffAndPatch(t *testing.T) { + t.Parallel() + diff := `diff --git a/core/a.go b/core/a.go +--- a/core/a.go ++++ b/core/a.go +@@ -10,2 +10,3 @@ +-old ++new ++newer + context +@@ -30 +31 @@ +-old ++new +diff --git a/core/a_test.go b/core/a_test.go +--- a/core/a_test.go ++++ b/core/a_test.go +@@ -1 +1 @@ +-old ++new +` + files := parseDiff(diff) + if len(files["core/a.go"]) != 2 { + t.Fatalf("unexpected ranges: %+v", files) + } + if r := files["core/a.go"][0]; r.Start != 10 || r.End != 12 { + t.Fatalf("unexpected first range: %+v", r) + } + + blocks := map[string]block{ + "one": {File: "core/a.go", StartLine: 10, EndLine: 11, NumStmt: 2, Count: 3}, + "two": {File: "core/a.go", StartLine: 12, EndLine: 12, NumStmt: 1, Count: 0}, + "three": {File: "core/a.go", StartLine: 31, EndLine: 31, NumStmt: 2, Count: 0}, + "outside": {File: "core/a.go", StartLine: 50, EndLine: 50, NumStmt: 9, Count: 1}, + } + patch := computePatch(blocks, files, func(path string) bool { return !excluded(path) && !hasTestSuffix(path) }) + if patch.ChangedFiles != 2 || patch.ChangedLines != 5 { + t.Fatalf("unexpected patch counts: %+v", patch) + } + if patch.Statements != 5 || patch.Covered != 2 || patch.Missed != 3 { + t.Fatalf("unexpected patch stats: %+v", patch) + } +} + +func TestLoadProfilesMergesDuplicates(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "coverage.txt") + content := `mode: atomic +github.com/smartcontractkit/chainlink/v2/core/a.go:1.1,1.10 2 0 +github.com/smartcontractkit/chainlink/v2/core/a.go:1.1,1.10 2 5 +github.com/smartcontractkit/chainlink/deployment/b.go:2.1,2.10 1 1 +` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + blocks, err := loadProfiles([]string{path}) + if err != nil { + t.Fatal(err) + } + if len(blocks) != 2 { + t.Fatalf("expected 2 merged blocks, got %d", len(blocks)) + } + for _, b := range blocks { + if b.File == "core/a.go" && b.Count != 5 { + t.Fatalf("duplicate block was not merged with max count: %+v", b) + } + } +} + +func hasTestSuffix(path string) bool { + return len(path) >= 8 && path[len(path)-8:] == "_test.go" +} diff --git a/tools/coverage/patch.sh b/tools/coverage/patch.sh new file mode 100755 index 00000000000..b6f65701fde --- /dev/null +++ b/tools/coverage/patch.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Fast patch-only coverage for the packages touched by a PR. +# +# It intentionally does NOT run the full suite or E2E. It runs the affected +# packages with -short, uses the normal Go build/module caches, computes patch +# coverage with tools/coverage, and updates a sticky PR comment. +# +# Required: +# BASE_SHA PR base SHA (the merge base is derived with git merge-base) +# HEAD_SHA PR head SHA +# Optional: +# OUT output directory (default: coverage-patch) +# PR_NUMBER PR number; when set, the report is commented on the PR +# GH_TOKEN token with pull-requests:write (defaults to gh auth) +# SHORT set to false to run the affected packages without -short +# TEST_TIMEOUT per-package test timeout (default: 10m) +set -euo pipefail + +BASE_SHA="${BASE_SHA:?BASE_SHA is required}" +HEAD_SHA="${HEAD_SHA:?HEAD_SHA is required}" +REPO_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPO_ROOT" + +OUT="${OUT:-coverage-patch}" +case "$OUT" in + /*) ;; + *) OUT="$REPO_ROOT/$OUT" ;; +esac +SHORT="${SHORT:-true}" +TEST_TIMEOUT="${TEST_TIMEOUT:-10m}" +COVDATA_ROOT="$OUT/covdata" +rm -rf "$OUT" +mkdir -p "$OUT" "$COVDATA_ROOT" + +CHANGED="$OUT/changed-files.txt" +git diff --name-only --diff-filter=ACMR "$BASE_SHA...$HEAD_SHA" -- '*.go' 2>/dev/null \ + | grep -vE '(^|/)(mocks|testdata|testutils|testhelpers|cltest|fixtures)/|_test\.go$|\.pb\.go$|_gen\.go$|_codecgen\.go$|^vendor/' \ + > "$CHANGED" || true + +if [[ ! -s "$CHANGED" ]]; then + echo "No changed production Go files." + exit 0 +fi + +# Map every changed file to the nearest Go module and package directory. +ROWS="$OUT/rows.tsv" +: > "$ROWS" +while IFS= read -r file; do + dir="$(dirname "$file")" + while [[ "$dir" != "." && ! -f "$dir/go.mod" ]]; do + dir="$(dirname "$dir")" + done + if [[ -f "$dir/go.mod" ]]; then + module_dir="$REPO_ROOT/$dir" + else + module_dir="$REPO_ROOT" + fi + printf '%s\t%s\n' "$module_dir" "$REPO_ROOT/$(dirname "$file")" >> "$ROWS" +done < "$CHANGED" + +modules="$(cut -f1 "$ROWS" | sort -u)" +tests_failed=0 +ran_any=0 + +while IFS= read -r module_dir; do + [[ -z "$module_dir" ]] && continue + map_file="$OUT/packages-$(echo "$module_dir" | tr '/' '_').tsv" + if ! (cd "$module_dir" && go list -e -f '{{.Dir}} {{.ImportPath}}' ./...) > "$map_file" 2> "$OUT/go-list.log"; then + echo "warning: could not list packages in $module_dir; skipping" >&2 + continue + fi + + packages=() + while IFS= read -r package_dir; do + [[ -z "$package_dir" ]] && continue + import_path="$(awk -F '\t' -v d="$package_dir" '$1 == d { print $2; exit }' "$map_file")" + [[ -z "$import_path" ]] && continue + packages+=("$import_path") + done < <(awk -F '\t' -v m="$module_dir" '$1 == m { print $2 }' "$ROWS" | sort -u) + + # Deduplicate test packages as well as the -coverpkg list. Several changed + # files in one package must not make go test run that package repeatedly. + unique_packages=() + while IFS= read -r import_path; do + [[ -n "$import_path" ]] && unique_packages+=("$import_path") + done < <(printf '%s\n' "${packages[@]}" | sort -u) + if [[ ${#unique_packages[@]} -eq 0 ]]; then + continue + fi + packages=("${unique_packages[@]}") + package_list="$(IFS=,; printf '%s' "${packages[*]}")" + + covdir="$COVDATA_ROOT/$(echo "$module_dir" | tr '/' '_')" + mkdir -p "$covdir" + short_flag=() + [[ "$SHORT" == "true" ]] && short_flag=(-short) + + echo "Testing affected packages in ${module_dir#"$REPO_ROOT"/}:" + echo " $package_list" + if ! (cd "$module_dir" && go test \ + -count=1 \ + -timeout "$TEST_TIMEOUT" \ + "${short_flag[@]}" \ + -cover \ + -covermode=atomic \ + -coverpkg="$package_list" \ + "${packages[@]}" \ + -args -test.gocoverdir="$covdir") >> "$OUT/test.log" 2>&1; then + tests_failed=1 + echo "warning: some affected-package tests failed in ${module_dir#"$REPO_ROOT"/}; see $OUT/test.log" >&2 + fi + ran_any=1 +done <<< "$modules" + +if [[ "$ran_any" == "0" ]]; then + echo "No runnable affected packages." + exit 0 +fi + +covdirs="$(find "$COVDATA_ROOT" -type f -name 'covmeta.*' -exec dirname {} \; 2>/dev/null | sort -u | paste -sd, -)" +if [[ -z "$covdirs" ]]; then + echo "No coverage data was produced for the affected packages." + exit 0 +fi + +go run ./tools/coverage merge --covdata "$covdirs" --out "$OUT/patch.txt" +merge_base="$(git merge-base "$BASE_SHA" "$HEAD_SHA")" +go run ./tools/coverage report \ + --profile "$OUT/patch.txt" \ + --base "$merge_base" \ + --head "$HEAD_SHA" \ + --category "patch (affected packages)" \ + --json "$OUT/patch.json" \ + --markdown "$OUT/patch.md" + +not_measured="$(jq -r '.Patch.Files[] | select(.TestFile == false and .Statements == 0) | .Path' "$OUT/patch.json" 2>/dev/null || true)" +if [[ -n "$not_measured" ]]; then + { + echo + echo "### Not measured" + echo + echo "These changed files had no tests in their own package, so the quick run could not measure them:" + echo + while IFS= read -r path; do + [[ -n "$path" ]] && echo "- \`$path\`" + done <<< "$not_measured" + echo + echo "The nightly full report covers these." + } >> "$OUT/patch.md" +fi + +{ + echo "" + echo + echo "### Quick patch coverage" + echo + if [[ "$SHORT" == "true" ]]; then + echo "Affected-package coverage for \`$HEAD_SHA\`, using \`-short\`. " + else + echo "Affected-package coverage for \`$HEAD_SHA\`. " + fi + echo "This is a fast PR signal, not the global coverage number; the nightly full report is authoritative." + echo + if [[ "$tests_failed" == "1" ]]; then + echo "> Some affected-package tests failed. The coverage below may be incomplete; check the job log." + echo + fi + cat "$OUT/patch.md" +} > "$OUT/comment.md" + +cat "$OUT/comment.md" + +if [[ -n "${PR_NUMBER:-}" && -n "${GITHUB_REPOSITORY:-}" ]]; then + marker="" + comment_id="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" 2>/dev/null \ + | jq -r --arg marker "$marker" '.[] | select(.user.login == "github-actions[bot]") | select(.body | contains($marker)) | .id' 2>/dev/null \ + | head -1 || true)" + if [[ -n "$comment_id" ]]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -F body="@$OUT/comment.md" + else + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$OUT/comment.md" + fi +fi + +if [[ "$tests_failed" == "1" ]]; then + exit 1 +fi diff --git a/tools/coverage/test_badge.py b/tools/coverage/test_badge.py new file mode 100644 index 00000000000..8767ef00c84 --- /dev/null +++ b/tools/coverage/test_badge.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +class BadgeTests(unittest.TestCase): + def test_badge_uses_source_only_and_writes_state(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + report = tmp / "any.json" + svg = tmp / "badge.svg" + state = tmp / "report.json" + report.write_text( + json.dumps( + { + "Global": {"Percent": 54.27}, + "GlobalSourceOnly": {"Percent": 60.92}, + "Patch": {"Percent": 43.84}, + } + ), + encoding="utf-8", + ) + subprocess.run( + [ + sys.executable, + str(HERE / "badge.py"), + str(report), + str(svg), + str(state), + "abc123", + "coverage-report-abc123-1", + "smartcontractkit/chainlink", + "https://example.invalid/run/1", + ], + check=True, + ) + svg_text = svg.read_text(encoding="utf-8") + state_text = json.loads(state.read_text(encoding="utf-8")) + self.assertIn("coverage: 60.9%", svg_text) + self.assertIn("60.9%", svg_text) + self.assertEqual(state_text["global_raw"], 54.27) + self.assertEqual(state_text["global_source_only"], 60.92) + self.assertEqual(state_text["patch"], 43.84) + self.assertEqual(state_text["sha"], "abc123") + + def test_badge_color_changes_with_coverage(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + report = tmp / "any.json" + svg = tmp / "badge.svg" + state = tmp / "report.json" + report.write_text( + json.dumps({"GlobalSourceOnly": {"Percent": 25.0}}), + encoding="utf-8", + ) + subprocess.run( + [ + sys.executable, + str(HERE / "badge.py"), + str(report), + str(svg), + str(state), + "abc123", + "artifact", + "org/repo", + "https://example.invalid/run/1", + ], + check=True, + ) + self.assertIn("#c62828", svg.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main()