From b6a798726734607d7800cb42f153d2c416cd5ab3 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:01:49 +1100 Subject: [PATCH 01/16] feat(ci): collect and publish test coverage artifacts Add covdata collection to unit, integration, deployment, tools, and E2E workflows. Add a coverage collector that merges artifacts, computes global and patch coverage, and publishes a report to the PR. Add a coverage flush hook for instrumented E2E nodes. --- .github/actions/collect-coverage/action.yml | 36 ++ .github/workflows/ccip-system-tests.yaml | 24 + .github/workflows/ci-core.yml | 36 ++ .github/workflows/ci-deployments.yml | 20 + .github/workflows/ci-tools.yml | 17 +- .github/workflows/coverage-collector.yml | 73 +++ .github/workflows/cre-local-env-tests.yaml | 15 + .github/workflows/cre-mixed-env-tests.yaml | 25 + .../cre-regression-system-tests.yaml | 25 + .github/workflows/cre-system-tests.yaml | 25 + .../workflows/integration-in-memory-tests.yml | 27 +- .github/workflows/integration-tests.yml | 18 +- .../workflows/legacy-non-functional-tests.yml | 16 + .github/workflows/legacy-system-tests.yml | 16 + GNUmakefile | 2 +- core/chainlink.Dockerfile | 5 +- core/coverage_flush.go | 27 + tools/bin/go_core_race_tests | 13 +- tools/bin/go_core_scripts_tests | 8 +- tools/bin/go_core_tests | 13 +- tools/bin/go_core_tests_integration | 17 +- tools/bin/go_deployment_tests | 14 +- tools/coverage/collect.sh | 149 +++++ tools/coverage/main.go | 552 ++++++++++++++++++ tools/coverage/main_test.go | 107 ++++ 25 files changed, 1254 insertions(+), 26 deletions(-) create mode 100644 .github/actions/collect-coverage/action.yml create mode 100644 .github/workflows/coverage-collector.yml create mode 100644 core/coverage_flush.go create mode 100755 tools/coverage/collect.sh create mode 100644 tools/coverage/main.go create mode 100644 tools/coverage/main_test.go diff --git a/.github/actions/collect-coverage/action.yml b/.github/actions/collect-coverage/action.yml new file mode 100644 index 00000000000..2646fdf6279 --- /dev/null +++ b/.github/actions/collect-coverage/action.yml @@ -0,0 +1,36 @@ +name: Collect container coverage +description: Copy Go coverage data from running or stopped containers. + +inputs: + destination: + description: Host directory that receives per-container coverage directories. + required: true + image: + description: Optional container image filter, for example chainlink-cover:local. + required: false + default: "" + +runs: + using: composite + steps: + - name: Copy coverage data + shell: bash + env: + DESTINATION: ${{ inputs.destination }} + IMAGE_FILTER: ${{ inputs.image }} + run: | + set -euo pipefail + mkdir -p "${DESTINATION}" + for container in $(docker ps -aq); do + image="$(docker inspect -f '{{.Config.Image}}' "${container}" 2>/dev/null || true)" + if [[ -n "${IMAGE_FILTER}" && "${image}" != "${IMAGE_FILTER}"* ]]; then + continue + fi + if [[ "${image}" == *"-coverage"* ]]; then + docker kill --signal=SIGUSR1 "${container}" >/dev/null 2>&1 || true + sleep 0.2 + fi + if docker cp "${container}:/var/tmp/go-coverage/." "${DESTINATION}/${container}" 2>/dev/null; then + echo "Copied coverage from ${container} (${image})" + fi + done diff --git a/.github/workflows/ccip-system-tests.yaml b/.github/workflows/ccip-system-tests.yaml index 191ba84227c..9806258df63 100644 --- a/.github/workflows/ccip-system-tests.yaml +++ b/.github/workflows/ccip-system-tests.yaml @@ -364,6 +364,30 @@ jobs: # unique name per matrix job to avoid 409 conflict when multiple jobs upload in the same workflow run artifact-name: ${{ matrix.tests.test_id }}_test_logs + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index e0190836946..13ec3c83f9c 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -323,6 +323,7 @@ jobs: setup-sui: "false" install-loopps: "false" build-cache-write: always + coverage: "true" - cmd: go_core_tests_integration os: @@ -341,6 +342,7 @@ jobs: setup-aptos: "false" setup-sui: "false" build-cache-write: always + coverage: "true" - cmd: go_core_fuzz os: @@ -379,6 +381,7 @@ jobs: install-loopps: "false" build-cache-version: go_core_race_tests build-cache-write: always + coverage: "true" name: Core Tests (${{ matrix.type.cmd }}) # Be careful modifying the job name, as it is used to fetch the job URL # We don't directly merge dependabot PRs, so let's not waste the resources @@ -493,6 +496,8 @@ jobs: shell: bash env: CL_DATABASE_URL: ${{ env.DB_URL }} + COVERAGE: ${{ matrix.type.coverage || 'false' }} + GOCOVERDIR: ${{ github.workspace }}/.covdata/${{ matrix.type.cmd }} OUTPUT_FILE: ./output.txt PRODUCE_JUNIT_XML: ${{ matrix.type.trunk-auto-quarantine }} TRUNK_AUTO_QUARANTINE: ${{ matrix.type.trunk-auto-quarantine }} @@ -564,6 +569,26 @@ jobs: ./junit.xml retention-days: 7 + - name: Store coverage artifacts + if: + ${{ always() && matrix.type.should-run == 'true' && matrix.type.coverage + == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: coverage-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/${{ matrix.type.cmd }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() && matrix.type.should-run == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: test-results-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }} + path: junit.xml + retention-days: 7 + if-no-files-found: ignore + - name: Notify Slack on Race Test Failure if: | failure() && @@ -599,6 +624,8 @@ jobs: - name: Run Tests env: OUTPUT_FILE: ./output.txt + COVERAGE: "true" + GOCOVERDIR: ${{ github.workspace }}/.covdata/scripts run: ./tools/bin/go_core_scripts_tests ./... - name: Store test report artifacts @@ -611,6 +638,15 @@ jobs: ./coverage.txt retention-days: 7 + - name: Store coverage artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-core-scripts-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/scripts + retention-days: 7 + if-no-files-found: warn + clean: name: Clean Go Tidy & Generate if: ${{ github.actor != 'dependabot[bot]' }} diff --git a/.github/workflows/ci-deployments.yml b/.github/workflows/ci-deployments.yml index 3d33e0a9cab..848a76b9060 100644 --- a/.github/workflows/ci-deployments.yml +++ b/.github/workflows/ci-deployments.yml @@ -199,6 +199,8 @@ jobs: shell: bash env: CL_DATABASE_URL: ${{ env.DB_URL }} + COVERAGE: "true" + GOCOVERDIR: ${{ github.workspace }}/.covdata/deployment-shard-${{ matrix.shard-id }} OUTPUT_FILE: ./output.txt PRODUCE_JUNIT_XML: "true" TRUNK_AUTO_QUARANTINE: "true" @@ -238,6 +240,24 @@ jobs: ./deployment/output-short.txt retention-days: 7 + - name: Store coverage artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-deployment-shard-${{ matrix.shard-id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/deployment-shard-${{ matrix.shard-id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-deployment-shard-${{ matrix.shard-id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: junit.xml + retention-days: 7 + if-no-files-found: ignore + deployment-tests-result: # Do not modify this job name, it is used by branch protection rules name: Core Tests (go_core_deployment_tests) diff --git a/.github/workflows/ci-tools.yml b/.github/workflows/ci-tools.yml index e05f54b96bd..0def74d55ff 100644 --- a/.github/workflows/ci-tools.yml +++ b/.github/workflows/ci-tools.yml @@ -72,4 +72,19 @@ jobs: - name: Run tests working-directory: ${{ matrix.target.dir }} - run: gotestsum --format github-actions -- ${{ matrix.target.packages }} + env: + GOCOVERDIR: ${{ github.workspace }}/.covdata/tools-${{ strategy.job-index }} + run: | + mkdir -p "$GOCOVERDIR" + gotestsum --format github-actions -- ${{ matrix.target.packages }} \ + -cover -covermode=atomic -coverpkg=./... \ + -args -test.gocoverdir="$GOCOVERDIR" + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-tools-${{ matrix.target.name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/tools-${{ strategy.job-index }} + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml new file mode 100644 index 00000000000..50da9777b17 --- /dev/null +++ b/.github/workflows/coverage-collector.yml @@ -0,0 +1,73 @@ +name: Coverage Collector + +on: + workflow_run: + workflows: + - CI Core + - Deployment Tests + - Integration In-Memory Tests + - Integration Tests + - CI Tools + - CRE System Tests + - CRE Regression System Tests + - CRE Mixed-Env Tests + - CRE Local Environment Tests + - CCIP System Tests + - Legacy System Tests + - Legacy Non Functional Tests + types: [completed] + # TEMP: manual trigger for pre-merge validation. Remove before opening the PR. + workflow_dispatch: + inputs: + sha: + description: "Commit SHA to collect" + required: true + type: string + pr: + description: "Optional PR number" + required: false + type: string + default: "" + +permissions: + actions: read + contents: read + pull-requests: write + checks: write + +concurrency: + group: coverage-${{ github.event.workflow_run.head_sha || inputs.sha }} + cancel-in-progress: true + +jobs: + collect: + name: Collect coverage + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Collect and publish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET_SHA: ${{ inputs.sha || github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ inputs.pr || (github.event.workflow_run.pull_requests[0].number || '') }} + run: ./tools/coverage/collect.sh + + - name: Upload merged report + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-report-${{ inputs.sha || github.event.workflow_run.head_sha }}-${{ github.run_id }} + path: coverage-report + retention-days: 30 + if-no-files-found: warn diff --git a/.github/workflows/cre-local-env-tests.yaml b/.github/workflows/cre-local-env-tests.yaml index 5a5fb7c280a..bc468ea4f42 100644 --- a/.github/workflows/cre-local-env-tests.yaml +++ b/.github/workflows/cre-local-env-tests.yaml @@ -324,6 +324,21 @@ jobs: docker logs "$c" > "logs/${c}.log" 2>&1 done + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-local-cre + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-local-cre-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-local-cre + retention-days: 7 + if-no-files-found: warn + - name: Upload all artifacts as single package if: failure() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/cre-mixed-env-tests.yaml b/.github/workflows/cre-mixed-env-tests.yaml index 2d5cd6441c6..0c3dee44e9c 100644 --- a/.github/workflows/cre-mixed-env-tests.yaml +++ b/.github/workflows/cre-mixed-env-tests.yaml @@ -454,6 +454,31 @@ jobs: trunk-upload-only: ${{ env.ENABLE_AUTO_QUARANTINE != 'true' }} artifact-name: mixed-env-${{ matrix.tests.test_id }}_test_logs + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + image: ${{ env.CHAINLINK_IMAGE_FULL }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit-report.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/cre-regression-system-tests.yaml b/.github/workflows/cre-regression-system-tests.yaml index 42bbb525b77..27133e87bd0 100644 --- a/.github/workflows/cre-regression-system-tests.yaml +++ b/.github/workflows/cre-regression-system-tests.yaml @@ -271,6 +271,31 @@ jobs: artifact-name: ${{ matrix.tests.test_id }}_test_logs trunk-test-collection-id: qt0j1cVm # chainlink-e2e + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + image: ${{ env.CHAINLINK_IMAGE_FULL }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit-report-regression.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 056f9866053..50ad8d8decd 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -411,6 +411,31 @@ jobs: artifact-name: ${{ matrix.tests.test_id }}_test_logs trunk-test-collection-id: qt0j1cVm # chainlink-e2e + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }} + image: ${{ env.CHAINLINK_IMAGE_FULL }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.tests.test_id }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit-report.xml + retention-days: 7 + if-no-files-found: ignore + - name: Show Docker containers status if: failure() || cancelled() shell: bash diff --git a/.github/workflows/integration-in-memory-tests.yml b/.github/workflows/integration-in-memory-tests.yml index f1e007a065d..ffbe9572c96 100644 --- a/.github/workflows/integration-in-memory-tests.yml +++ b/.github/workflows/integration-in-memory-tests.yml @@ -233,7 +233,9 @@ jobs: run: | set -euo pipefail mkdir -p integration-tests/bin - (cd integration-tests && go test -c -ldflags="-s -w" -o ./bin/ccip-inmemory.test ./smoke/ccip) + (cd integration-tests && go test -c -cover -covermode=atomic \ + -coverpkg=github.com/smartcontractkit/chainlink/v2/...,github.com/smartcontractkit/chainlink/deployment/... \ + -ldflags="-s -w" -o ./bin/ccip-inmemory.test ./smoke/ccip) # Pre-build preparetest so matrix jobs skip ~1min of module downloads + compile each go build -o integration-tests/bin/preparetest ./core/store/cmd/preparetest GOBIN="$PWD/integration-tests/bin" go install gotest.tools/gotestsum@v1.13.0 @@ -344,12 +346,15 @@ jobs: TEST_RUN: ${{ matrix.tests.test }} TEST_TIMEOUT: ${{ matrix.tests.timeout }} PARALLEL: ${{ matrix.tests.parallel }} + GOCOVERDIR: ${{ github.workspace }}/.covdata/inmemory-${{ strategy.job-index }} run: | + mkdir -p "${GOCOVERDIR}" gotestsum --jsonfile=/tmp/gotest.log --junitfile=/tmp/junit.xml \ --format=github-actions --raw-command -- \ go tool test2json -t -p github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip \ ../../bin/ccip-inmemory.test -test.v -test.run "${TEST_RUN}" \ - -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL}" + -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL}" \ + -test.gocoverdir="${GOCOVERDIR}" - name: Ensure Junit XML file exists if: always() @@ -371,6 +376,24 @@ jobs: trunk-token: ${{ secrets.TRUNK_API_KEY }} artifact-name: ${{ matrix.tests.test_id }}_test_logs + - name: Store coverage artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-inmemory-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/inmemory-${{ strategy.job-index }} + retention-days: 7 + if-no-files-found: warn + + - name: Store test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: test-results-inmemory-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/junit.xml + retention-days: 7 + if-no-files-found: ignore + - name: Collect Diagnostic Logs on Failure if: ${{ steps.run-tests.outcome != 'success' || cancelled() }} shell: bash diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 955230c1dad..a851ee29128 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,6 +30,11 @@ on: required: false type: string default: "chainlink" + coverage: + description: "Build the Chainlink image with Go coverage instrumentation and use the -coverage tag." + required: false + type: boolean + default: false # Only run 1 of this workflow at a time per PR, but don't cancel runs on develop concurrency: @@ -312,7 +317,7 @@ jobs: uses: smartcontractkit/.github/actions/ecr-image-exists@ecr-image-exists/0.2.0 with: repository: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} - tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }} + tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && inputs.coverage && '-coverage' || '' }} aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Explain build/no-build decision @@ -358,7 +363,8 @@ jobs: # Rebuild the final-stage apt layer daily so `apt-get upgrade` isn't served stale from cache. docker-additional-build-args: | APT_CACHE_BUST=${{ steps.apt-cache-bust.outputs.key }} - image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }} + ${{ matrix.image.name == '' && format('GO_COVER_FLAG={0}', inputs.coverage) || '' }} + image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && inputs.coverage && '-coverage' || '' }} dockerfile: ${{ matrix.image.dockerfile }} docker-registry-url: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com docker-repository-name: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} @@ -501,7 +507,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} secrets: inherit run-core-cre-e2e-regression-tests: @@ -519,7 +525,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} secrets: inherit run-core-cre-mixed-env-tests: @@ -537,7 +543,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} # Base-branch commit this PR is built on top of, so the mixed-env baseline is the exact # branch the PR image derives from (develop or a release branch) and the 2-2 split # isolates only the PR's changes. Empty for non-PR events; the workflow then falls back @@ -845,7 +851,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} baseline_ref: ${{ github.event.pull_request.base.sha }} base_ref_name: ${{ github.event.pull_request.base.ref }} secrets: inherit diff --git a/.github/workflows/legacy-non-functional-tests.yml b/.github/workflows/legacy-non-functional-tests.yml index 7c6e647ca2c..836d5685960 100644 --- a/.github/workflows/legacy-non-functional-tests.yml +++ b/.github/workflows/legacy-non-functional-tests.yml @@ -188,6 +188,22 @@ jobs: echo "Running tests for: $CHAINLINK_IMAGE, test: $TESTCMD" eval "$TESTCMD" + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.logs_archive_name }} + image: ${{ env.CHAINLINK_IMAGE }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.logs_archive_name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.logs_archive_name }} + retention-days: 7 + if-no-files-found: warn + - name: Upload Logs if: always() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/legacy-system-tests.yml b/.github/workflows/legacy-system-tests.yml index c7bbd997d74..c984a9034b6 100644 --- a/.github/workflows/legacy-system-tests.yml +++ b/.github/workflows/legacy-system-tests.yml @@ -257,6 +257,22 @@ jobs: echo "Running tests for: $CHAINLINK_IMAGE, test: $TESTCMD" eval "$TESTCMD" + - name: Collect coverage + if: ${{ always() }} + uses: $/.github/actions/collect-coverage + with: + destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.logs_archive_name }} + image: ${{ env.CHAINLINK_IMAGE }} + + - name: Upload coverage + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-e2e-${{ matrix.logs_archive_name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .covdata/e2e-${{ matrix.logs_archive_name }} + retention-days: 7 + if-no-files-found: warn + - name: Upload Logs if: always() uses: actions/upload-artifact@v7 diff --git a/GNUmakefile b/GNUmakefile index 1f178d09597..94b3c8ceca0 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -84,7 +84,7 @@ install-chainlink-dev: operator-ui ## Install the chainlink binary. .PHONY: install-chainlink-cover install-chainlink-cover: operator-ui ## Install the chainlink binary with cover flag. - go install -cover $(GOFLAGS) . + go install -tags coverage -cover -covermode=atomic $(GOFLAGS) . .PHONY: chainlink chainlink: ## Build the chainlink binary. diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index e7024c030ce..f7c71ebc509 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -97,11 +97,14 @@ FROM deps AS build-chainlink ARG COMMIT_SHA ARG VERSION_TAG ARG CL_IS_PROD_BUILD=true +ARG GO_COVER_FLAG=false RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ mkdir -p /gobins && \ - if [ "$CL_IS_PROD_BUILD" = "false" ]; then \ + if [ "$GO_COVER_FLAG" = "true" ]; then \ + GOBIN=/gobins make install-chainlink-cover; \ + elif [ "$CL_IS_PROD_BUILD" = "false" ]; then \ GOBIN=/gobins make install-chainlink-dev; \ else \ GOBIN=/gobins make install-chainlink; \ diff --git a/core/coverage_flush.go b/core/coverage_flush.go new file mode 100644 index 00000000000..4b2f8639f65 --- /dev/null +++ b/core/coverage_flush.go @@ -0,0 +1,27 @@ +//go:build coverage + +package core + +import ( + "os" + "os/signal" + "runtime/coverage" + "syscall" +) + +// init installs a SIGUSR1 handler used by E2E coverage collection. The handler +// flushes coverage counters while the process is still running, so coverage is +// not lost when a container is later stopped with SIGKILL. +func init() { + dir := os.Getenv("GOCOVERDIR") + if dir == "" { + return + } + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGUSR1) + go func() { + for range ch { + _ = coverage.WriteCountersDir(dir) + } + }() +} diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index 467dda4e92f..83ecf24ce67 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -3,10 +3,21 @@ set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} TIMEOUT="${TIMEOUT:-10s}" COUNT="${COUNT:-5}" +COVERAGE=${COVERAGE:-false} + +COVERAGE_FLAGS="" +COVERAGE_ARGS="" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + COVERAGE_FLAGS="-covermode=atomic -cover -coverpkg=${COVERPKG:-./...}" + COVERAGE_ARGS="-args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi echo "Test execution results: ---------------------" echo "" -GORACE="log_path=$PWD/race" go test -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE +GORACE="log_path=$PWD/race" go test -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $COVERAGE_FLAGS "$1" $COVERAGE_ARGS | cat > $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} diff --git a/tools/bin/go_core_scripts_tests b/tools/bin/go_core_scripts_tests index e4380264215..11d346f298f 100755 --- a/tools/bin/go_core_scripts_tests +++ b/tools/bin/go_core_scripts_tests @@ -5,14 +5,18 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} EXTRA_FLAGS="" +COVERAGE=${COVERAGE:-false} cd ./core/scripts || exit go mod download echo "Test execution results: ---------------------" echo "" -if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then - EXTRA_FLAGS="-covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/coverage} + mkdir -p "$COVERAGE_DIR" + EXTRA_FLAGS="-covermode=atomic -cover -coverpkg=${COVERPKG:-./...} -args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" fi go test ./... $EXTRA_FLAGS | tee $OUTPUT_FILE | grep -Ev '\[no test files\]|\[no tests to run\]' EXITCODE=${PIPESTATUS[0]} diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index f2ad773f820..d4473a02cba 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -6,6 +6,7 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"} GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"15m"} +PACKAGES=${PACKAGES:-"./..."} echo "Installing gotestsum..." go install gotest.tools/gotestsum@v1.13.0 @@ -16,8 +17,6 @@ echo "Running go_core_tests for event: $GITHUB_EVENT_NAME" GO_TEST_FLAGS="-timeout=${GO_TEST_TIMEOUT}" if [[ "$GITHUB_EVENT_NAME" == 'push' || "$GITHUB_EVENT_NAME" == 'workflow_dispatch' || "$GITHUB_EVENT_NAME" == 'schedule' ]]; then GO_TEST_FLAGS="$GO_TEST_FLAGS -count=1" -elif [[ "$GITHUB_EVENT_NAME" == 'schedule' ]]; then - GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge_group' ]]; then RERUN_FLAGS="" if [[ "$TRUNK_AUTO_QUARANTINE" != "true" ]]; then @@ -28,6 +27,14 @@ elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge fi fi +COVERAGE=${COVERAGE:-false} +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -cover -coverpkg=${COVERPKG:-./...} -args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi + if [[ "$PRODUCE_JUNIT_XML" == "true" ]]; then JUNIT_FLAG="--junitfile=$JUNIT_FILE" fi @@ -45,7 +52,7 @@ gotestsum \ --format='standard-quiet' \ --hide-summary=skipped \ $RERUN_FLAGS \ - --packages='./...' \ + --packages="$PACKAGES" \ --jsonfile "$OUTPUT_FILE" \ --post-run-command "sh -c \"echo; echo 'Slowest tests'; echo '----------------------------------------------'; echo; gotestsum tool slowest --num 20 --jsonfile ${OUTPUT_FILE}\"" \ "$JUNIT_FLAG" \ diff --git a/tools/bin/go_core_tests_integration b/tools/bin/go_core_tests_integration index aaaa7fe7769..3850dc96199 100755 --- a/tools/bin/go_core_tests_integration +++ b/tools/bin/go_core_tests_integration @@ -19,11 +19,6 @@ echo "Running go_core_tests_integration for event: $GITHUB_EVENT_NAME" GO_TEST_FLAGS="-timeout=${GO_TEST_TIMEOUT}" if [[ "$GITHUB_EVENT_NAME" == 'push' ]]; then GO_TEST_FLAGS="$GO_TEST_FLAGS -count=1" -elif [[ "$GITHUB_EVENT_NAME" == 'schedule' ]]; then - # Experimental code to minimize size of this coverage report - # ALL_IMPORTS=$(go list -f '{{ join .Imports "\n" }}' $INTEGRATION_TEST_DIRS | sort -u) - # COVERPKG_DIRS=$(echo "$INTEGRATION_TEST_DIRS $ALL_IMPORTS" | grep "smartcontractkit/chainlink" | tr '\n' ',') - GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge_group' ]]; then RERUN_FLAGS="" if [[ "$TRUNK_AUTO_QUARANTINE" != "true" ]]; then @@ -34,6 +29,16 @@ elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge fi fi +COVERAGE=${COVERAGE:-false} +COVERAGE_ARGS="" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -cover -coverpkg=${COVERPKG:-./...}" + COVERAGE_ARGS="-args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi + if [[ "$PRODUCE_JUNIT_XML" == "true" ]]; then JUNIT_FLAG="--junitfile=$JUNIT_FILE" fi @@ -66,7 +71,7 @@ gotestsum \ --jsonfile "$OUTPUT_FILE" \ --post-run-command "sh -c \"echo; echo 'Slowest tests'; echo '----------------------------------------------'; echo; gotestsum tool slowest --num 20 --jsonfile ${OUTPUT_FILE}\"" \ "$JUNIT_FLAG" \ - -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED + -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED $COVERAGE_ARGS EXITCODE=${PIPESTATUS[0]} diff --git a/tools/bin/go_deployment_tests b/tools/bin/go_deployment_tests index b5c350c65d7..9dca828cc34 100755 --- a/tools/bin/go_deployment_tests +++ b/tools/bin/go_deployment_tests @@ -32,8 +32,6 @@ GO_TEST_FLAGS="-timeout=${GO_TEST_TIMEOUT}" GO_TEST_FLAGS="$GO_TEST_FLAGS -p=2 -parallel=1" if [[ "$GITHUB_EVENT_NAME" == 'push' || "$GITHUB_EVENT_NAME" == 'workflow_dispatch' || "$GITHUB_EVENT_NAME" == 'schedule' ]]; then GO_TEST_FLAGS="$GO_TEST_FLAGS -count=1" -elif [[ "$GITHUB_EVENT_NAME" == 'schedule' ]]; then - GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -coverpkg=./... -coverprofile=coverage.txt" elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge_group' ]]; then RERUN_FLAGS="" if [[ "$TRUNK_AUTO_QUARANTINE" != "true" ]]; then @@ -44,6 +42,16 @@ elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'merge fi fi +COVERAGE=${COVERAGE:-false} +COVERAGE_ARGS="" +if [[ "$COVERAGE" == "true" ]]; then + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} + mkdir -p "$COVERAGE_DIR" + GO_TEST_FLAGS="$GO_TEST_FLAGS -covermode=atomic -cover -coverpkg=${COVERPKG:-./...}" + COVERAGE_ARGS="-args -test.gocoverdir=$COVERAGE_DIR" + echo "Coverage enabled: $COVERAGE_DIR" +fi + if [[ "$PRODUCE_JUNIT_XML" == "true" ]]; then JUNIT_FLAG="--junitfile=$JUNIT_FILE" fi @@ -88,7 +96,7 @@ gotestsum \ --jsonfile "$OUTPUT_FILE" \ --post-run-command "sh -c \"echo; echo 'Slowest tests'; echo '----------------------------------------------'; echo; gotestsum tool slowest --num 20 --jsonfile ${OUTPUT_FILE}\"" \ "$JUNIT_FLAG" \ - -- $GO_TEST_FLAGS "${SHARD_PACKAGES[@]}" + -- $GO_TEST_FLAGS "${SHARD_PACKAGES[@]}" $COVERAGE_ARGS EXITCODE=${PIPESTATUS[0]} diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh new file mode 100755 index 00000000000..8a9831ff818 --- /dev/null +++ b/tools/coverage/collect.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Collect coverage artifacts for a commit and publish a PR report. +# +# Required environment: +# GITHUB_REPOSITORY owner/repo +# GITHUB_TOKEN token with actions:read, pull-requests:write +# TARGET_SHA commit whose workflow runs should be collected +# +# Optional: +# PR_NUMBER PR number; inferred from TARGET_SHA when empty +# OUTPUT_DIR report output directory (default: coverage-report) +set -euo pipefail + +REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +TARGET_SHA="${TARGET_SHA:?TARGET_SHA is required}" +OUT="${OUTPUT_DIR:-coverage-report}" +ARTIFACTS="${OUT}/artifacts" +mkdir -p "$ARTIFACTS" + +if [[ -z "${PR_NUMBER:-}" ]]; then + PR_NUMBER="$(gh api "repos/${REPO}/commits/${TARGET_SHA}/pulls" --jq '.[0].number // empty' 2>/dev/null || true)" +fi +BASE_SHA="" +if [[ -n "${PR_NUMBER:-}" ]]; then + BASE_SHA="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.base.sha' 2>/dev/null || true)" +fi + +if [[ -n "$BASE_SHA" ]]; then + git fetch --no-tags origin "$BASE_SHA" "$TARGET_SHA" >/dev/null 2>&1 || true +fi + +echo "Collecting coverage for ${REPO}@${TARGET_SHA} (PR ${PR_NUMBER:-none}, base ${BASE_SHA:-none})" + +runs="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 100 --json databaseId --jq '.[].databaseId')" +for run in $runs; do + names="$(gh api "repos/${REPO}/actions/runs/${run}/artifacts?per_page=100" --jq '.artifacts[].name' 2>/dev/null | grep -E '^(coverage-(core|deployment|inmemory|tools|e2e)-|test-results-)' || true)" + while IFS= read -r name; do + [[ -z "$name" ]] && continue + dest="${ARTIFACTS}/${run}/${name}" + mkdir -p "$dest" + echo "Downloading ${name} from run ${run}" + gh run download "$run" --repo "$REPO" -n "$name" -D "$dest" || true + done <<< "$names" +done + +all_dirs="" +unit_dirs="" +integration_dirs="" +e2e_dirs="" + +while IFS= read -r meta; do + [[ -z "$meta" ]] && continue + dir="$(dirname "$meta")" + all_dirs="${all_dirs}${all_dirs:+,}${dir}" + case "$meta" in + */coverage-core-go_core_tests-*/*|*/coverage-core-go_core_race_tests-*/*|*/coverage-core-scripts-*/*|*/coverage-tools-*/*) + unit_dirs="${unit_dirs}${unit_dirs:+,}${dir}" ;; + */coverage-core-go_core_tests_integration-*/*|*/coverage-deployment-*/*|*/coverage-inmemory-*/*) + integration_dirs="${integration_dirs}${integration_dirs:+,}${dir}" ;; + */coverage-e2e-*/*) + e2e_dirs="${e2e_dirs}${e2e_dirs:+,}${dir}" ;; + esac +done < <(find "$ARTIFACTS" -type f -name 'covmeta.*' 2>/dev/null | sort -u) + +merge_and_report() { + local category="$1" dirs="$2" + [[ -z "$dirs" ]] && return 0 + echo "Merging ${category}: ${dirs}" + go run ./tools/coverage merge --covdata "$dirs" --out "${OUT}/${category}.txt" + local args=(--profile "${OUT}/${category}.txt" --category "$category" + --json "${OUT}/${category}.json" --markdown "${OUT}/${category}.md") + if [[ -n "$BASE_SHA" ]]; then + args+=(--base "$BASE_SHA" --head "$TARGET_SHA") + fi + go run ./tools/coverage report "${args[@]}" +} + +merge_and_report unit "$unit_dirs" +merge_and_report integration "$integration_dirs" +merge_and_report e2e "$e2e_dirs" +merge_and_report any "$all_dirs" + +test_xmls="$(find "$ARTIFACTS" -type f -name '*.xml' -path '*test-results-*' 2>/dev/null | sort -u)" +if [[ -n "$test_xmls" ]]; then + python3 - "$OUT/test-results.json" $test_xmls <<'PY' +import json, sys, xml.etree.ElementTree as ET +files = sys.argv[2:] +totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0, "files": len(files)} +for path in files: + try: + root = ET.parse(path).getroot() + except Exception: + continue + suites = root.findall("testsuite") if root.tag == "testsuites" else [root] + for suite in suites: + for key in ("tests", "failures", "errors", "skipped"): + try: + totals[key] += int(suite.get(key, "0")) + except ValueError: + pass +with open(sys.argv[1], "w", encoding="utf-8") as f: + json.dump(totals, f) +PY +fi + +{ + echo "" + echo "# Coverage report for \`${TARGET_SHA:0:9}\`" + echo + echo "Commit: \`${TARGET_SHA}\`" + if [[ -n "${PR_NUMBER:-}" ]]; then echo "PR: #${PR_NUMBER}"; fi + if [[ -n "$BASE_SHA" ]]; then echo "Base: \`${BASE_SHA}\`"; fi + echo + echo "Targets: project coverage should not drop more than 1%; patch target is 80%. This report is informational." + echo + if [[ -f "${OUT}/test-results.json" ]]; then + echo "### Test results" + echo + echo '```json' + cat "${OUT}/test-results.json" + echo + echo '```' + echo + fi + for category in any unit integration e2e; do + if [[ -f "${OUT}/${category}.md" ]]; then + cat "${OUT}/${category}.md" + echo + fi + done +} > "${OUT}/summary.md" + +cat "${OUT}/summary.md" + +if [[ -n "${PR_NUMBER:-}" ]]; then + marker="" + comment_id="$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \ + | jq -r --arg marker "$marker" '.[] | select(.body | contains($marker)) | .id' 2>/dev/null \ + | head -1 || true)" + if [[ -n "$comment_id" ]]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body="@${OUT}/summary.md" + else + gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "${OUT}/summary.md" + fi +fi + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + cat "${OUT}/summary.md" >> "$GITHUB_STEP_SUMMARY" +fi diff --git a/tools/coverage/main.go b/tools/coverage/main.go new file mode 100644 index 00000000000..13785af3430 --- /dev/null +++ b/tools/coverage/main.go @@ -0,0 +1,552 @@ +// Command coverage computes Go coverage summaries and PR patch coverage. +// +// It is intentionally dependency-free so it can run in CI jobs that only have +// the Go toolchain and a coverage profile. +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" +) + +type block struct { + File string + StartLine int + StartCol int + EndLine int + EndCol int + NumStmt int + Count int64 +} + +type lineRange struct { + Start int + End int +} + +type filePatch struct { + Path string + TestFile bool + Added []lineRange + Statements int + Covered int + Missed int + MissedSpans []string +} + +type stats struct { + Statements int + Covered int + Missed int + Percent float64 +} + +type patchStats struct { + ChangedFiles int + ChangedLines int + Statements int + Covered int + Missed int + Percent float64 + Files []filePatch +} + +type report struct { + Category string + Profile []string + Base string + Head string + Global stats + GlobalSourceOnly stats + Patch *patchStats + PatchSourceOnly *patchStats +} + +var modulePrefixes = []string{ + "github.com/smartcontractkit/chainlink/v2/", + "github.com/smartcontractkit/chainlink/", +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + var err error + switch os.Args[1] { + case "report": + err = runReport(os.Args[2:]) + case "merge": + err = runMerge(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + err = fmt.Errorf("unknown subcommand %q", os.Args[1]) + } + if err != nil { + fmt.Fprintln(os.Stderr, "coverage:", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, `usage: + coverage report --profile file[,file...] [--base SHA --head SHA] [--repo DIR] + [--category NAME] [--json FILE] [--markdown FILE] + [--min-global PCT] [--min-patch PCT] + coverage merge --covdata dir[,dir...] --out coverage.txt`) +} + +func runMerge(args []string) error { + fs := flag.NewFlagSet("merge", flag.ExitOnError) + covdata := fs.String("covdata", "", "comma-separated covdata directories") + out := fs.String("out", "coverage.txt", "text profile output path") + if err := fs.Parse(args); err != nil { + return err + } + dirs := splitList(*covdata) + if len(dirs) == 0 { + return fmt.Errorf("--covdata is required") + } + for _, dir := range dirs { + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + return fmt.Errorf("covdata directory %q is not accessible", dir) + } + } + tmp, err := os.MkdirTemp("", "coverage-merge-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + merge := exec.Command("go", "tool", "covdata", "merge", "-i="+strings.Join(dirs, ","), "-o="+tmp) + merge.Stdout = os.Stdout + merge.Stderr = os.Stderr + if err := merge.Run(); err != nil { + return fmt.Errorf("covdata merge: %w", err) + } + textfmt := exec.Command("go", "tool", "covdata", "textfmt", "-i="+tmp, "-o="+*out) + textfmt.Stdout = os.Stdout + textfmt.Stderr = os.Stderr + if err := textfmt.Run(); err != nil { + return fmt.Errorf("covdata textfmt: %w", err) + } + return nil +} + +func runReport(args []string) error { + fs := flag.NewFlagSet("report", flag.ExitOnError) + profiles := fs.String("profile", "", "comma-separated text coverage profiles") + base := fs.String("base", "", "base git revision for patch coverage") + head := fs.String("head", "", "head git revision for patch coverage") + repo := fs.String("repo", ".", "repository root") + category := fs.String("category", "all", "coverage category label") + jsonOut := fs.String("json", "", "JSON output path (default stdout when --markdown is empty)") + mdOut := fs.String("markdown", "", "Markdown output path") + minGlobal := fs.Float64("min-global", -1, "fail if global coverage is below this percentage") + minPatch := fs.Float64("min-patch", -1, "fail if patch coverage is below this percentage") + if err := fs.Parse(args); err != nil { + return err + } + profilePaths := splitList(*profiles) + if len(profilePaths) == 0 { + return fmt.Errorf("--profile is required") + } + + blocks, err := loadProfiles(profilePaths) + if err != nil { + return err + } + rep := report{ + Category: *category, + Profile: profilePaths, + Base: *base, + Head: *head, + } + rep.Global = summarize(blocks, func(string) bool { return true }) + rep.GlobalSourceOnly = summarize(blocks, func(path string) bool { return !excluded(path) }) + + if *base != "" || *head != "" { + if *base == "" || *head == "" { + return fmt.Errorf("--base and --head must be provided together") + } + diff, err := gitDiff(*repo, *base, *head) + if err != nil { + return err + } + rep.Patch = computePatch(blocks, diff, func(string) bool { return true }) + rep.PatchSourceOnly = computePatch(blocks, diff, func(path string) bool { + return !excluded(path) && !strings.HasSuffix(path, "_test.go") + }) + } + + md := markdown(rep) + if *mdOut != "" { + if err := os.WriteFile(*mdOut, []byte(md), 0o644); err != nil { + return err + } + } + if *jsonOut != "" { + data, err := json.MarshalIndent(rep, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(*jsonOut, append(data, '\n'), 0o644); err != nil { + return err + } + } + if *mdOut == "" && *jsonOut == "" { + fmt.Print(md) + } + + var failures []string + if *minGlobal >= 0 && rep.Global.Percent < *minGlobal { + failures = append(failures, fmt.Sprintf("global coverage %.2f%% is below %.2f%%", rep.Global.Percent, *minGlobal)) + } + if *minPatch >= 0 { + if rep.Patch == nil { + failures = append(failures, "patch coverage requested but --base/--head not provided") + } else if rep.Patch.Percent < *minPatch { + failures = append(failures, fmt.Sprintf("patch coverage %.2f%% is below %.2f%%", rep.Patch.Percent, *minPatch)) + } + } + if len(failures) > 0 { + return fmt.Errorf("%s", strings.Join(failures, "; ")) + } + return nil +} + +func loadProfiles(paths []string) (map[string]block, error) { + merged := make(map[string]block) + for _, path := range paths { + f, err := os.Open(path) + if err != nil { + return nil, err + } + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "mode:") { + continue + } + b, err := parseBlock(line) + if err != nil { + f.Close() + return nil, fmt.Errorf("%s: %w", path, err) + } + key := blockKey(b) + if old, ok := merged[key]; !ok || b.Count > old.Count { + merged[key] = b + } + } + if err := sc.Err(); err != nil { + f.Close() + return nil, err + } + f.Close() + } + return merged, nil +} + +func parseBlock(line string) (block, error) { + fields := strings.Fields(line) + if len(fields) != 3 { + return block{}, fmt.Errorf("malformed coverage line %q", line) + } + loc := fields[0] + colon := strings.LastIndex(loc, ":") + if colon < 0 { + return block{}, fmt.Errorf("malformed coverage location %q", loc) + } + file := normalizePath(loc[:colon]) + span := loc[colon+1:] + comma := strings.Index(span, ",") + if comma < 0 { + return block{}, fmt.Errorf("malformed coverage span %q", span) + } + start, err := parsePos(span[:comma]) + if err != nil { + return block{}, err + } + end, err := parsePos(span[comma+1:]) + if err != nil { + return block{}, err + } + numStmt, err := strconv.Atoi(fields[1]) + if err != nil { + return block{}, err + } + count, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return block{}, err + } + return block{ + File: file, + StartLine: start[0], + StartCol: start[1], + EndLine: end[0], + EndCol: end[1], + NumStmt: numStmt, + Count: count, + }, nil +} + +func parsePos(s string) ([2]int, error) { + var pos [2]int + dot := strings.Index(s, ".") + if dot < 0 { + return pos, fmt.Errorf("malformed position %q", s) + } + line, err := strconv.Atoi(s[:dot]) + if err != nil { + return pos, err + } + col, err := strconv.Atoi(s[dot+1:]) + if err != nil { + return pos, err + } + pos[0], pos[1] = line, col + return pos, nil +} + +func blockKey(b block) string { + return fmt.Sprintf("%s:%d.%d,%d.%d", b.File, b.StartLine, b.StartCol, b.EndLine, b.EndCol) +} + +func normalizePath(path string) string { + for _, prefix := range modulePrefixes { + if strings.HasPrefix(path, prefix) { + path = strings.TrimPrefix(path, prefix) + break + } + } + path = strings.TrimPrefix(path, "./") + return path +} + +func excluded(path string) bool { + base := filepath.Base(path) + parts := strings.Split(path, "/") + for _, part := range parts { + switch part { + case "mocks", "testutils", "testhelpers", "cltest", "testdata", "fixtures": + return true + } + } + if strings.HasPrefix(base, "mock_") || strings.HasSuffix(base, "_mock.go") { + return true + } + for _, suffix := range []string{".pb.go", "_gen.go", "_codecgen.go"} { + if strings.HasSuffix(base, suffix) { + return true + } + } + return false +} + +func summarize(blocks map[string]block, include func(string) bool) stats { + var s stats + for _, b := range blocks { + if !include(b.File) { + continue + } + s.Statements += b.NumStmt + if b.Count > 0 { + s.Covered += b.NumStmt + } + } + s.Missed = s.Statements - s.Covered + s.Percent = percent(s.Covered, s.Statements) + return s +} + +func computePatch(blocks map[string]block, diff map[string][]lineRange, include func(string) bool) *patchStats { + ps := &patchStats{} + paths := make([]string, 0, len(diff)) + for path := range diff { + paths = append(paths, path) + } + sort.Strings(paths) + ps.ChangedFiles = len(paths) + for _, path := range paths { + ranges := diff[path] + for _, r := range ranges { + ps.ChangedLines += r.End - r.Start + 1 + } + fp := filePatch{ + Path: path, + TestFile: strings.HasSuffix(path, "_test.go"), + Added: ranges, + } + if include(path) && !fp.TestFile { + seen := make(map[string]bool) + missed := make(map[string]bool) + for key, b := range blocks { + if b.File != path || seen[key] || !intersects(b, ranges) { + continue + } + seen[key] = true + fp.Statements += b.NumStmt + if b.Count > 0 { + fp.Covered += b.NumStmt + } else { + fp.Missed += b.NumStmt + missed[fmt.Sprintf("%d-%d", b.StartLine, b.EndLine)] = true + } + } + for span := range missed { + fp.MissedSpans = append(fp.MissedSpans, span) + } + sort.Strings(fp.MissedSpans) + ps.Statements += fp.Statements + ps.Covered += fp.Covered + ps.Missed += fp.Missed + } + ps.Files = append(ps.Files, fp) + } + ps.Percent = percent(ps.Covered, ps.Statements) + return ps +} + +func intersects(b block, ranges []lineRange) bool { + for _, r := range ranges { + if b.EndLine >= r.Start && b.StartLine <= r.End { + return true + } + } + return false +} + +func percent(covered, total int) float64 { + if total == 0 { + return 0 + } + return float64(int(10000*float64(covered)/float64(total)+0.5)) / 100 +} + +func gitDiff(repo, base, head string) (map[string][]lineRange, error) { + cmd := exec.Command("git", "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff", base+".."+head, "--", "*.go") + cmd.Dir = repo + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git diff: %w", err) + } + return parseDiff(string(out)), nil +} + +func parseDiff(diff string) map[string][]lineRange { + files := make(map[string][]lineRange) + var current string + sc := bufio.NewScanner(strings.NewReader(diff)) + sc.Buffer(make([]byte, 64*1024), 8*1024*1024) + for sc.Scan() { + line := sc.Text() + switch { + case strings.HasPrefix(line, "+++ "): + path := strings.TrimPrefix(line, "+++ ") + path = strings.TrimPrefix(path, "b/") + if path == "/dev/null" { + current = "" + } else { + current = path + } + case strings.HasPrefix(line, "@@ ") && current != "": + if r, ok := parseHunk(line); ok { + files[current] = append(files[current], r) + } + } + } + return files +} + +func parseHunk(line string) (lineRange, bool) { + plus := strings.Index(line, " +") + if plus < 0 { + return lineRange{}, false + } + rest := line[plus+1:] + end := strings.Index(rest, " @@") + if end < 0 { + return lineRange{}, false + } + rest = rest[:end] + startStr := rest + count := 1 + if comma := strings.Index(rest, ","); comma >= 0 { + startStr = rest[:comma] + n, err := strconv.Atoi(rest[comma+1:]) + if err != nil { + return lineRange{}, false + } + count = n + } + start, err := strconv.Atoi(startStr) + if err != nil || count <= 0 || start <= 0 { + return lineRange{}, false + } + return lineRange{Start: start, End: start + count - 1}, true +} + +func splitList(value string) []string { + var out []string + for _, item := range strings.Split(value, ",") { + if trimmed := strings.TrimSpace(item); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +func markdown(rep report) string { + var b strings.Builder + fmt.Fprintf(&b, "## Coverage — %s\n\n", rep.Category) + if rep.Base != "" || rep.Head != "" { + fmt.Fprintf(&b, "`%s..%s`\n\n", rep.Base, rep.Head) + } + b.WriteString("### Global\n\n") + b.WriteString("| Scope | Statements | Covered | Missed | Coverage |\n") + b.WriteString("|---|---:|---:|---:|---:|\n") + writeStatsRow(&b, "raw", rep.Global) + writeStatsRow(&b, "source-only", rep.GlobalSourceOnly) + if rep.Patch != nil { + b.WriteString("\n### Patch\n\n") + b.WriteString("| Scope | Changed files | Changed lines | Statements | Covered | Missed | Coverage |\n") + b.WriteString("|---|---:|---:|---:|---:|---:|---:|\n") + writePatchRow(&b, "raw", rep.Patch) + if rep.PatchSourceOnly != nil { + writePatchRow(&b, "source-only", rep.PatchSourceOnly) + } + if len(rep.Patch.Files) > 0 { + b.WriteString("\n### Changed files\n\n") + b.WriteString("| File | Statements | Covered | Missed | Coverage | Missed spans |\n") + b.WriteString("|---|---:|---:|---:|---:|---|\n") + for _, f := range rep.Patch.Files { + if f.TestFile || f.Statements == 0 { + continue + } + fmt.Fprintf(&b, "| `%s` | %d | %d | %d | %.2f%% | %s |\n", + f.Path, f.Statements, f.Covered, f.Missed, percent(f.Covered, f.Statements), strings.Join(f.MissedSpans, ", ")) + } + } + } + return b.String() +} + +func writeStatsRow(b *strings.Builder, name string, s stats) { + fmt.Fprintf(b, "| %s | %d | %d | %d | **%.2f%%** |\n", name, s.Statements, s.Covered, s.Missed, s.Percent) +} + +func writePatchRow(b *strings.Builder, name string, p *patchStats) { + fmt.Fprintf(b, "| %s | %d | %d | %d | %d | %d | **%.2f%%** |\n", + name, p.ChangedFiles, p.ChangedLines, p.Statements, p.Covered, p.Missed, p.Percent) +} diff --git a/tools/coverage/main_test.go b/tools/coverage/main_test.go new file mode 100644 index 00000000000..f541b3d00c7 --- /dev/null +++ b/tools/coverage/main_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseBlockAndNormalize(t *testing.T) { + b, err := parseBlock("github.com/smartcontractkit/chainlink/v2/core/foo.go:10.2,12.4 3 7") + if err != nil { + t.Fatal(err) + } + if b.File != "core/foo.go" || b.StartLine != 10 || b.EndLine != 12 || b.NumStmt != 3 || b.Count != 7 { + t.Fatalf("unexpected block: %+v", b) + } + if got := normalizePath("github.com/smartcontractkit/chainlink/deployment/foo.go"); got != "deployment/foo.go" { + t.Fatalf("unexpected deployment path: %q", got) + } +} + +func TestSummarizeAndExclude(t *testing.T) { + blocks := map[string]block{ + "a": {File: "core/a.go", NumStmt: 4, Count: 1}, + "b": {File: "core/a.go", NumStmt: 2, Count: 0}, + "c": {File: "core/mocks/m.go", NumStmt: 10, Count: 0}, + } + raw := summarize(blocks, func(string) bool { return true }) + if raw.Statements != 16 || raw.Covered != 4 || raw.Missed != 12 { + t.Fatalf("unexpected raw stats: %+v", raw) + } + source := summarize(blocks, func(path string) bool { return !excluded(path) }) + if source.Statements != 6 || source.Covered != 4 || source.Missed != 2 { + t.Fatalf("unexpected source stats: %+v", source) + } +} + +func TestParseDiffAndPatch(t *testing.T) { + diff := `diff --git a/core/a.go b/core/a.go +--- a/core/a.go ++++ b/core/a.go +@@ -10,2 +10,3 @@ +-old ++new ++newer + context +@@ -30 +31 @@ +-old ++new +diff --git a/core/a_test.go b/core/a_test.go +--- a/core/a_test.go ++++ b/core/a_test.go +@@ -1 +1 @@ +-old ++new +` + files := parseDiff(diff) + if len(files["core/a.go"]) != 2 { + t.Fatalf("unexpected ranges: %+v", files) + } + if r := files["core/a.go"][0]; r.Start != 10 || r.End != 12 { + t.Fatalf("unexpected first range: %+v", r) + } + + blocks := map[string]block{ + "one": {File: "core/a.go", StartLine: 10, EndLine: 11, NumStmt: 2, Count: 3}, + "two": {File: "core/a.go", StartLine: 12, EndLine: 12, NumStmt: 1, Count: 0}, + "three": {File: "core/a.go", StartLine: 31, EndLine: 31, NumStmt: 2, Count: 0}, + "outside": {File: "core/a.go", StartLine: 50, EndLine: 50, NumStmt: 9, Count: 1}, + } + patch := computePatch(blocks, files, func(path string) bool { return !excluded(path) && !hasTestSuffix(path) }) + if patch.ChangedFiles != 2 || patch.ChangedLines != 5 { + t.Fatalf("unexpected patch counts: %+v", patch) + } + if patch.Statements != 5 || patch.Covered != 2 || patch.Missed != 3 { + t.Fatalf("unexpected patch stats: %+v", patch) + } +} + +func TestLoadProfilesMergesDuplicates(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "coverage.txt") + content := `mode: atomic +github.com/smartcontractkit/chainlink/v2/core/a.go:1.1,1.10 2 0 +github.com/smartcontractkit/chainlink/v2/core/a.go:1.1,1.10 2 5 +github.com/smartcontractkit/chainlink/deployment/b.go:2.1,2.10 1 1 +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + blocks, err := loadProfiles([]string{path}) + if err != nil { + t.Fatal(err) + } + if len(blocks) != 2 { + t.Fatalf("expected 2 merged blocks, got %d", len(blocks)) + } + for _, b := range blocks { + if b.File == "core/a.go" && b.Count != 5 { + t.Fatalf("duplicate block was not merged with max count: %+v", b) + } + } +} + +func hasTestSuffix(path string) bool { + return len(path) >= 8 && path[len(path)-8:] == "_test.go" +} From 7cad100ae178c71441ee7c1d9e2abc954c8a2af2 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:53:31 +1100 Subject: [PATCH 02/16] fix(ci): flush E2E coverage before teardown Send SIGUSR1 to coverage-instrumented containers while tests run so counters survive container teardown. Fix coverage CLI lint findings and sanitize the CI Tools artifact name. --- .../actions/start-coverage-flusher/action.yml | 40 +++++++++++++++++++ .github/workflows/ccip-system-tests.yaml | 3 ++ .github/workflows/ci-tools.yml | 2 +- .github/workflows/cre-local-env-tests.yaml | 3 ++ .github/workflows/cre-mixed-env-tests.yaml | 3 ++ .../cre-regression-system-tests.yaml | 3 ++ .github/workflows/cre-system-tests.yaml | 3 ++ .../workflows/legacy-non-functional-tests.yml | 3 ++ .github/workflows/legacy-system-tests.yml | 3 ++ tools/coverage/main.go | 7 ++-- 10 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 .github/actions/start-coverage-flusher/action.yml diff --git a/.github/actions/start-coverage-flusher/action.yml b/.github/actions/start-coverage-flusher/action.yml new file mode 100644 index 00000000000..e1c21eae1dc --- /dev/null +++ b/.github/actions/start-coverage-flusher/action.yml @@ -0,0 +1,40 @@ +name: Start coverage flusher +description: Periodically sends SIGUSR1 to coverage-instrumented containers so covcounters are written before teardown. + +inputs: + interval: + description: Seconds between flushes. + required: false + default: "15" + pidfile: + description: File used to record the background process ID. + required: false + default: /tmp/chainlink-coverage-flusher.pid + +runs: + using: composite + steps: + - name: Start flusher + shell: bash + env: + INTERVAL: ${{ inputs.interval }} + PIDFILE: ${{ inputs.pidfile }} + run: | + set -euo pipefail + cat > /tmp/chainlink-coverage-flusher.sh <<'EOF' + #!/usr/bin/env bash + set -u + interval="${INTERVAL:-15}" + while true; do + for container in $(docker ps -q 2>/dev/null || true); do + image="$(docker inspect -f '{{.Config.Image}}' "${container}" 2>/dev/null || true)" + if [[ "${image}" == *"-coverage"* ]]; then + docker kill --signal=SIGUSR1 "${container}" >/dev/null 2>&1 || true + fi + done + sleep "${interval}" + done + EOF + chmod +x /tmp/chainlink-coverage-flusher.sh + nohup env INTERVAL="${INTERVAL}" /tmp/chainlink-coverage-flusher.sh >/tmp/chainlink-coverage-flusher.log 2>&1 & + echo $! > "${PIDFILE}" diff --git a/.github/workflows/ccip-system-tests.yaml b/.github/workflows/ccip-system-tests.yaml index 9806258df63..460486b7f56 100644 --- a/.github/workflows/ccip-system-tests.yaml +++ b/.github/workflows/ccip-system-tests.yaml @@ -296,6 +296,9 @@ jobs: echo "Baseline image: ${image}:${version}" fi + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CCIP smoke tests id: run-tests shell: bash diff --git a/.github/workflows/ci-tools.yml b/.github/workflows/ci-tools.yml index 0def74d55ff..99613ba12a7 100644 --- a/.github/workflows/ci-tools.yml +++ b/.github/workflows/ci-tools.yml @@ -84,7 +84,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: coverage-tools-${{ matrix.target.name }}-${{ github.run_id }}-${{ github.run_attempt }} + name: coverage-tools-${{ strategy.job-index }}-${{ github.run_id }}-${{ github.run_attempt }} path: .covdata/tools-${{ strategy.job-index }} retention-days: 7 if-no-files-found: warn diff --git a/.github/workflows/cre-local-env-tests.yaml b/.github/workflows/cre-local-env-tests.yaml index bc468ea4f42..5f16d439cd8 100644 --- a/.github/workflows/cre-local-env-tests.yaml +++ b/.github/workflows/cre-local-env-tests.yaml @@ -196,6 +196,9 @@ jobs: total_seconds=$(echo "$minutes * 60 + $seconds" | bc) echo "execution_time=$total_seconds" >> "${GITHUB_OUTPUT}" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Deploy Cron workflow (Go) shell: bash working-directory: core/scripts/cre/environment diff --git a/.github/workflows/cre-mixed-env-tests.yaml b/.github/workflows/cre-mixed-env-tests.yaml index 0c3dee44e9c..8ab4c92fbd6 100644 --- a/.github/workflows/cre-mixed-env-tests.yaml +++ b/.github/workflows/cre-mixed-env-tests.yaml @@ -373,6 +373,9 @@ jobs: aws-role-duration-seconds: "1800" set-git-config: "true" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CRE Smoke system tests (mixed-env) id: run-tests if: ${{ steps.baseline.outputs.skip != 'true' }} diff --git a/.github/workflows/cre-regression-system-tests.yaml b/.github/workflows/cre-regression-system-tests.yaml index 27133e87bd0..5f6daa42c4d 100644 --- a/.github/workflows/cre-regression-system-tests.yaml +++ b/.github/workflows/cre-regression-system-tests.yaml @@ -212,6 +212,9 @@ jobs: cleanup-on-error: "false" working-directory: core/scripts/cre/environment + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CRE Regression system tests id: run-regression-tests shell: bash diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 50ad8d8decd..0aafeb101d2 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -351,6 +351,9 @@ jobs: aws-role-duration-seconds: "1800" set-git-config: "true" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run CRE Smoke system tests id: run-tests shell: bash diff --git a/.github/workflows/legacy-non-functional-tests.yml b/.github/workflows/legacy-non-functional-tests.yml index 836d5685960..7da7e5f082b 100644 --- a/.github/workflows/legacy-non-functional-tests.yml +++ b/.github/workflows/legacy-non-functional-tests.yml @@ -179,6 +179,9 @@ jobs: echo "Setting up environment with: $ENV_CMD" eval "$ENV_CMD" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run tests working-directory: devenv/tests/${{ matrix.tests_dir }} env: diff --git a/.github/workflows/legacy-system-tests.yml b/.github/workflows/legacy-system-tests.yml index c984a9034b6..583fdbc0ca5 100644 --- a/.github/workflows/legacy-system-tests.yml +++ b/.github/workflows/legacy-system-tests.yml @@ -248,6 +248,9 @@ jobs: echo "Setting up environment with: $ENV_CMD" eval "$ENV_CMD" + - name: Start coverage flusher + uses: $/.github/actions/start-coverage-flusher + - name: Run tests working-directory: devenv/tests/${{ matrix.tests_dir }} env: diff --git a/tools/coverage/main.go b/tools/coverage/main.go index 13785af3430..b1b9ca4cc1b 100644 --- a/tools/coverage/main.go +++ b/tools/coverage/main.go @@ -7,6 +7,7 @@ package main import ( "bufio" "encoding/json" + "errors" "flag" "fmt" "os" @@ -115,7 +116,7 @@ func runMerge(args []string) error { } dirs := splitList(*covdata) if len(dirs) == 0 { - return fmt.Errorf("--covdata is required") + return errors.New("--covdata is required") } for _, dir := range dirs { if info, err := os.Stat(dir); err != nil || !info.IsDir() { @@ -159,7 +160,7 @@ func runReport(args []string) error { } profilePaths := splitList(*profiles) if len(profilePaths) == 0 { - return fmt.Errorf("--profile is required") + return errors.New("--profile is required") } blocks, err := loadProfiles(profilePaths) @@ -177,7 +178,7 @@ func runReport(args []string) error { if *base != "" || *head != "" { if *base == "" || *head == "" { - return fmt.Errorf("--base and --head must be provided together") + return errors.New("--base and --head must be provided together") } diff, err := gitDiff(*repo, *base, *head) if err != nil { From 974cf972a8dfa7fce7d2dab829c3bd8aeb6da08a Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:42:00 +1100 Subject: [PATCH 03/16] fix(coverage): flush counters periodically in coverage builds Coverage-instrumented E2E nodes can be SIGKILLed before graceful shutdown. Flush counters every 10 seconds in coverage-tagged builds so counter data survives container teardown. --- core/coverage_flush.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/coverage_flush.go b/core/coverage_flush.go index 4b2f8639f65..d3de0e6703d 100644 --- a/core/coverage_flush.go +++ b/core/coverage_flush.go @@ -7,11 +7,12 @@ import ( "os/signal" "runtime/coverage" "syscall" + "time" ) -// init installs a SIGUSR1 handler used by E2E coverage collection. The handler -// flushes coverage counters while the process is still running, so coverage is -// not lost when a container is later stopped with SIGKILL. +// init installs a SIGUSR1 handler and a periodic flush used by E2E coverage +// collection. Flushing while the process is running means coverage is not lost +// when a container is later stopped with SIGKILL. func init() { dir := os.Getenv("GOCOVERDIR") if dir == "" { @@ -24,4 +25,11 @@ func init() { _ = coverage.WriteCountersDir(dir) } }() + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + _ = coverage.WriteCountersDir(dir) + } + }() } From e22872a8a57edad14ebbba39aca6d220607388e0 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:11:08 +1100 Subject: [PATCH 04/16] feat(coverage): merge category profiles for union reporting Add merge-profiles for fast text-profile union and let the collector compute patch coverage for a separate merged-PR range. Avoids merging hundreds of covdata directories at once. --- .github/workflows/coverage-collector.yml | 12 +++++ tools/coverage/collect.sh | 26 +++++++++-- tools/coverage/main.go | 57 +++++++++++++++++++++++- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index 50da9777b17..cbed3c633ce 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -28,6 +28,16 @@ on: required: false type: string default: "" + patch_base: + description: "Optional base SHA for patch coverage" + required: false + type: string + default: "" + patch_head: + description: "Optional head SHA for patch coverage" + required: false + type: string + default: "" permissions: actions: read @@ -61,6 +71,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TARGET_SHA: ${{ inputs.sha || github.event.workflow_run.head_sha }} PR_NUMBER: ${{ inputs.pr || (github.event.workflow_run.pull_requests[0].number || '') }} + PATCH_BASE: ${{ inputs.patch_base }} + PATCH_HEAD: ${{ inputs.patch_head }} run: ./tools/coverage/collect.sh - name: Upload merged report diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh index 8a9831ff818..9bb5ebc22e1 100755 --- a/tools/coverage/collect.sh +++ b/tools/coverage/collect.sh @@ -24,10 +24,15 @@ BASE_SHA="" if [[ -n "${PR_NUMBER:-}" ]]; then BASE_SHA="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.base.sha' 2>/dev/null || true)" fi +PATCH_BASE="${PATCH_BASE:-$BASE_SHA}" +PATCH_HEAD="${PATCH_HEAD:-$TARGET_SHA}" if [[ -n "$BASE_SHA" ]]; then git fetch --no-tags origin "$BASE_SHA" "$TARGET_SHA" >/dev/null 2>&1 || true fi +if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + git fetch --no-tags origin "$PATCH_BASE" "$PATCH_HEAD" >/dev/null 2>&1 || true +fi echo "Collecting coverage for ${REPO}@${TARGET_SHA} (PR ${PR_NUMBER:-none}, base ${BASE_SHA:-none})" @@ -69,8 +74,8 @@ merge_and_report() { go run ./tools/coverage merge --covdata "$dirs" --out "${OUT}/${category}.txt" local args=(--profile "${OUT}/${category}.txt" --category "$category" --json "${OUT}/${category}.json" --markdown "${OUT}/${category}.md") - if [[ -n "$BASE_SHA" ]]; then - args+=(--base "$BASE_SHA" --head "$TARGET_SHA") + if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") fi go run ./tools/coverage report "${args[@]}" } @@ -78,7 +83,22 @@ merge_and_report() { merge_and_report unit "$unit_dirs" merge_and_report integration "$integration_dirs" merge_and_report e2e "$e2e_dirs" -merge_and_report any "$all_dirs" + +profile_list="" +for category in unit integration e2e; do + if [[ -f "${OUT}/${category}.txt" ]]; then + profile_list="${profile_list}${profile_list:+,}${OUT}/${category}.txt" + fi +done +if [[ -n "$profile_list" ]]; then + go run ./tools/coverage merge-profiles --profile "$profile_list" --out "${OUT}/any.txt" + args=(--profile "${OUT}/any.txt" --category any + --json "${OUT}/any.json" --markdown "${OUT}/any.md") + if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") + fi + go run ./tools/coverage report "${args[@]}" +fi test_xmls="$(find "$ARTIFACTS" -type f -name '*.xml' -path '*test-results-*' 2>/dev/null | sort -u)" if [[ -n "$test_xmls" ]]; then diff --git a/tools/coverage/main.go b/tools/coverage/main.go index b1b9ca4cc1b..259545e3b1e 100644 --- a/tools/coverage/main.go +++ b/tools/coverage/main.go @@ -87,6 +87,8 @@ func main() { err = runReport(os.Args[2:]) case "merge": err = runMerge(os.Args[2:]) + case "merge-profiles": + err = runMergeProfiles(os.Args[2:]) case "-h", "--help", "help": usage() return @@ -104,7 +106,8 @@ func usage() { coverage report --profile file[,file...] [--base SHA --head SHA] [--repo DIR] [--category NAME] [--json FILE] [--markdown FILE] [--min-global PCT] [--min-patch PCT] - coverage merge --covdata dir[,dir...] --out coverage.txt`) + coverage merge --covdata dir[,dir...] --out coverage.txt + coverage merge-profiles --profile file[,file...] --out coverage.txt`) } func runMerge(args []string) error { @@ -144,6 +147,58 @@ func runMerge(args []string) error { return nil } +func runMergeProfiles(args []string) error { + fs := flag.NewFlagSet("merge-profiles", flag.ExitOnError) + profiles := fs.String("profile", "", "comma-separated text coverage profiles") + out := fs.String("out", "coverage.txt", "text profile output path") + if err := fs.Parse(args); err != nil { + return err + } + paths := splitList(*profiles) + if len(paths) == 0 { + return errors.New("--profile is required") + } + blocks, err := loadProfiles(paths) + if err != nil { + return err + } + f, err := os.Create(*out) + if err != nil { + return err + } + defer f.Close() + if _, err := fmt.Fprintln(f, "mode: atomic"); err != nil { + return err + } + keys := make([]string, 0, len(blocks)) + for key := range blocks { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + a, b := blocks[keys[i]], blocks[keys[j]] + if a.File != b.File { + return a.File < b.File + } + if a.StartLine != b.StartLine { + return a.StartLine < b.StartLine + } + if a.StartCol != b.StartCol { + return a.StartCol < b.StartCol + } + if a.EndLine != b.EndLine { + return a.EndLine < b.EndLine + } + return a.EndCol < b.EndCol + }) + for _, key := range keys { + b := blocks[key] + if _, err := fmt.Fprintf(f, "%s:%d.%d,%d.%d %d %d\n", b.File, b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt, b.Count); err != nil { + return err + } + } + return nil +} + func runReport(args []string) error { fs := flag.NewFlagSet("report", flag.ExitOnError) profiles := fs.String("profile", "", "comma-separated text coverage profiles") From d353c5f84ce42150d6a3238ee79b9e0c79278e53 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:12:02 +1100 Subject: [PATCH 05/16] test(ci): run coverage collector on pull requests Temporary pull_request trigger so the collector can be validated before the workflow exists on the default branch. Remove the manual dispatch trigger before the final PR. --- .github/workflows/coverage-collector.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index cbed3c633ce..77baf83211f 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -1,6 +1,8 @@ name: Coverage Collector on: + pull_request: + types: [opened, synchronize, reopened] workflow_run: workflows: - CI Core @@ -52,7 +54,7 @@ concurrency: jobs: collect: name: Collect coverage - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion != 'skipped' }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion != 'skipped' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -66,11 +68,15 @@ jobs: go-version-file: go.mod cache: true + - name: Wait for PR test artifacts + if: ${{ github.event_name == 'pull_request' }} + run: sleep 120 + - name: Collect and publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TARGET_SHA: ${{ inputs.sha || github.event.workflow_run.head_sha }} - PR_NUMBER: ${{ inputs.pr || (github.event.workflow_run.pull_requests[0].number || '') }} + TARGET_SHA: ${{ github.event.pull_request.head.sha || inputs.sha || github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr || (github.event.workflow_run.pull_requests[0].number || '') }} PATCH_BASE: ${{ inputs.patch_base }} PATCH_HEAD: ${{ inputs.patch_head }} run: ./tools/coverage/collect.sh From 8431b9bdfaec9ef0a594853085448b0ee9aefb84 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:54:39 +1100 Subject: [PATCH 06/16] fix(coverage): use merge-base for PR patch coverage The PR base SHA can be ahead of the branch point. Use git merge-base so patch coverage only includes the PR's own changes. --- tools/coverage/collect.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh index 9bb5ebc22e1..593950790a1 100755 --- a/tools/coverage/collect.sh +++ b/tools/coverage/collect.sh @@ -24,7 +24,7 @@ BASE_SHA="" if [[ -n "${PR_NUMBER:-}" ]]; then BASE_SHA="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.base.sha' 2>/dev/null || true)" fi -PATCH_BASE="${PATCH_BASE:-$BASE_SHA}" +PATCH_BASE="${PATCH_BASE:-}" PATCH_HEAD="${PATCH_HEAD:-$TARGET_SHA}" if [[ -n "$BASE_SHA" ]]; then @@ -33,6 +33,9 @@ fi if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then git fetch --no-tags origin "$PATCH_BASE" "$PATCH_HEAD" >/dev/null 2>&1 || true fi +if [[ -z "$PATCH_BASE" && -n "$BASE_SHA" ]]; then + PATCH_BASE="$(git merge-base "$BASE_SHA" "$TARGET_SHA" 2>/dev/null || echo "$BASE_SHA")" +fi echo "Collecting coverage for ${REPO}@${TARGET_SHA} (PR ${PR_NUMBER:-none}, base ${BASE_SHA:-none})" From aef9be736d42780e084ce060ec76246a970a5d00 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:55:14 +1100 Subject: [PATCH 07/16] chore(ci): limit coverage collector to workflow_run Remove the temporary pull-request and manual dispatch triggers used to validate the collector before merge. The final collector is triggered by completed test workflows. --- .github/workflows/coverage-collector.yml | 42 ++++-------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index 77baf83211f..7e222a8e82a 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -1,8 +1,6 @@ name: Coverage Collector on: - pull_request: - types: [opened, synchronize, reopened] workflow_run: workflows: - CI Core @@ -18,28 +16,6 @@ on: - Legacy System Tests - Legacy Non Functional Tests types: [completed] - # TEMP: manual trigger for pre-merge validation. Remove before opening the PR. - workflow_dispatch: - inputs: - sha: - description: "Commit SHA to collect" - required: true - type: string - pr: - description: "Optional PR number" - required: false - type: string - default: "" - patch_base: - description: "Optional base SHA for patch coverage" - required: false - type: string - default: "" - patch_head: - description: "Optional head SHA for patch coverage" - required: false - type: string - default: "" permissions: actions: read @@ -48,13 +24,13 @@ permissions: checks: write concurrency: - group: coverage-${{ github.event.workflow_run.head_sha || inputs.sha }} + group: coverage-${{ github.event.workflow_run.head_sha }} cancel-in-progress: true jobs: collect: name: Collect coverage - if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion != 'skipped' }} + if: ${{ github.event.workflow_run.conclusion != 'skipped' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -68,24 +44,20 @@ jobs: go-version-file: go.mod cache: true - - name: Wait for PR test artifacts - if: ${{ github.event_name == 'pull_request' }} - run: sleep 120 - - name: Collect and publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TARGET_SHA: ${{ github.event.pull_request.head.sha || inputs.sha || github.event.workflow_run.head_sha }} - PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr || (github.event.workflow_run.pull_requests[0].number || '') }} - PATCH_BASE: ${{ inputs.patch_base }} - PATCH_HEAD: ${{ inputs.patch_head }} + TARGET_SHA: ${{ github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number || '' }} + PATCH_BASE: "" + PATCH_HEAD: "" run: ./tools/coverage/collect.sh - name: Upload merged report if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: coverage-report-${{ inputs.sha || github.event.workflow_run.head_sha }}-${{ github.run_id }} + name: coverage-report-${{ github.event.workflow_run.head_sha }}-${{ github.run_id }} path: coverage-report retention-days: 30 if-no-files-found: warn From 8183b6cd4115bd0266d7f25a47a1ea456aa7d51c Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:58:07 +1100 Subject: [PATCH 08/16] fix(coverage): address golangci-lint findings Use context-aware exec, tighten output permissions, apply modernize simplifications, and mark tests parallel. --- tools/coverage/main.go | 43 ++++++++++++++++++------------------- tools/coverage/main_test.go | 6 +++++- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/tools/coverage/main.go b/tools/coverage/main.go index 259545e3b1e..fd2a41e8e54 100644 --- a/tools/coverage/main.go +++ b/tools/coverage/main.go @@ -6,6 +6,7 @@ package main import ( "bufio" + "context" "encoding/json" "errors" "flag" @@ -132,13 +133,13 @@ func runMerge(args []string) error { } defer os.RemoveAll(tmp) - merge := exec.Command("go", "tool", "covdata", "merge", "-i="+strings.Join(dirs, ","), "-o="+tmp) + merge := exec.CommandContext(context.Background(), "go", "tool", "covdata", "merge", "-i="+strings.Join(dirs, ","), "-o="+tmp) //nolint:gosec // paths are validated above merge.Stdout = os.Stdout merge.Stderr = os.Stderr if err := merge.Run(); err != nil { return fmt.Errorf("covdata merge: %w", err) } - textfmt := exec.Command("go", "tool", "covdata", "textfmt", "-i="+tmp, "-o="+*out) + textfmt := exec.CommandContext(context.Background(), "go", "tool", "covdata", "textfmt", "-i="+tmp, "-o="+*out) //nolint:gosec // output path is user-provided CLI input textfmt.Stdout = os.Stdout textfmt.Stderr = os.Stderr if err := textfmt.Run(); err != nil { @@ -247,7 +248,7 @@ func runReport(args []string) error { md := markdown(rep) if *mdOut != "" { - if err := os.WriteFile(*mdOut, []byte(md), 0o644); err != nil { + if err := os.WriteFile(*mdOut, []byte(md), 0o600); err != nil { return err } } @@ -256,7 +257,7 @@ func runReport(args []string) error { if err != nil { return err } - if err := os.WriteFile(*jsonOut, append(data, '\n'), 0o644); err != nil { + if err := os.WriteFile(*jsonOut, append(data, '\n'), 0o600); err != nil { return err } } @@ -320,21 +321,20 @@ func parseBlock(line string) (block, error) { return block{}, fmt.Errorf("malformed coverage line %q", line) } loc := fields[0] - colon := strings.LastIndex(loc, ":") - if colon < 0 { + filePart, span, ok := strings.CutLast(loc, ":") + if !ok { return block{}, fmt.Errorf("malformed coverage location %q", loc) } - file := normalizePath(loc[:colon]) - span := loc[colon+1:] - comma := strings.Index(span, ",") - if comma < 0 { + file := normalizePath(filePart) + startPart, endPart, ok := strings.Cut(span, ",") + if !ok { return block{}, fmt.Errorf("malformed coverage span %q", span) } - start, err := parsePos(span[:comma]) + start, err := parsePos(startPart) if err != nil { return block{}, err } - end, err := parsePos(span[comma+1:]) + end, err := parsePos(endPart) if err != nil { return block{}, err } @@ -359,15 +359,15 @@ func parseBlock(line string) (block, error) { func parsePos(s string) ([2]int, error) { var pos [2]int - dot := strings.Index(s, ".") - if dot < 0 { + linePart, colPart, ok := strings.Cut(s, ".") + if !ok { return pos, fmt.Errorf("malformed position %q", s) } - line, err := strconv.Atoi(s[:dot]) + line, err := strconv.Atoi(linePart) if err != nil { return pos, err } - col, err := strconv.Atoi(s[dot+1:]) + col, err := strconv.Atoi(colPart) if err != nil { return pos, err } @@ -381,8 +381,8 @@ func blockKey(b block) string { func normalizePath(path string) string { for _, prefix := range modulePrefixes { - if strings.HasPrefix(path, prefix) { - path = strings.TrimPrefix(path, prefix) + if stripped, ok := strings.CutPrefix(path, prefix); ok { + path = stripped break } } @@ -392,8 +392,7 @@ func normalizePath(path string) string { func excluded(path string) bool { base := filepath.Base(path) - parts := strings.Split(path, "/") - for _, part := range parts { + for part := range strings.SplitSeq(path, "/") { switch part { case "mocks", "testutils", "testhelpers", "cltest", "testdata", "fixtures": return true @@ -491,7 +490,7 @@ func percent(covered, total int) float64 { } func gitDiff(repo, base, head string) (map[string][]lineRange, error) { - cmd := exec.Command("git", "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff", base+".."+head, "--", "*.go") + cmd := exec.CommandContext(context.Background(), "git", "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff", base+".."+head, "--", "*.go") //nolint:gosec // revisions come from the caller cmd.Dir = repo out, err := cmd.Output() if err != nil { @@ -555,7 +554,7 @@ func parseHunk(line string) (lineRange, bool) { func splitList(value string) []string { var out []string - for _, item := range strings.Split(value, ",") { + for item := range strings.SplitSeq(value, ",") { if trimmed := strings.TrimSpace(item); trimmed != "" { out = append(out, trimmed) } diff --git a/tools/coverage/main_test.go b/tools/coverage/main_test.go index f541b3d00c7..5e07ddbd411 100644 --- a/tools/coverage/main_test.go +++ b/tools/coverage/main_test.go @@ -7,6 +7,7 @@ import ( ) func TestParseBlockAndNormalize(t *testing.T) { + t.Parallel() b, err := parseBlock("github.com/smartcontractkit/chainlink/v2/core/foo.go:10.2,12.4 3 7") if err != nil { t.Fatal(err) @@ -20,6 +21,7 @@ func TestParseBlockAndNormalize(t *testing.T) { } func TestSummarizeAndExclude(t *testing.T) { + t.Parallel() blocks := map[string]block{ "a": {File: "core/a.go", NumStmt: 4, Count: 1}, "b": {File: "core/a.go", NumStmt: 2, Count: 0}, @@ -36,6 +38,7 @@ func TestSummarizeAndExclude(t *testing.T) { } func TestParseDiffAndPatch(t *testing.T) { + t.Parallel() diff := `diff --git a/core/a.go b/core/a.go --- a/core/a.go +++ b/core/a.go @@ -78,6 +81,7 @@ diff --git a/core/a_test.go b/core/a_test.go } func TestLoadProfilesMergesDuplicates(t *testing.T) { + t.Parallel() dir := t.TempDir() path := filepath.Join(dir, "coverage.txt") content := `mode: atomic @@ -85,7 +89,7 @@ github.com/smartcontractkit/chainlink/v2/core/a.go:1.1,1.10 2 0 github.com/smartcontractkit/chainlink/v2/core/a.go:1.1,1.10 2 5 github.com/smartcontractkit/chainlink/deployment/b.go:2.1,2.10 1 1 ` - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } blocks, err := loadProfiles([]string{path}) From e4ac673585a0991906f90712d316cded5bf977ff Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:26:13 +1100 Subject: [PATCH 09/16] fix(ci): enable label-triggered E2E coverage and harden collector Allow E2E coverage via the run-e2e-coverage PR label, wait for all relevant workflow runs before reporting, keep raw artifacts out of the report artifact, degrade gracefully, dedupe covdata dirs, paginate run lookups, and restrict comment updates to the collector bot. --- .github/workflows/ci-core.yml | 1 - .github/workflows/coverage-collector.yml | 3 +- .github/workflows/integration-tests.yml | 15 ++-- core/coverage_flush.go | 11 ++- tools/bin/go_core_scripts_tests | 2 +- tools/coverage/collect.sh | 106 ++++++++++++++++------- 6 files changed, 94 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 13ec3c83f9c..9bdeee249e1 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -381,7 +381,6 @@ jobs: install-loopps: "false" build-cache-version: go_core_race_tests build-cache-write: always - coverage: "true" name: Core Tests (${{ matrix.type.cmd }}) # Be careful modifying the job name, as it is used to fetch the job URL # We don't directly merge dependabot PRs, so let's not waste the resources diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index 7e222a8e82a..30a372c994a 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -21,7 +21,6 @@ permissions: actions: read contents: read pull-requests: write - checks: write concurrency: group: coverage-${{ github.event.workflow_run.head_sha }} @@ -51,6 +50,8 @@ jobs: PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number || '' }} PATCH_BASE: "" PATCH_HEAD: "" + WAIT_FOR_RUNS: "true" + WAIT_TIMEOUT: "2700" run: ./tools/coverage/collect.sh - name: Upload merged report diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a851ee29128..6029304c43f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -46,6 +46,7 @@ env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name || 'chainlink-integration-tests' }} CHAINLINK_REF: ${{ inputs.cl_ref || github.sha }} TEST_SUITE: smoke + E2E_COVERAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage')) }} jobs: enforce-ctf-version: @@ -317,7 +318,7 @@ jobs: uses: smartcontractkit/.github/actions/ecr-image-exists@ecr-image-exists/0.2.0 with: repository: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} - tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && inputs.coverage && '-coverage' || '' }} + tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && env.E2E_COVERAGE && '-coverage' || '' }} aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Explain build/no-build decision @@ -363,8 +364,8 @@ jobs: # Rebuild the final-stage apt layer daily so `apt-get upgrade` isn't served stale from cache. docker-additional-build-args: | APT_CACHE_BUST=${{ steps.apt-cache-bust.outputs.key }} - ${{ matrix.image.name == '' && format('GO_COVER_FLAG={0}', inputs.coverage) || '' }} - image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && inputs.coverage && '-coverage' || '' }} + ${{ matrix.image.name == '' && format('GO_COVER_FLAG={0}', env.E2E_COVERAGE) || '' }} + image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && env.E2E_COVERAGE && '-coverage' || '' }} dockerfile: ${{ matrix.image.dockerfile }} docker-registry-url: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com docker-repository-name: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} @@ -507,7 +508,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} secrets: inherit run-core-cre-e2e-regression-tests: @@ -525,7 +526,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} secrets: inherit run-core-cre-mixed-env-tests: @@ -543,7 +544,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} # Base-branch commit this PR is built on top of, so the mixed-env baseline is the exact # branch the PR image derives from (develop or a release branch) and the 2-2 split # isolates only the PR's changes. Empty for non-PR events; the workflow then falls back @@ -851,7 +852,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ inputs.coverage && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} baseline_ref: ${{ github.event.pull_request.base.sha }} base_ref_name: ${{ github.event.pull_request.base.ref }} secrets: inherit diff --git a/core/coverage_flush.go b/core/coverage_flush.go index d3de0e6703d..656f01d34d9 100644 --- a/core/coverage_flush.go +++ b/core/coverage_flush.go @@ -6,6 +6,7 @@ import ( "os" "os/signal" "runtime/coverage" + "sync" "syscall" "time" ) @@ -18,18 +19,24 @@ func init() { if dir == "" { return } + var mu sync.Mutex + flush := func() { + mu.Lock() + defer mu.Unlock() + _ = coverage.WriteCountersDir(dir) + } ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGUSR1) go func() { for range ch { - _ = coverage.WriteCountersDir(dir) + flush() } }() go func() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for range ticker.C { - _ = coverage.WriteCountersDir(dir) + flush() } }() } diff --git a/tools/bin/go_core_scripts_tests b/tools/bin/go_core_scripts_tests index 11d346f298f..2adc76fd87a 100755 --- a/tools/bin/go_core_scripts_tests +++ b/tools/bin/go_core_scripts_tests @@ -13,7 +13,7 @@ echo "Test execution results: ---------------------" echo "" if [[ "$COVERAGE" == "true" ]]; then - COVERAGE_DIR=${GOCOVERDIR:-$PWD/coverage} + COVERAGE_DIR=${GOCOVERDIR:-$PWD/.covdata} mkdir -p "$COVERAGE_DIR" EXTRA_FLAGS="-covermode=atomic -cover -coverpkg=${COVERPKG:-./...} -args -test.gocoverdir=$COVERAGE_DIR" echo "Coverage enabled: $COVERAGE_DIR" diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh index 593950790a1..736d2e3497d 100755 --- a/tools/coverage/collect.sh +++ b/tools/coverage/collect.sh @@ -9,16 +9,21 @@ # Optional: # PR_NUMBER PR number; inferred from TARGET_SHA when empty # OUTPUT_DIR report output directory (default: coverage-report) +# ARTIFACTS_DIR raw artifact download directory (default: $RUNNER_TEMP/coverage-artifacts) +# WAIT_FOR_RUNS wait for other workflow runs for TARGET_SHA (true/false) +# WAIT_TIMEOUT seconds to wait for runs (default: 2700) set -euo pipefail REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" TARGET_SHA="${TARGET_SHA:?TARGET_SHA is required}" OUT="${OUTPUT_DIR:-coverage-report}" -ARTIFACTS="${OUT}/artifacts" -mkdir -p "$ARTIFACTS" +ARTIFACTS="${ARTIFACTS_DIR:-${RUNNER_TEMP:-/tmp}/coverage-artifacts}" +rm -rf "$ARTIFACTS" +mkdir -p "$OUT" "$ARTIFACTS" if [[ -z "${PR_NUMBER:-}" ]]; then - PR_NUMBER="$(gh api "repos/${REPO}/commits/${TARGET_SHA}/pulls" --jq '.[0].number // empty' 2>/dev/null || true)" + PR_NUMBER="$(gh api "repos/${REPO}/commits/${TARGET_SHA}/pulls" \ + --jq '[.[] | select(.state == "open")][0].number // empty' 2>/dev/null || true)" fi BASE_SHA="" if [[ -n "${PR_NUMBER:-}" ]]; then @@ -27,6 +32,25 @@ fi PATCH_BASE="${PATCH_BASE:-}" PATCH_HEAD="${PATCH_HEAD:-$TARGET_SHA}" +if [[ "${WAIT_FOR_RUNS:-false}" == "true" ]]; then + deadline=$((SECONDS + ${WAIT_TIMEOUT:-2700})) + sleep 30 + while (( SECONDS < deadline )); do + runs_json="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 500 \ + --json workflowName,status 2>/dev/null || echo '[]')" + pending="$(printf '%s' "$runs_json" | jq -r \ + '.[] | select((.status == "queued" or .status == "in_progress") and .workflowName != "Coverage Collector") | .workflowName' \ + 2>/dev/null | sort -u || true)" + if [[ -z "$pending" ]]; then + break + fi + echo "Waiting for: ${pending//$'\n'/, }" + sleep 30 + done +fi + +echo "Collecting coverage for ${REPO}@${TARGET_SHA} (PR ${PR_NUMBER:-none}, base ${BASE_SHA:-none})" + if [[ -n "$BASE_SHA" ]]; then git fetch --no-tags origin "$BASE_SHA" "$TARGET_SHA" >/dev/null 2>&1 || true fi @@ -37,50 +61,53 @@ if [[ -z "$PATCH_BASE" && -n "$BASE_SHA" ]]; then PATCH_BASE="$(git merge-base "$BASE_SHA" "$TARGET_SHA" 2>/dev/null || echo "$BASE_SHA")" fi -echo "Collecting coverage for ${REPO}@${TARGET_SHA} (PR ${PR_NUMBER:-none}, base ${BASE_SHA:-none})" - -runs="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 100 --json databaseId --jq '.[].databaseId')" +runs="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 500 --json databaseId --jq '.[].databaseId' 2>/dev/null || true)" for run in $runs; do - names="$(gh api "repos/${REPO}/actions/runs/${run}/artifacts?per_page=100" --jq '.artifacts[].name' 2>/dev/null | grep -E '^(coverage-(core|deployment|inmemory|tools|e2e)-|test-results-)' || true)" + names="$(gh api "repos/${REPO}/actions/runs/${run}/artifacts?per_page=100" --jq '.artifacts[].name' 2>/dev/null \ + | grep -E '^(coverage-(core|deployment|inmemory|tools|e2e)-|test-results-)' || true)" while IFS= read -r name; do [[ -z "$name" ]] && continue dest="${ARTIFACTS}/${run}/${name}" mkdir -p "$dest" echo "Downloading ${name} from run ${run}" - gh run download "$run" --repo "$REPO" -n "$name" -D "$dest" || true + gh run download "$run" --repo "$REPO" -n "$name" -D "$dest" || \ + echo "::warning::Failed to download ${name} from ${run}" done <<< "$names" done -all_dirs="" unit_dirs="" integration_dirs="" e2e_dirs="" - -while IFS= read -r meta; do - [[ -z "$meta" ]] && continue - dir="$(dirname "$meta")" - all_dirs="${all_dirs}${all_dirs:+,}${dir}" - case "$meta" in - */coverage-core-go_core_tests-*/*|*/coverage-core-go_core_race_tests-*/*|*/coverage-core-scripts-*/*|*/coverage-tools-*/*) +while IFS= read -r dir; do + case "$dir" in + */coverage-core-go_core_tests-*|*/coverage-core-go_core_race_tests-*|*/coverage-core-scripts-*|*/coverage-tools-*) unit_dirs="${unit_dirs}${unit_dirs:+,}${dir}" ;; - */coverage-core-go_core_tests_integration-*/*|*/coverage-deployment-*/*|*/coverage-inmemory-*/*) + */coverage-core-go_core_tests_integration-*|*/coverage-deployment-*|*/coverage-inmemory-*) integration_dirs="${integration_dirs}${integration_dirs:+,}${dir}" ;; - */coverage-e2e-*/*) + */coverage-e2e-*) e2e_dirs="${e2e_dirs}${e2e_dirs:+,}${dir}" ;; esac -done < <(find "$ARTIFACTS" -type f -name 'covmeta.*' 2>/dev/null | sort -u) +done < <(find "$ARTIFACTS" -type f -name 'covmeta.*' -exec dirname {} \; 2>/dev/null | sort -u) merge_and_report() { local category="$1" dirs="$2" [[ -z "$dirs" ]] && return 0 - echo "Merging ${category}: ${dirs}" - go run ./tools/coverage merge --covdata "$dirs" --out "${OUT}/${category}.txt" + echo "Merging ${category}" + if ! go run ./tools/coverage merge --covdata "$dirs" --out "${OUT}/${category}.txt"; then + echo "::warning::Failed to merge ${category} coverage" + return 0 + fi local args=(--profile "${OUT}/${category}.txt" --category "$category" --json "${OUT}/${category}.json" --markdown "${OUT}/${category}.md") if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") fi - go run ./tools/coverage report "${args[@]}" + if ! go run ./tools/coverage report "${args[@]}"; then + echo "::warning::Failed to compute ${category} patch coverage; retrying without patch" + args=(--profile "${OUT}/${category}.txt" --category "$category" + --json "${OUT}/${category}.json" --markdown "${OUT}/${category}.md") + go run ./tools/coverage report "${args[@]}" || echo "::warning::Failed to report ${category} coverage" + fi } merge_and_report unit "$unit_dirs" @@ -94,18 +121,26 @@ for category in unit integration e2e; do fi done if [[ -n "$profile_list" ]]; then - go run ./tools/coverage merge-profiles --profile "$profile_list" --out "${OUT}/any.txt" - args=(--profile "${OUT}/any.txt" --category any - --json "${OUT}/any.json" --markdown "${OUT}/any.md") - if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then - args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") + if go run ./tools/coverage merge-profiles --profile "$profile_list" --out "${OUT}/any.txt"; then + args=(--profile "${OUT}/any.txt" --category any + --json "${OUT}/any.json" --markdown "${OUT}/any.md") + if [[ -n "$PATCH_BASE" && -n "$PATCH_HEAD" ]]; then + args+=(--base "$PATCH_BASE" --head "$PATCH_HEAD") + fi + if ! go run ./tools/coverage report "${args[@]}"; then + echo "::warning::Failed to compute any patch coverage; retrying without patch" + args=(--profile "${OUT}/any.txt" --category any + --json "${OUT}/any.json" --markdown "${OUT}/any.md") + go run ./tools/coverage report "${args[@]}" || echo "::warning::Failed to report any coverage" + fi + else + echo "::warning::Failed to merge any coverage" fi - go run ./tools/coverage report "${args[@]}" fi test_xmls="$(find "$ARTIFACTS" -type f -name '*.xml' -path '*test-results-*' 2>/dev/null | sort -u)" if [[ -n "$test_xmls" ]]; then - python3 - "$OUT/test-results.json" $test_xmls <<'PY' + python3 - "$OUT/test-results.json" $test_xmls <<'PY' || echo "::warning::Failed to parse test results" import json, sys, xml.etree.ElementTree as ET files = sys.argv[2:] totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0, "files": len(files)} @@ -126,6 +161,11 @@ with open(sys.argv[1], "w", encoding="utf-8") as f: PY fi +if ! compgen -G "${OUT}/*.md" >/dev/null; then + echo "No coverage profiles were produced for ${TARGET_SHA}" + exit 0 +fi + { echo "" echo "# Coverage report for \`${TARGET_SHA:0:9}\`" @@ -158,12 +198,14 @@ cat "${OUT}/summary.md" if [[ -n "${PR_NUMBER:-}" ]]; then marker="" comment_id="$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \ - | jq -r --arg marker "$marker" '.[] | select(.body | contains($marker)) | .id' 2>/dev/null \ + | jq -r --arg marker "$marker" '.[] | select(.user.login == "github-actions[bot]") | select(.body | contains($marker)) | .id' 2>/dev/null \ | head -1 || true)" if [[ -n "$comment_id" ]]; then - gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body="@${OUT}/summary.md" + gh api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -F body="@${OUT}/summary.md" || \ + echo "::warning::Failed to update coverage comment" else - gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "${OUT}/summary.md" + gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "${OUT}/summary.md" || \ + echo "::warning::Failed to create coverage comment" fi fi From 1aa3d16ff681a211ba3af2669ca4afc43cf69326 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:27:04 +1100 Subject: [PATCH 10/16] fix(ci): run collector on PRs after workflows finish Keep a pull_request trigger for pre-merge reporting and wait for all relevant runs before collecting. The collector remains workflow_run-driven after merge. --- .github/workflows/coverage-collector.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index 30a372c994a..ac8249078d5 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -1,6 +1,8 @@ name: Coverage Collector on: + pull_request: + types: [opened, synchronize, reopened, labeled] workflow_run: workflows: - CI Core @@ -23,13 +25,13 @@ permissions: pull-requests: write concurrency: - group: coverage-${{ github.event.workflow_run.head_sha }} + group: coverage-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} cancel-in-progress: true jobs: collect: name: Collect coverage - if: ${{ github.event.workflow_run.conclusion != 'skipped' }} + if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion != 'skipped' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -43,11 +45,15 @@ jobs: go-version-file: go.mod cache: true + - name: Wait for PR workflows to start + if: ${{ github.event_name == 'pull_request' }} + run: sleep 60 + - name: Collect and publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TARGET_SHA: ${{ github.event.workflow_run.head_sha }} - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number || '' }} + TARGET_SHA: ${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ github.event.pull_request.number || (github.event.workflow_run.pull_requests[0].number || '') }} PATCH_BASE: "" PATCH_HEAD: "" WAIT_FOR_RUNS: "true" @@ -58,7 +64,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: coverage-report-${{ github.event.workflow_run.head_sha }}-${{ github.run_id }} + name: coverage-report-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }}-${{ github.run_id }} path: coverage-report retention-days: 30 if-no-files-found: warn From 9866db130bb82df60a9744db707a61f44c5ab9d7 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:47:31 +1100 Subject: [PATCH 11/16] fix(coverage): only wait for coverage-relevant workflows Avoid blocking the collector on unrelated PR workflows such as CodeQL, UI, and vault checks. --- tools/coverage/collect.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh index 736d2e3497d..e3b1bdcf257 100755 --- a/tools/coverage/collect.sh +++ b/tools/coverage/collect.sh @@ -38,8 +38,9 @@ if [[ "${WAIT_FOR_RUNS:-false}" == "true" ]]; then while (( SECONDS < deadline )); do runs_json="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 500 \ --json workflowName,status 2>/dev/null || echo '[]')" - pending="$(printf '%s' "$runs_json" | jq -r \ - '.[] | select((.status == "queued" or .status == "in_progress") and .workflowName != "Coverage Collector") | .workflowName' \ + watched='["CI Core","Deployment Tests","Integration In-Memory Tests","Integration Tests","CI Tools","CRE System Tests","CRE Regression System Tests","CRE Mixed-Env Tests","CRE Local Environment Tests","CCIP System Tests","Legacy System Tests","Legacy Non Functional Tests"]' + pending="$(printf '%s' "$runs_json" | jq -r --argjson watched "$watched" \ + '.[] | select(.workflowName as $n | $watched | index($n)) | select(.status == "queued" or .status == "in_progress") | .workflowName' \ 2>/dev/null | sort -u || true)" if [[ -z "$pending" ]]; then break From bd0ea6c8d04ac6f5947a575f6c709a770d971fc0 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:19:21 +1100 Subject: [PATCH 12/16] ci(coverage): publish nightly race coverage as a 7-day artifact Enable coverage only on scheduled (or manually forced) race runs and upload coverage-core-go_core_race_tests for 7 days. The collector skips that artifact so -race instrumentation cannot skew project or patch coverage. Add a workflow_dispatch hook to recalculate coverage for a SHA. --- .github/workflows/ci-core.yml | 22 ++++++++++++++----- .github/workflows/coverage-collector.yml | 27 ++++++++++++++++++------ tools/coverage/collect.sh | 9 ++++++-- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 9bdeee249e1..988262e07cb 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -16,6 +16,12 @@ on: schedule: - cron: "0 0,1,2,3,4,5,6,20,21,22,23 * * *" # Run regularly at night to get better flaky test data workflow_dispatch: + inputs: + race_coverage: + description: "Run the nightly race-coverage pass on this manual run (test hook)." + required: false + default: false + type: boolean jobs: filter: @@ -381,12 +387,17 @@ jobs: install-loopps: "false" build-cache-version: go_core_race_tests build-cache-write: always + # Nightly race runs also produce a coverage artifact so a race- + # instrumented run can be inspected later. It is deliberately not + # merged into the project coverage calculation; see collect.sh. + coverage_on_schedule: "true" name: Core Tests (${{ matrix.type.cmd }}) # Be careful modifying the job name, as it is used to fetch the job URL # We don't directly merge dependabot PRs, so let's not waste the resources if: ${{ github.actor != 'dependabot[bot]' }} needs: [filter] - timeout-minutes: 60 + # Scheduled race runs with coverage are much slower than PR runs. + timeout-minutes: 120 # Use ubuntu-latest for jobs that will be skipped runs-on: ${{ matrix.type.should-run == 'true' && matrix.type.os || 'ubuntu-latest' }} @@ -489,13 +500,13 @@ jobs: - name: Run tests if: ${{ matrix.type.should-run == 'true' }} - timeout-minutes: 50 + timeout-minutes: 90 continue-on-error: ${{ matrix.type.trunk-auto-quarantine == 'true' }} id: run-tests shell: bash env: CL_DATABASE_URL: ${{ env.DB_URL }} - COVERAGE: ${{ matrix.type.coverage || 'false' }} + COVERAGE: ${{ matrix.type.coverage || (matrix.type.coverage_on_schedule == 'true' && (github.event_name == 'schedule' || inputs.race_coverage)) || 'false' }} GOCOVERDIR: ${{ github.workspace }}/.covdata/${{ matrix.type.cmd }} OUTPUT_FILE: ./output.txt PRODUCE_JUNIT_XML: ${{ matrix.type.trunk-auto-quarantine }} @@ -570,8 +581,9 @@ jobs: - name: Store coverage artifacts if: - ${{ always() && matrix.type.should-run == 'true' && matrix.type.coverage - == 'true' }} + ${{ always() && matrix.type.should-run == 'true' && (matrix.type.coverage + == 'true' || (matrix.type.coverage_on_schedule == 'true' && + (github.event_name == 'schedule' || inputs.race_coverage))) }} uses: actions/upload-artifact@v7 with: name: coverage-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index ac8249078d5..454fc8db3b7 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -3,6 +3,21 @@ name: Coverage Collector on: pull_request: types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + target_sha: + description: Commit SHA whose coverage artifacts should be collected + required: true + type: string + pr_number: + description: PR number to comment on (optional) + required: false + type: string + wait_for_runs: + description: Wait for the other watched workflows for this SHA + required: false + default: true + type: boolean workflow_run: workflows: - CI Core @@ -25,13 +40,13 @@ permissions: pull-requests: write concurrency: - group: coverage-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} + group: coverage-${{ inputs.target_sha || github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} cancel-in-progress: true jobs: collect: name: Collect coverage - if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion != 'skipped' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' || github.event.workflow_run.conclusion != 'skipped' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -52,11 +67,11 @@ jobs: - name: Collect and publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TARGET_SHA: ${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} - PR_NUMBER: ${{ github.event.pull_request.number || (github.event.workflow_run.pull_requests[0].number || '') }} + TARGET_SHA: ${{ inputs.target_sha || github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number || (github.event.workflow_run.pull_requests[0].number || '') }} PATCH_BASE: "" PATCH_HEAD: "" - WAIT_FOR_RUNS: "true" + WAIT_FOR_RUNS: ${{ github.event_name != 'workflow_dispatch' && 'true' || (inputs.wait_for_runs == true && 'true' || 'false') }} WAIT_TIMEOUT: "2700" run: ./tools/coverage/collect.sh @@ -64,7 +79,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: coverage-report-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }}-${{ github.run_id }} + name: coverage-report-${{ inputs.target_sha || github.event.pull_request.head.sha || github.event.workflow_run.head_sha }}-${{ github.run_id }} path: coverage-report retention-days: 30 if-no-files-found: warn diff --git a/tools/coverage/collect.sh b/tools/coverage/collect.sh index e3b1bdcf257..4970a737daa 100755 --- a/tools/coverage/collect.sh +++ b/tools/coverage/collect.sh @@ -65,7 +65,12 @@ fi runs="$(gh run list --repo "$REPO" --commit "$TARGET_SHA" -L 500 --json databaseId --jq '.[].databaseId' 2>/dev/null || true)" for run in $runs; do names="$(gh api "repos/${REPO}/actions/runs/${run}/artifacts?per_page=100" --jq '.artifacts[].name' 2>/dev/null \ - | grep -E '^(coverage-(core|deployment|inmemory|tools|e2e)-|test-results-)' || true)" + | grep -E '^(coverage-(core|deployment|inmemory|tools|e2e)-|test-results-)' \ + | grep -v '^coverage-core-go_core_race_tests-' || true)" + # Nightly race coverage is a separate 7-day artifact for race-instrumented + # inspection. It is intentionally excluded from the project and patch + # coverage calculation: -race changes runtime/timing and would make the + # normal coverage trend incomparable. while IFS= read -r name; do [[ -z "$name" ]] && continue dest="${ARTIFACTS}/${run}/${name}" @@ -81,7 +86,7 @@ integration_dirs="" e2e_dirs="" while IFS= read -r dir; do case "$dir" in - */coverage-core-go_core_tests-*|*/coverage-core-go_core_race_tests-*|*/coverage-core-scripts-*|*/coverage-tools-*) + */coverage-core-go_core_tests-*|*/coverage-core-scripts-*|*/coverage-tools-*) unit_dirs="${unit_dirs}${unit_dirs:+,}${dir}" ;; */coverage-core-go_core_tests_integration-*|*/coverage-deployment-*|*/coverage-inmemory-*) integration_dirs="${integration_dirs}${integration_dirs:+,}${dir}" ;; From a7ffd442631ac626cab8076c98e242dc99decc5a Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:19:21 +1100 Subject: [PATCH 13/16] build: add dev and race instrumentation to coverage/debug images install-chainlink-cover now includes the dev tag, matching the normal E2E image. Add install-chainlink-debug (dev + coverage + -race) and a GO_DEBUG_FLAG Docker build path. Wire a debug input and run-e2e-debug label into integration-tests so the debug image is selectable. --- .github/workflows/integration-tests.yml | 19 +++++++++++++------ GNUmakefile | 6 +++++- core/chainlink.Dockerfile | 5 ++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6029304c43f..d3fb9272c9e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -35,6 +35,11 @@ on: required: false type: boolean default: false + debug: + description: "Build the Chainlink image with dev + coverage + race instrumentation and use the -debug tag." + required: false + type: boolean + default: false # Only run 1 of this workflow at a time per PR, but don't cancel runs on develop concurrency: @@ -47,6 +52,7 @@ env: CHAINLINK_REF: ${{ inputs.cl_ref || github.sha }} TEST_SUITE: smoke E2E_COVERAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage')) }} + E2E_DEBUG: ${{ (github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug')) }} jobs: enforce-ctf-version: @@ -318,7 +324,7 @@ jobs: uses: smartcontractkit/.github/actions/ecr-image-exists@ecr-image-exists/0.2.0 with: repository: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} - tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && env.E2E_COVERAGE && '-coverage' || '' }} + tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG && '-debug' || env.E2E_COVERAGE && '-coverage' || '') || '' }} aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Explain build/no-build decision @@ -365,7 +371,8 @@ jobs: docker-additional-build-args: | APT_CACHE_BUST=${{ steps.apt-cache-bust.outputs.key }} ${{ matrix.image.name == '' && format('GO_COVER_FLAG={0}', env.E2E_COVERAGE) || '' }} - image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && env.E2E_COVERAGE && '-coverage' || '' }} + ${{ matrix.image.name == '' && format('GO_DEBUG_FLAG={0}', env.E2E_DEBUG) || '' }} + image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG && '-debug' || env.E2E_COVERAGE && '-coverage' || '') || '' }} dockerfile: ${{ matrix.image.dockerfile }} docker-registry-url: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com docker-repository-name: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} @@ -508,7 +515,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} secrets: inherit run-core-cre-e2e-regression-tests: @@ -526,7 +533,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} secrets: inherit run-core-cre-mixed-env-tests: @@ -544,7 +551,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} # Base-branch commit this PR is built on top of, so the mixed-env baseline is the exact # branch the PR image derives from (develop or a release branch) and the 2-2 split # isolates only the PR's changes. Empty for non-PR events; the workflow then falls back @@ -852,7 +859,7 @@ jobs: ecr: "sdlc" chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }} - chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '' }} + chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}${{ ((github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug'))) && '-debug' || (((github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage'))) && '-coverage' || '') }} baseline_ref: ${{ github.event.pull_request.base.sha }} base_ref_name: ${{ github.event.pull_request.base.ref }} secrets: inherit diff --git a/GNUmakefile b/GNUmakefile index 94b3c8ceca0..9eac2444615 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -84,7 +84,11 @@ install-chainlink-dev: operator-ui ## Install the chainlink binary. .PHONY: install-chainlink-cover install-chainlink-cover: operator-ui ## Install the chainlink binary with cover flag. - go install -tags coverage -cover -covermode=atomic $(GOFLAGS) . + go install -tags "dev coverage" -cover -covermode=atomic $(GOFLAGS) . + +.PHONY: install-chainlink-debug +install-chainlink-debug: operator-ui ## Install the chainlink binary with dev, coverage, and race instrumentation. + go install -tags "dev coverage" -cover -covermode=atomic -race $(GCFLAGS) $(GOFLAGS) . .PHONY: chainlink chainlink: ## Build the chainlink binary. diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index f7c71ebc509..9bfaeb24768 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -98,11 +98,14 @@ ARG COMMIT_SHA ARG VERSION_TAG ARG CL_IS_PROD_BUILD=true ARG GO_COVER_FLAG=false +ARG GO_DEBUG_FLAG=false RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ mkdir -p /gobins && \ - if [ "$GO_COVER_FLAG" = "true" ]; then \ + if [ "$GO_DEBUG_FLAG" = "true" ]; then \ + CGO_ENABLED=1 GOBIN=/gobins make install-chainlink-debug; \ + elif [ "$GO_COVER_FLAG" = "true" ]; then \ GOBIN=/gobins make install-chainlink-cover; \ elif [ "$CL_IS_PROD_BUILD" = "false" ]; then \ GOBIN=/gobins make install-chainlink-dev; \ From a286ce01d53be98b8101ac1b275c28a8b90ab3d3 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:31:22 +1100 Subject: [PATCH 14/16] fix(ci): compare E2E debug/coverage env strings when choosing image tag --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d3fb9272c9e..91f7173c958 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -324,7 +324,7 @@ jobs: uses: smartcontractkit/.github/actions/ecr-image-exists@ecr-image-exists/0.2.0 with: repository: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} - tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG && '-debug' || env.E2E_COVERAGE && '-coverage' || '') || '' }} + tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG == 'true' && '-debug' || env.E2E_COVERAGE == 'true' && '-coverage' || '') || '' }} aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Explain build/no-build decision @@ -372,7 +372,7 @@ jobs: APT_CACHE_BUST=${{ steps.apt-cache-bust.outputs.key }} ${{ matrix.image.name == '' && format('GO_COVER_FLAG={0}', env.E2E_COVERAGE) || '' }} ${{ matrix.image.name == '' && format('GO_DEBUG_FLAG={0}', env.E2E_DEBUG) || '' }} - image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG && '-debug' || env.E2E_COVERAGE && '-coverage' || '') || '' }} + image-tag: ${{ inputs.evm-ref || env.CHAINLINK_REF }}${{ matrix.image.tag-suffix }}${{ matrix.image.name == '' && (env.E2E_DEBUG == 'true' && '-debug' || env.E2E_COVERAGE == 'true' && '-coverage' || '') || '' }} dockerfile: ${{ matrix.image.dockerfile }} docker-registry-url: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com docker-repository-name: ${{ inputs.ecr_name || 'chainlink-integration-tests' }} From c31dad49aa229084e5e0cf8f4cd5ee603a352993 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:35:34 +1100 Subject: [PATCH 15/16] feat(coverage): add quick patch coverage and nightly badge --- .github/workflows/coverage-collector.yml | 91 +++++++++++- .github/workflows/coverage-patch.yml | 67 +++++++++ .github/workflows/integration-tests.yml | 5 +- README.md | 1 + tools/coverage/badge.py | 78 ++++++++++ tools/coverage/patch.sh | 179 +++++++++++++++++++++++ 6 files changed, 419 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/coverage-patch.yml create mode 100644 tools/coverage/badge.py create mode 100755 tools/coverage/patch.sh diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index 454fc8db3b7..2b33584f417 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -18,6 +18,11 @@ on: required: false default: true type: boolean + update_badge: + description: Update the coverage badge branch from this manual run (develop only) + required: false + default: false + type: boolean workflow_run: workflows: - CI Core @@ -36,7 +41,8 @@ on: permissions: actions: read - contents: read + # The badge job writes a tiny SVG/JSON to the orphan `coverage` branch. + contents: write pull-requests: write concurrency: @@ -83,3 +89,86 @@ jobs: path: coverage-report retention-days: 30 if-no-files-found: warn + badge: + name: Update coverage badge + needs: [collect] + if: | + (github.event_name == 'workflow_run' && + github.event.workflow_run.name == 'Integration Tests' && + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.head_branch == 'develop' && + github.event.workflow_run.conclusion == 'success') || + (github.event_name == 'workflow_dispatch' && + inputs.update_badge && + github.ref == 'refs/heads/develop') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Download coverage report + uses: actions/download-artifact@v7 + with: + name: coverage-report-${{ github.event.workflow_run.head_sha || inputs.target_sha }}-${{ github.run_id }} + path: ${{ runner.temp }}/coverage-report + + - name: Require a full report + id: full + run: | + set -euo pipefail + for file in unit.txt integration.txt e2e.txt any.json; do + if [[ ! -f "$RUNNER_TEMP/coverage-report/$file" ]]; then + echo "Missing $file; not updating the badge from a partial report." + echo "full=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + done + echo "full=true" >> "$GITHUB_OUTPUT" + + - name: Generate badge and state + if: steps.full.outputs.full == 'true' + env: + SHA: ${{ github.event.workflow_run.head_sha || inputs.target_sha }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ARTIFACT: coverage-report-${{ github.event.workflow_run.head_sha || inputs.target_sha }}-${{ github.run_id }} + run: | + python3 tools/coverage/badge.py \ + "$RUNNER_TEMP/coverage-report/any.json" \ + "$RUNNER_TEMP/coverage-badge.svg" \ + "$RUNNER_TEMP/coverage-state.json" \ + "$SHA" "$ARTIFACT" "$REPO" "$RUN_URL" + + - name: Publish coverage branch + if: steps.full.outputs.full == 'true' + env: + SHA: ${{ github.event.workflow_run.head_sha || inputs.target_sha }} + run: | + set -euo pipefail + worktree="$RUNNER_TEMP/coverage-branch" + rm -rf "$worktree" + if git ls-remote --exit-code --heads origin coverage >/dev/null 2>&1; then + git fetch --no-tags origin coverage:refs/remotes/origin/coverage + git worktree add --detach "$worktree" refs/remotes/origin/coverage + else + git worktree add --detach "$worktree" + cd "$worktree" + git checkout --orphan coverage + git rm -rf . >/dev/null 2>&1 || true + fi + cd "$worktree" + mkdir -p latest + cp "$RUNNER_TEMP/coverage-badge.svg" latest/badge.svg + cp "$RUNNER_TEMP/coverage-state.json" latest/report.json + git add latest + if git diff --cached --quiet; then + echo "Coverage badge unchanged; not committing." + exit 0 + fi + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -m "coverage: update badge for ${SHA}" + git push origin HEAD:refs/heads/coverage diff --git a/.github/workflows/coverage-patch.yml b/.github/workflows/coverage-patch.yml new file mode 100644 index 00000000000..57b2914fdfb --- /dev/null +++ b/.github/workflows/coverage-patch.yml @@ -0,0 +1,67 @@ +name: Coverage Patch + +# Fast, non-blocking patch coverage for the packages touched by a PR. +# This is intentionally not the full coverage suite: it runs affected packages +# with -short, reuses the normal Go build/module caches, and posts a sticky PR +# comment. The nightly full report remains the authoritative number. + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "**/*.go" + - "**/go.mod" + - "**/go.sum" + +permissions: + contents: read + pull-requests: write + +concurrency: + group: coverage-patch-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + patch: + name: Patch coverage + if: ${{ !github.event.pull_request.draft && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version-file: go.mod + go-module-file: go.sum + build-cache-version: coverage-patch + # Upsert the branch-specific build cache so later pushes on this PR + # reuse the coverage build outputs. The cache version is unique to + # this job, so concurrent PRs cannot race on one key. + build-cache-write: always + + - name: Run affected-package tests and publish patch coverage + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_REPOSITORY: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + OUT: coverage-patch + SHORT: "true" + TEST_TIMEOUT: "10m" + run: ./tools/coverage/patch.sh + + - name: Upload patch report + if: always() + uses: actions/upload-artifact@v7 + with: + name: coverage-patch-${{ github.event.pull_request.head.sha }} + path: coverage-patch + retention-days: 1 + if-no-files-found: warn diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 91f7173c958..385f3447348 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,6 +11,9 @@ on: - "*" branches: - develop + schedule: + # Nightly full E2E coverage for the full coverage review/badge. + - cron: "0 3 * * *" workflow_dispatch: inputs: cl_ref: @@ -51,7 +54,7 @@ env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name || 'chainlink-integration-tests' }} CHAINLINK_REF: ${{ inputs.cl_ref || github.sha }} TEST_SUITE: smoke - E2E_COVERAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage')) }} + E2E_COVERAGE: ${{ (github.event_name == 'workflow_dispatch' && inputs.coverage) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-coverage')) || github.event_name == 'schedule' }} E2E_DEBUG: ${{ (github.event_name == 'workflow_dispatch' && inputs.debug) || (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-e2e-debug')) }} jobs: diff --git a/README.md b/README.md index d003c1bd806..8cdf07dec46 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ [![GitHub contributors](https://img.shields.io/github/contributors-anon/smartcontractkit/chainlink?style=flat-square)](https://github.com/smartcontractkit/chainlink/graphs/contributors) [![GitHub commit activity](https://img.shields.io/github/commit-activity/y/smartcontractkit/chainlink?style=flat-square)](https://github.com/smartcontractkit/chainlink/commits/master) [![Official documentation](https://img.shields.io/static/v1?label=docs&message=latest&color=blue)](https://docs.chain.link/) +[![Coverage](https://raw.githubusercontent.com/smartcontractkit/chainlink/coverage/latest/badge.svg)](https://github.com/smartcontractkit/chainlink/actions/workflows/coverage-collector.yml) [Chainlink](https://chain.link/) expands the capabilities of smart contracts by enabling access to real-world data and off-chain computation while maintaining the security and reliability guarantees inherent to blockchain technology. diff --git a/tools/coverage/badge.py b/tools/coverage/badge.py new file mode 100644 index 00000000000..5a7706e0c85 --- /dev/null +++ b/tools/coverage/badge.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Generate a small coverage badge and latest-report JSON from tools/coverage output. + +Usage: + badge.py any.json badge.svg report.json SHA artifact_name repo run_url + +The badge uses the source-only global percentage, which excludes tests, +generated code, and mocks. The report JSON is the only state kept in the +orphan `coverage` branch. +""" + +from __future__ import annotations + +import datetime as dt +import json +import sys +from pathlib import Path + + +def color_for(percent: float) -> str: + if percent >= 70: + return "#2e7d32" + if percent >= 55: + return "#f9a825" + if percent >= 45: + return "#ef6c00" + return "#c62828" + + +def main() -> int: + if len(sys.argv) != 8: + print(__doc__, file=sys.stderr) + return 2 + + report_path, svg_path, state_path, sha, artifact, repo, run_url = sys.argv[1:] + report = json.loads(Path(report_path).read_text(encoding="utf-8")) + source = report.get("GlobalSourceOnly") or report.get("Global") or {} + percent = float(source.get("Percent", 0.0)) + raw = float((report.get("Global") or {}).get("Percent", 0.0)) + patch = report.get("Patch") or {} + patch_percent = patch.get("Percent") + + value = f"{percent:.1f}%" + color = color_for(percent) + svg = ( + '' + '' + f'' + f'' + '' + 'coverage' + f'{value}' + "\n" + ) + Path(svg_path).write_text(svg, encoding="utf-8") + + state = { + "sha": sha, + "repo": repo, + "global_raw": raw, + "global_source_only": percent, + "patch": patch_percent, + "report_artifact": artifact, + "run_url": run_url, + "updated_at": dt.datetime.now(dt.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + } + Path(state_path).write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + print(f"coverage badge: {value} ({color})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/coverage/patch.sh b/tools/coverage/patch.sh new file mode 100755 index 00000000000..0b899f1f0bf --- /dev/null +++ b/tools/coverage/patch.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Fast patch-only coverage for the packages touched by a PR. +# +# It intentionally does NOT run the full suite or E2E. It runs the affected +# packages with -short, uses the normal Go build/module caches, computes patch +# coverage with tools/coverage, and updates a sticky PR comment. +# +# Required: +# BASE_SHA PR base SHA (the merge base is derived with git merge-base) +# HEAD_SHA PR head SHA +# Optional: +# OUT output directory (default: coverage-patch) +# PR_NUMBER PR number; when set, the report is commented on the PR +# GH_TOKEN token with pull-requests:write (defaults to gh auth) +# SHORT set to false to run the affected packages without -short +# TEST_TIMEOUT per-package test timeout (default: 10m) +set -euo pipefail + +BASE_SHA="${BASE_SHA:?BASE_SHA is required}" +HEAD_SHA="${HEAD_SHA:?HEAD_SHA is required}" +REPO_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPO_ROOT" + +OUT="${OUT:-coverage-patch}" +case "$OUT" in + /*) ;; + *) OUT="$REPO_ROOT/$OUT" ;; +esac +SHORT="${SHORT:-true}" +TEST_TIMEOUT="${TEST_TIMEOUT:-10m}" +COVDATA_ROOT="$OUT/covdata" +rm -rf "$OUT" +mkdir -p "$OUT" "$COVDATA_ROOT" + +CHANGED="$OUT/changed-files.txt" +git diff --name-only --diff-filter=ACMR "$BASE_SHA...$HEAD_SHA" -- '*.go' 2>/dev/null \ + | grep -vE '(^|/)(mocks|testdata|testutils|testhelpers|cltest|fixtures)/|_test\.go$|\.pb\.go$|_gen\.go$|_codecgen\.go$|^vendor/' \ + > "$CHANGED" || true + +if [[ ! -s "$CHANGED" ]]; then + echo "No changed production Go files." + exit 0 +fi + +# Map every changed file to the nearest Go module and package directory. +ROWS="$OUT/rows.tsv" +: > "$ROWS" +while IFS= read -r file; do + dir="$(dirname "$file")" + while [[ "$dir" != "." && ! -f "$dir/go.mod" ]]; do + dir="$(dirname "$dir")" + done + if [[ -f "$dir/go.mod" ]]; then + module_dir="$REPO_ROOT/$dir" + else + module_dir="$REPO_ROOT" + fi + printf '%s\t%s\n' "$module_dir" "$REPO_ROOT/$(dirname "$file")" >> "$ROWS" +done < "$CHANGED" + +modules="$(cut -f1 "$ROWS" | sort -u)" +tests_failed=0 +ran_any=0 + +while IFS= read -r module_dir; do + [[ -z "$module_dir" ]] && continue + map_file="$OUT/packages-$(echo "$module_dir" | tr '/' '_').tsv" + if ! (cd "$module_dir" && go list -e -f '{{.Dir}} {{.ImportPath}}' ./...) > "$map_file" 2> "$OUT/go-list.log"; then + echo "warning: could not list packages in $module_dir; skipping" >&2 + continue + fi + + packages=() + while IFS= read -r package_dir; do + [[ -z "$package_dir" ]] && continue + import_path="$(awk -F '\t' -v d="$package_dir" '$1 == d { print $2; exit }' "$map_file")" + [[ -z "$import_path" ]] && continue + packages+=("$import_path") + done < <(awk -F '\t' -v m="$module_dir" '$1 == m { print $2 }' "$ROWS" | sort -u) + + # Deduplicate package import paths while preserving a stable order. + package_list="$(printf '%s\n' "${packages[@]}" | sort -u | paste -sd, -)" + [[ -z "$package_list" ]] && continue + + covdir="$COVDATA_ROOT/$(echo "$module_dir" | tr '/' '_')" + mkdir -p "$covdir" + short_flag=() + [[ "$SHORT" == "true" ]] && short_flag=(-short) + + echo "Testing affected packages in ${module_dir#"$REPO_ROOT"/}:" + echo " $package_list" + if ! (cd "$module_dir" && go test \ + -count=1 \ + -timeout "$TEST_TIMEOUT" \ + "${short_flag[@]}" \ + -cover \ + -covermode=atomic \ + -coverpkg="$package_list" \ + "${packages[@]}" \ + -args -test.gocoverdir="$covdir") >> "$OUT/test.log" 2>&1; then + tests_failed=1 + echo "warning: some affected-package tests failed in ${module_dir#"$REPO_ROOT"/}; see $OUT/test.log" >&2 + fi + ran_any=1 +done <<< "$modules" + +if [[ "$ran_any" == "0" ]]; then + echo "No runnable affected packages." + exit 0 +fi + +covdirs="$(find "$COVDATA_ROOT" -type f -name 'covmeta.*' -exec dirname {} \; 2>/dev/null | sort -u | paste -sd, -)" +if [[ -z "$covdirs" ]]; then + echo "No coverage data was produced for the affected packages." + exit 0 +fi + +go run ./tools/coverage merge --covdata "$covdirs" --out "$OUT/patch.txt" +merge_base="$(git merge-base "$BASE_SHA" "$HEAD_SHA")" +go run ./tools/coverage report \ + --profile "$OUT/patch.txt" \ + --base "$merge_base" \ + --head "$HEAD_SHA" \ + --category "patch (affected packages)" \ + --json "$OUT/patch.json" \ + --markdown "$OUT/patch.md" + +not_measured="$(jq -r '.Patch.Files[] | select(.TestFile == false and .Statements == 0) | .Path' "$OUT/patch.json" 2>/dev/null || true)" +if [[ -n "$not_measured" ]]; then + { + echo + echo "### Not measured" + echo + echo "These changed files had no tests in their own package, so the quick run could not measure them:" + echo + while IFS= read -r path; do + [[ -n "$path" ]] && echo "- \`$path\`" + done <<< "$not_measured" + echo + echo "The nightly full report covers these." + } >> "$OUT/patch.md" +fi + +{ + echo "" + echo + echo "### Quick patch coverage" + echo + if [[ "$SHORT" == "true" ]]; then + echo "Affected-package coverage for \`$HEAD_SHA\`, using \`-short\`. " + else + echo "Affected-package coverage for \`$HEAD_SHA\`. " + fi + echo "This is a fast PR signal, not the global coverage number; the nightly full report is authoritative." + echo + if [[ "$tests_failed" == "1" ]]; then + echo "> Some affected-package tests failed. The coverage below may be incomplete; check the job log." + echo + fi + cat "$OUT/patch.md" +} > "$OUT/comment.md" + +cat "$OUT/comment.md" + +if [[ -n "${PR_NUMBER:-}" && -n "${GITHUB_REPOSITORY:-}" ]]; then + marker="" + comment_id="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" 2>/dev/null \ + | jq -r --arg marker "$marker" '.[] | select(.user.login == "github-actions[bot]") | select(.body | contains($marker)) | .id' 2>/dev/null \ + | head -1 || true)" + if [[ -n "$comment_id" ]]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -F body="@$OUT/comment.md" + else + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$OUT/comment.md" + fi +fi + +if [[ "$tests_failed" == "1" ]]; then + exit 1 +fi From 2803024b589923fc3f2d3528de0bbb5a8e0a40a8 Mon Sep 17 00:00:00 2001 From: "ben.notyourbusiness" <12837887+007vasy@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:53:15 +1100 Subject: [PATCH 16/16] fix(coverage): restrict badge writes and harden quick patch tests --- .github/workflows/coverage-collector.yml | 7 +- .github/workflows/coverage-patch.yml | 3 + tools/coverage/patch.sh | 14 +++- tools/coverage/test_badge.py | 81 ++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 tools/coverage/test_badge.py diff --git a/.github/workflows/coverage-collector.yml b/.github/workflows/coverage-collector.yml index 2b33584f417..57e1a23e0aa 100644 --- a/.github/workflows/coverage-collector.yml +++ b/.github/workflows/coverage-collector.yml @@ -41,8 +41,7 @@ on: permissions: actions: read - # The badge job writes a tiny SVG/JSON to the orphan `coverage` branch. - contents: write + contents: read pull-requests: write concurrency: @@ -103,6 +102,10 @@ jobs: github.ref == 'refs/heads/develop') runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + # Only this trusted, scheduled badge job may write the coverage branch. + actions: read + contents: write steps: - name: Checkout uses: actions/checkout@v7 diff --git a/.github/workflows/coverage-patch.yml b/.github/workflows/coverage-patch.yml index 57b2914fdfb..245df854f31 100644 --- a/.github/workflows/coverage-patch.yml +++ b/.github/workflows/coverage-patch.yml @@ -45,6 +45,9 @@ jobs: # this job, so concurrent PRs cannot race on one key. build-cache-write: always + - name: Test coverage tooling + run: python3 -m unittest discover -s tools/coverage -p 'test_*.py' -v + - name: Run affected-package tests and publish patch coverage env: BASE_SHA: ${{ github.event.pull_request.base.sha }} diff --git a/tools/coverage/patch.sh b/tools/coverage/patch.sh index 0b899f1f0bf..b6f65701fde 100755 --- a/tools/coverage/patch.sh +++ b/tools/coverage/patch.sh @@ -78,9 +78,17 @@ while IFS= read -r module_dir; do packages+=("$import_path") done < <(awk -F '\t' -v m="$module_dir" '$1 == m { print $2 }' "$ROWS" | sort -u) - # Deduplicate package import paths while preserving a stable order. - package_list="$(printf '%s\n' "${packages[@]}" | sort -u | paste -sd, -)" - [[ -z "$package_list" ]] && continue + # Deduplicate test packages as well as the -coverpkg list. Several changed + # files in one package must not make go test run that package repeatedly. + unique_packages=() + while IFS= read -r import_path; do + [[ -n "$import_path" ]] && unique_packages+=("$import_path") + done < <(printf '%s\n' "${packages[@]}" | sort -u) + if [[ ${#unique_packages[@]} -eq 0 ]]; then + continue + fi + packages=("${unique_packages[@]}") + package_list="$(IFS=,; printf '%s' "${packages[*]}")" covdir="$COVDATA_ROOT/$(echo "$module_dir" | tr '/' '_')" mkdir -p "$covdir" diff --git a/tools/coverage/test_badge.py b/tools/coverage/test_badge.py new file mode 100644 index 00000000000..8767ef00c84 --- /dev/null +++ b/tools/coverage/test_badge.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +class BadgeTests(unittest.TestCase): + def test_badge_uses_source_only_and_writes_state(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + report = tmp / "any.json" + svg = tmp / "badge.svg" + state = tmp / "report.json" + report.write_text( + json.dumps( + { + "Global": {"Percent": 54.27}, + "GlobalSourceOnly": {"Percent": 60.92}, + "Patch": {"Percent": 43.84}, + } + ), + encoding="utf-8", + ) + subprocess.run( + [ + sys.executable, + str(HERE / "badge.py"), + str(report), + str(svg), + str(state), + "abc123", + "coverage-report-abc123-1", + "smartcontractkit/chainlink", + "https://example.invalid/run/1", + ], + check=True, + ) + svg_text = svg.read_text(encoding="utf-8") + state_text = json.loads(state.read_text(encoding="utf-8")) + self.assertIn("coverage: 60.9%", svg_text) + self.assertIn("60.9%", svg_text) + self.assertEqual(state_text["global_raw"], 54.27) + self.assertEqual(state_text["global_source_only"], 60.92) + self.assertEqual(state_text["patch"], 43.84) + self.assertEqual(state_text["sha"], "abc123") + + def test_badge_color_changes_with_coverage(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + report = tmp / "any.json" + svg = tmp / "badge.svg" + state = tmp / "report.json" + report.write_text( + json.dumps({"GlobalSourceOnly": {"Percent": 25.0}}), + encoding="utf-8", + ) + subprocess.run( + [ + sys.executable, + str(HERE / "badge.py"), + str(report), + str(svg), + str(state), + "abc123", + "artifact", + "org/repo", + "https://example.invalid/run/1", + ], + check=True, + ) + self.assertIn("#c62828", svg.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main()