Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/actions/collect-coverage/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: Collect container coverage
description: Copy Go coverage data from running or stopped containers.

inputs:
destination:
description: Host directory that receives per-container coverage directories.
required: true
image:
description: Optional container image filter, for example chainlink-cover:local.
required: false
default: ""

runs:
using: composite
steps:
- name: Copy coverage data
shell: bash
env:
DESTINATION: ${{ inputs.destination }}
IMAGE_FILTER: ${{ inputs.image }}
run: |
set -euo pipefail
mkdir -p "${DESTINATION}"
for container in $(docker ps -aq); do
image="$(docker inspect -f '{{.Config.Image}}' "${container}" 2>/dev/null || true)"
if [[ -n "${IMAGE_FILTER}" && "${image}" != "${IMAGE_FILTER}"* ]]; then
continue
fi
if [[ "${image}" == *"-coverage"* ]]; then
docker kill --signal=SIGUSR1 "${container}" >/dev/null 2>&1 || true
sleep 0.2
fi
if docker cp "${container}:/var/tmp/go-coverage/." "${DESTINATION}/${container}" 2>/dev/null; then
echo "Copied coverage from ${container} (${image})"
fi
done
40 changes: 40 additions & 0 deletions .github/actions/start-coverage-flusher/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: Start coverage flusher
description: Periodically sends SIGUSR1 to coverage-instrumented containers so covcounters are written before teardown.

inputs:
interval:
description: Seconds between flushes.
required: false
default: "15"
pidfile:
description: File used to record the background process ID.
required: false
default: /tmp/chainlink-coverage-flusher.pid

runs:
using: composite
steps:
- name: Start flusher
shell: bash
env:
INTERVAL: ${{ inputs.interval }}
PIDFILE: ${{ inputs.pidfile }}
run: |
set -euo pipefail
cat > /tmp/chainlink-coverage-flusher.sh <<'EOF'
#!/usr/bin/env bash
set -u
interval="${INTERVAL:-15}"
while true; do
for container in $(docker ps -q 2>/dev/null || true); do
image="$(docker inspect -f '{{.Config.Image}}' "${container}" 2>/dev/null || true)"
if [[ "${image}" == *"-coverage"* ]]; then
docker kill --signal=SIGUSR1 "${container}" >/dev/null 2>&1 || true
fi
done
sleep "${interval}"
done
EOF
chmod +x /tmp/chainlink-coverage-flusher.sh
nohup env INTERVAL="${INTERVAL}" /tmp/chainlink-coverage-flusher.sh >/tmp/chainlink-coverage-flusher.log 2>&1 &
echo $! > "${PIDFILE}"
27 changes: 27 additions & 0 deletions .github/workflows/ccip-system-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,9 @@ jobs:
echo "Baseline image: ${image}:${version}"
fi

- name: Start coverage flusher
uses: $/.github/actions/start-coverage-flusher

- name: Run CCIP smoke tests
id: run-tests
shell: bash
Expand Down Expand Up @@ -364,6 +367,30 @@ jobs:
# unique name per matrix job to avoid 409 conflict when multiple jobs upload in the same workflow run
artifact-name: ${{ matrix.tests.test_id }}_test_logs

- name: Collect coverage
if: ${{ always() }}
uses: $/.github/actions/collect-coverage
with:
destination: ${{ github.workspace }}/.covdata/e2e-${{ matrix.tests.test_id }}

- name: Upload coverage
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: coverage-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }}
path: .covdata/e2e-${{ matrix.tests.test_id }}
retention-days: 7
if-no-files-found: warn

- name: Store test results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: test-results-e2e-${{ matrix.tests.test_id }}-${{ github.run_id }}-${{ github.run_attempt }}
path: /tmp/junit.xml
retention-days: 7
if-no-files-found: ignore

- name: Show Docker containers status
if: failure() || cancelled()
shell: bash
Expand Down
51 changes: 49 additions & 2 deletions .github/workflows/ci-core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ on:
schedule:
- cron: "0 0,1,2,3,4,5,6,20,21,22,23 * * *" # Run regularly at night to get better flaky test data
workflow_dispatch:
inputs:
race_coverage:
description: "Run the nightly race-coverage pass on this manual run (test hook)."
required: false
default: false
type: boolean

jobs:
filter:
Expand Down Expand Up @@ -323,6 +329,7 @@ jobs:
setup-sui: "false"
install-loopps: "false"
build-cache-write: always
coverage: "true"

- cmd: go_core_tests_integration
os:
Expand All @@ -341,6 +348,7 @@ jobs:
setup-aptos: "false"
setup-sui: "false"
build-cache-write: always
coverage: "true"

- cmd: go_core_fuzz
os:
Expand Down Expand Up @@ -379,12 +387,17 @@ jobs:
install-loopps: "false"
build-cache-version: go_core_race_tests
build-cache-write: always
# Nightly race runs also produce a coverage artifact so a race-
# instrumented run can be inspected later. It is deliberately not
# merged into the project coverage calculation; see collect.sh.
coverage_on_schedule: "true"

name: Core Tests (${{ matrix.type.cmd }}) # Be careful modifying the job name, as it is used to fetch the job URL
# We don't directly merge dependabot PRs, so let's not waste the resources
if: ${{ github.actor != 'dependabot[bot]' }}
needs: [filter]
timeout-minutes: 60
# Scheduled race runs with coverage are much slower than PR runs.
timeout-minutes: 120
# Use ubuntu-latest for jobs that will be skipped
runs-on: ${{ matrix.type.should-run == 'true' && matrix.type.os ||
'ubuntu-latest' }}
Expand Down Expand Up @@ -487,12 +500,14 @@ jobs:

- name: Run tests
if: ${{ matrix.type.should-run == 'true' }}
timeout-minutes: 50
timeout-minutes: 90
continue-on-error: ${{ matrix.type.trunk-auto-quarantine == 'true' }}
id: run-tests
shell: bash
env:
CL_DATABASE_URL: ${{ env.DB_URL }}
COVERAGE: ${{ matrix.type.coverage || (matrix.type.coverage_on_schedule == 'true' && (github.event_name == 'schedule' || inputs.race_coverage)) || 'false' }}
GOCOVERDIR: ${{ github.workspace }}/.covdata/${{ matrix.type.cmd }}
OUTPUT_FILE: ./output.txt
PRODUCE_JUNIT_XML: ${{ matrix.type.trunk-auto-quarantine }}
TRUNK_AUTO_QUARANTINE: ${{ matrix.type.trunk-auto-quarantine }}
Expand Down Expand Up @@ -564,6 +579,27 @@ jobs:
./junit.xml
retention-days: 7

- name: Store coverage artifacts
if:
${{ always() && matrix.type.should-run == 'true' && (matrix.type.coverage
== 'true' || (matrix.type.coverage_on_schedule == 'true' &&
(github.event_name == 'schedule' || inputs.race_coverage))) }}
uses: actions/upload-artifact@v7
with:
name: coverage-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }}
path: .covdata/${{ matrix.type.cmd }}
retention-days: 7
if-no-files-found: warn

- name: Store test results
if: ${{ always() && matrix.type.should-run == 'true' }}
uses: actions/upload-artifact@v7
with:
name: test-results-core-${{ matrix.type.cmd }}-${{ github.run_id }}-${{ github.run_attempt }}
path: junit.xml
retention-days: 7
if-no-files-found: ignore

- name: Notify Slack on Race Test Failure
if: |
failure() &&
Expand Down Expand Up @@ -599,6 +635,8 @@ jobs:
- name: Run Tests
env:
OUTPUT_FILE: ./output.txt
COVERAGE: "true"
GOCOVERDIR: ${{ github.workspace }}/.covdata/scripts
run: ./tools/bin/go_core_scripts_tests ./...

- name: Store test report artifacts
Expand All @@ -611,6 +649,15 @@ jobs:
./coverage.txt
retention-days: 7

- name: Store coverage artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: coverage-core-scripts-${{ github.run_id }}-${{ github.run_attempt }}
path: .covdata/scripts
retention-days: 7
if-no-files-found: warn

clean:
name: Clean Go Tidy & Generate
if: ${{ github.actor != 'dependabot[bot]' }}
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/ci-deployments.yml
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ jobs:
shell: bash
env:
CL_DATABASE_URL: ${{ env.DB_URL }}
COVERAGE: "true"
GOCOVERDIR: ${{ github.workspace }}/.covdata/deployment-shard-${{ matrix.shard-id }}
OUTPUT_FILE: ./output.txt
PRODUCE_JUNIT_XML: "true"
TRUNK_AUTO_QUARANTINE: "true"
Expand Down Expand Up @@ -238,6 +240,24 @@ jobs:
./deployment/output-short.txt
retention-days: 7

- name: Store coverage artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: coverage-deployment-shard-${{ matrix.shard-id }}-${{ github.run_id }}-${{ github.run_attempt }}
path: .covdata/deployment-shard-${{ matrix.shard-id }}
retention-days: 7
if-no-files-found: warn

- name: Store test results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: test-results-deployment-shard-${{ matrix.shard-id }}-${{ github.run_id }}-${{ github.run_attempt }}
path: junit.xml
retention-days: 7
if-no-files-found: ignore

deployment-tests-result:
# Do not modify this job name, it is used by branch protection rules
name: Core Tests (go_core_deployment_tests)
Expand Down
17 changes: 16 additions & 1 deletion .github/workflows/ci-tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,19 @@ jobs:

- name: Run tests
working-directory: ${{ matrix.target.dir }}
run: gotestsum --format github-actions -- ${{ matrix.target.packages }}
env:
GOCOVERDIR: ${{ github.workspace }}/.covdata/tools-${{ strategy.job-index }}
run: |
mkdir -p "$GOCOVERDIR"
gotestsum --format github-actions -- ${{ matrix.target.packages }} \
-cover -covermode=atomic -coverpkg=./... \
-args -test.gocoverdir="$GOCOVERDIR"

- name: Upload coverage
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: coverage-tools-${{ strategy.job-index }}-${{ github.run_id }}-${{ github.run_attempt }}
path: .covdata/tools-${{ strategy.job-index }}
retention-days: 7
if-no-files-found: warn
Loading
Loading