Skip to content

CI Core

CI Core #76257

Workflow file for this run

name: CI Core
run-name: CI Core ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }}
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.distinct_run_name }}
cancel-in-progress: true
# Run on key branches to make sure integration is good, otherwise run on all PR's
on:
push:
branches:
- develop
- "release/*"
merge_group:
pull_request:
schedule:
- cron: "0 0,6,12,18 * * *"
workflow_dispatch:
inputs:
distinct_run_name:
description: "A unique identifier for this run, used when running from other repos"
required: false
type: string
evm-ref:
description: The chainlink-evm reference to use when testing against a specific version for compatibliity
required: false
default: ""
type: string
jobs:
filter:
name: Detect Changes
permissions:
pull-requests: read
outputs:
deployment-changes: ${{ steps.match-some.outputs.deployment == 'true' }}
should-run-ci-core: ${{ steps.match-some.outputs.core-ci == 'true' || steps.match-every.outputs.non-ignored == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
should-run-golangci: ${{ steps.match-some.outputs.golang-ci == 'true' || steps.match-every.outputs.non-ignored == 'true' || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
steps:
- name: Checkout the repo
uses: actions/[email protected]
with:
repository: smartcontractkit/chainlink
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: match-some
with:
# "if any changed file matches one or more of the conditions" (https://github.com/dorny/paths-filter/issues/225)
predicate-quantifier: some
# deployment - any changes to files in `deployments/`
# core-ci - any changes that could affect this workflow definition
# golang-ci - any changes that could affect the linting result
filters: |
deployment:
- 'deployment/**'
core-ci:
- '.github/workflows/ci-core.yml'
- '.github/actions/**'
golang-ci:
- '.golangci.yml'
- '.github/workflows/ci-core.yml'
- '.github/actions/**'
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: match-every
with:
# "if any changed file match all of the conditions" (https://github.com/dorny/paths-filter/issues/225)
predicate-quantifier: every
# non-integration-tests - only changes made outside of the `integration-tests` directory
# non-ignored - only changes except for the negated ones
# - This is opt-in on purpose. To be safe, new files are assumed to have an affect on CI Core unless listed here specifically.
filters: |
non-integration-tests:
- '**'
- '!integration-tests/**'
non-ignored:
- '**'
- '!docs/**'
- '!integration-tests/**'
- '!tools/secrets/**'
- '!tools/goreleaser-config/**'
- '!tools/docker/**'
- '!tools/benchmark/**'
- '!**/README.md'
- '!**/CHANGELOG.md'
- '!.goreleaser.develop.yaml'
- '!.goreleaser.devspace.yaml'
- '!.goreleaser.production.yaml'
- '!*.nix'
- '!sonar-project.properties'
- '!nix.conf'
- '!nix-darwin-shell-hook.sh'
- '!LICENSE'
- '!.github/**'
golangci:
# We don't directly merge dependabot PRs, so let's not waste the resources
if: ${{ (github.event_name == 'pull_request' || github.event_name == 'schedule') && github.actor != 'dependabot[bot]' }}
name: lint
permissions:
# For golangci-lint-actions to annotate code in the PR.
checks: write
contents: read
# For golangci-lint-action's `only-new-issues` option.
pull-requests: read
runs-on: ubuntu-24.04-8cores-32GB-ARM
needs: [filter, run-frequency]
steps:
- uses: actions/[email protected]
- name: Golang Lint
uses: ./.github/actions/golangci-lint
if: ${{ needs.filter.outputs.should-run-golangci == 'true' }}
- name: Notify Slack
if: ${{ failure() && needs.run-frequency.outputs.one-per-day-frequency == 'true' }}
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
with:
channel-id: "#team-core"
slack-message: "golangci-lint failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
core:
env:
# We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests
# when they should not be using it, while still allowing us to DRY up the setup
DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
strategy:
fail-fast: false
matrix:
type:
- cmd: go_core_tests
os: ubuntu22.04-32cores-128GB
printResults: true
- cmd: go_core_tests_integration
os: ubuntu22.04-32cores-128GB
printResults: true
- cmd: go_core_ccip_deployment_tests
os: ubuntu22.04-32cores-128GB
printResults: true
- cmd: go_core_race_tests
# use 64cores for certain scheduled runs only
os: ${{ needs.run-frequency.outputs.two-per-day-frequency == 'true' && 'ubuntu-latest-64cores-256GB' || 'ubuntu-latest-32cores-128GB' }}
- cmd: go_core_fuzz
os: ubuntu22.04-8cores-32GB
name: Core Tests (${{ matrix.type.cmd }})
# We don't directly merge dependabot PRs, so let's not waste the resources
if: ${{ github.actor != 'dependabot[bot]' }}
needs: [filter, run-frequency]
runs-on: ${{ matrix.type.os }}
permissions:
id-token: write
contents: read
steps:
- name: Checkout the repo
uses: actions/[email protected]
- name: Change Modtime of Files (cache optimization)
shell: bash
run: |
find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true
- name: Setup NodeJS
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
uses: ./.github/actions/setup-nodejs
with:
prod: "true"
- name: Setup Go
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
uses: ./.github/actions/setup-go
with:
# race/fuzz tests don't benefit repeated caching, so restore from develop's build cache
restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }}
build-cache-version: ${{ matrix.type.cmd }}
- name: Replace chainlink-evm deps
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' && inputs.evm-ref != ''}}
shell: bash
run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }}
- name: Setup Solana
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
uses: ./.github/actions/setup-solana
- name: Setup wasmd
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
uses: ./.github/actions/setup-wasmd
- name: Setup Postgres
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
uses: ./.github/actions/setup-postgres
- name: Touching core/web/assets/index.html
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
run: mkdir -p core/web/assets && touch core/web/assets/index.html
- name: Download Go vendor packages
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
run: go mod download
- name: Build binary
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
run: go build -o chainlink.test .
- name: Setup DB
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
run: ./chainlink.test local db preparetest
env:
CL_DATABASE_URL: ${{ env.DB_URL }}
- name: Install LOOP Plugins
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
run: |
pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds)
go install ./cmd/chainlink-feeds
popd
pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-data-streams)
go install ./mercury/cmd/chainlink-mercury
popd
pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-solana)
go install ./pkg/solana/cmd/chainlink-solana
popd
pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer)
go install ./pkg/chainlink/cmd/chainlink-starknet
popd
- name: Increase Timeouts for Fuzz/Race
# Increase timeouts for scheduled runs only
if: ${{ github.event.schedule != '' && needs.filter.outputs.should-run-ci-core == 'true' }}
run: |
echo "TIMEOUT=10m" >> $GITHUB_ENV
echo "COUNT=50" >> $GITHUB_ENV
echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV
- name: Install gotestloghelper
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/[email protected]
- name: Run tests
if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
id: run-tests
env:
OUTPUT_FILE: ./output.txt
USE_TEE: false
CL_DATABASE_URL: ${{ env.DB_URL }}
run: ./tools/bin/${{ matrix.type.cmd }} ./...
- name: Print Filtered Test Results
if: ${{ failure() && needs.filter.outputs.should-run-ci-core == 'true' && steps.run-tests.conclusion == 'failure' }}
run: |
if [[ "${{ matrix.type.printResults }}" == "true" ]]; then
cat output.txt | gotestloghelper -ci
fi
- name: Print Races
id: print-races
if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && needs.filter.outputs.should-run-ci-core == 'true' }}
run: |
find race.* | xargs cat > race.txt
if [[ -s race.txt ]]; then
cat race.txt
echo "post_to_slack=true" >> $GITHUB_OUTPUT
else
echo "post_to_slack=false" >> $GITHUB_OUTPUT
fi
echo "github.event_name: ${{ github.event_name }}"
echo "github.ref: ${{ github.ref }}"
- name: Print postgres logs
if: ${{ always() && needs.filter.outputs.should-run-ci-core == 'true' }}
run: docker compose logs postgres | tee ../../../postgres_logs.txt
working-directory: ./.github/actions/setup-postgres
- name: Store logs artifacts
if: ${{ always() && needs.filter.outputs.should-run-ci-core == 'true' }}
uses: actions/[email protected]
with:
name: ${{ matrix.type.cmd }}_logs
path: |
./output.txt
./output-short.txt
./race.*
./coverage.txt
./postgres_logs.txt
retention-days: 7
- name: Notify Slack on Race Test Failure
if: |
failure() &&
matrix.type.cmd == 'go_core_race_tests' &&
steps.print-races.outputs.post_to_slack == 'true' &&
(github.event_name == 'merge_group' || github.ref == 'refs/heads/develop') &&
needs.filter.outputs.should-run-ci-core == 'true'
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
with:
channel-id: "#topic-data-races"
slack-message: "Race tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}"
detect-flakey-tests:
needs: [filter, core]
name: Flakey Test Detection
runs-on: ubuntu-latest
if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
env:
CL_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
permissions:
id-token: write
contents: read
steps:
- name: Checkout the repo
uses: actions/[email protected]
- name: Setup node
uses: actions/[email protected]
- name: Setup NodeJS
uses: ./.github/actions/setup-nodejs
with:
prod: "true"
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Postgres
uses: ./.github/actions/setup-postgres
- name: Touching core/web/assets/index.html
run: mkdir -p core/web/assets && touch core/web/assets/index.html
- name: Download Go vendor packages
run: go mod download
- name: Replace chainlink-evm deps
if: ${{ github.event_name == 'workflow_dispatch' && inputs.evm-ref != ''}}
shell: bash
run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }}
- name: Build binary
run: go build -o chainlink.test .
- name: Setup DB
run: ./chainlink.test local db preparetest
- name: Load test outputs
uses: actions/[email protected]
with:
name: go_core_tests_logs
path: ./artifacts
- name: Delete go_core_tests_logs/coverage.txt
shell: bash
run: |
# Need to delete coverage.txt so the disk doesn't fill up
rm -f ./artifacts/go_core_tests_logs/coverage.txt
- name: Build flakey test runner
run: go build ./tools/flakeytests/cmd/runner
- name: Re-run tests
env:
GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
GITHUB_EVENT_PATH: ${{ github.event_path }}
GITHUB_EVENT_NAME: ${{ github.event_name }}
GITHUB_REPO: ${{ github.repository }}
GITHUB_RUN_ID: ${{ github.run_id }}
run: |
./runner \
-grafana_auth=$GRAFANA_INTERNAL_BASIC_AUTH \
-grafana_host=$GRAFANA_INTERNAL_HOST \
-gh_sha=$GITHUB_SHA \
-gh_event_path=$GITHUB_EVENT_PATH \
-gh_event_name=$GITHUB_EVENT_NAME \
-gh_run_id=$GITHUB_RUN_ID \
-gh_repo=$GITHUB_REPO \
-command=./tools/bin/go_core_tests \
`ls -R ./artifacts/output.txt`
- name: Store logs artifacts
if: ${{ always() }}
uses: actions/[email protected]
with:
name: flakey_test_runner_logs
path: |
./output.txt
retention-days: 7
scan:
name: SonarQube Scan
needs: [core, run-frequency]
if: ${{ always() && needs.run-frequency.outputs.four-per-day-frequency == 'true' && github.actor != 'dependabot[bot]' }}
runs-on: ubuntu-latest
steps:
- name: Checkout the repo
uses: actions/[email protected]
with:
fetch-depth: 0 # fetches all history for all tags and branches to provide more metadata for sonar reports
- name: Download all workflow run artifacts
uses: actions/[email protected]
- name: Check and Set SonarQube Report Paths
shell: bash
run: |
# Check and assign paths for coverage/test reports in go_core_tests_logs
if [ -d "go_core_tests_logs" ]; then
sonarqube_coverage_report_paths=$(find go_core_tests_logs -name coverage.txt | paste -sd "," -)
sonarqube_tests_report_paths=$(find go_core_tests_logs -name output.txt | paste -sd "," -)
else
sonarqube_coverage_report_paths=""
sonarqube_tests_report_paths=""
fi
# Check and assign paths for coverage/test reports in go_core_tests_integration_logs
if [ -d "go_core_tests_integration_logs" ]; then
integration_coverage_paths=$(find go_core_tests_integration_logs -name coverage.txt | paste -sd "," -)
integration_tests_paths=$(find go_core_tests_integration_logs -name output.txt | paste -sd "," -)
# Append to existing paths if they are set, otherwise assign directly
sonarqube_coverage_report_paths="${sonarqube_coverage_report_paths:+$sonarqube_coverage_report_paths,}$integration_coverage_paths"
sonarqube_tests_report_paths="${sonarqube_tests_report_paths:+$sonarqube_tests_report_paths,}$integration_tests_paths"
fi
# Check and assign paths for lint reports
if [ -d "golangci-lint-report" ]; then
sonarqube_lint_report_paths=$(find golangci-lint-report -name golangci-lint-report.xml | paste -sd "," -)
else
sonarqube_lint_report_paths=""
fi
ARGS=""
if [[ -z "$sonarqube_tests_report_paths" ]]; then
echo "::warning::No test report paths found, will not pass to sonarqube"
else
echo "Found test report paths: $sonarqube_tests_report_paths"
ARGS="$ARGS -Dsonar.go.tests.reportPaths=$sonarqube_tests_report_paths"
fi
if [[ -z "$sonarqube_coverage_report_paths" ]]; then
echo "::warning::No coverage report paths found, will not pass to sonarqube"
else
echo "Found coverage report paths: $sonarqube_coverage_report_paths"
ARGS="$ARGS -Dsonar.go.coverage.reportPaths=$sonarqube_coverage_report_paths"
fi
if [[ -z "$sonarqube_lint_report_paths" ]]; then
echo "::warning::No lint report paths found, will not pass to sonarqube"
else
echo "Found lint report paths: $sonarqube_lint_report_paths"
ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=$sonarqube_lint_report_paths"
fi
echo "Final SONARQUBE_ARGS: $ARGS"
echo "SONARQUBE_ARGS=$ARGS" >> $GITHUB_ENV
- name: SonarQube Scan
if: ${{ env.SONARQUBE_ARGS != '' }}
uses: sonarsource/sonarqube-scan-action@aecaf43ae57e412bd97d70ef9ce6076e672fe0a9 # v2.3.0
with:
args: ${{ env.SONARQUBE_ARGS }}
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
SONAR_SCANNER_OPTS: "-Xms6g -Xmx8g"
trigger-flaky-test-detection-for-root-project:
name: Find New Flaky Tests In Chainlink Project
uses: ./.github/workflows/find-new-flaky-tests.yml
if: ${{ github.event_name == 'pull_request' }}
with:
repoUrl: 'https://github.com/smartcontractkit/chainlink'
projectPath: '.'
baseRef: ${{ github.base_ref }}
headRef: ${{ github.head_ref }}
runThreshold: '0.99'
findByTestFilesDiff: true
findByAffectedPackages: false
slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications
extraArgs: '{ "skipped_tests": "TestChainComponents", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "3", "min_pass_ratio": "0.01" }'
secrets:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
trigger-flaky-test-detection-for-deployment-project:
name: Find New Flaky Tests In Deployment Project
uses: ./.github/workflows/find-new-flaky-tests.yml
needs: [filter]
if: ${{ github.event_name == 'pull_request' && needs.filter.outputs.deployment-changes == 'true'}}
with:
repoUrl: 'https://github.com/smartcontractkit/chainlink'
projectPath: 'deployment'
baseRef: ${{ github.base_ref }}
headRef: ${{ github.head_ref }}
runThreshold: '0.99'
findByTestFilesDiff: true
findByAffectedPackages: false
slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications
extraArgs: '{ "skipped_tests": "TestAddLane", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "3", "min_pass_ratio": "0.01" }'
secrets:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
clean:
name: Clean Go Tidy & Generate
if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') && github.actor != 'dependabot[bot]' }}
runs-on: ubuntu22.04-8cores-32GB
defaults:
run:
shell: bash
steps:
- name: Check for Skip Tests Label
if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests')
run: |
echo "## \`skip-smoke-tests\` label is active, skipping E2E smoke tests" >>$GITHUB_STEP_SUMMARY
exit 0
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Setup Go
uses: ./.github/actions/setup-go
with:
only-modules: "true"
- name: Install protoc-gen-go-wsrpc
run: curl https://github.com/smartcontractkit/wsrpc/raw/main/cmd/protoc-gen-go-wsrpc/protoc-gen-go-wsrpc --output $HOME/go/bin/protoc-gen-go-wsrpc && chmod +x $HOME/go/bin/protoc-gen-go-wsrpc
- name: Setup NodeJS
uses: ./.github/actions/setup-nodejs
- name: make generate
run: |
make rm-mocked
make generate
- name: Ensure clean after generate
run: |
git add --all
git diff --stat --cached --exit-code
- run: make gomodtidy
- name: Ensure clean after tidy
run: |
git add --all
git diff --minimal --cached --exit-code
run-frequency:
name: Scheduled Run Frequency
outputs:
one-per-day-frequency: ${{ steps.check-time.outputs.one-per-day-frequency || 'false' }}
two-per-day-frequency: ${{ steps.check-time.outputs.two-per-day-frequency || 'false' }}
four-per-day-frequency: ${{ steps.check-time.outputs.four-per-day-frequency || 'false' }}
six-per-day-frequency: ${{ steps.check-time.outputs.six-per-day-frequency || 'false' }}
runs-on: ubuntu-latest
steps:
- name: Check time and set frequencies
id: check-time
shell: bash
run: |
if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then
# Not a scheduled event, set all frequencies to false
echo "one-per-day-frequency=false" >> $GITHUB_OUTPUT
echo "two-per-day-frequency=false" >> $GITHUB_OUTPUT
echo "four-per-day-frequency=false" >> $GITHUB_OUTPUT
echo "six-per-day-frequency=false" >> $GITHUB_OUTPUT
else
# Scheduled event, check current time for frequencies
current_hour=$(date +"%H")
# Check if the current hour is 00 (one per day)
if [ "$current_hour" -eq "00" ]; then
echo "one-per-day-frequency=true" >> $GITHUB_OUTPUT
fi
# Check if the current hour is 00 or 12 (twice per day)
if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "12" ]; then
echo "two-per-day-frequency=true" >> $GITHUB_OUTPUT
fi
# Check if the current hour is 00, 06, 12, or 18 (four times per day)
if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "06" ] || [ "$current_hour" -eq "12" ] || [ "$current_hour" -eq "18" ]; then
echo "four-per-day-frequency=true" >> $GITHUB_OUTPUT
fi
# Check if the current hour is one of 00, 04, 08, 12, 16, or 20 (six times per day)
if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "04" ] || [ "$current_hour" -eq "08" ] || [ "$current_hour" -eq "12" ] || [ "$current_hour" -eq "16" ] || [ "$current_hour" -eq "20" ]; then
echo "six-per-day-frequency=true" >> $GITHUB_OUTPUT
fi
fi