Tests

chore(deps-dev): bump aiohttp from 3.11.10 to 3.11.11 in /_integration-tests/utils/agent #1206

Workflow file for this run

.github/workflows/validate.yml at 14e6bfe

	name: Tests
	on:
	pull_request:
	branches: ['**']
	merge_group:
	branches: [main]
	push:
	branches: [main]
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.number \|\| ((github.event_name == 'push' && github.sha) \|\| github.ref) }}
	cancel-in-progress: true

	permissions: read-all

	env:
	# Make sure we're actually testing with the intended Go release (i.e, ensure
	# no automatic toolchain download happens).
	GOTOOLCHAIN: local

	jobs:
	##############################################################################
	# Run all the code generators; and refresh the LICENSES-3rdparty.csv file
	generate:
	needs: coverage-preflight
	runs-on: ubuntu-latest
	name: Run all generators
	outputs:
	has-patch: ${{ steps.is-tree-dirty.outputs.result }}
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Setup go
	id: setup-go
	uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
	with:
	go-version: stable
	cache-dependency-path: '**/go.mod'

	- name: Run 'go generate ./...'
	run: \|-
	mkdir -p ${GOCOVERDIR}
	find . -name go.mod -execdir go generate ./... \;
	env:
	GOFLAGS: -cover -covermode=atomic -coverpkg=github.com/DataDog/orchestrion/...,./...
	GOCOVERDIR: ${{ github.workspace }}/coverage
	- name: Consolidate coverage report
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	run: go tool covdata textfmt -i ./coverage -o ./coverage/generator.out
	- name: Determine simple go version
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	id: go
	run: \|-
	set -euo pipefail
	echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' \| cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
	- name: Upload coverage report
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
	with:
	name: coverage-generators+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
	path: ./coverage/generator.out
	- name: Run 'go mod tidy'
	# Don't run for push, it's not necessary
	if: github.event_name != 'push'
	run: find . -name go.mod -execdir go mod tidy \;

	- name: Refresh LICENSE-3rdparty.csv
	run: ./_tools/make-licenses.sh
	env:
	TMPDIR: ${{ runner.temp }}

	- name: Check if working tree is dirty
	# Don't run for push, it's not necessary
	if: github.event_name != 'push'
	id: is-tree-dirty
	run: \|-
	set -euxo pipefail
	git add .
	git status
	git diff --staged --patch --exit-code > .repo.patch \|\| echo 'result=true' >> "${GITHUB_OUTPUT}"
	- name: Upload patch
	if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
	uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
	with:
	if-no-files-found: error
	include-hidden-files: true
	name: repo.patch
	path: .repo.patch
	- name: Fail build if working tree is dirty
	if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
	run: \|-
	echo "::error::Files have been modified by 'go generate ./...' (see logs)."
	cat .repo.patch
	exit 1
	##############################################################################
	# If the generators changed anything, and we can update the PR, then we'll
	# proactively do it with the mutator token.
	self-mutation:
	needs: generate
	runs-on: ubuntu-latest
	name: Update PR with generated files
	if: always() && needs.generate.outputs.has-patch == 'true' && github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name == github.repository \|\| github.event.pull_request.maintainer_can_modify)
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	with:
	ref: ${{ github.event.pull_request.head.ref }}
	repository: ${{ github.event.pull_request.head.repo.full_name }}
	- name: Download patch
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
	with:
	name: repo.patch
	path: ${{ runner.temp }}
	- name: Apply patch
	run: \|-
	[ -s '${{ runner.temp }}/.repo.patch' ] && git apply '${{ runner.temp }}/.repo.patch' \|\| echo 'Empty patch. Skipping.'
	# We use ghcommit to create signed commits directly using the GitHub API
	- name: Push changes
	uses: planetscale/ghcommit-action@d4176bfacef926cc2db351eab20398dfc2f593b5 # v0.2.0
	with:
	commit_message: "chore: update generated files"
	repo: ${{ github.event.pull_request.head.repo.full_name }}
	branch: ${{ github.event.pull_request.head.ref }}
	env:
	GITHUB_TOKEN: ${{ secrets.MUTATOR_GITHUB_TOKEN }}

	##############################################################################
	# Run the various linters we have set up...
	lint:
	needs: generate
	runs-on: ubuntu-latest
	name: Go Linters
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Setup go
	uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
	with:
	go-version: stable
	cache-dependency-path: "**/go.mod"
	- name: Lint main module
	uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6
	with:
	version: v1.60.3
	- name: Lint integration tests
	uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6
	with:
	version: v1.60.3
	working-directory: _integration-tests
	- name: Verify license headers
	run: go run ./_tools/headercheck/header_check.go
	- name: vet
	run: go vet ./...

	##############################################################################
	# Verify all GitHub workflows have hash-pinned actions
	lint-workflows:
	runs-on: ubuntu-latest
	name: GitHub Workflow Linters
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Ensure SHA pinned actions
	uses: zgosalvez/github-actions-ensure-sha-pinned-actions@5d6ac37a4cef8b8df67f482a8e384987766f0213 # v3

	##############################################################################
	# Run all unit tests with coverage enabled
	unit-tests:
	needs: generate
	runs-on: ubuntu-latest
	strategy:
	fail-fast: ${{ github.event_name == 'merge_group' }}
	matrix:
	go-version: [oldstable, stable]
	name: Unit tests (go ${{ matrix.go-version }})
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Setup Go
	id: setup-go
	uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
	with:
	go-version: ${{ matrix.go-version }}
	cache-dependency-path: "**/go.mod"
	- name: Run unit tests
	shell: bash
	run: \|-
	mkdir -p coverage
	test_args=("-shuffle=on" "-race" "-v")
	if [ "${{ github.event_name }}" != "merge_group" ]; then
	test_args+=("-cover" "-covermode=atomic" "-coverpkg=./...,github.com/DataDog/orchestrion/..." "-coverprofile=${{ github.workspace }}/coverage/unit.out")
	fi
	go test "${test_args[@]}" ./...
	- name: Run integraton suite unit tests
	shell: bash
	run: \|-
	mkdir -p coverage
	test_args=("-shuffle=on" "-race" "-v")
	if [ "${{ github.event_name }}" != "merge_group" ]; then
	test_args+=("-cover" "-covermode=atomic" "-coverpkg=./...,github.com/DataDog/orchestrion/..." "-coverprofile=${{ github.workspace }}/coverage/integration.out")
	fi
	go -C _integration-tests test "${test_args[@]}" ./...
	- name: Determine simple go version
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	id: go
	shell: bash
	run: \|-
	set -euo pipefail
	echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' \| cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
	- name: Upload coverage report
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
	with:
	name: coverage-unit+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
	path: \|-
	./coverage/unit.out
	./coverage/integration.out

	##############################################################################
	# Run all benchmarks and generate report
	benchmark:
	needs: generate
	runs-on: arm-8core-linux
	name: Benchmarks
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Setup go
	id: setup-go
	uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
	with:
	go-version: stable
	cache-dependency-path: "**/go.mod"
	- name: Run benchmarks
	run: \|-
	set -euo pipefail
	go test -bench=. -timeout=1h -run=^$ . \| tee ${{ runner.temp }}/benchmarks.txt
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Upload benchmark report (raw)
	uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
	with:
	if-no-files-found: error
	name: Benchmark Report
	path: ${{ runner.temp }}/benchmarks.txt
	- name: Format Report
	run: \|-
	go -C _tools run golang.org/x/perf/cmd/benchstat \
	-table=.name -row=/repo -col=/variant \
	${{ runner.temp }}/benchmarks.txt \
	\| tee ${{ runner.temp }}/benchmarks-formatted.txt
	- name: Setting Job Summary
	run: \|-
	echo "### Benchmark Report" >> "${GITHUB_STEP_SUMMARY}"
	echo '```' >> "${GITHUB_STEP_SUMMARY}"
	cat ${{ runner.temp }}/benchmarks-formatted.txt >> "${GITHUB_STEP_SUMMARY}"
	echo '```' >> "${GITHUB_STEP_SUMMARY}"


	##############################################################################
	# Run all integration tests and gather extensive coverage
	integration-tests:
	needs: generate
	strategy:
	fail-fast: ${{ github.event_name == 'merge_group' }}
	matrix:
	runs-on:
	- macos
	- ubuntu
	- windows
	go-version: [oldstable, stable, '~1.24.0-rc.1']
	build-mode: [DRIVER]
	include:
	# Alternate build modes (only on ubuntu, latest go; to save CI time)
	- runs-on: ubuntu
	go-version: oldstable
	build-mode: TOOLEXEC
	- runs-on: ubuntu
	go-version: oldstable
	build-mode: GOFLAGS
	runs-on: ${{ matrix.runs-on == 'ubuntu' && fromJson('{"labels":"ubuntu-16-core-latest","group":"Large Runner Shared Public"}') \|\| (matrix.runs-on == 'windows' && fromJson('{"labels":"windows-shared-8core","group":"LARGE WINDOWS SHARED"}')) \|\| format('{0}-latest', matrix.runs-on) }}
	env:
	# Ryuk is problematic with concurrent executions, and unnecessary in ephemeral environments like GHA.
	TESTCONTAINERS_RYUK_DISABLED: true
	name: Integration tests (go ${{ matrix.go-version }}, ${{ matrix.runs-on }}, ${{ matrix.build-mode }})
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Setup go
	id: setup-go
	uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
	with:
	go-version: ${{ matrix.go-version }}
	cache-dependency-path: "**/go.mod"
	- name: Setup python
	uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
	with:
	python-version: 3.x
	cache: pip
	cache-dependency-path: _integration-tests/utils/agent/requirements-dev.txt
	- name: Install python dependencies
	run: pip install -r _integration-tests/utils/agent/requirements-dev.txt
	- name: Build orchestrion binary
	shell: bash
	run: \|-
	build_args=()
	if [ "${{ github.event_name }}" != "merge_group" ]; then
	build_args+=("-cover" "-covermode=atomic" "-coverpkg=./...")
	fi
	go build ${build_args[@]+"${build_args[@]}"} -o="bin/orchestrion.exe" .
	- name: Run Integration Tests
	shell: bash
	run: \|-
	mkdir -p "${GOCOVERDIR}"
	test_args=("-shuffle=on")
	if [ "${{ github.event_name }}" != "merge_group" ]; then
	test_args+=("-coverpkg=./...,github.com/DataDog/orchestrion/..." "-covermode=atomic" "-cover" "-coverprofile=${{ github.workspace }}/coverage/integration.run.out")
	fi

	case "${{ matrix.build-mode }}" in
	"DRIVER")
	bin/orchestrion.exe -C=_integration-tests go test "${test_args[@]}" -a ./...
	;;
	"TOOLEXEC")
	go -C=_integration-tests test "${test_args[@]}" -toolexec="${{ github.workspace }}/bin/orchestrion.exe toolexec" ./...
	;;
	"GOFLAGS")
	export GOFLAGS="'-toolexec=${{ github.workspace }}/bin/orchestrion.exe toolexec' ${GOFLAGS}"
	go -C=_integration-tests test "${test_args[@]}" ./...
	;;
	*)
	echo "Unknown build mode: ${{ matrix.build-mode }}"
	exit 1
	;;
	esac
	env:
	GOCOVERDIR: ${{ github.workspace }}/coverage/raw
	GOFLAGS: ${{ matrix.runs-on == 'ubuntu' && '-p=4' \|\| ''}} -tags=githubci,integration,buildtag # Globally set build tags (buildtag is used by the dd-span test)
	- name: Consolidate coverage report
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	run: go tool covdata textfmt -i ./coverage/raw -o ./coverage/integration.out
	- name: Determine go minor version
	id: go
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	shell: bash
	run: \|-
	set -euo pipefail
	echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' \| cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
	- name: Upload coverage report
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
	with:
	name: coverage-integration+${{ matrix.build-mode }}+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
	path: \|-
	./coverage/integration.out
	./coverage/integration.run.out

	##############################################################################
	# Assert everything is complete. This simplifies branch protection settings
	# and allows us to have one single trigger for CodeCov reporting.
	complete:
	runs-on: ubuntu-latest
	name: Complete
	needs:
	- generate
	- lint
	- lint-workflows
	- unit-tests
	- integration-tests
	- benchmark
	if: '!cancelled()'
	steps:
	- name: Success
	if: needs.generate.result != 'failure' && needs.lint.result != 'failure' && needs.lint-workflows.result != 'failure' && needs.unit-tests.result != 'failure' && needs.integration-tests.result != 'failure'
	run: echo "OK"
	- name: Failed
	if: needs.generate.result == 'failure' \|\| needs.lint.result == 'failure' \|\| needs.lint-workflows.result == 'failure' \|\| needs.unit-tests.result == 'failure' \|\| needs.integration-tests.result == 'failure'
	run: \|-
	echo "Failed!"
	exit 1

	##############################################################################
	# Produce a CodeCov coverage report with all uploaded code coverage data.
	coverage-preflight:
	runs-on: ubuntu-latest
	name: CodeCov pre-flight
	steps:
	- name: Checkout
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Download codecov CLI
	id: codecov-cli
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	uses: ./.github/actions/codecov-cli
	- name: Register commit with CodeCov
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	shell: bash
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
	run: \|-
	set -euo pipefail

	pr=()
	sha="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha \|\| github.sha }}"
	parentsha="${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha \|\| github.event.before }}"
	if [ "${{ github.event_name }}" == "pull_request" ]; then
	pr+=("--pr=${{ github.event.number }}")
	fi

	echo "::group::Register commit metadata with CodeCov"
	${{ steps.codecov-cli.outputs.codecov }} \
	--auto-load-params-from=GithubActions \
	--verbose \
	create-commit \
	--parent-sha="${parentsha}" \
	${pr[@]+"${pr[@]}"} \
	--sha="${sha}" \
	--fail-on-error \
	--git-service=github \
	--token="${CODECOV_TOKEN}" \
	--slug="${{ github.repository }}"
	echo "::endgroup::"

	echo "::group::Create a new blank CodeCov report"
	${{ steps.codecov-cli.outputs.codecov }} \
	--auto-load-params-from=GithubActions \
	--verbose \
	create-report \
	${pr[@]+"${pr[@]}"} \
	--sha="${sha}" \
	--fail-on-error \
	--git-service=github \
	--token="${CODECOV_TOKEN}" \
	--slug="${{ github.repository }}"
	echo "::endgroup::"

	coverage-matrix:
	runs-on: ubuntu-latest
	name: Compute Coverage Matrix
	needs:
	- coverage-preflight
	- unit-tests
	- integration-tests
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	outputs:
	artifacts: ${{ steps.compute.outputs.artifacts }}
	files: ${{ steps.compute.outputs.files }}
	matrix: ${{ steps.compute.outputs.matrix }}
	steps:
	- name: Setup Node
	uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
	with:
	node-version: latest
	- name: Download Artifacts
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
	with:
	pattern: coverage-*
	- name: Compute Matrix
	id: compute
	run: \|-
	node <<-EOF
	const fs = require('node:fs');
	const path = require('node:path');
	const process = require('node:process');

	const flags = [];
	const flagFiles = {};
	for (const dirname of fs.readdirSync(process.cwd())) {
	const prefix = 'coverage-';
	if (!dirname.startsWith(prefix)) {
	continue;
	}

	const files = fs.globSync(path.join(process.cwd(), dirname, '*', '.out'));
	console.log('Found asset named ' + dirname + ' with ' + files.length + ' report files.');
	if (files.length == 0) {
	continue;
	}

	for (const flag of dirname.substring(prefix.length).split('+')) {
	if (!flags.includes(flag)) {
	flags.push(flag);
	}

	flagFiles[flag] ??= [];
	flagFiles[flag].push(...files);
	}
	}

	console.log('Flags:', flags);
	console.log('Files:', flagFiles);

	// Join the lists because the workflow subsequently expects a whitespace-separted list.
	for (const [flag, list] of Object.entries(flagFiles)) {
	flagFiles[flag] = list.join(' ');
	}

	fs.writeFileSync(
	path.join(process.env.GITHUB_OUTPUT),
	[
	"matrix=" + JSON.stringify({ flag: flags }),
	"files=" + JSON.stringify(flagFiles),
	].join('\n'),
	);
	EOF


	coverage-upload:
	runs-on: ubuntu-latest
	name: Upload report to CodeCov (${{ matrix.flag }})
	needs: [coverage-matrix]
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	strategy:
	fail-fast: true
	matrix: ${{ fromJson(needs.coverage-matrix.outputs.matrix) }}
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Download Artifacts
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
	with:
	pattern: coverage-*
	- name: Upload Reports
	uses: ./.github/actions/codecov-upload
	with:
	name: ${{ matrix.flag }}
	flags: ${{ matrix.flag }}
	files: ${{ fromJson(needs.coverage-matrix.outputs.files)[matrix.flag] }}
	token: ${{ secrets.CODECOV_TOKEN }}


	coverage-finalize:
	runs-on: ubuntu-latest
	name: Create CodeCov report
	needs: [coverage-upload]
	if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
	steps:
	- name: Checkout
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
	- name: Download codecov CLI
	id: codecov-cli
	uses: ./.github/actions/codecov-cli
	- name: Create CodeCov report
	shell: bash
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
	run: \|-
	set -euo pipefail
	sha="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha \|\| github.sha }}"

	echo "::group::Create CodeCov report results"
	${{ steps.codecov-cli.outputs.codecov }} \
	--auto-load-params-from=GithubActions \
	--verbose \
	create-report-results \
	--sha="${sha}" \
	--fail-on-error \
	--git-service=github \
	--token="${CODECOV_TOKEN}" \
	--slug="${{ github.repository }}"
	echo "::endgroup::"

	echo "::group::Issue GitHub notifications"
	${{ steps.codecov-cli.outputs.codecov }} \
	--auto-load-params-from=GithubActions \
	--verbose \
	send-notifications \
	--sha="${sha}" \
	--fail-on-error \
	--git-service=github \
	--token="${CODECOV_TOKEN}" \
	--slug=${{ github.repository }}
	echo "::endgroup::"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

chore(deps-dev): bump aiohttp from 3.11.10 to 3.11.11 in /_integration-tests/utils/agent #1206

Workflow file

chore(deps-dev): bump aiohttp from 3.11.10 to 3.11.11 in /_integration-tests/utils/agent #1206

Jobs

Run details

Workflow file for this run