Skip to content

Commit

Permalink
Merge branch 'main' into mackjmr/refactor-is-enabled-otlp
Browse files Browse the repository at this point in the history
  • Loading branch information
mackjmr authored Nov 28, 2024
2 parents 5595fa7 + 4cfa2da commit a8965e0
Show file tree
Hide file tree
Showing 223 changed files with 6,053 additions and 1,765 deletions.
72 changes: 72 additions & 0 deletions .github/workflows/collector-generate-and-update.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: Update OTel Collector Dependencies and Generate OTel Agent

on:
schedule:
- cron: '0 12 * * 3'
workflow_dispatch:

jobs:
update-and-generate:
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Set date
id: date
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT

- name: Checkout repository
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

- name: Set up Python
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
python-version: '3.12.6'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run update task
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: inv -e collector.update

- name: Run generate task
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: inv -e collector.generate

- name: Check for changes
id: check_changes
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git add .
if git diff-index --quiet HEAD; then
echo "No changes detected"
echo "changes_detected=false" >> $GITHUB_OUTPUT
else
echo "Changes detected"
echo "changes_detected=true" >> $GITHUB_OUTPUT
fi
- name: Commit changes
if: steps.check_changes.outputs.changes_detected == 'true'
run: |
git switch -c update-otel-collector-dependencies-${{ steps.date.outputs.date }}
git commit -m "Update OTel Collector dependencies and generate OTel Agent"
git push -u origin update-otel-collector-dependencies-${{ steps.date.outputs.date }}
- name: Install GitHub CLI
if: steps.check_changes.outputs.changes_detected == 'true'
run: |
sudo apt-get update
sudo apt-get install gh
- name: Create draft pull request
if: steps.check_changes.outputs.changes_detected == 'true'
run: |
gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }}
gh pr create --title "Update OTel collector dependencies" --body "This PR updates the OTel Collector dependencies to the latest version. Please ensure that all tests pass before marking ready for review." --base main --head update-otel-collector-dependencies-${{ steps.date.outputs.date }} --draft
44 changes: 22 additions & 22 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -173,49 +173,49 @@ variables:
# To use images from datadog-agent-buildimages dev branches, set the corresponding
# SUFFIX variable to _test_only
DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_BUILDIMAGES: v48372186-ff395e52
DATADOG_AGENT_BUILDIMAGES: v48815877-9bfad02c
DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_WINBUILDIMAGES: v48372186-ff395e52
DATADOG_AGENT_WINBUILDIMAGES: v48815877-9bfad02c
DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_ARMBUILDIMAGES: v48372186-ff395e52
DATADOG_AGENT_ARMBUILDIMAGES: v48815877-9bfad02c
DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48372186-ff395e52
DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48815877-9bfad02c
DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48372186-ff395e52
DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48815877-9bfad02c
# New images to enable different version per image - not used yet
CI_IMAGE_BTF_GEN: v48372186-ff395e52
CI_IMAGE_BTF_GEN: v48815877-9bfad02c
CI_IMAGE_BTF_GEN_SUFFIX: ""
CI_IMAGE_DEB_X64: v48372186-ff395e52
CI_IMAGE_DEB_X64: v48815877-9bfad02c
CI_IMAGE_DEB_X64_SUFFIX: ""
CI_IMAGE_DEB_ARM64: v48372186-ff395e52
CI_IMAGE_DEB_ARM64: v48815877-9bfad02c
CI_IMAGE_DEB_ARM64_SUFFIX: ""
CI_IMAGE_DEB_ARMHF: v48372186-ff395e52
CI_IMAGE_DEB_ARMHF: v48815877-9bfad02c
CI_IMAGE_DEB_ARMHF_SUFFIX: ""
CI_IMAGE_DD_AGENT_TESTING: v48372186-ff395e52
CI_IMAGE_DD_AGENT_TESTING: v48815877-9bfad02c
CI_IMAGE_DD_AGENT_TESTING_SUFFIX: ""
CI_IMAGE_DOCKER_X64: v48372186-ff395e52
CI_IMAGE_DOCKER_X64: v48815877-9bfad02c
CI_IMAGE_DOCKER_X64_SUFFIX: ""
CI_IMAGE_DOCKER_ARM64: v48372186-ff395e52
CI_IMAGE_DOCKER_ARM64: v48815877-9bfad02c
CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
CI_IMAGE_GITLAB_AGENT_DEPLOY: v48372186-ff395e52
CI_IMAGE_GITLAB_AGENT_DEPLOY: v48815877-9bfad02c
CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: ""
CI_IMAGE_LINUX_GLIBC_2_17_X64: v48372186-ff395e52
CI_IMAGE_LINUX_GLIBC_2_17_X64: v48815877-9bfad02c
CI_IMAGE_LINUX_GLIBC_2_17_X64_SUFFIX: ""
CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48372186-ff395e52
CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48815877-9bfad02c
CI_IMAGE_LINUX_GLIBC_2_23_ARM64_SUFFIX: ""
CI_IMAGE_SYSTEM_PROBE_X64: v48372186-ff395e52
CI_IMAGE_SYSTEM_PROBE_X64: v48815877-9bfad02c
CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: ""
CI_IMAGE_SYSTEM_PROBE_ARM64: v48372186-ff395e52
CI_IMAGE_SYSTEM_PROBE_ARM64: v48815877-9bfad02c
CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: ""
CI_IMAGE_RPM_X64: v48372186-ff395e52
CI_IMAGE_RPM_X64: v48815877-9bfad02c
CI_IMAGE_RPM_X64_SUFFIX: ""
CI_IMAGE_RPM_ARM64: v48372186-ff395e52
CI_IMAGE_RPM_ARM64: v48815877-9bfad02c
CI_IMAGE_RPM_ARM64_SUFFIX: ""
CI_IMAGE_RPM_ARMHF: v48372186-ff395e52
CI_IMAGE_RPM_ARMHF: v48815877-9bfad02c
CI_IMAGE_RPM_ARMHF_SUFFIX: ""
CI_IMAGE_WIN_1809_X64: v48372186-ff395e52
CI_IMAGE_WIN_1809_X64: v48815877-9bfad02c
CI_IMAGE_WIN_1809_X64_SUFFIX: ""
CI_IMAGE_WIN_LTSC2022_X64: v48372186-ff395e52
CI_IMAGE_WIN_LTSC2022_X64: v48815877-9bfad02c
CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""

DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/common/test_infra_version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ variables:
# and check the job creating the image to make sure you have the right SHA prefix
TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
# Make sure to update test-infra-definitions version in go.mod as well
TEST_INFRA_DEFINITIONS_BUILDIMAGES: 7cd5e8a62570
TEST_INFRA_DEFINITIONS_BUILDIMAGES: 047dd64128b6
53 changes: 53 additions & 0 deletions .gitlab/container_build/docker_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,33 @@ docker_build_agent7_arm64:
TAG_SUFFIX: -7
BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-arm64.tar.xz

# build agent7 fips image
docker_build_fips_agent7:
extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
rules:
- !reference [.except_mergequeue]
- when: on_success
needs:
- job: datadog-agent-7-x64-fips
variables:
IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
BUILD_CONTEXT: Dockerfiles/agent
TAG_SUFFIX: -7-fips
BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz

docker_build_fips_agent7_arm64:
extends: [.docker_build_job_definition_arm64, .docker_build_artifact]
rules:
- !reference [.except_mergequeue]
- when: on_success
needs:
- job: datadog-agent-7-arm64-fips
variables:
IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
BUILD_CONTEXT: Dockerfiles/agent
TAG_SUFFIX: -7-fips
BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz

# build agent7 jmx image
docker_build_agent7_jmx:
extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
Expand Down Expand Up @@ -116,6 +143,32 @@ docker_build_agent7_jmx_arm64:
TAG_SUFFIX: -7-jmx
BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-arm64.tar.xz

docker_build_fips_agent7_jmx:
extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
rules:
- !reference [.except_mergequeue]
- when: on_success
needs:
- job: datadog-agent-7-x64-fips
variables:
IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
BUILD_CONTEXT: Dockerfiles/agent
TAG_SUFFIX: -7-fips-jmx
BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz

docker_build_fips_agent7_arm64_jmx:
extends: [.docker_build_job_definition_arm64, .docker_build_artifact]
rules:
- !reference [.except_mergequeue]
- when: on_success
needs:
- job: datadog-agent-7-arm64-fips
variables:
IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
BUILD_CONTEXT: Dockerfiles/agent
TAG_SUFFIX: -7-fips-jmx
BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz

# build agent7 UA image
docker_build_ot_agent7:
extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
Expand Down
36 changes: 36 additions & 0 deletions .gitlab/deploy_packages/nix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ deploy_packages_deb-arm64-7:
variables:
PACKAGE_ARCH: arm64

deploy_packages_deb-x64-7-fips:
extends: .deploy_packages_deb-7
needs: [ agent_deb-x64-a7-fips ]
variables:
PACKAGE_ARCH: amd64

deploy_packages_deb-arm64-7-fips:
extends: .deploy_packages_deb-7
needs: [ agent_deb-arm64-a7-fips ]
variables:
PACKAGE_ARCH: arm64

deploy_packages_heroku_deb-x64-7:
extends: .deploy_packages_deb-7
needs: [ agent_heroku_deb-x64-a7 ]
Expand Down Expand Up @@ -62,6 +74,18 @@ deploy_packages_rpm-arm64-7:
variables:
PACKAGE_ARCH: aarch64

deploy_packages_rpm-x64-7-fips:
extends: .deploy_packages_rpm-7
needs: [ agent_rpm-x64-a7-fips ]
variables:
PACKAGE_ARCH: x86_64

deploy_packages_rpm-arm64-7-fips:
extends: .deploy_packages_rpm-7
needs: [ agent_rpm-arm64-a7-fips ]
variables:
PACKAGE_ARCH: aarch64

deploy_packages_iot_rpm-x64-7:
extends: .deploy_packages_rpm-7
needs: [ iot_agent_rpm-x64 ]
Expand Down Expand Up @@ -98,6 +122,18 @@ deploy_packages_suse_rpm-arm64-7:
variables:
PACKAGE_ARCH: aarch64

deploy_packages_suse_rpm-x64-7-fips:
extends: .deploy_packages_suse_rpm-7
needs: [ agent_suse-x64-a7-fips ]
variables:
PACKAGE_ARCH: x86_64

deploy_packages_suse_rpm-arm64-7-fips:
extends: .deploy_packages_suse_rpm-7
needs: [ agent_suse-arm64-a7-fips ]
variables:
PACKAGE_ARCH: aarch64

deploy_packages_iot_suse_rpm-x64-7:
extends: .deploy_packages_suse_rpm-7
needs: [ iot_agent_suse-x64 ]
Expand Down
18 changes: 18 additions & 0 deletions .gitlab/dev_container_deploy/docker_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,24 @@ dev_branch_multiarch-a7:
- IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64
IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx

dev_branch_multiarch-fips:
extends: .docker_publish_job_definition
stage: dev_container_deploy
rules: !reference [.manual]
needs:
- docker_build_fips_agent7
- docker_build_fips_agent7_arm64
- docker_build_fips_agent7_jmx
- docker_build_fips_agent7_arm64_jmx
variables:
IMG_REGISTRIES: dev
parallel:
matrix:
- IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-arm64
IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-fips
- IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-arm64
IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-fips-jmx

dev_branch_multiarch-dogstatsd:
extends: .docker_publish_job_definition
stage: dev_container_deploy
Expand Down
17 changes: 17 additions & 0 deletions .gitlab/dev_container_deploy/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,23 @@ qa_agent:
IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-arm64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-win1809-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-winltsc2022-amd64
IMG_DESTINATIONS: agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}

qa_agent_jmx:
extends: .docker_publish_job_definition
stage: dev_container_deploy
rules:
- !reference [.except_mergequeue]
- !reference [.except_disable_e2e_tests]
- when: on_success
needs:
- docker_build_agent7_jmx
- docker_build_agent7_jmx_arm64
- docker_build_agent7_windows1809_jmx
- docker_build_agent7_windows2022_jmx
variables:
IMG_REGISTRIES: agent-qa
IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-win1809-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-winltsc2022-amd64
IMG_DESTINATIONS: agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-jmx

qa_agent_ot:
extends: .docker_publish_job_definition
stage: dev_container_deploy
Expand Down
19 changes: 19 additions & 0 deletions .gitlab/e2e/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
needs:
- !reference [.needs_new_e2e_template]
- qa_agent
- qa_agent_jmx
- qa_dca
- qa_dogstatsd

Expand Down Expand Up @@ -410,6 +411,20 @@ new-e2e-installer-windows:
TARGETS: ./tests/installer/windows
TEAM: fleet
FLEET_INSTALL_METHOD: "windows"
parallel:
matrix:
# agent-package
- EXTRA_PARAMS: --run "TestAgentInstallsWithAgentUser$"
- EXTRA_PARAMS: --run "TestAgentInstalls$"
- EXTRA_PARAMS: --run "TestAgentUpgrades$"
# install-script
- EXTRA_PARAMS: --run "TestInstallScriptWithAgentUser$"
# installer-package
- EXTRA_PARAMS: --run "TestInstaller$"
- EXTRA_PARAMS: --run "TestInstallerRollback$"
- EXTRA_PARAMS: --run "TestInstallerSystemIntegrity$"
- EXTRA_PARAMS: --run "TestInstallerUpgrades$"


new-e2e-installer-ansible:
extends: .new_e2e_template
Expand Down Expand Up @@ -468,6 +483,10 @@ new-e2e-windows-systemprobe:
TARGETS: ./tests/sysprobe-functional
TEAM: windows-kernel-integrations
SHOULD_RUN_IN_FLAKES_FINDER: "false" # Currently broken in flake finder ADXT-687
parallel:
matrix:
- EXTRA_PARAMS: --run TestUSMAutoTaggingSuite
- EXTRA_PARAMS: --run TestVMSuite

new-e2e-windows-security-agent:
extends: .new_e2e_template
Expand Down
Loading

0 comments on commit a8965e0

Please sign in to comment.