From f8fc54248833ddf52ec0300afed0c85097f5658d Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 1 Mar 2023 13:23:51 +0100 Subject: [PATCH 01/33] Update buildimages to v13735072-2b0fd7c test images --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 261f3d1a254e0..63b8b4ecd997d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,12 +143,12 @@ variables: # Build images versions # To use images from datadog-agent-buildimages dev branches, set the corresponding # SUFFIX variable to _test_only - DATADOG_AGENT_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BUILDIMAGES: v15101851-1eecc1f + DATADOG_AGENT_BUILDIMAGES_SUFFIX: "_test_only" + DATADOG_AGENT_BUILDIMAGES: v15269855-5199fa1 DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" DATADOG_AGENT_WINBUILDIMAGES: v15101851-1eecc1f - DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_ARMBUILDIMAGES: v15101851-1eecc1f + DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "_test_only" + DATADOG_AGENT_ARMBUILDIMAGES: v15269855-5199fa1 DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v15101851-1eecc1f DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX: "" From 732cc5cb2d2734b37788d309e99e2f661a2a99c6 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 1 Mar 2023 13:24:30 +0100 Subject: [PATCH 02/33] Update prep line --- .gitlab/binary_build/cluster_agent.yml | 4 ++-- .gitlab/binary_build/cluster_agent_cloudfoundry.yml | 2 +- .gitlab/binary_build/linux.yml | 8 ++++---- .gitlab/binary_build/serverless.yml | 1 + .gitlab/deploy_6/container.yml | 1 + .gitlab/deploy_7/container.yml | 2 ++ .gitlab/deploy_dca.yml | 1 + .gitlab/deps_fetch.yml | 2 ++ .gitlab/docker_common/publish_job_templates.yml | 1 + .gitlab/integration_test.yml | 2 +- .gitlab/internal_image_deploy.yml | 2 ++ .gitlab/internal_kubernetes_deploy.yml | 1 + .gitlab/notify.yml | 2 +- .gitlab/package_build/deb.yml | 12 ++++++------ .gitlab/package_build/dmg.yml | 1 + .gitlab/package_build/rpm.yml | 10 +++++----- .gitlab/package_build/suse_rpm.yml | 8 ++++---- .gitlab/pkg_metrics.yml | 9 ++++++--- .gitlab/source_test/golang_deps_generate.yml | 1 + .gitlab/source_test/linux.yml | 3 ++- .gitlab/source_test/macos.yml | 1 + .gitlab/trigger_release.yml | 1 + 22 files changed, 47 insertions(+), 28 deletions(-) diff --git a/.gitlab/binary_build/cluster_agent.yml b/.gitlab/binary_build/cluster_agent.yml index 113ae06679094..3c29fdce8e0ee 100644 --- a/.gitlab/binary_build/cluster_agent.yml +++ b/.gitlab/binary_build/cluster_agent.yml @@ -19,8 +19,8 @@ cluster_agent-build_amd64: variables: ARCH: amd64 before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 cluster_agent-build_arm64: extends: .cluster_agent-build_common @@ -32,5 +32,5 @@ cluster_agent-build_arm64: variables: ARCH: arm64 before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 diff --git a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml index 1046527ad405e..3f98c6413c0ab 100644 --- a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml +++ b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml @@ -13,8 +13,8 @@ cluster_agent_cloudfoundry-build_amd64: variables: ARCH: amd64 before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: - inv check-go-version - inv -e cluster-agent-cloudfoundry.build diff --git a/.gitlab/binary_build/linux.yml b/.gitlab/binary_build/linux.yml index 3061b5d36b088..8f3fb8e242445 100644 --- a/.gitlab/binary_build/linux.yml +++ b/.gitlab/binary_build/linux.yml @@ -10,8 +10,8 @@ build_dogstatsd_static-binary_x64: variables: ARCH: amd64 before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: - inv check-go-version - inv -e dogstatsd.build --static --major-version 7 @@ -27,8 +27,8 @@ build_dogstatsd_static-binary_arm64: variables: ARCH: arm64 before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: - inv check-go-version - inv -e dogstatsd.build --static --major-version 7 @@ -42,8 +42,8 @@ build_dogstatsd-binary_x64: tags: ["runner:main"] needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"] before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: - inv check-go-version - inv -e dogstatsd.build --major-version 7 @@ -59,8 +59,8 @@ build_dogstatsd-binary_arm64: variables: ARCH: arm64 before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: - inv check-go-version - inv -e dogstatsd.build --major-version 7 diff --git a/.gitlab/binary_build/serverless.yml b/.gitlab/binary_build/serverless.yml index 0196a1e4bc62b..03a142a919694 100644 --- a/.gitlab/binary_build/serverless.yml +++ b/.gitlab/binary_build/serverless.yml @@ -2,6 +2,7 @@ .build_serverless_common: stage: binary_build before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] script: - inv check-go-version diff --git a/.gitlab/deploy_6/container.yml b/.gitlab/deploy_6/container.yml index a50e7ad1e058b..1ac81f7696b28 100644 --- a/.gitlab/deploy_6/container.yml +++ b/.gitlab/deploy_6/container.yml @@ -14,6 +14,7 @@ stage: deploy6 dependencies: [] before_script: + - source /root/.bashrc - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv -e agent.version --major-version 6 --url-safe)"; fi - export IMG_SOURCES="${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-arm64" - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${JMX}" diff --git a/.gitlab/deploy_7/container.yml b/.gitlab/deploy_7/container.yml index 8425e0b4d7e4d..9b2d4f00b5f3c 100644 --- a/.gitlab/deploy_7/container.yml +++ b/.gitlab/deploy_7/container.yml @@ -14,6 +14,7 @@ stage: deploy7 dependencies: [] before_script: + - source /root/.bashrc - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv -e agent.version --major-version 7 --url-safe)"; fi - export IMG_BASE_SRC="${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7${JMX}-amd64,${IMG_BASE_SRC}-7${JMX}-arm64" @@ -55,6 +56,7 @@ deploy_containers-dogstatsd: !reference [.on_deploy_a7_manual_auto_on_rc] dependencies: [] before_script: + - source /root/.bashrc - export VERSION="$(inv -e agent.version --major-version 7 --url-safe)" - export IMG_SOURCES="${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64" - export IMG_DESTINATIONS="${DSD_REPOSITORY}:${VERSION}" diff --git a/.gitlab/deploy_dca.yml b/.gitlab/deploy_dca.yml index a3dea18471df7..1643b67d79a63 100644 --- a/.gitlab/deploy_dca.yml +++ b/.gitlab/deploy_dca.yml @@ -14,6 +14,7 @@ stage: deploy_dca dependencies: [] before_script: + - source /root/.bashrc - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv -e agent.version --major-version 7 --url-safe)"; fi - if [[ "$CLUSTER_AGENT_REPOSITORY" == "" ]]; then export CLUSTER_AGENT_REPOSITORY="cluster-agent"; fi - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" diff --git a/.gitlab/deps_fetch.yml b/.gitlab/deps_fetch.yml index 9dfbeefe0956a..6d7bc6e309193 100644 --- a/.gitlab/deps_fetch.yml +++ b/.gitlab/deps_fetch.yml @@ -17,6 +17,7 @@ go_deps: tags: ["runner:main"] needs: [] script: + - source /root/.bashrc - inv -e deps --verbose - cd $GOPATH/pkg/mod/ && tar czf $CI_PROJECT_DIR/modcache.tar.gz . artifacts: @@ -31,6 +32,7 @@ go_tools_deps: tags: ["runner:main"] needs: [] script: + - source /root/.bashrc - inv -e download-tools - cd $GOPATH/pkg/mod/ && tar czf $CI_PROJECT_DIR/modcache_tools.tar.gz . artifacts: diff --git a/.gitlab/docker_common/publish_job_templates.yml b/.gitlab/docker_common/publish_job_templates.yml index f6eb8e19425c2..144e9ce5416aa 100644 --- a/.gitlab/docker_common/publish_job_templates.yml +++ b/.gitlab/docker_common/publish_job_templates.yml @@ -12,6 +12,7 @@ IMG_VARIABLES: "" IMG_SIGNING: "" script: # We can't use the 'trigger' keyword on manual jobs, otherwise they can't be run if the pipeline fails and is retried + - source /root/.bashrc - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - ECR_RELEASE_SUFFIX="${CI_COMMIT_TAG+-release}" - IMG_VARIABLES="$(sed -E "s#(${SRC_AGENT}|${SRC_DSD}|${SRC_DCA})#\1${ECR_RELEASE_SUFFIX}#g" <<<"$IMG_VARIABLES")" diff --git a/.gitlab/integration_test.yml b/.gitlab/integration_test.yml index d58299e7f8990..c2740355dbf96 100644 --- a/.gitlab/integration_test.yml +++ b/.gitlab/integration_test.yml @@ -10,7 +10,7 @@ dogstatsd_x64_size_test: tags: ["runner:main"] needs: ["build_dogstatsd_static-binary_x64"] before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - mkdir -p $STATIC_BINARIES_DIR - $S3_CP_CMD $S3_ARTIFACTS_URI/static/dogstatsd.amd64 $STATIC_BINARIES_DIR/dogstatsd script: diff --git a/.gitlab/internal_image_deploy.yml b/.gitlab/internal_image_deploy.yml index d1f0fa7051e91..76783740a4bbb 100644 --- a/.gitlab/internal_image_deploy.yml +++ b/.gitlab/internal_image_deploy.yml @@ -23,6 +23,7 @@ docker_trigger_internal: RELEASE_STAGING: "true" RELEASE_PROD: "true" script: + - source /root/.bashrc - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi @@ -50,6 +51,7 @@ docker_trigger_cluster_agent_internal: RELEASE_STAGING: "true" RELEASE_PROD: "true" script: + - source /root/.bashrc - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi diff --git a/.gitlab/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy.yml index b5002a111f19e..dd630696f5876 100644 --- a/.gitlab/internal_kubernetes_deploy.yml +++ b/.gitlab/internal_kubernetes_deploy.yml @@ -27,5 +27,6 @@ internal_kubernetes_deploy_experimental: WORKFLOW: "agents" FILTER: "cluster.env == 'experimental' and cluster.shortName == 'snowver'" script: + - source /root/.bashrc - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text) - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,WORKFLOW,OPTION_PRE_SCRIPT,FILTER,SKIP_PLAN_CHECK" diff --git a/.gitlab/notify.yml b/.gitlab/notify.yml index 422e7525c870a..2d84c4ef958db 100644 --- a/.gitlab/notify.yml +++ b/.gitlab/notify.yml @@ -48,7 +48,7 @@ send_pipeline_stats: when: always dependencies: [] script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - set +x - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_read_api_token --with-decryption --query "Parameter.Value" --out text) - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text) diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 1270f62402554..c03f5e62db8de 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -73,7 +73,7 @@ agent_deb-x64-a6: DESTINATION_DEB: 'datadog-agent_6_amd64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_amd64.deb' before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 agent_deb-x64-a7: @@ -92,7 +92,7 @@ agent_deb-x64-a7: DESTINATION_DEB: 'datadog-agent_7_amd64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_amd64.deb' before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 agent_deb-arm64-a6: @@ -110,7 +110,7 @@ agent_deb-arm64-a6: DESTINATION_DEB: 'datadog-agent_6_arm64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_arm64.deb' before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 agent_deb-arm64-a7: @@ -128,7 +128,7 @@ agent_deb-arm64-a7: DESTINATION_DEB: 'datadog-agent_7_arm64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_arm64.deb' before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 .iot_agent_build_common_deb: @@ -202,8 +202,8 @@ dogstatsd_deb-x64: variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR/* @@ -229,8 +229,8 @@ dogstatsd_deb-arm64: tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR/* diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml index ccfa168e69fdb..09acd7e90767e 100644 --- a/.gitlab/package_build/dmg.yml +++ b/.gitlab/package_build/dmg.yml @@ -31,4 +31,5 @@ agent_dmg-x64-a7: AGENT_MAJOR_VERSION: 7 PYTHON_RUNTIMES: '3' before_script: + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml index bdc48ba153379..8f9bb53852275 100644 --- a/.gitlab/package_build/rpm.yml +++ b/.gitlab/package_build/rpm.yml @@ -68,7 +68,7 @@ agent_rpm-x64-a6: PYTHON_RUNTIMES: '2,3' PACKAGE_ARCH: amd64 before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 # build Agent package for rpm-x64 @@ -86,7 +86,7 @@ agent_rpm-x64-a7: PYTHON_RUNTIMES: '3' PACKAGE_ARCH: amd64 before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 # build Agent package for rpm-arm64 @@ -103,7 +103,7 @@ agent_rpm-arm64-a6: PYTHON_RUNTIMES: '2,3' PACKAGE_ARCH: arm64 before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 # build Agent package for rpm-arm64 @@ -120,7 +120,7 @@ agent_rpm-arm64-a7: PYTHON_RUNTIMES: '3' PACKAGE_ARCH: arm64 before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 .iot_agent_build_common_rpm: @@ -191,8 +191,8 @@ dogstatsd_rpm-x64: variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR/* diff --git a/.gitlab/package_build/suse_rpm.yml b/.gitlab/package_build/suse_rpm.yml index 514faa911f7c1..e8ef1523e15da 100644 --- a/.gitlab/package_build/suse_rpm.yml +++ b/.gitlab/package_build/suse_rpm.yml @@ -70,7 +70,7 @@ agent_suse-x64-a6: PYTHON_RUNTIMES: '2,3' PACKAGE_ARCH: amd64 before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 # build Agent package for suse-x64 @@ -88,7 +88,7 @@ agent_suse-x64-a7: PYTHON_RUNTIMES: '3' PACKAGE_ARCH: amd64 before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 iot_agent_suse-x64: @@ -99,7 +99,7 @@ iot_agent_suse-x64: tags: ["runner:main"] needs: ["go_mod_tidy_check", "go_deps"] before_script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc script: - echo "About to build iot agent for $RELEASE_VERSION_7" - !reference [.setup_ruby_mirror_linux] @@ -138,8 +138,8 @@ dogstatsd_suse-x64: variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] - - source /root/.bashrc && conda activate ddpy3 script: # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR_SUSE/* diff --git a/.gitlab/pkg_metrics.yml b/.gitlab/pkg_metrics.yml index d25ac0f58e385..466756668c7fe 100644 --- a/.gitlab/pkg_metrics.yml +++ b/.gitlab/pkg_metrics.yml @@ -51,7 +51,7 @@ send_pkg_size-a6: - ls -l $OMNIBUS_PACKAGE_DIR - ls -l $OMNIBUS_PACKAGE_DIR_SUSE script: - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - mkdir -p /tmp/deb/amd64-agent - mkdir -p /tmp/deb/arm64-agent - mkdir -p /tmp/rpm/amd64-agent @@ -119,9 +119,10 @@ send_pkg_size-a7: - ls -l $OMNIBUS_PACKAGE_DIR - ls -l $OMNIBUS_PACKAGE_DIR_SUSE script: + - source /root/.bashrc - !reference [.add_metric_func, script] - - source /root/.bashrc && conda activate ddpy3 + - source /root/.bashrc - mkdir -p /tmp/amd64-deb/agent /tmp/amd64-deb/dogstatsd /tmp/amd64-deb/iot-agent /tmp/amd64-deb/heroku-agent - mkdir -p /tmp/arm64-deb/agent /tmp/arm64-deb/dogstatsd /tmp/arm64-deb/iot-agent - mkdir -p /tmp/amd64-rpm/agent /tmp/amd64-rpm/dogstatsd /tmp/amd64-rpm/iot-agent @@ -190,11 +191,13 @@ send_pkg_size-a7: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["runner:main"] script: + - source /root/.bashrc - !reference [.add_metric_func, script] - ls -l $OMNIBUS_PACKAGE_DIR - if [[ "${ARCH}" == "amd64" ]]; then ls -l $OMNIBUS_PACKAGE_DIR_SUSE; fi - - source /root/.bashrc && conda activate ddpy3 + + - source /root/.bashrc - export failures=0 - export last_stable=$(inv release.get-release-json-value "last_stable::${MAJOR_VERSION}") # Get stable packages from S3 buckets, send new package sizes & compare stable and new package sizes diff --git a/.gitlab/source_test/golang_deps_generate.yml b/.gitlab/source_test/golang_deps_generate.yml index b906fefd6cd4c..6f4ac5db622df 100644 --- a/.gitlab/source_test/golang_deps_generate.yml +++ b/.gitlab/source_test/golang_deps_generate.yml @@ -9,6 +9,7 @@ golang_deps_generate: tags: ["runner:main"] needs: ["go_deps"] before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] script: - inv agent.build-dep-tree diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 3a3705df7e792..6b1f8a72165e4 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -2,8 +2,8 @@ .rtloader_tests: stage: source_test before_script: - - !reference [.retrieve_linux_go_deps] - source /root/.bashrc && conda activate $CONDA_ENV + - !reference [.retrieve_linux_go_deps] - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "$PYTHON_RUNTIMES" - inv -e rtloader.install - inv -e rtloader.format --raise-if-changed @@ -212,6 +212,7 @@ go_mod_tidy_check: tags: ["runner:main"] needs: ["go_deps"] before_script: + - source /root/.bashrc - !reference [.retrieve_linux_go_deps] script: - inv -e check-mod-tidy diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index a416002836e5e..dd4f94732e573 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -9,6 +9,7 @@ tests_macos: variables: PYTHON_RUNTIMES: '3' script: + - source /root/.bashrc - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text) - export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_app_id --with-decryption --query "Parameter.Value" --out text) - export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_installation_id --with-decryption --query "Parameter.Value" --out text) diff --git a/.gitlab/trigger_release.yml b/.gitlab/trigger_release.yml index 4bd6d8b4c0b16..df8b6c2a0b1a4 100644 --- a/.gitlab/trigger_release.yml +++ b/.gitlab/trigger_release.yml @@ -18,6 +18,7 @@ script: # agent-release-management creates pipeline for both Agent 6 and Agent 7 # when triggered with major version 7 + - source /root/.bashrc - export RELEASE_VERSION=$(inv -e agent.version --major-version 7 --url-safe --omnibus-format)-1 - inv pipeline.trigger-child-pipeline --no-follow --project-name "DataDog/agent-release-management" --git-ref "main" --variables "ACTION,AUTO_RELEASE,BUILD_PIPELINE_ID,RELEASE_PRODUCT,RELEASE_VERSION,TARGET_REPO,TARGET_REPO_BRANCH" From bfb30d5668a9787e746b4a8c23dcf680eadbe521 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 1 Mar 2023 15:30:15 +0100 Subject: [PATCH 03/33] Fix remaining build jobs --- .gitlab/binary_build/linux.yml | 2 ++ .gitlab/package_build/deb.yml | 1 + .gitlab/package_build/rpm.yml | 1 + 3 files changed, 4 insertions(+) diff --git a/.gitlab/binary_build/linux.yml b/.gitlab/binary_build/linux.yml index 8f3fb8e242445..6ebe481f13851 100644 --- a/.gitlab/binary_build/linux.yml +++ b/.gitlab/binary_build/linux.yml @@ -77,6 +77,7 @@ build_iot_agent-binary_x64: before_script: - !reference [.retrieve_linux_go_deps] script: + - source /root/.bashrc - inv check-go-version - inv -e agent.build --flavor iot --major-version 7 - $S3_CP_CMD $CI_PROJECT_DIR/$AGENT_BINARIES_DIR/agent $S3_ARTIFACTS_URI/iot/agent @@ -93,5 +94,6 @@ build_iot_agent-binary_arm64: before_script: - !reference [.retrieve_linux_go_deps] script: + - source /root/.bashrc - inv check-go-version - inv -e agent.build --flavor iot --major-version 7 diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index c03f5e62db8de..c5b02aeee4b66 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -133,6 +133,7 @@ agent_deb-arm64-a7: .iot_agent_build_common_deb: script: + - source /root/.bashrc - !reference [.setup_ruby_mirror_linux] - !reference [.setup_python_mirror_linux] - !reference [.retrieve_linux_go_deps] diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml index 8f9bb53852275..bfea3cfe5b4e4 100644 --- a/.gitlab/package_build/rpm.yml +++ b/.gitlab/package_build/rpm.yml @@ -126,6 +126,7 @@ agent_rpm-arm64-a7: .iot_agent_build_common_rpm: script: - echo "About to build iot agent for $RELEASE_VERSION_7" + - source /root/.bashrc - !reference [.setup_ruby_mirror_linux] - !reference [.setup_python_mirror_linux] - !reference [.retrieve_linux_go_deps] From 65e34b85fb277148ecb380c617aaab6006796b23 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 12:27:09 +0100 Subject: [PATCH 04/33] Fix newly added jobs --- .gitlab/source_test/slack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/source_test/slack.yml b/.gitlab/source_test/slack.yml index 79ba633dd21bc..e01a9846cd86a 100644 --- a/.gitlab/source_test/slack.yml +++ b/.gitlab/source_test/slack.yml @@ -5,6 +5,7 @@ slack_teams_channels_check: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["runner:main"] script: + - source /root/.bashrc - python3 -m pip install codeowners - inv -e pipeline.check-notify-teams From efccf06afe79e844acd3e2adb8de7a1100506db2 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 14:42:11 +0100 Subject: [PATCH 05/33] Try migrating deps_fetch, source_test, binary_build and package_build jobs --- .gitlab/binary_build/linux.yml | 12 +++++------ .gitlab/binary_build/serverless.yml | 4 ++-- .gitlab/deps_fetch.yml | 4 ++-- .gitlab/package_build/deb.yml | 18 ++++++++-------- .gitlab/package_build/dmg.yml | 2 +- .gitlab/package_build/rpm.yml | 16 +++++++------- .gitlab/package_build/suse_rpm.yml | 8 +++---- .gitlab/source_test/golang_deps_generate.yml | 2 +- .gitlab/source_test/linux.yml | 22 ++++++++++---------- .gitlab/source_test/macos.yml | 2 +- .gitlab/source_test/slack.yml | 2 +- 11 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.gitlab/binary_build/linux.yml b/.gitlab/binary_build/linux.yml index 6ebe481f13851..a5e6d0d142b99 100644 --- a/.gitlab/binary_build/linux.yml +++ b/.gitlab/binary_build/linux.yml @@ -5,7 +5,7 @@ build_dogstatsd_static-binary_x64: rules: !reference [.on_a7] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"] variables: ARCH: amd64 @@ -22,7 +22,7 @@ build_dogstatsd_static-binary_arm64: rules: !reference [.on_a7] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["lint_deb-arm64", "tests_deb-arm64-py3", "go_deps"] variables: ARCH: arm64 @@ -39,7 +39,7 @@ build_dogstatsd-binary_x64: rules: !reference [.on_a7] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"] before_script: - source /root/.bashrc @@ -54,7 +54,7 @@ build_dogstatsd-binary_arm64: !reference [.on_all_builds_a7] stage: binary_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["lint_deb-arm64", "tests_deb-arm64-py3", "go_deps"] variables: ARCH: arm64 @@ -72,7 +72,7 @@ build_iot_agent-binary_x64: rules: !reference [.on_a7] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"] before_script: - !reference [.retrieve_linux_go_deps] @@ -87,7 +87,7 @@ build_iot_agent-binary_arm64: !reference [.on_all_builds_a7] stage: binary_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["lint_deb-arm64", "tests_deb-arm64-py3", "go_deps"] variables: ARCH: arm64 diff --git a/.gitlab/binary_build/serverless.yml b/.gitlab/binary_build/serverless.yml index 03a142a919694..61ad89c721589 100644 --- a/.gitlab/binary_build/serverless.yml +++ b/.gitlab/binary_build/serverless.yml @@ -13,7 +13,7 @@ build_serverless-deb_x64: variables: BINARY_NAME: datadog-agent-x64 image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps"] artifacts: expire_in: 1 day @@ -25,5 +25,5 @@ build_serverless-deb_arm64: variables: BINARY_NAME: datadog-agent-arm64 image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_deps"] diff --git a/.gitlab/deps_fetch.yml b/.gitlab/deps_fetch.yml index 6d7bc6e309193..5fb0ebc885f73 100644 --- a/.gitlab/deps_fetch.yml +++ b/.gitlab/deps_fetch.yml @@ -14,7 +14,7 @@ go_deps: stage: deps_fetch image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: [] script: - source /root/.bashrc @@ -29,7 +29,7 @@ go_deps: go_tools_deps: stage: deps_fetch image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: [] script: - source /root/.bashrc diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index c5b02aeee4b66..7b4d0985bd3e5 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -63,7 +63,7 @@ agent_deb-x64-a6: !reference [.on_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -82,7 +82,7 @@ agent_deb-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -101,7 +101,7 @@ agent_deb-arm64-a6: !reference [.on_all_builds_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 6 @@ -119,7 +119,7 @@ agent_deb-arm64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 7 @@ -162,7 +162,7 @@ iot_agent_deb-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: amd64 @@ -174,7 +174,7 @@ iot_agent_deb-arm64: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: arm64 @@ -187,7 +187,7 @@ iot_agent_deb-armhf: stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_armhf$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES # Run with platform:arm64 since no platform:armhf exists and arm64 should be backwards compatible - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: armhf @@ -198,7 +198,7 @@ dogstatsd_deb-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -227,7 +227,7 @@ dogstatsd_deb-arm64: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] before_script: - source /root/.bashrc diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml index 09acd7e90767e..34c17b4562060 100644 --- a/.gitlab/package_build/dmg.yml +++ b/.gitlab/package_build/dmg.yml @@ -24,7 +24,7 @@ agent_dmg-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml index bfea3cfe5b4e4..f06e15f3ea26a 100644 --- a/.gitlab/package_build/rpm.yml +++ b/.gitlab/package_build/rpm.yml @@ -60,7 +60,7 @@ agent_rpm-x64-a6: !reference [.on_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -78,7 +78,7 @@ agent_rpm-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -96,7 +96,7 @@ agent_rpm-arm64-a6: !reference [.on_all_builds_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 6 @@ -113,7 +113,7 @@ agent_rpm-arm64-a7: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 7 @@ -157,7 +157,7 @@ iot_agent_rpm-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "go_deps"] iot_agent_rpm-arm64: @@ -166,7 +166,7 @@ iot_agent_rpm-arm64: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "go_deps"] iot_agent_rpm-armhf: @@ -176,7 +176,7 @@ iot_agent_rpm-armhf: stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_armhf$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES # Run with platform:arm64 since no platform:armhf exists and arm64 should be backwards compatible - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "go_deps"] before_script: # Ensures uname -m reports armv7l @@ -187,7 +187,7 @@ dogstatsd_rpm-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials diff --git a/.gitlab/package_build/suse_rpm.yml b/.gitlab/package_build/suse_rpm.yml index e8ef1523e15da..527de597036dc 100644 --- a/.gitlab/package_build/suse_rpm.yml +++ b/.gitlab/package_build/suse_rpm.yml @@ -62,7 +62,7 @@ agent_suse-x64-a6: !reference [.on_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -80,7 +80,7 @@ agent_suse-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -96,7 +96,7 @@ iot_agent_suse-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "go_deps"] before_script: - source /root/.bashrc @@ -133,7 +133,7 @@ dogstatsd_suse-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials diff --git a/.gitlab/source_test/golang_deps_generate.yml b/.gitlab/source_test/golang_deps_generate.yml index 6f4ac5db622df..db182cbdcf25d 100644 --- a/.gitlab/source_test/golang_deps_generate.yml +++ b/.gitlab/source_test/golang_deps_generate.yml @@ -6,7 +6,7 @@ golang_deps_generate: !reference [.on_main_or_release_branch] stage: source_test image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps"] before_script: - source /root/.bashrc diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 6b1f8a72165e4..c9cb593efab89 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -45,7 +45,7 @@ tests_deb-x64-py2: rules: !reference [.on_a6] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps"] variables: PYTHON_RUNTIMES: '2' @@ -56,7 +56,7 @@ tests_deb-x64-py3: - .rtloader_tests - .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -74,7 +74,7 @@ tests_flavor_iot_deb-x64: - .rtloader_tests - .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -95,7 +95,7 @@ tests_flavor_dogstatsd_deb-x64: - .rtloader_tests - .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -119,7 +119,7 @@ tests_rpm-x64-py2: rules: !reference [.on_a6] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64_testing$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps"] variables: PYTHON_RUNTIMES: '2' @@ -131,7 +131,7 @@ tests_rpm-x64-py3: - .rtloader_tests - .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64_testing$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -152,7 +152,7 @@ tests_deb-arm64-py2: rules: !reference [.on_a6] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_deps"] variables: PYTHON_RUNTIMES: '2' @@ -164,7 +164,7 @@ tests_deb-arm64-py3: - .linux_tests needs: ["go_deps", "go_tools_deps"] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] variables: PYTHON_RUNTIMES: '3' CONDA_ENV: ddpy3 @@ -181,7 +181,7 @@ tests_rpm-arm64-py2: rules: !reference [.on_a6] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_deps"] variables: PYTHON_RUNTIMES: '2' @@ -192,7 +192,7 @@ tests_rpm-arm64-py3: - .rtloader_tests - .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -209,7 +209,7 @@ lint_rpm-arm64: go_mod_tidy_check: stage: source_test image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps"] before_script: - source /root/.bashrc diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index dd4f94732e573..0e68692439d9a 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -4,7 +4,7 @@ tests_macos: rules: !reference [.on_a6] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: [] variables: PYTHON_RUNTIMES: '3' diff --git a/.gitlab/source_test/slack.yml b/.gitlab/source_test/slack.yml index e01a9846cd86a..26c2f351d9fa7 100644 --- a/.gitlab/source_test/slack.yml +++ b/.gitlab/source_test/slack.yml @@ -3,7 +3,7 @@ slack_teams_channels_check: stage: source_test image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] script: - source /root/.bashrc - python3 -m pip install codeowners From 71262c1f61f44cd8347ebfe6525985daf37fc7d4 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 15:21:56 +0100 Subject: [PATCH 06/33] Set concurrency for golangci-lint --- .gitlab/source_test/linux.yml | 2 +- tasks/test.py | 49 +++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index c9cb593efab89..8959c264b86c8 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -38,7 +38,7 @@ - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "3" - inv -e rtloader.install - inv -e install-tools - - inv -e lint-go $FLAVORS $EXTRA_OPTS + - inv -e lint-go --cpus 4 $FLAVORS $EXTRA_OPTS tests_deb-x64-py2: extends: .rtloader_tests diff --git a/tasks/test.py b/tasks/test.py index 48d5cd1fd10a3..f6fdd71a3e866 100644 --- a/tasks/test.py +++ b/tasks/test.py @@ -270,7 +270,13 @@ def get_failure(self, flavor): def lint_flavor( - ctx, modules: List[GoModule], flavor: AgentFlavor, build_tags: List[str], arch: str, rtloader_root: bool + ctx, + modules: List[GoModule], + flavor: AgentFlavor, + build_tags: List[str], + arch: str, + rtloader_root: bool, + concurrency: int, ): """ Runs linters for given flavor, build tags, and modules. @@ -279,7 +285,12 @@ def lint_flavor( def command(module_results, module, module_result): with ctx.cd(module.full_path()): lint_results = run_golangci_lint( - ctx, targets=module.targets, rtloader_root=rtloader_root, build_tags=build_tags, arch=arch + ctx, + targets=module.targets, + rtloader_root=rtloader_root, + build_tags=build_tags, + arch=arch, + concurrency=concurrency, ) for lint_result in lint_results: module_result.lint_outputs.append(lint_result) @@ -455,7 +466,7 @@ def test( rtloader_root=None, python_home_2=None, python_home_3=None, - cpus=0, + cpus=None, major_version='7', python_runtimes='3', timeout=180, @@ -497,7 +508,15 @@ def test( if not skip_linters: modules_results_per_phase["lint"] = run_lint_go( - ctx, module, targets, flavors, build_include, build_exclude, rtloader_root, arch + ctx, + module, + targets, + flavors, + build_include, + build_exclude, + rtloader_root, + arch, + cpus, ) # Process input arguments @@ -513,8 +532,6 @@ def test( timeout = int(timeout) - # Lint - ldflags, gcflags, env = get_build_flags( ctx, rtloader_root=rtloader_root, @@ -644,6 +661,7 @@ def run_lint_go( build_exclude=None, rtloader_root=None, arch="x64", + cpus=None, ): modules, flavors = process_input_args(module, targets, flavors) @@ -660,7 +678,13 @@ def run_lint_go( for flavor, build_tags in linter_tags.items(): modules_lint_results_per_flavor[flavor] = lint_flavor( - ctx, modules=modules, flavor=flavor, build_tags=build_tags, arch=arch, rtloader_root=rtloader_root + ctx, + modules=modules, + flavor=flavor, + build_tags=build_tags, + arch=arch, + rtloader_root=rtloader_root, + concurrency=cpus, ) return modules_lint_results_per_flavor @@ -676,6 +700,7 @@ def lint_go( build_exclude=None, rtloader_root=None, arch="x64", + cpus=None, ): """ Run go linters on the given module and targets. @@ -702,7 +727,15 @@ def lint_go( modules_results_per_phase = defaultdict(dict) modules_results_per_phase["lint"] = run_lint_go( - ctx, module, targets, flavors, build_include, build_exclude, rtloader_root, arch + ctx, + module, + targets, + flavors, + build_include, + build_exclude, + rtloader_root, + arch, + cpus, ) success = process_module_results(modules_results_per_phase) From 183e258bc290f6a20b2805c49602bbeb0e96d612 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 17:15:44 +0100 Subject: [PATCH 07/33] Set empty detected features in pkg/metadata/host tests --- pkg/metadata/host/host_tags_test.go | 21 +++++++++++++++++++++ pkg/metadata/host/host_test.go | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/pkg/metadata/host/host_tags_test.go b/pkg/metadata/host/host_tags_test.go index 1d49ed4279d5c..03ecc6ae90f72 100644 --- a/pkg/metadata/host/host_tags_test.go +++ b/pkg/metadata/host/host_tags_test.go @@ -21,6 +21,9 @@ func init() { } func TestGetHostTags(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() mockConfig := config.Mock(t) mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag3"}) @@ -32,6 +35,9 @@ func TestGetHostTags(t *testing.T) { } func TestGetEmptyHostTags(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() // getHostTags should never return a nil value under System even when there are no host tags hostTags := GetHostTags(ctx, false) @@ -40,6 +46,9 @@ func TestGetEmptyHostTags(t *testing.T) { } func TestGetHostTagsWithSplits(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() mockConfig := config.Mock(t) mockConfig.Set("tag_value_split_separator", map[string]string{"kafka_partition": ","}) @@ -52,6 +61,9 @@ func TestGetHostTagsWithSplits(t *testing.T) { } func TestGetHostTagsWithoutSplits(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() mockConfig := config.Mock(t) mockConfig.Set("tag_value_split_separator", map[string]string{"kafka_partition": ";"}) @@ -64,6 +76,9 @@ func TestGetHostTagsWithoutSplits(t *testing.T) { } func TestGetHostTagsWithEnv(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() mockConfig := config.Mock(t) mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag3", "env:prod"}) @@ -89,6 +104,9 @@ func TestMarshalEmptyHostTags(t *testing.T) { } func TestCombineExtraTags(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() mockConfig := config.Mock(t) mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag4"}) @@ -102,6 +120,9 @@ func TestCombineExtraTags(t *testing.T) { } func TestHostTagsCache(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() mockConfig := config.Mock(t) mockConfig.Set("collect_gce_tags", false) diff --git a/pkg/metadata/host/host_test.go b/pkg/metadata/host/host_test.go index 310ce8b652b5d..fb053bfe21f19 100644 --- a/pkg/metadata/host/host_test.go +++ b/pkg/metadata/host/host_test.go @@ -29,6 +29,9 @@ import ( ) func TestGetPayload(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() p := GetPayload(ctx, hostname.Data{Hostname: "myhostname", Provider: ""}) assert.NotEmpty(t, p.Os) @@ -76,6 +79,9 @@ func TestGetHostInfo(t *testing.T) { } func TestGetMeta(t *testing.T) { + config.SetDetectedFeatures(config.FeatureMap{}) + defer config.SetDetectedFeatures(nil) + ctx := context.Background() meta := getMeta(ctx, hostname.Data{}) assert.NotEmpty(t, meta.SocketHostname) From 61b4cec2b0f3b67baab059968586ec03705d1261 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 17:58:58 +0100 Subject: [PATCH 08/33] Try using autoconfig_from_environment --- pkg/metadata/host/host_tags_test.go | 44 +++++++++++++++-------------- pkg/metadata/host/host_test.go | 6 ---- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/pkg/metadata/host/host_tags_test.go b/pkg/metadata/host/host_tags_test.go index 03ecc6ae90f72..bc0e11ca33f7f 100644 --- a/pkg/metadata/host/host_tags_test.go +++ b/pkg/metadata/host/host_tags_test.go @@ -21,11 +21,11 @@ func init() { } func TestGetHostTags(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag3"}) defer mockConfig.Set("tags", nil) @@ -35,10 +35,12 @@ func TestGetHostTags(t *testing.T) { } func TestGetEmptyHostTags(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() + + mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + // getHostTags should never return a nil value under System even when there are no host tags hostTags := GetHostTags(ctx, false) assert.NotNil(t, hostTags.System) @@ -46,11 +48,11 @@ func TestGetEmptyHostTags(t *testing.T) { } func TestGetHostTagsWithSplits(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + mockConfig.Set("tag_value_split_separator", map[string]string{"kafka_partition": ","}) mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag3", "kafka_partition:0,1,2"}) defer mockConfig.Set("tags", nil) @@ -61,11 +63,11 @@ func TestGetHostTagsWithSplits(t *testing.T) { } func TestGetHostTagsWithoutSplits(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + mockConfig.Set("tag_value_split_separator", map[string]string{"kafka_partition": ";"}) mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag3", "kafka_partition:0,1,2"}) defer mockConfig.Set("tags", nil) @@ -76,11 +78,11 @@ func TestGetHostTagsWithoutSplits(t *testing.T) { } func TestGetHostTagsWithEnv(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag3", "env:prod"}) mockConfig.Set("env", "preprod") defer mockConfig.Set("tags", nil) @@ -104,11 +106,11 @@ func TestMarshalEmptyHostTags(t *testing.T) { } func TestCombineExtraTags(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + mockConfig.Set("tags", []string{"tag1:value1", "tag2", "tag4"}) mockConfig.Set("extra_tags", []string{"tag1:value2", "tag3", "tag4"}) defer mockConfig.Set("tags", nil) @@ -120,11 +122,11 @@ func TestCombineExtraTags(t *testing.T) { } func TestHostTagsCache(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() mockConfig := config.Mock(t) + mockConfig.Set("autoconfig_from_environment", false) + defer mockConfig.Set("autoconfig_from_environment", true) + mockConfig.Set("collect_gce_tags", false) fooTags := []string{"foo1:value1"} diff --git a/pkg/metadata/host/host_test.go b/pkg/metadata/host/host_test.go index fb053bfe21f19..310ce8b652b5d 100644 --- a/pkg/metadata/host/host_test.go +++ b/pkg/metadata/host/host_test.go @@ -29,9 +29,6 @@ import ( ) func TestGetPayload(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() p := GetPayload(ctx, hostname.Data{Hostname: "myhostname", Provider: ""}) assert.NotEmpty(t, p.Os) @@ -79,9 +76,6 @@ func TestGetHostInfo(t *testing.T) { } func TestGetMeta(t *testing.T) { - config.SetDetectedFeatures(config.FeatureMap{}) - defer config.SetDetectedFeatures(nil) - ctx := context.Background() meta := getMeta(ctx, hostname.Data{}) assert.NotEmpty(t, meta.SocketHostname) From 0934006b70db3a86c120d98b5273f8df8069fbc7 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 17:59:22 +0100 Subject: [PATCH 09/33] Try modifying orderof script runs --- .gitlab/package_build/deb.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 7b4d0985bd3e5..ad2f01530a56c 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -8,10 +8,11 @@ .agent_build_common_deb: script: - - echo "About to build for $RELEASE_VERSION" + - source /root/.bashrc - !reference [.setup_ruby_mirror_linux] - !reference [.setup_python_mirror_linux] - !reference [.retrieve_linux_go_deps] + - echo "About to build for $RELEASE_VERSION" # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR/* # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. @@ -73,7 +74,6 @@ agent_deb-x64-a6: DESTINATION_DEB: 'datadog-agent_6_amd64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_amd64.deb' before_script: - - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 agent_deb-x64-a7: @@ -92,7 +92,6 @@ agent_deb-x64-a7: DESTINATION_DEB: 'datadog-agent_7_amd64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_amd64.deb' before_script: - - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 agent_deb-arm64-a6: @@ -110,7 +109,6 @@ agent_deb-arm64-a6: DESTINATION_DEB: 'datadog-agent_6_arm64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_arm64.deb' before_script: - - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_6 agent_deb-arm64-a7: @@ -128,7 +126,6 @@ agent_deb-arm64-a7: DESTINATION_DEB: 'datadog-agent_7_arm64.deb' DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_arm64.deb' before_script: - - source /root/.bashrc - export RELEASE_VERSION=$RELEASE_VERSION_7 .iot_agent_build_common_deb: From f59039b8f86746557c91d143f91dcdf63b3d2712 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 18:29:17 +0100 Subject: [PATCH 10/33] Debug --- .gitlab/source_test/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 8959c264b86c8..96c848f92b310 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -19,6 +19,8 @@ - !reference [.retrieve_linux_go_tools_deps] - inv -e install-tools - inv -e invoke-unit-tests + - echo "Env - $DD_ENV" + - echo "Site - $DD_SITE" - inv -e test $FLAVORS --skip-linters --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus 4 $EXTRA_OPTS --save-result-json test_output.json --junit-tar "junit-${CI_JOB_NAME}.tgz" artifacts: expire_in: 2 weeks From 1f8a13c305d0e3665dffc37d9c1ec41ef2ce878d Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 23:49:21 +0100 Subject: [PATCH 11/33] Make inv test ignore DD_ variables --- tasks/test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tasks/test.py b/tasks/test.py index f6fdd71a3e866..e74f37a6e6670 100644 --- a/tasks/test.py +++ b/tasks/test.py @@ -504,6 +504,13 @@ def test( # } modules_results_per_phase = defaultdict(dict) + # Sanitize environment variables + # We want to ignore all `DD_` variables, as they will interfere with the behavior + # of some unit tests + for env in os.environ.keys(): + if env.startswith("DD_"): + del os.environ[env] + # Run linters first if not skip_linters: From c706f15c44baa16b59cfe53ebbf3b0cbef856f58 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 22 Mar 2023 23:50:13 +0100 Subject: [PATCH 12/33] Debug --- .gitlab/source_test/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 96c848f92b310..2b2cacc7478bc 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -22,6 +22,8 @@ - echo "Env - $DD_ENV" - echo "Site - $DD_SITE" - inv -e test $FLAVORS --skip-linters --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus 4 $EXTRA_OPTS --save-result-json test_output.json --junit-tar "junit-${CI_JOB_NAME}.tgz" + - echo "Env - $DD_ENV" + - echo "Site - $DD_SITE" artifacts: expire_in: 2 weeks when: always From 1e30f377d3468b43ba6525dc0d883736616ce9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 23 Mar 2023 11:12:27 +0100 Subject: [PATCH 13/33] Fix `pkg/clusteragent/admission/controllers` tests when run in a k8s pod Fix `pkg/util/kubernetes/apiserver` tests when run in a k8s pod Prefer mock config --- .../controllers/webhook/controller_v1_test.go | 15 +++++--- .../webhook/controller_v1beta1_test.go | 15 +++++--- .../admission/mutate/config_test.go | 3 +- .../apiserver/hpa_controller_test.go | 35 +++++++++++-------- .../apiserver/wpa_controller_test.go | 23 +++++++----- 5 files changed, 60 insertions(+), 31 deletions(-) diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go index 53330c7f87820..51f99f4634b9c 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go @@ -185,7 +185,7 @@ func TestGenerateTemplatesV1(t *testing.T) { Name: name, ClientConfig: admiv1.WebhookClientConfig{ Service: &admiv1.ServiceReference{ - Namespace: "default", + Namespace: "nsfoo", Name: "datadog-admission-controller", Port: &port, Path: &path, @@ -485,6 +485,9 @@ func TestGenerateTemplatesV1(t *testing.T) { }, }, } + + mockConfig.Set("kube_resources_namespace", "nsfoo") + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setupConfig() @@ -500,6 +503,7 @@ func TestGenerateTemplatesV1(t *testing.T) { } func TestGetWebhookSkeletonV1(t *testing.T) { + mockConfig := config.Mock(t) defaultReinvocationPolicy := admiv1.IfNeededReinvocationPolicy failurePolicy := admiv1.Ignore matchPolicy := admiv1.Exact @@ -515,7 +519,7 @@ func TestGetWebhookSkeletonV1(t *testing.T) { Name: "datadog.webhook.foo", ClientConfig: admiv1.WebhookClientConfig{ Service: &admiv1.ServiceReference{ - Namespace: "default", + Namespace: "nsfoo", Name: "datadog-admission-controller", Port: &port, Path: &path, @@ -583,11 +587,14 @@ func TestGetWebhookSkeletonV1(t *testing.T) { want: webhook(&customTimeout, objectSelector, nil), }, } + + mockConfig.Set("kube_resources_namespace", "nsfoo") + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.timeout != nil { - config.Datadog.Set("admission_controller.timeout_seconds", *tt.timeout) - defer config.Datadog.SetDefault("admission_controller.timeout_seconds", defaultTimeout) + mockConfig.Set("admission_controller.timeout_seconds", *tt.timeout) + defer mockConfig.SetDefault("admission_controller.timeout_seconds", defaultTimeout) } c := &ControllerV1{} diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go index 6396d43e487b6..fb1869ad24283 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go @@ -185,7 +185,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { Name: name, ClientConfig: admiv1beta1.WebhookClientConfig{ Service: &admiv1beta1.ServiceReference{ - Namespace: "default", + Namespace: "nsfoo", Name: "datadog-admission-controller", Port: &port, Path: &path, @@ -486,6 +486,9 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { }, }, } + + mockConfig.Set("kube_resources_namespace", "nsfoo") + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setupConfig() @@ -501,6 +504,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { } func TestGetWebhookSkeletonV1beta1(t *testing.T) { + mockConfig := config.Mock(t) failurePolicy := admiv1beta1.Ignore matchPolicy := admiv1beta1.Exact sideEffects := admiv1beta1.SideEffectClassNone @@ -516,7 +520,7 @@ func TestGetWebhookSkeletonV1beta1(t *testing.T) { Name: "datadog.webhook.foo", ClientConfig: admiv1beta1.WebhookClientConfig{ Service: &admiv1beta1.ServiceReference{ - Namespace: "default", + Namespace: "nsfoo", Name: "datadog-admission-controller", Port: &port, Path: &path, @@ -584,11 +588,14 @@ func TestGetWebhookSkeletonV1beta1(t *testing.T) { want: webhook(&customTimeout, objectSelector, nil), }, } + + mockConfig.Set("kube_resources_namespace", "nsfoo") + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.timeout != nil { - config.Datadog.Set("admission_controller.timeout_seconds", *tt.timeout) - defer config.Datadog.SetDefault("admission_controller.timeout_seconds", defaultTimeout) + mockConfig.Set("admission_controller.timeout_seconds", *tt.timeout) + defer mockConfig.SetDefault("admission_controller.timeout_seconds", defaultTimeout) } c := &ControllerV1beta1{} diff --git a/pkg/clusteragent/admission/mutate/config_test.go b/pkg/clusteragent/admission/mutate/config_test.go index 24a0d04c7a79d..fd1814664fcc5 100644 --- a/pkg/clusteragent/admission/mutate/config_test.go +++ b/pkg/clusteragent/admission/mutate/config_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/common" "github.com/DataDog/datadog-agent/pkg/util/pointer" "github.com/stretchr/testify/assert" @@ -126,7 +127,7 @@ func TestInjectService(t *testing.T) { pod = withLabels(pod, map[string]string{"admission.datadoghq.com/enabled": "true", "admission.datadoghq.com/config.mode": "service"}) err := injectConfig(pod, "", nil) assert.Nil(t, err) - assert.Contains(t, pod.Spec.Containers[0].Env, fakeEnvWithValue("DD_AGENT_HOST", "datadog.default.svc.cluster.local")) + assert.Contains(t, pod.Spec.Containers[0].Env, fakeEnvWithValue("DD_AGENT_HOST", "datadog."+common.GetMyNamespace()+".svc.cluster.local")) } func TestInjectSocket(t *testing.T) { diff --git a/pkg/util/kubernetes/apiserver/hpa_controller_test.go b/pkg/util/kubernetes/apiserver/hpa_controller_test.go index 91e64fabe8efa..b0c8e6e3e7bc9 100644 --- a/pkg/util/kubernetes/apiserver/hpa_controller_test.go +++ b/pkg/util/kubernetes/apiserver/hpa_controller_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/record" "github.com/DataDog/datadog-agent/pkg/clusteragent/custommetrics" + "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/autoscalers" "github.com/DataDog/datadog-agent/pkg/util/pointer" @@ -184,8 +185,11 @@ func makeAnnotations(metricName string, labels map[string]string) map[string]str // TestupdateExternalMetrics checks the reconciliation between the local cache and the global store logic func TestUpdate(t *testing.T) { + mockConfig := config.Mock(t) + mockConfig.Set("kube_resources_namespace", "nsfoo") + name := custommetrics.GetConfigmapName() - store, client := newFakeConfigMapStore(t, "default", name, nil) + store, client := newFakeConfigMapStore(t, "nsfoo", name, nil) d := &fakeDatadogClient{} p := &fakeProcessor{ @@ -215,13 +219,13 @@ func TestUpdate(t *testing.T) { // Start the DCA with already existing Data // Check if nothing in local store and Global Store is full we update the Global Store metrics correctly metricsToStore := map[string]custommetrics.ExternalMetricValue{ - "external_metric-horizontal-default-foo-metric1": { + "external_metric-horizontal-nsfoo-foo-metric1": { MetricName: "metric1", Labels: map[string]string{"foo": "bar"}, Ref: custommetrics.ObjectReference{ Type: "horizontal", Name: "foo", - Namespace: "default", + Namespace: "nsfoo", }, Value: 1.3, Valid: true, @@ -248,13 +252,13 @@ func TestUpdate(t *testing.T) { // Fresh start // Check if local store is not empty hctrl.toStore.m.Lock() - hctrl.toStore.data["external_metric-horizontal-default-foo-metric2"] = custommetrics.ExternalMetricValue{ + hctrl.toStore.data["external_metric-horizontal-nsfoo-foo-metric2"] = custommetrics.ExternalMetricValue{ MetricName: "metric2", Labels: map[string]string{"foo": "bar"}, Ref: custommetrics.ObjectReference{ Type: "horizontal", Name: "foo", - Namespace: "default", + Namespace: "nsfoo", }, } require.Len(t, hctrl.toStore.data, 1) @@ -269,13 +273,13 @@ func TestUpdate(t *testing.T) { // Check that if there is conflicting info from the local store and the Global Store that we merge correctly // Check conflict on metric name and labels hctrl.toStore.m.Lock() - hctrl.toStore.data["external_metric-horizontal-default-foo-metric2"] = custommetrics.ExternalMetricValue{ + hctrl.toStore.data["external_metric-horizontal-nsfoo-foo-metric2"] = custommetrics.ExternalMetricValue{ MetricName: "metric2", Labels: map[string]string{"foo": "baz"}, Ref: custommetrics.ObjectReference{ Type: "horizontal", Name: "foo", - Namespace: "default", + Namespace: "nsfoo", }, } require.Len(t, hctrl.toStore.data, 1) @@ -295,9 +299,12 @@ func TestUpdate(t *testing.T) { // TestAutoscalerController is an integration test of the AutoscalerController func TestAutoscalerController(t *testing.T) { + mockConfig := config.Mock(t) + mockConfig.Set("kube_resources_namespace", "nsfoo") + penTime := (int(time.Now().Unix()) - int(maxAge.Seconds()/2)) * 1000 name := custommetrics.GetConfigmapName() - store, client := newFakeConfigMapStore(t, "default", name, nil) + store, client := newFakeConfigMapStore(t, "nsfoo", name, nil) metricName := "foo" ddSeries := []datadog.Series{ { @@ -333,14 +340,14 @@ func TestAutoscalerController(t *testing.T) { mockedHPA := newFakeHorizontalPodAutoscaler( "hpa_1", - "default", + "nsfoo", "1", "foo", map[string]string{"foo": "bar"}, ) mockedHPA.Annotations = makeAnnotations("foo", map[string]string{"foo": "bar"}) - _, err := c.HorizontalPodAutoscalers("default").Create(context.TODO(), mockedHPA, metav1.CreateOptions{}) + _, err := c.HorizontalPodAutoscalers("nsfoo").Create(context.TODO(), mockedHPA, metav1.CreateOptions{}) require.NoError(t, err) timeout := time.NewTimer(5 * time.Second) @@ -454,14 +461,14 @@ func TestAutoscalerController(t *testing.T) { newMockedHPA := newFakeHorizontalPodAutoscaler( "hpa_2", - "default", + "nsfoo", "1", "foo", map[string]string{"foo": "bar"}, ) mockedHPA.Annotations = makeAnnotations("foo", map[string]string{"foo": "bar"}) - _, err = c.HorizontalPodAutoscalers("default").Create(context.TODO(), newMockedHPA, metav1.CreateOptions{}) + _, err = c.HorizontalPodAutoscalers("nsfoo").Create(context.TODO(), newMockedHPA, metav1.CreateOptions{}) require.NoError(t, err) select { case key := <-hctrl.autoscalers: @@ -471,7 +478,7 @@ func TestAutoscalerController(t *testing.T) { } // Verify that a Delete removes the Data from the Global Store and decreases metricsProcessdCount - err = c.HorizontalPodAutoscalers("default").Delete(context.TODO(), newMockedHPA.Name, metav1.DeleteOptions{}) + err = c.HorizontalPodAutoscalers("nsfoo").Delete(context.TODO(), newMockedHPA.Name, metav1.DeleteOptions{}) require.NoError(t, err) select { case <-ticker.C: @@ -480,7 +487,7 @@ func TestAutoscalerController(t *testing.T) { require.FailNow(t, "Timeout waiting for HPAs to update") } // Verify that a Delete removes the Data from the Global Store - err = c.HorizontalPodAutoscalers("default").Delete(context.TODO(), mockedHPA.Name, metav1.DeleteOptions{}) + err = c.HorizontalPodAutoscalers("nsfoo").Delete(context.TODO(), mockedHPA.Name, metav1.DeleteOptions{}) require.NoError(t, err) select { case <-ticker.C: diff --git a/pkg/util/kubernetes/apiserver/wpa_controller_test.go b/pkg/util/kubernetes/apiserver/wpa_controller_test.go index 7602e7fb2db7b..bfc26e489f998 100644 --- a/pkg/util/kubernetes/apiserver/wpa_controller_test.go +++ b/pkg/util/kubernetes/apiserver/wpa_controller_test.go @@ -41,6 +41,7 @@ import ( "github.com/DataDog/watermarkpodautoscaler/api/v1alpha1" "github.com/DataDog/datadog-agent/pkg/clusteragent/custommetrics" + "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/autoscalers" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -58,8 +59,11 @@ func init() { // TestupdateExternalMetrics checks the reconciliation between the local cache and the global store logic func TestUpdateWPA(t *testing.T) { + mockConfig := config.Mock(t) + mockConfig.Set("kube_resources_namespace", "nsfoo") + name := custommetrics.GetConfigmapName() - store, client := newFakeConfigMapStore(t, "default", name, nil) + store, client := newFakeConfigMapStore(t, "nsfoo", name, nil) d := &fakeDatadogClient{} p := &fakeProcessor{ @@ -89,13 +93,13 @@ func TestUpdateWPA(t *testing.T) { // Start the DCA with already existing Data // Check if nothing in local store and Global Store is full we update the Global Store metrics correctly metricsToStore := map[string]custommetrics.ExternalMetricValue{ - "external_metric-watermark-default-foo-metric1": { + "external_metric-watermark-nsfoo-foo-metric1": { MetricName: "metric1", Labels: map[string]string{"foo": "bar"}, Ref: custommetrics.ObjectReference{ Type: "watermark", Name: "foo", - Namespace: "default", + Namespace: "nsfoo", }, Value: 1.3, Valid: true, @@ -122,13 +126,13 @@ func TestUpdateWPA(t *testing.T) { // Fresh start // Check if local store is not empty hctrl.toStore.m.Lock() - hctrl.toStore.data["external_metric-watermark-default-foo-metric2"] = custommetrics.ExternalMetricValue{ + hctrl.toStore.data["external_metric-watermark-nsfoo-foo-metric2"] = custommetrics.ExternalMetricValue{ MetricName: "metric2", Labels: map[string]string{"foo": "bar"}, Ref: custommetrics.ObjectReference{ Type: "watermark", Name: "foo", - Namespace: "default", + Namespace: "nsfoo", }, } require.Len(t, hctrl.toStore.data, 1) @@ -143,13 +147,13 @@ func TestUpdateWPA(t *testing.T) { // Check that if there is conflicting info from the local store and the Global Store that we merge correctly // Check conflict on metric name and labels hctrl.toStore.m.Lock() - hctrl.toStore.data["external_metric-watermark-default-foo-metric2"] = custommetrics.ExternalMetricValue{ + hctrl.toStore.data["external_metric-watermark-nsfoo-foo-metric2"] = custommetrics.ExternalMetricValue{ MetricName: "metric2", Labels: map[string]string{"foo": "baz"}, Ref: custommetrics.ObjectReference{ Type: "watermark", Name: "foo", - Namespace: "default", + Namespace: "nsfoo", }, } require.Len(t, hctrl.toStore.data, 1) @@ -226,9 +230,12 @@ func TestWPAController(t *testing.T) { logFlush := configureLoggerForTest(t) defer logFlush() metricName := "foo" - namespace := "default" + namespace := "nsfoo" wpaName := "wpa_1" + mockConfig := config.Mock(t) + mockConfig.Set("kube_resources_namespace", "nsfoo") + penTime := (int(time.Now().Unix()) - int(maxAge.Seconds()/2)) * 1000 name := custommetrics.GetConfigmapName() store, client := newFakeConfigMapStore(t, namespace, name, nil) From bdf344946afe4eda09095fe97f761131a1264d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Mon, 3 Apr 2023 12:51:04 +0200 Subject: [PATCH 14/33] Fix `pkg/util/kubernetes/kubelet` tests when run in a k8s pod --- pkg/util/kubernetes/kubelet/kubelet_test.go | 2 ++ pkg/util/kubernetes/kubelet/testdata/ca.crt | 29 +++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 pkg/util/kubernetes/kubelet/testdata/ca.crt diff --git a/pkg/util/kubernetes/kubelet/kubelet_test.go b/pkg/util/kubernetes/kubelet/kubelet_test.go index c782d0f266f62..989d487f7dfd3 100644 --- a/pkg/util/kubernetes/kubelet/kubelet_test.go +++ b/pkg/util/kubernetes/kubelet/kubelet_test.go @@ -560,6 +560,7 @@ func (suite *KubeletTestSuite) TestKubeletInitTokenHttps() { mockConfig.Set("kubelet_auth_token_path", "./testdata/fakeBearerToken") mockConfig.Set("kubelet_tls_verify", false) mockConfig.Set("kubernetes_kubelet_host", "127.0.0.1") + mockConfig.Set("kubelet_client_ca", "./testdata/ca.crt") ku := NewKubeUtil() err = ku.init() @@ -578,6 +579,7 @@ func (suite *KubeletTestSuite) TestKubeletInitTokenHttps() { map[string]string{ "url": fmt.Sprintf("https://127.0.0.1:%d", kubeletPort), "verify_tls": "false", + "ca_cert": "./testdata/ca.crt", "token": "fakeBearerToken", }, ku.GetRawConnectionInfo()) } diff --git a/pkg/util/kubernetes/kubelet/testdata/ca.crt b/pkg/util/kubernetes/kubelet/testdata/ca.crt new file mode 100644 index 0000000000000..f8ee20cedcfd6 --- /dev/null +++ b/pkg/util/kubernetes/kubelet/testdata/ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE+zCCAuOgAwIBAgIULxSv8YbxTVq2/XMtdHjOR6zjnRMwDQYJKoZIhvcNAQEL +BQAwDTELMAkGA1UEBhMCRlIwHhcNMjMwNDAzMTA0MjMyWhcNMzMwMzMxMTA0MjMy +WjANMQswCQYDVQQGEwJGUjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +AKZOZvYrSTYvladnE5tSxJQy4iePH75FX9TfzTA14vFR/hBj4800KRoPMZ0RuPnm +Xj8tgaRQ4cW8vMiQumCorQbH8uRVcJ6bIexfEKIcvsXvQPQTDoXoGWgvQE0+aVM8 +0EmWecrD9QoAyx0wVtsv/rn9l1d9TVTmcphIMCnXzZ6/4qXqW9uiMWcrSM9vd4Kk ++hfSv1naogjGQoU6aiR/F4admifGU2ubF8ajTjrweLP++uxk5kK4TSCGkBYbPqjR +ZM962cnJ+0PsHZ2qABDzWrltqzRIgtPubVO5AMpVMf0n8wF90ZHfeGcYiNi7FrCX +vwdsKB9QQ3w8b/l4Oa7P4gmHBosvBjLqpecwfmsdf6T3rxHfmVAafrLguwhyZEq9 +ZlWiIcdox47384A79hspFCB57xjHDjikWOwu7QZFWUWC7GemaTDt6pcVaNAcl7Pk +wADDs8tgsMdrCqg7cMovrYeRo0yRuljJmqYWd86W329X8l4/uiKzW7PMLFNsij6M +ujs6G9GNlNKNYx0Or0tf5jUVmoL+MA6sOc6qPk6e1m5oXQwYfT3kplSRI1KgBFRI +6Z+eVEdlz4yrFTidni7+pKL+9HzeRL2pyXO3KrhDa3svNvCym46Y0E37t2xOH6/Y +KO2QRrrnt/pf8jCjePx1Wn9RFd2LdBRTlOw4vUgQqUhHAgMBAAGjUzBRMB0GA1Ud +DgQWBBTgTQTss0gITdgLd55jerdV69tJdjAfBgNVHSMEGDAWgBTgTQTss0gITdgL +d55jerdV69tJdjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCQ +P+JQsEVxD8cU1jrjF0s3QCut4t6rusXw1B8vWBZGCR0dokhILqhDcoutqge8kpBX +SFBSRfJvYMPu9ySlOJ3jPBQ/ekU6EbScowkX+gLSGnYUZ9muvxkYtWGBWxsBv7OC +Lw4hRA50XfszEGkY0lafYZoBqbvWKJboM7ssr5y2cDcHJ8c4XNbn36nqMRXbgfdi +bSleo9yki/Vu3LkzTOdsG9CBrZ0padmuvNCW5wafqzSpjA4JBM/lQDWWruAL1LvV +vyPuH0KYhJ88RVIYgQGRSD7pXLH+IG45Fuj5HUQMq8ayQEv2zNrYee0w4eG8XWd8 ++8DeyIpnutI6i9goFtbEXrA8SyUvkaI1Qc6kgHj+ofSYZRA+jbz7/u4FzV9iaeAM +nnSFnDp0hZiiNfJkoVXalljHHoEFZVO2A4vS3ojyc42snH7A/6u4kdBQIRGjluTU +mDQZf5mhZaLWjaCSLOsmc07aNPMNrSa6BTITnWUvA/1zdTxavnkym+T0BHB5SPL8 +q+a0ivVw/kr9HZYR7C/69qtUp+dgiUGlUu+ogEGBjgZsLLwli0PewVzACJAtmYDw +PtHo/Ho4mDzt2BE5pcRO2aMUERJX7SU+0zNyVdG48nVryD8L6PJvq6EyIFK5XiCI +nleTIrTrQpY1luBWe1sJ9QUJrBLmGoU572CXXhqc/w== +-----END CERTIFICATE----- From 07fcae1f67092cecae88a00bc0d714fd2d61933d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Mon, 3 Apr 2023 15:28:13 +0200 Subject: [PATCH 15/33] Fix `pkg/util/kubernetes/kubelet` tests when run in a k8s pod --- pkg/util/kubernetes/kubelet/kubelet_test.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/util/kubernetes/kubelet/kubelet_test.go b/pkg/util/kubernetes/kubelet/kubelet_test.go index 989d487f7dfd3..223655edfe939 100644 --- a/pkg/util/kubernetes/kubelet/kubelet_test.go +++ b/pkg/util/kubernetes/kubelet/kubelet_test.go @@ -600,7 +600,7 @@ func (suite *KubeletTestSuite) TestKubeletInitHttpsCerts() { mockConfig.Set("kubernetes_https_kubelet_port", kubeletPort) mockConfig.Set("kubernetes_http_kubelet_port", -1) - mockConfig.Set("kubelet_auth_token_path", "") + mockConfig.Set("kubelet_auth_token_path", "./testdata/fakeBearerToken") mockConfig.Set("kubelet_tls_verify", true) mockConfig.Set("kubelet_client_crt", k.testingCertificate) mockConfig.Set("kubelet_client_key", k.testingPrivateKey) @@ -613,16 +613,20 @@ func (suite *KubeletTestSuite) TestKubeletInitHttpsCerts() { <-k.Requests // Throwing away first GET assert.Equal(suite.T(), fmt.Sprintf("https://127.0.0.1:%d", kubeletPort), ku.kubeletClient.kubeletURL) - assert.False(suite.T(), ku.kubeletClient.client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) + if transport, ok := ku.kubeletClient.client.Transport.(*http.Transport); ok { + assert.False(suite.T(), transport.TLSClientConfig.InsecureSkipVerify) + } b, code, err := ku.QueryKubelet(ctx, "/healthz") assert.Nil(suite.T(), err) assert.Equal(suite.T(), "ok", string(b)) assert.Equal(suite.T(), 200, code) r := <-k.Requests - assert.Equal(suite.T(), "", r.Header.Get(authorizationHeaderKey)) - clientCerts := ku.kubeletClient.client.Transport.(*http.Transport).TLSClientConfig.Certificates - require.Equal(suite.T(), 1, len(clientCerts)) - assert.Equal(suite.T(), clientCerts, s.TLS.Certificates) + assert.Equal(suite.T(), "Bearer fakeBearerToken", r.Header.Get(authorizationHeaderKey)) + if transport, ok := ku.kubeletClient.client.Transport.(*http.Transport); ok { + clientCerts := transport.TLSClientConfig.Certificates + require.Equal(suite.T(), 1, len(clientCerts)) + assert.Equal(suite.T(), clientCerts, s.TLS.Certificates) + } require.EqualValues(suite.T(), map[string]string{ @@ -631,6 +635,7 @@ func (suite *KubeletTestSuite) TestKubeletInitHttpsCerts() { "client_crt": k.testingCertificate, "client_key": k.testingPrivateKey, "ca_cert": k.testingCertificate, + "token": "fakeBearerToken", }, ku.GetRawConnectionInfo()) } From 390feb2d2803cbf821ce22e9163052a081aa489d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Mon, 3 Apr 2023 22:22:28 +0200 Subject: [PATCH 16/33] Fix `pkg/util/kubernetes/apiserver/common` tests when run in a k8s pod --- pkg/util/kubernetes/apiserver/common/common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/kubernetes/apiserver/common/common_test.go b/pkg/util/kubernetes/apiserver/common/common_test.go index 3a420dcb4e431..d65c8e3f54abe 100644 --- a/pkg/util/kubernetes/apiserver/common/common_test.go +++ b/pkg/util/kubernetes/apiserver/common/common_test.go @@ -25,7 +25,7 @@ func TestGetOrCreateClusterID(t *testing.T) { // kube-system doesn't exist GetOrCreateClusterID(client) - _, err := client.ConfigMaps("default").Get(context.TODO(), defaultClusterIDMap, metav1.GetOptions{}) + _, err := client.ConfigMaps(GetMyNamespace()).Get(context.TODO(), defaultClusterIDMap, metav1.GetOptions{}) assert.True(t, errors.IsNotFound(err)) // kube-system does exist @@ -40,7 +40,7 @@ func TestGetOrCreateClusterID(t *testing.T) { GetOrCreateClusterID(client) - cm, err := client.ConfigMaps("default").Get(context.TODO(), defaultClusterIDMap, metav1.GetOptions{}) + cm, err := client.ConfigMaps(GetMyNamespace()).Get(context.TODO(), defaultClusterIDMap, metav1.GetOptions{}) assert.Nil(t, err) id, found := cm.Data["id"] assert.True(t, found) From 670a4edb365351034dd5aa72c1936a21709ad8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Wed, 5 Apr 2023 15:03:51 +0200 Subject: [PATCH 17/33] Ignore race in `pkg/util/kubernetes/apiserver` tests They are caused by the mock config the state of which is automatically restored at the end of the test. --- pkg/util/kubernetes/apiserver/hpa_controller_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/kubernetes/apiserver/hpa_controller_test.go b/pkg/util/kubernetes/apiserver/hpa_controller_test.go index b0c8e6e3e7bc9..25354049bae14 100644 --- a/pkg/util/kubernetes/apiserver/hpa_controller_test.go +++ b/pkg/util/kubernetes/apiserver/hpa_controller_test.go @@ -3,8 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build kubeapiserver -// +build kubeapiserver +//go:build !race && kubeapiserver +// +build !race,kubeapiserver package apiserver From 25951b7d03aea7ab1cbf0b9d91021c7416b3741b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Tue, 11 Apr 2023 12:34:12 +0200 Subject: [PATCH 18/33] Fix `pkg/compliance/agent` tests when run in a k8s pod --- pkg/compliance/agent/agent_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/compliance/agent/agent_test.go b/pkg/compliance/agent/agent_test.go index 4ac4e429fe99a..a68ae56838444 100644 --- a/pkg/compliance/agent/agent_test.go +++ b/pkg/compliance/agent/agent_test.go @@ -173,6 +173,9 @@ func TestRunK8s(t *testing.T) { } func TestRunDocker(t *testing.T) { + // Prevent config.IsKubernetes() from returning true on CI Kubernetes workers + os.Unsetenv("KUBERNETES_SERVICE_PORT") + assert := assert.New(t) opts := aggregator.DefaultAgentDemultiplexerOptions() From 5babefa32d263b1ee7a182293087977dd41e6d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Tue, 11 Apr 2023 14:39:51 +0200 Subject: [PATCH 19/33] fixup! Ignore race in `pkg/util/kubernetes/apiserver` tests --- pkg/util/kubernetes/apiserver/wpa_controller_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/kubernetes/apiserver/wpa_controller_test.go b/pkg/util/kubernetes/apiserver/wpa_controller_test.go index bfc26e489f998..2e53539d9944d 100644 --- a/pkg/util/kubernetes/apiserver/wpa_controller_test.go +++ b/pkg/util/kubernetes/apiserver/wpa_controller_test.go @@ -3,8 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build kubeapiserver -// +build kubeapiserver +//go:build !race && kubeapiserver +// +build !race,kubeapiserver package apiserver From ba7b0fd5555f43372a42a1c3cfac35728d09e70e Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Fri, 28 Apr 2023 11:13:49 +0200 Subject: [PATCH 20/33] Fix docker_x64 suffix --- .gitlab/kernel_version_testing/system_probe.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml index 773f278f180f4..43049bb457f22 100644 --- a/.gitlab/kernel_version_testing/system_probe.yml +++ b/.gitlab/kernel_version_testing/system_probe.yml @@ -1,5 +1,5 @@ .pull_test_dockers: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64:$DATADOG_AGENT_BUILDIMAGES + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES needs: [] tags: ["runner:docker"] rules: From b836c771eb9f40aa3fac2aaaa4028ce0de83853a Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Fri, 28 Apr 2023 11:19:32 +0200 Subject: [PATCH 21/33] Use k8s runners for linters too --- .gitlab/source_test/linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 2b2cacc7478bc..07801e8ebb633 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -70,7 +70,7 @@ lint_deb-x64: extends: - .linux_lint image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] tests_flavor_iot_deb-x64: @@ -89,7 +89,7 @@ lint_flavor_iot_deb-x64: extends: - .linux_lint image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: FLAVORS: '--flavors iot' @@ -110,7 +110,7 @@ lint_flavor_dogstatsd_deb-x64: extends: - .linux_lint image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: FLAVORS: '--flavors dogstatsd' @@ -146,7 +146,7 @@ lint_rpm-x64: extends: - .linux_lint image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: EXTRA_OPTS: '--build-exclude=systemd' @@ -178,7 +178,7 @@ lint_deb-arm64: - .linux_lint needs: ["go_deps", "go_tools_deps"] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] tests_rpm-arm64-py2: extends: .rtloader_tests @@ -206,7 +206,7 @@ lint_rpm-arm64: extends: - .linux_lint image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:docker-arm", "platform:arm64"] + tags: ["arch:arm64"] needs: ["go_deps", "go_tools_deps"] # Check consistency of go.mod file with project imports From 7501b1c990102e492c79281203a5a42cce5ae15e Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Fri, 28 Apr 2023 14:26:50 +0200 Subject: [PATCH 22/33] Debug failing gohai test --- .gitlab/source_test/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 07801e8ebb633..c5a9a55f27e95 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -21,6 +21,8 @@ - inv -e invoke-unit-tests - echo "Env - $DD_ENV" - echo "Site - $DD_SITE" + # Debug: gohai + - df -l -k - inv -e test $FLAVORS --skip-linters --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus 4 $EXTRA_OPTS --save-result-json test_output.json --junit-tar "junit-${CI_JOB_NAME}.tgz" - echo "Env - $DD_ENV" - echo "Site - $DD_SITE" From 22bac00b9ae5b318e4569ac605a21052a67304b7 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Fri, 28 Apr 2023 15:24:37 +0200 Subject: [PATCH 23/33] Debug --- .gitlab/source_test/linux.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index c5a9a55f27e95..93b95fc321a4c 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -22,7 +22,10 @@ - echo "Env - $DD_ENV" - echo "Site - $DD_SITE" # Debug: gohai - - df -l -k + - cat /etc/mtab || true + - cat /etc/fstab || true + - strace df -l -k || true + - df -l -k || true - inv -e test $FLAVORS --skip-linters --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus 4 $EXTRA_OPTS --save-result-json test_output.json --junit-tar "junit-${CI_JOB_NAME}.tgz" - echo "Env - $DD_ENV" - echo "Site - $DD_SITE" From a661ee443cc2b848f8738c2e8510e80b6192f8aa Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 16 May 2023 14:58:03 +0200 Subject: [PATCH 24/33] Migrate setup_agent_version --- .gitlab/setup/setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml index 5ab044f7541f2..471b0aa69699b 100644 --- a/.gitlab/setup/setup.yml +++ b/.gitlab/setup/setup.yml @@ -2,8 +2,9 @@ setup_agent_version: stage: setup image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] script: + - source /root/.bashrc - inv -e agent.version --version-cached - $S3_CP_CMD $CI_PROJECT_DIR/agent-version.cache $S3_ARTIFACTS_URI/agent-version.cache From 66b366a510cf6fc21f7253033b2fedac9cc0a031 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 16 May 2023 16:41:55 +0200 Subject: [PATCH 25/33] Disable gohai test relying on df on Linux arm64 --- pkg/gohai/filesystem/filesystem_nix_test.go | 6 ++++++ pkg/gohai/gohai_test.go | 5 +++++ pkg/metadata/internal/gohai/gohai_test.go | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/pkg/gohai/filesystem/filesystem_nix_test.go b/pkg/gohai/filesystem/filesystem_nix_test.go index 9e233ab38a73c..f182cbc232347 100644 --- a/pkg/gohai/filesystem/filesystem_nix_test.go +++ b/pkg/gohai/filesystem/filesystem_nix_test.go @@ -9,6 +9,8 @@ package filesystem import ( + "os" + "runtime" "testing" "time" @@ -134,6 +136,10 @@ func TestFaileDfWithData(t *testing.T) { } func TestGetFileSystemInfo(t *testing.T) { + if os.Getenv("CI") != "" && runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + t.Skip("Test disabled on arm64 Linux CI runners, as df doesn't work") + } + out, err := getFileSystemInfo() require.NoError(t, err) outArray := out.([]interface{}) diff --git a/pkg/gohai/gohai_test.go b/pkg/gohai/gohai_test.go index 155ec96aa583c..c31cd1e48b24f 100644 --- a/pkg/gohai/gohai_test.go +++ b/pkg/gohai/gohai_test.go @@ -8,6 +8,7 @@ package main import ( "encoding/json" "net" + "os" "runtime" "testing" @@ -89,6 +90,10 @@ type gohaiPayload struct { } func TestGohaiSerialization(t *testing.T) { + if os.Getenv("CI") != "" && runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + t.Skip("Test disabled on arm64 Linux CI runners, as df doesn't work") + } + gohai, err := Collect() assert.NoError(t, err) diff --git a/pkg/metadata/internal/gohai/gohai_test.go b/pkg/metadata/internal/gohai/gohai_test.go index 3d1e3fa24bf78..08267875b5c44 100644 --- a/pkg/metadata/internal/gohai/gohai_test.go +++ b/pkg/metadata/internal/gohai/gohai_test.go @@ -6,12 +6,18 @@ package gohai import ( + "os" + "runtime" "testing" "github.com/stretchr/testify/assert" ) func TestGetPayload(t *testing.T) { + if os.Getenv("CI") != "" && runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + t.Skip("Test disabled on arm64 Linux CI runners, as df doesn't work") + } + gohai := GetPayload() assert.NotNil(t, gohai.Gohai.CPU) @@ -22,6 +28,10 @@ func TestGetPayload(t *testing.T) { } func TestGetPayloadContainerized(t *testing.T) { + if os.Getenv("CI") != "" && runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + t.Skip("Test disabled on arm64 Linux CI runners, as df doesn't work") + } + t.Setenv("DOCKER_DD_AGENT", "true") detectDocker0() @@ -39,6 +49,10 @@ func TestGetPayloadContainerized(t *testing.T) { } func TestGetPayloadContainerizedWithDocker0(t *testing.T) { + if os.Getenv("CI") != "" && runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + t.Skip("Test disabled on arm64 Linux CI runners, as df doesn't work") + } + t.Setenv("DOCKER_DD_AGENT", "true") detectDocker0() From baa5c1d6dccd0d9134d3eb302c355966884c9779 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 16 May 2023 18:52:42 +0200 Subject: [PATCH 26/33] Revert changes to package builds --- .gitlab/package_build/deb.yml | 18 +++++++++--------- .gitlab/package_build/dmg.yml | 2 +- .gitlab/package_build/rpm.yml | 16 ++++++++-------- .gitlab/package_build/suse_rpm.yml | 8 ++++---- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml index 8e3965d97e9ad..701c545981951 100644 --- a/.gitlab/package_build/deb.yml +++ b/.gitlab/package_build/deb.yml @@ -68,7 +68,7 @@ agent_deb-x64-a6: !reference [.on_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -86,7 +86,7 @@ agent_deb-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -104,7 +104,7 @@ agent_deb-arm64-a6: !reference [.on_all_builds_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 6 @@ -121,7 +121,7 @@ agent_deb-arm64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 7 @@ -163,7 +163,7 @@ iot_agent_deb-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: amd64 @@ -175,7 +175,7 @@ iot_agent_deb-arm64: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: arm64 @@ -188,7 +188,7 @@ iot_agent_deb-armhf: stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_armhf$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES # Run with platform:arm64 since no platform:armhf exists and arm64 should be backwards compatible - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: armhf @@ -199,7 +199,7 @@ dogstatsd_deb-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -228,7 +228,7 @@ dogstatsd_deb-arm64: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] before_script: - source /root/.bashrc diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml index 834dbc72d922b..066f61c103091 100644 --- a/.gitlab/package_build/dmg.yml +++ b/.gitlab/package_build/dmg.yml @@ -26,7 +26,7 @@ agent_dmg-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml index beed87a407e0c..4a80e234d2efc 100644 --- a/.gitlab/package_build/rpm.yml +++ b/.gitlab/package_build/rpm.yml @@ -64,7 +64,7 @@ agent_rpm-x64-a6: !reference [.on_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -82,7 +82,7 @@ agent_rpm-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -100,7 +100,7 @@ agent_rpm-arm64-a6: !reference [.on_all_builds_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 6 @@ -117,7 +117,7 @@ agent_rpm-arm64-a7: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "build_system-probe-arm64", "go_deps", "generate_minimized_btfs_arm64"] variables: AGENT_MAJOR_VERSION: 7 @@ -161,7 +161,7 @@ iot_agent_rpm-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "go_deps"] iot_agent_rpm-arm64: @@ -170,7 +170,7 @@ iot_agent_rpm-arm64: !reference [.on_all_builds_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "go_deps"] iot_agent_rpm-armhf: @@ -180,7 +180,7 @@ iot_agent_rpm-armhf: stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_armhf$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES # Run with platform:arm64 since no platform:armhf exists and arm64 should be backwards compatible - tags: ["arch:arm64"] + tags: ["runner:docker-arm", "platform:arm64"] needs: ["go_mod_tidy_check", "go_deps"] before_script: # Ensures uname -m reports armv7l @@ -191,7 +191,7 @@ dogstatsd_rpm-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials diff --git a/.gitlab/package_build/suse_rpm.yml b/.gitlab/package_build/suse_rpm.yml index 8971b914fbd2a..859c354bc991b 100644 --- a/.gitlab/package_build/suse_rpm.yml +++ b/.gitlab/package_build/suse_rpm.yml @@ -66,7 +66,7 @@ agent_suse-x64-a6: !reference [.on_a6] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -84,7 +84,7 @@ agent_suse-x64-a7: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_system-probe-x64", "go_deps", "generate_minimized_btfs_x64"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials @@ -100,7 +100,7 @@ iot_agent_suse-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "go_deps"] before_script: - source /root/.bashrc @@ -137,7 +137,7 @@ dogstatsd_suse-x64: !reference [.on_a7] stage: package_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + tags: ["runner:main"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] variables: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials From e86de909646db215493bcbf71b0b188b729bf742 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 16 May 2023 19:15:48 +0200 Subject: [PATCH 27/33] Debug: check state of df during tests --- .gitlab/source_test/linux.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 93b95fc321a4c..9c1eb7baa040e 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -2,12 +2,28 @@ .rtloader_tests: stage: source_test before_script: + # Debug: gohai + - cat /etc/mtab || true + - cat /etc/fstab || true + - df -l -k || true - source /root/.bashrc && conda activate $CONDA_ENV + # Debug: gohai + - cat /etc/mtab || true + - cat /etc/fstab || true + - df -l -k || true - !reference [.retrieve_linux_go_deps] + # Debug: gohai + - cat /etc/mtab || true + - cat /etc/fstab || true + - df -l -k || true - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "$PYTHON_RUNTIMES" - inv -e rtloader.install - inv -e rtloader.format --raise-if-changed - inv -e rtloader.test + # Debug: gohai + - cat /etc/mtab || true + - cat /etc/fstab || true + - df -l -k || true # Placeholder script, overridden by .linux_tests when running all go tests script: [ "# Skipping go tests" ] @@ -17,6 +33,10 @@ FLAVORS: '--flavors base' script: - !reference [.retrieve_linux_go_tools_deps] + # Debug: gohai + - cat /etc/mtab || true + - cat /etc/fstab || true + - df -l -k || true - inv -e install-tools - inv -e invoke-unit-tests - echo "Env - $DD_ENV" @@ -24,7 +44,6 @@ # Debug: gohai - cat /etc/mtab || true - cat /etc/fstab || true - - strace df -l -k || true - df -l -k || true - inv -e test $FLAVORS --skip-linters --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus 4 $EXTRA_OPTS --save-result-json test_output.json --junit-tar "junit-${CI_JOB_NAME}.tgz" - echo "Env - $DD_ENV" From eb88a0ee15de1ba34b4cc042e6e71931c86dc519 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 17 May 2023 10:57:34 +0200 Subject: [PATCH 28/33] Cleanup debug steps --- .gitlab/source_test/linux.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 9c1eb7baa040e..e9ec97a471d81 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -2,28 +2,12 @@ .rtloader_tests: stage: source_test before_script: - # Debug: gohai - - cat /etc/mtab || true - - cat /etc/fstab || true - - df -l -k || true - source /root/.bashrc && conda activate $CONDA_ENV - # Debug: gohai - - cat /etc/mtab || true - - cat /etc/fstab || true - - df -l -k || true - !reference [.retrieve_linux_go_deps] - # Debug: gohai - - cat /etc/mtab || true - - cat /etc/fstab || true - - df -l -k || true - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "$PYTHON_RUNTIMES" - inv -e rtloader.install - inv -e rtloader.format --raise-if-changed - inv -e rtloader.test - # Debug: gohai - - cat /etc/mtab || true - - cat /etc/fstab || true - - df -l -k || true # Placeholder script, overridden by .linux_tests when running all go tests script: [ "# Skipping go tests" ] @@ -33,21 +17,9 @@ FLAVORS: '--flavors base' script: - !reference [.retrieve_linux_go_tools_deps] - # Debug: gohai - - cat /etc/mtab || true - - cat /etc/fstab || true - - df -l -k || true - inv -e install-tools - inv -e invoke-unit-tests - - echo "Env - $DD_ENV" - - echo "Site - $DD_SITE" - # Debug: gohai - - cat /etc/mtab || true - - cat /etc/fstab || true - - df -l -k || true - inv -e test $FLAVORS --skip-linters --race --profile --rerun-fails=2 --python-runtimes "$PYTHON_RUNTIMES" --cpus 4 $EXTRA_OPTS --save-result-json test_output.json --junit-tar "junit-${CI_JOB_NAME}.tgz" - - echo "Env - $DD_ENV" - - echo "Site - $DD_SITE" artifacts: expire_in: 2 weeks when: always From 4f04d9de26dc12069cbf4bccc2eb7e70f4708171 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 17 May 2023 11:24:59 +0200 Subject: [PATCH 29/33] Fix btf-gen image suffix --- .gitlab/kernel_version_testing/system_probe.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml index 76c7c70ff47b6..a07c5bca988a5 100644 --- a/.gitlab/kernel_version_testing/system_probe.yml +++ b/.gitlab/kernel_version_testing/system_probe.yml @@ -33,7 +33,7 @@ pull_test_dockers_arm64: stage: kernel_matrix_testing rules: !reference [.on_system_probe_changes_or_manual] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES tags: ["runner:main"] script: # Build dependencies directory From ed1abe8ae5cd4f7e9275f74fb9b94afaa7ddf47c Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Sat, 10 Jun 2023 12:50:28 -0400 Subject: [PATCH 30/33] Update buildimages to v16680833-5ecd649 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5ae6d6c0a909..76352af244190 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -146,11 +146,11 @@ variables: # To use images from datadog-agent-buildimages dev branches, set the corresponding # SUFFIX variable to _test_only DATADOG_AGENT_BUILDIMAGES_SUFFIX: "_test_only" - DATADOG_AGENT_BUILDIMAGES: v15830939-045ac73 + DATADOG_AGENT_BUILDIMAGES: v16680833-5ecd649 DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" DATADOG_AGENT_WINBUILDIMAGES: v16026304-782441d DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "_test_only" - DATADOG_AGENT_ARMBUILDIMAGES: v15830939-045ac73 + DATADOG_AGENT_ARMBUILDIMAGES: v16680833-5ecd649 DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v16026304-782441d DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX: "" From 46c671ea779299cb821080b9bf4a3dfd146dc183 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 13 Jun 2023 17:18:00 +0200 Subject: [PATCH 31/33] Update to stable images --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 388720e3995ea..1f1fcf2f9d4a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -146,17 +146,17 @@ variables: # To use images from datadog-agent-buildimages dev branches, set the corresponding # SUFFIX variable to _test_only DATADOG_AGENT_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BUILDIMAGES: v16690913-5148788 + DATADOG_AGENT_BUILDIMAGES: v16742019-a2675dc DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_WINBUILDIMAGES: v16690913-5148788 + DATADOG_AGENT_WINBUILDIMAGES: v16742019-a2675dc DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_ARMBUILDIMAGES: v16690913-5148788 + DATADOG_AGENT_ARMBUILDIMAGES: v16742019-a2675dc DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v16026304-782441d + DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v16742019-a2675dc DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_NIKOS_BUILDIMAGES: v16026304-782441d + DATADOG_AGENT_NIKOS_BUILDIMAGES: v16742019-a2675dc DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v16026304-782441d + DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v16742019-a2675dc DATADOG_AGENT_BUILDERS: v9930706-ef9d493 DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded From 802357959b87ca4b54c706c1156498cc7d57e136 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 13 Jun 2023 20:04:13 +0200 Subject: [PATCH 32/33] Convert new heroku tests to k8s Gitlab runners --- .gitlab/source_test/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 64adc1361a586..61006a3e20c95 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -116,7 +116,7 @@ tests_flavor_heroku_deb-x64: - .rtloader_tests - .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -127,7 +127,7 @@ lint_flavor_heroku_deb-x64: extends: - .linux_lint image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["runner:main"] + tags: ["arch:amd64"] needs: ["go_deps", "go_tools_deps"] variables: FLAVORS: '--flavors heroku' From 7d640d6a9305ffa95ac2d65dcfb4b3e74b6098fb Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Wed, 14 Jun 2023 14:35:25 +0200 Subject: [PATCH 33/33] Fix merge --- .gitlab/source_test/macos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index fbb2abe3f33fa..3d2c3908ddc24 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -5,7 +5,6 @@ tests_macos: !reference [.on_a6] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - tags: ["runner:main"] variables: PYTHON_RUNTIMES: '3' script: