[gitlab] Migration of unit tests CI jobs to k8s Gitlab runners

.gitlab-ci.yml

@@ -145,12 +145,12 @@ variables:
   # Build images versions
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
-  DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v16026304-782441d
+  DATADOG_AGENT_BUILDIMAGES_SUFFIX: "_test_only"
+  DATADOG_AGENT_BUILDIMAGES: v15830939-045ac73
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
   DATADOG_AGENT_WINBUILDIMAGES: v16026304-782441d
-  DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v16026304-782441d
+  DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "_test_only"
+  DATADOG_AGENT_ARMBUILDIMAGES: v15830939-045ac73
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v16026304-782441d
   DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX: ""

.gitlab/binary_build/cluster_agent.yml

@@ -19,8 +19,8 @@ cluster_agent-build_amd64:
   variables:
     ARCH: amd64
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
 
 cluster_agent-build_arm64:
   extends: .cluster_agent-build_common
@@ -32,5 +32,5 @@ cluster_agent-build_arm64:
   variables:
     ARCH: arm64
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3

.gitlab/binary_build/cluster_agent_cloudfoundry.yml

@@ -13,8 +13,8 @@ cluster_agent_cloudfoundry-build_amd64:
   variables:
     ARCH: amd64
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     - inv check-go-version
     - inv -e cluster-agent-cloudfoundry.build

.gitlab/binary_build/linux.yml

@@ -5,13 +5,13 @@ build_dogstatsd_static-binary_x64:
   rules:
     !reference [.on_a7]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:main"]
+  tags: ["arch:amd64"]
   needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"]
   variables:
     ARCH: amd64
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     - inv check-go-version
     - inv -e dogstatsd.build --static --major-version 7
@@ -22,13 +22,13 @@ build_dogstatsd_static-binary_arm64:
   rules:
     !reference [.on_a7]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
-  tags: ["runner:docker-arm", "platform:arm64"]
+  tags: ["arch:arm64"]
   needs: ["lint_deb-arm64", "tests_deb-arm64-py3", "go_deps"]
   variables:
     ARCH: arm64
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     - inv check-go-version
     - inv -e dogstatsd.build --static --major-version 7
@@ -39,11 +39,11 @@ build_dogstatsd-binary_x64:
   rules:
     !reference [.on_a7]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:main"]
+  tags: ["arch:amd64"]
   needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"]
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     - inv check-go-version
     - inv -e dogstatsd.build --major-version 7
@@ -54,13 +54,13 @@ build_dogstatsd-binary_arm64:
     !reference [.on_all_builds_a7]
   stage: binary_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
-  tags: ["runner:docker-arm", "platform:arm64"]
+  tags: ["arch:arm64"]
   needs: ["lint_deb-arm64", "tests_deb-arm64-py3", "go_deps"]
   variables:
     ARCH: arm64
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     - inv check-go-version
     - inv -e dogstatsd.build --major-version 7
@@ -72,11 +72,12 @@ build_iot_agent-binary_x64:
   rules:
     !reference [.on_a7]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:main"]
+  tags: ["arch:amd64"]
   needs: ["lint_deb-x64", "tests_deb-x64-py3", "go_deps"]
   before_script:
     - !reference [.retrieve_linux_go_deps]
   script:
+    - source /root/.bashrc
     - inv check-go-version
     - inv -e agent.build --flavor iot --major-version 7
     - $S3_CP_CMD $CI_PROJECT_DIR/$AGENT_BINARIES_DIR/agent $S3_ARTIFACTS_URI/iot/agent
@@ -86,12 +87,13 @@ build_iot_agent-binary_arm64:
     !reference [.on_all_builds_a7]
   stage: binary_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:docker-arm", "platform:arm64"]
+  tags: ["arch:arm64"]
   needs: ["lint_deb-arm64", "tests_deb-arm64-py3", "go_deps"]
   variables:
     ARCH: arm64
   before_script:
     - !reference [.retrieve_linux_go_deps]
   script:
+    - source /root/.bashrc
     - inv check-go-version
     - inv -e agent.build --flavor iot --major-version 7

.gitlab/binary_build/serverless.yml

@@ -2,6 +2,7 @@
 .build_serverless_common:
   stage: binary_build
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
   script:
     - inv check-go-version
@@ -12,7 +13,7 @@ build_serverless-deb_x64:
   variables:
     BINARY_NAME: datadog-agent-x64
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:main"]
+  tags: ["arch:amd64"]
   needs: ["go_deps"]
   artifacts:
     expire_in: 1 day
@@ -24,5 +25,5 @@ build_serverless-deb_arm64:
   variables:
     BINARY_NAME: datadog-agent-arm64
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:docker-arm", "platform:arm64"]
+  tags: ["arch:arm64"]
   needs: ["go_deps"]

.gitlab/deploy_6/container.yml

@@ -14,6 +14,7 @@
   stage: deploy6
   dependencies: []
   before_script:
+    - source /root/.bashrc
     - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 6 --url-safe)"; fi
     - export IMG_SOURCES="${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-arm64"
     - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${JMX}"

.gitlab/deploy_7/container.yml

@@ -14,6 +14,7 @@
   stage: deploy7
   dependencies: []
   before_script:
+    - source /root/.bashrc
     - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi
     - export IMG_BASE_SRC="${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
     - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7${JMX}-amd64,${IMG_BASE_SRC}-7${JMX}-arm64"
@@ -55,6 +56,7 @@ deploy_containers-dogstatsd:
     !reference [.on_deploy_a7_manual_auto_on_rc]
   dependencies: []
   before_script:
+    - source /root/.bashrc
     - export VERSION="$(inv agent.version --major-version 7 --url-safe)"
     - export IMG_SOURCES="${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64"
     - export IMG_DESTINATIONS="${DSD_REPOSITORY}:${VERSION}"

.gitlab/deploy_dca.yml

@@ -14,6 +14,7 @@
   stage: deploy_dca
   dependencies: []
   before_script:
+    - source /root/.bashrc
     - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi
     - if [[ "$CLUSTER_AGENT_REPOSITORY" == "" ]]; then export CLUSTER_AGENT_REPOSITORY="cluster-agent"; fi
     - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"

.gitlab/deps_fetch.yml

@@ -14,9 +14,10 @@
 go_deps:
   stage: deps_fetch
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:main"]
+  tags: ["arch:amd64"]
   needs: []
   script:
+    - source /root/.bashrc
     - inv -e deps --verbose
     - cd $GOPATH/pkg/mod/ && tar czf $CI_PROJECT_DIR/modcache.tar.gz .
   artifacts:
@@ -28,9 +29,10 @@ go_deps:
 go_tools_deps:
   stage: deps_fetch
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["runner:main"]
+  tags: ["arch:amd64"]
   needs: []
   script:
+    - source /root/.bashrc
     - inv -e download-tools
     - cd $GOPATH/pkg/mod/ && tar czf $CI_PROJECT_DIR/modcache_tools.tar.gz .
   artifacts:

.gitlab/docker_common/publish_job_templates.yml

@@ -12,6 +12,7 @@
     IMG_VARIABLES: ""
     IMG_SIGNING: ""
   script: # We can't use the 'trigger' keyword on manual jobs, otherwise they can't be run if the pipeline fails and is retried
+    - source /root/.bashrc
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - ECR_RELEASE_SUFFIX="${CI_COMMIT_TAG+-release}"
     - IMG_VARIABLES="$(sed -E "s#(${SRC_AGENT}|${SRC_DSD}|${SRC_DCA})#\1${ECR_RELEASE_SUFFIX}#g" <<<"$IMG_VARIABLES")"

.gitlab/integration_test.yml

@@ -10,7 +10,7 @@ dogstatsd_x64_size_test:
   tags: ["runner:main"]
   needs: ["build_dogstatsd_static-binary_x64"]
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
+    - source /root/.bashrc
     - mkdir -p $STATIC_BINARIES_DIR
     - $S3_CP_CMD $S3_ARTIFACTS_URI/static/dogstatsd.amd64 $STATIC_BINARIES_DIR/dogstatsd
   script:

.gitlab/internal_image_deploy.yml

@@ -22,6 +22,7 @@ docker_trigger_internal:
     TMPL_SRC_REPO: ci/datadog-agent/agent
     RELEASE_STAGING: "true"
   script:
+    - source /root/.bashrc
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi
     - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi
@@ -49,6 +50,7 @@ docker_trigger_cluster_agent_internal:
     RELEASE_STAGING: "true"
     RELEASE_PROD: "true"
   script:
+    - source /root/.bashrc
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi
     - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi

.gitlab/internal_kubernetes_deploy.yml

@@ -27,5 +27,6 @@ internal_kubernetes_deploy_experimental:
     WORKFLOW: "agents"
     FILTER: "cluster.env == 'experimental' and cluster.shortName == 'snowver'"
   script:
+    - source /root/.bashrc
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,WORKFLOW,OPTION_PRE_SCRIPT,FILTER,SKIP_PLAN_CHECK"

.gitlab/notify.yml

@@ -48,7 +48,7 @@ send_pipeline_stats:
   when: always
   dependencies: []
   script:
-    - source /root/.bashrc && conda activate ddpy3
+    - source /root/.bashrc
     - set +x
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_read_api_token --with-decryption --query "Parameter.Value" --out text)
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)

.gitlab/package_build/deb.yml

@@ -8,10 +8,11 @@
 
 .agent_build_common_deb:
   script:
-    - echo "About to build for $RELEASE_VERSION"
+    - source /root/.bashrc
     - !reference [.setup_ruby_mirror_linux]
     - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
+    - echo "About to build for $RELEASE_VERSION"
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
     # Artifacts and cache must live within project directory but we run omnibus in a neutral directory.
@@ -77,7 +78,6 @@ agent_deb-x64-a6:
     DESTINATION_DEB: 'datadog-agent_6_amd64.deb'
     DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_amd64.deb'
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
     - export RELEASE_VERSION=$RELEASE_VERSION_6
 
 agent_deb-x64-a7:
@@ -96,7 +96,6 @@ agent_deb-x64-a7:
     DESTINATION_DEB: 'datadog-agent_7_amd64.deb'
     DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_amd64.deb'
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
 agent_deb-arm64-a6:
@@ -114,7 +113,6 @@ agent_deb-arm64-a6:
     DESTINATION_DEB: 'datadog-agent_6_arm64.deb'
     DESTINATION_DBG_DEB: 'datadog-agent-dbg_6_arm64.deb'
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
     - export RELEASE_VERSION=$RELEASE_VERSION_6
 
 agent_deb-arm64-a7:
@@ -132,11 +130,11 @@ agent_deb-arm64-a7:
     DESTINATION_DEB: 'datadog-agent_7_arm64.deb'
     DESTINATION_DBG_DEB: 'datadog-agent-dbg_7_arm64.deb'
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
 .iot_agent_build_common_deb:
   script:
+    - source /root/.bashrc
     - !reference [.setup_ruby_mirror_linux]
     - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
@@ -206,8 +204,8 @@ dogstatsd_deb-x64:
   variables:
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
@@ -233,8 +231,8 @@ dogstatsd_deb-arm64:
   tags: ["runner:docker-arm", "platform:arm64"]
   needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"]
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*

.gitlab/package_build/dmg.yml

@@ -33,4 +33,5 @@ agent_dmg-x64-a7:
     AGENT_MAJOR_VERSION: 7
     PYTHON_RUNTIMES: '3'
   before_script:
+    - source /root/.bashrc
     - export RELEASE_VERSION=$RELEASE_VERSION_7

.gitlab/package_build/rpm.yml

@@ -72,7 +72,7 @@ agent_rpm-x64-a6:
     PYTHON_RUNTIMES: '2,3'
     PACKAGE_ARCH: amd64
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
+    - source /root/.bashrc
     - export RELEASE_VERSION=$RELEASE_VERSION_6
 
 # build Agent package for rpm-x64
@@ -90,7 +90,7 @@ agent_rpm-x64-a7:
     PYTHON_RUNTIMES: '3'
     PACKAGE_ARCH: amd64
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
+    - source /root/.bashrc
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
 # build Agent package for rpm-arm64
@@ -107,7 +107,7 @@ agent_rpm-arm64-a6:
     PYTHON_RUNTIMES: '2,3'
     PACKAGE_ARCH: arm64
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
+    - source /root/.bashrc
     - export RELEASE_VERSION=$RELEASE_VERSION_6
 
 # build Agent package for rpm-arm64
@@ -124,12 +124,13 @@ agent_rpm-arm64-a7:
     PYTHON_RUNTIMES: '3'
     PACKAGE_ARCH: arm64
   before_script:
-    - source /root/.bashrc && conda activate ddpy3
+    - source /root/.bashrc
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
 .iot_agent_build_common_rpm:
   script:
     - echo "About to build iot agent for $RELEASE_VERSION_7"
+    - source /root/.bashrc
     - !reference [.setup_ruby_mirror_linux]
     - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
@@ -195,8 +196,8 @@ dogstatsd_rpm-x64:
   variables:
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
   before_script:
+    - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
-    - source /root/.bashrc && conda activate ddpy3
   script:
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*