Reapply "feat(vault): Add e2e test secrets to vault" (#30328) (#31368)

DataDog · Nov 22, 2024 · b84c7ee · b84c7ee
1 parent e410256
commit b84c7ee
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 23 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -300,7 +300,11 @@ variables:
   ATLASSIAN_WRITE: atlassian-write  # agent-devx-infra
   CODECOV: codecov  # agent-devx-infra
   DOCKER_REGISTRY_RO: dockerhub-readonly  # agent-delivery
+  E2E_AZURE: e2e-azure  # agent-devx-loops
+  E2E_GCP: e2e-gcp  # agent-devx-loops
   GITLAB_TOKEN: gitlab-token  # agent-devx-infra
+  KITCHEN_AWS: kitchen-aws  # agent-devx-loops
+  KITCHEN_AZURE: kitchen-azure  # agent-devx-loops
   INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2  # agent-devx-infra
   MACOS_GITHUB_APP_1: macos-github-app-one  # agent-devx-infra
   MACOS_GITHUB_APP_2: macos-github-app-two  # agent-devx-infra

diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml
@@ -20,13 +20,13 @@
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
     # Setup Azure credentials. https://www.pulumi.com/registry/packages/azure-native/installation-configuration/#set-configuration-using-pulumi-config
     # The app is called `agent-e2e-tests`
-    - ARM_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_CLIENT_ID) || exit $?; export ARM_CLIENT_ID
-    - ARM_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_CLIENT_SECRET) || exit $?; export ARM_CLIENT_SECRET
-    - ARM_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_TENANT_ID) || exit $?; export ARM_TENANT_ID
-    - ARM_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_SUBSCRIPTION_ID) || exit $?; export ARM_SUBSCRIPTION_ID
-    # Setup GCP credentials. https://www.pulumi.com/registry/packages/gcp/service-account/
+    - ARM_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE client_id) || exit $?; export ARM_CLIENT_ID
+    - ARM_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE token) || exit $?; export ARM_CLIENT_SECRET
+    - ARM_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE tenant_id) || exit $?; export ARM_TENANT_ID
+    - ARM_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE subscription_id) || exit $?; export ARM_SUBSCRIPTION_ID
+    # Setup GCP credentials. https://www.pulumi.com/registry/packages/gcp/installation-configuration/
     # The service account is called `agent-e2e-tests`
-    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_GCP_CREDENTIALS > ~/gcp-credentials.json || exit $?
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_GCP credentials_json > ~/gcp-credentials.json || exit $?
     - export GOOGLE_APPLICATION_CREDENTIALS=~/gcp-credentials.json
     # Generate external links to CI VISIBILITY, used by artifacts:reports:annotations
     - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH

diff --git a/.gitlab/maintenance_jobs/kitchen.yml b/.gitlab/maintenance_jobs/kitchen.yml
@@ -26,10 +26,10 @@ periodic_kitchen_cleanup_azure:
   # the job to be run one at a time.
   resource_group: azure_cleanup
   script:
-    - ARM_SUBSCRIPTION_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID` || exit $?; export ARM_SUBSCRIPTION_ID
-    - ARM_CLIENT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID` || exit $?; export ARM_CLIENT_ID
-    - ARM_CLIENT_SECRET=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET` || exit $?; export ARM_CLIENT_SECRET
-    - ARM_TENANT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID` || exit $?; export ARM_TENANT_ID
+    - ARM_SUBSCRIPTION_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id` || exit $?; export ARM_SUBSCRIPTION_ID
+    - ARM_CLIENT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id` || exit $?; export ARM_CLIENT_ID
+    - ARM_CLIENT_SECRET=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token` || exit $?; export ARM_CLIENT_SECRET
+    - ARM_TENANT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id` || exit $?; export ARM_TENANT_ID
     # Remove kitchen resources for all existing test suite prefixes
     - RESOURCE_GROUP_PREFIX=kitchen-chef python3 /deploy_scripts/cleanup_azure.py
     - RESOURCE_GROUP_PREFIX=kitchen-win python3 /deploy_scripts/cleanup_azure.py

diff --git a/test/kitchen/tasks/clean.sh b/test/kitchen/tasks/clean.sh
@@ -8,19 +8,19 @@ set -euo pipefail
 
 # These should not be printed out
 if [ -z ${AZURE_CLIENT_ID+x} ]; then
-  AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID)
+  AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id)
   export AZURE_CLIENT_ID
 fi
 if [ -z ${AZURE_CLIENT_SECRET+x} ]; then
-  AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET)
+  AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token)
   export AZURE_CLIENT_SECRET
 fi
 if [ -z ${AZURE_TENANT_ID+x} ]; then
-  AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID)
+  AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id)
   export AZURE_TENANT_ID
 fi
 if [ -z ${AZURE_SUBSCRIPTION_ID+x} ]; then
-  AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID)
+  AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id)
   export AZURE_SUBSCRIPTION_ID
 fi
 if [ -z ${DD_PIPELINE_ID+x} ]; then

diff --git a/test/kitchen/tasks/run-test-kitchen.sh b/test/kitchen/tasks/run-test-kitchen.sh
@@ -54,25 +54,25 @@ if [ "$KITCHEN_PROVIDER" == "azure" ]; then
   # These should not be printed out
   set +x
   if [ -z ${AZURE_CLIENT_ID+x} ]; then
-    AZURE_CLIENT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID)
+    AZURE_CLIENT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id)
     # make sure whitespace is removed
     AZURE_CLIENT_ID="$(echo -e "${AZURE_CLIENT_ID}" | tr -d '[:space:]')"
     export AZURE_CLIENT_ID
   fi
   if [ -z ${AZURE_CLIENT_SECRET+x} ]; then
-    AZURE_CLIENT_SECRET=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET)
+    AZURE_CLIENT_SECRET=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token)
     # make sure whitespace is removed
     AZURE_CLIENT_SECRET="$(echo -e "${AZURE_CLIENT_SECRET}" | tr -d '[:space:]')"
     export AZURE_CLIENT_SECRET
   fi
   if [ -z ${AZURE_TENANT_ID+x} ]; then
-    AZURE_TENANT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID)
+    AZURE_TENANT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id)
     # make sure whitespace is removed
     AZURE_TENANT_ID="$(echo -e "${AZURE_TENANT_ID}" | tr -d '[:space:]')"
     export AZURE_TENANT_ID
   fi
   if [ -z ${AZURE_SUBSCRIPTION_ID+x} ]; then
-    AZURE_SUBSCRIPTION_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID)
+    AZURE_SUBSCRIPTION_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id)
     # make sure whitespace is removed
     AZURE_SUBSCRIPTION_ID="$(echo -e "${AZURE_SUBSCRIPTION_ID}" | tr -d '[:space:]')"
     export AZURE_SUBSCRIPTION_ID
@@ -101,7 +101,7 @@ elif [ "$KITCHEN_PROVIDER" == "ec2" ]; then
     export KITCHEN_EC2_SSH_KEY_ID="datadog-agent-kitchen"
     export KITCHEN_EC2_SSH_KEY_PATH="$(pwd)/aws-ssh-key"
     touch $KITCHEN_EC2_SSH_KEY_PATH && chmod 600 $KITCHEN_EC2_SSH_KEY_PATH
-    $PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_EC2_SSH_KEY > $KITCHEN_EC2_SSH_KEY_PATH
+    $PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AWS ssh_private_key > $KITCHEN_EC2_SSH_KEY_PATH
   fi
 fi
 

diff --git a/test/kitchen/tasks/show-strays.sh b/test/kitchen/tasks/show-strays.sh
@@ -10,19 +10,19 @@ set -euo pipefail
 # These should not be printed out
 set +x
 if [ -z ${AZURE_CLIENT_ID+x} ]; then
-  AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID)
+  AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id)
   export AZURE_CLIENT_ID
 fi
 if [ -z ${AZURE_CLIENT_SECRET+x} ]; then
-  AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET)
+  AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token)
   export AZURE_CLIENT_SECRET
 fi
 if [ -z ${AZURE_TENANT_ID+x} ]; then
-  AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID)
+  AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id)
   export AZURE_TENANT_ID
 fi
 if [ -z ${AZURE_SUBSCRIPTION_ID+x} ]; then
-  AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID)
+  AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id)
   export AZURE_SUBSCRIPTION_ID
 fi
 if [ -z ${DD_PIPELINE_ID+x} ]; then