From b84c7ee4267ba166c77c4d47263ad5e0f1ab8353 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 22 Nov 2024 11:11:44 +0100 Subject: [PATCH] Reapply "feat(vault): Add e2e test secrets to vault" (#30328) (#31368) --- .gitlab-ci.yml | 4 ++++ .gitlab/e2e/e2e.yml | 12 ++++++------ .gitlab/maintenance_jobs/kitchen.yml | 8 ++++---- test/kitchen/tasks/clean.sh | 8 ++++---- test/kitchen/tasks/run-test-kitchen.sh | 10 +++++----- test/kitchen/tasks/show-strays.sh | 8 ++++---- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fab353404816d..a1beca34642b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -300,7 +300,11 @@ variables: ATLASSIAN_WRITE: atlassian-write # agent-devx-infra CODECOV: codecov # agent-devx-infra DOCKER_REGISTRY_RO: dockerhub-readonly # agent-delivery + E2E_AZURE: e2e-azure # agent-devx-loops + E2E_GCP: e2e-gcp # agent-devx-loops GITLAB_TOKEN: gitlab-token # agent-devx-infra + KITCHEN_AWS: kitchen-aws # agent-devx-loops + KITCHEN_AZURE: kitchen-azure # agent-devx-loops INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2 # agent-devx-infra MACOS_GITHUB_APP_1: macos-github-app-one # agent-devx-infra MACOS_GITHUB_APP_2: macos-github-app-two # agent-devx-infra diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml index d5a5b7a8ce3f6..e3e09a264c733 100644 --- a/.gitlab/e2e/e2e.yml +++ b/.gitlab/e2e/e2e.yml @@ -20,13 +20,13 @@ - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" # Setup Azure credentials. https://www.pulumi.com/registry/packages/azure-native/installation-configuration/#set-configuration-using-pulumi-config # The app is called `agent-e2e-tests` - - ARM_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_CLIENT_ID) || exit $?; export ARM_CLIENT_ID - - ARM_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_CLIENT_SECRET) || exit $?; export ARM_CLIENT_SECRET - - ARM_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_TENANT_ID) || exit $?; export ARM_TENANT_ID - - ARM_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_AZURE_SUBSCRIPTION_ID) || exit $?; export ARM_SUBSCRIPTION_ID - # Setup GCP credentials. https://www.pulumi.com/registry/packages/gcp/service-account/ + - ARM_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE client_id) || exit $?; export ARM_CLIENT_ID + - ARM_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE token) || exit $?; export ARM_CLIENT_SECRET + - ARM_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE tenant_id) || exit $?; export ARM_TENANT_ID + - ARM_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE subscription_id) || exit $?; export ARM_SUBSCRIPTION_ID + # Setup GCP credentials. https://www.pulumi.com/registry/packages/gcp/installation-configuration/ # The service account is called `agent-e2e-tests` - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_GCP_CREDENTIALS > ~/gcp-credentials.json || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_GCP credentials_json > ~/gcp-credentials.json || exit $? - export GOOGLE_APPLICATION_CREDENTIALS=~/gcp-credentials.json # Generate external links to CI VISIBILITY, used by artifacts:reports:annotations - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH diff --git a/.gitlab/maintenance_jobs/kitchen.yml b/.gitlab/maintenance_jobs/kitchen.yml index 91b9f01c766b0..3eea73295c63c 100644 --- a/.gitlab/maintenance_jobs/kitchen.yml +++ b/.gitlab/maintenance_jobs/kitchen.yml @@ -26,10 +26,10 @@ periodic_kitchen_cleanup_azure: # the job to be run one at a time. resource_group: azure_cleanup script: - - ARM_SUBSCRIPTION_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID` || exit $?; export ARM_SUBSCRIPTION_ID - - ARM_CLIENT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID` || exit $?; export ARM_CLIENT_ID - - ARM_CLIENT_SECRET=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET` || exit $?; export ARM_CLIENT_SECRET - - ARM_TENANT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID` || exit $?; export ARM_TENANT_ID + - ARM_SUBSCRIPTION_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id` || exit $?; export ARM_SUBSCRIPTION_ID + - ARM_CLIENT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id` || exit $?; export ARM_CLIENT_ID + - ARM_CLIENT_SECRET=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token` || exit $?; export ARM_CLIENT_SECRET + - ARM_TENANT_ID=`$CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id` || exit $?; export ARM_TENANT_ID # Remove kitchen resources for all existing test suite prefixes - RESOURCE_GROUP_PREFIX=kitchen-chef python3 /deploy_scripts/cleanup_azure.py - RESOURCE_GROUP_PREFIX=kitchen-win python3 /deploy_scripts/cleanup_azure.py diff --git a/test/kitchen/tasks/clean.sh b/test/kitchen/tasks/clean.sh index c351060cc08ca..b05c08fa71f7d 100755 --- a/test/kitchen/tasks/clean.sh +++ b/test/kitchen/tasks/clean.sh @@ -8,19 +8,19 @@ set -euo pipefail # These should not be printed out if [ -z ${AZURE_CLIENT_ID+x} ]; then - AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID) + AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id) export AZURE_CLIENT_ID fi if [ -z ${AZURE_CLIENT_SECRET+x} ]; then - AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET) + AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token) export AZURE_CLIENT_SECRET fi if [ -z ${AZURE_TENANT_ID+x} ]; then - AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID) + AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id) export AZURE_TENANT_ID fi if [ -z ${AZURE_SUBSCRIPTION_ID+x} ]; then - AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID) + AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id) export AZURE_SUBSCRIPTION_ID fi if [ -z ${DD_PIPELINE_ID+x} ]; then diff --git a/test/kitchen/tasks/run-test-kitchen.sh b/test/kitchen/tasks/run-test-kitchen.sh index 6e51d4013da71..795ee25dc4ff1 100755 --- a/test/kitchen/tasks/run-test-kitchen.sh +++ b/test/kitchen/tasks/run-test-kitchen.sh @@ -54,25 +54,25 @@ if [ "$KITCHEN_PROVIDER" == "azure" ]; then # These should not be printed out set +x if [ -z ${AZURE_CLIENT_ID+x} ]; then - AZURE_CLIENT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID) + AZURE_CLIENT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id) # make sure whitespace is removed AZURE_CLIENT_ID="$(echo -e "${AZURE_CLIENT_ID}" | tr -d '[:space:]')" export AZURE_CLIENT_ID fi if [ -z ${AZURE_CLIENT_SECRET+x} ]; then - AZURE_CLIENT_SECRET=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET) + AZURE_CLIENT_SECRET=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token) # make sure whitespace is removed AZURE_CLIENT_SECRET="$(echo -e "${AZURE_CLIENT_SECRET}" | tr -d '[:space:]')" export AZURE_CLIENT_SECRET fi if [ -z ${AZURE_TENANT_ID+x} ]; then - AZURE_TENANT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID) + AZURE_TENANT_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id) # make sure whitespace is removed AZURE_TENANT_ID="$(echo -e "${AZURE_TENANT_ID}" | tr -d '[:space:]')" export AZURE_TENANT_ID fi if [ -z ${AZURE_SUBSCRIPTION_ID+x} ]; then - AZURE_SUBSCRIPTION_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID) + AZURE_SUBSCRIPTION_ID=$($PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id) # make sure whitespace is removed AZURE_SUBSCRIPTION_ID="$(echo -e "${AZURE_SUBSCRIPTION_ID}" | tr -d '[:space:]')" export AZURE_SUBSCRIPTION_ID @@ -101,7 +101,7 @@ elif [ "$KITCHEN_PROVIDER" == "ec2" ]; then export KITCHEN_EC2_SSH_KEY_ID="datadog-agent-kitchen" export KITCHEN_EC2_SSH_KEY_PATH="$(pwd)/aws-ssh-key" touch $KITCHEN_EC2_SSH_KEY_PATH && chmod 600 $KITCHEN_EC2_SSH_KEY_PATH - $PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_EC2_SSH_KEY > $KITCHEN_EC2_SSH_KEY_PATH + $PARENT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AWS ssh_private_key > $KITCHEN_EC2_SSH_KEY_PATH fi fi diff --git a/test/kitchen/tasks/show-strays.sh b/test/kitchen/tasks/show-strays.sh index 996c95f6ac04b..f1d2eff4df7a8 100755 --- a/test/kitchen/tasks/show-strays.sh +++ b/test/kitchen/tasks/show-strays.sh @@ -10,19 +10,19 @@ set -euo pipefail # These should not be printed out set +x if [ -z ${AZURE_CLIENT_ID+x} ]; then - AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_ID) + AZURE_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE client_id) export AZURE_CLIENT_ID fi if [ -z ${AZURE_CLIENT_SECRET+x} ]; then - AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_CLIENT_SECRET) + AZURE_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE token) export AZURE_CLIENT_SECRET fi if [ -z ${AZURE_TENANT_ID+x} ]; then - AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_TENANT_ID) + AZURE_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE tenant_id) export AZURE_TENANT_ID fi if [ -z ${AZURE_SUBSCRIPTION_ID+x} ]; then - AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE_SUBSCRIPTION_ID) + AZURE_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $KITCHEN_AZURE subscription_id) export AZURE_SUBSCRIPTION_ID fi if [ -z ${DD_PIPELINE_ID+x} ]; then