From 8726e216b0e50edb7df696199ba6e92028a11a18 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 30 Sep 2024 10:03:24 +0200 Subject: [PATCH] feat(security): Migrate some credentials to vault (#29626) --- .gitlab-ci.yml | 5 +++++ .gitlab/.pre/create_release_qa_cards.yml | 4 ++-- .gitlab/common/shared.yml | 18 +++++++++--------- .gitlab/post_rc_build/post_rc_tasks.yml | 4 ++-- .gitlab/setup/setup.yml | 12 ++++++------ 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59034636c55827..c520ed00508851 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -253,6 +253,11 @@ variables: VCPKG_BLOB_SAS_URL: ci.datadog-agent-buildimages.vcpkg_blob_sas_url # windows-agent WINGET_PAT: ci.datadog-agent.winget_pat # windows-agent + ATLASSIAN_WRITE: atlassian-write + AGENT_GITHUB_APP: agent-github-app + MACOS_GITHUB_APP_1: macos-github-app-one + MACOS_GITHUB_APP_2: macos-github-app-two + DD_PKG_VERSION: "latest" # Job stage attempts (see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts) diff --git a/.gitlab/.pre/create_release_qa_cards.yml b/.gitlab/.pre/create_release_qa_cards.yml index d6343b073fd15e..d22c256bb3609e 100644 --- a/.gitlab/.pre/create_release_qa_cards.yml +++ b/.gitlab/.pre/create_release_qa_cards.yml @@ -7,8 +7,8 @@ create_release_qa_cards: - !reference [.on_deploy_rc] script: - !reference [.setup_agent_github_app] - - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $JIRA_READ_API_TOKEN) || exit $?; export ATLASSIAN_PASSWORD - - export ATLASSIAN_USERNAME=robot-jira-agentplatform@datadoghq.com + - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD + - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME - pip install ddqa - inv release.create-qa-cards -t ${CI_COMMIT_REF_NAME} allow_failure: true diff --git a/.gitlab/common/shared.yml b/.gitlab/common/shared.yml index 8a16870d9fac25..b00293abc750ee 100644 --- a/.gitlab/common/shared.yml +++ b/.gitlab/common/shared.yml @@ -30,21 +30,21 @@ # This balances the requests made to GitHub between the two apps we have set up. - | if [[ "$(( RANDOM % 2 ))" == "1" ]]; then - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID + GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 + GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID + GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID echo "Using GitHub App instance 1" else - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY_2) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID_2) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID_2) || exit $?; export GITHUB_INSTALLATION_ID + GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 + GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID + GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID echo "Using GitHub App instance 2" fi .setup_agent_github_app: - - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_KEY) || exit $?; export GITHUB_KEY_B64 - - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP_ID) || exit $?; export GITHUB_APP_ID - - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID + - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP key_b64) || exit $?; export GITHUB_KEY_B64 + - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP app_id) || exit $?; export GITHUB_APP_ID + - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP installation_id) || exit $?; export GITHUB_INSTALLATION_ID - echo "Using agent GitHub App" # Install `dd-pkg` and lint packages produced by Omnibus, supports only deb and rpm packages diff --git a/.gitlab/post_rc_build/post_rc_tasks.yml b/.gitlab/post_rc_build/post_rc_tasks.yml index 3c9ca13b377f42..f02bda3516650e 100644 --- a/.gitlab/post_rc_build/post_rc_tasks.yml +++ b/.gitlab/post_rc_build/post_rc_tasks.yml @@ -11,8 +11,8 @@ update_rc_build_links: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] script: - - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $JIRA_READ_API_TOKEN) || exit $?; export ATLASSIAN_PASSWORD - - ATLASSIAN_USERNAME=robot-jira-agentplatform@datadoghq.com; export ATLASSIAN_USERNAME + - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD + - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME - python3 -m pip install -r tasks/requirements_release_tasks.txt - PATCH=$(echo "$CI_COMMIT_REF_NAME" | cut -d'.' -f3 | cut -c1) - if [[ "$PATCH" == "0" ]]; then PATCH_OPTION=""; else PATCH_OPTION="-p"; fi diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml index 39b6c2a20b0dd1..28779481845c80 100644 --- a/.gitlab/setup/setup.yml +++ b/.gitlab/setup/setup.yml @@ -18,15 +18,15 @@ github_rate_limit_info: script: - python3 -m pip install -r tasks/libs/requirements-github.txt datadog_api_client # Send stats for app 1 - - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY) || exit $?; export GITHUB_KEY_B64 - - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID) || exit $?; export GITHUB_APP_ID - - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID + - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 + - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID + - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 1 # Send stats for app 2 - - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY_2) || exit $?; export GITHUB_KEY_B64 - - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID_2) || exit $?; export GITHUB_APP_ID - - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID_2) || exit $?; export GITHUB_INSTALLATION_ID + - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 + - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID + - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 2 allow_failure: true