From cc79200cea2d7164491bd0a58d9c69e404c2f9d2 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 9 Dec 2024 12:07:42 +0100 Subject: [PATCH] fix(ci): [agent6] Remove internal image/k8s deploy stages (#31875) --- .gitlab-ci.yml | 4 - .gitlab/JOBOWNERS | 8 - .../internal_image_deploy.yml | 138 ------------------ .../internal_kubernetes_deploy/include.yml | 7 - .../internal_kubernetes_deploy.yml | 65 --------- .../rc_kubernetes_deploy.yml | 38 ----- .gitlab/post_rc_build/post_rc_tasks.yml | 4 +- tasks/release.py | 4 +- 8 files changed, 3 insertions(+), 265 deletions(-) delete mode 100644 .gitlab/internal_image_deploy/internal_image_deploy.yml delete mode 100644 .gitlab/internal_kubernetes_deploy/include.yml delete mode 100644 .gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml delete mode 100644 .gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4047cdd1563f..73bc51414eecf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,8 +23,6 @@ include: - .gitlab/functional_test_cleanup/functional_test_cleanup.yml - .gitlab/install_script_testing/install_script_testing.yml - .gitlab/integration_test/include.yml - - .gitlab/internal_image_deploy/internal_image_deploy.yml - - .gitlab/internal_kubernetes_deploy/include.yml - .gitlab/junit_upload/junit_upload.yml - .gitlab/kitchen_cleanup/include.yml - .gitlab/kitchen_deploy/kitchen_deploy.yml @@ -87,7 +85,6 @@ stages: - trigger_release - choco_build - choco_deploy - - internal_image_deploy - install_script_testing - e2e - kitchen_cleanup @@ -95,7 +92,6 @@ stages: - functional_test_sysprobe - functional_test_cleanup - junit_upload - - internal_kubernetes_deploy - lint - post_rc_build - check_merge diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index 648a6d4a6146c..bde89e851baaa 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -78,14 +78,6 @@ dev_branch* @DataDog/container-integrations dev_master* @DataDog/container-integrations dev_nightly* @DataDog/container-integrations -# Internal image deploy -docker_trigger_internal* @DataDog/container-integrations -docker_trigger_cluster_agent_internal* @DataDog/container-integrations -docker_trigger_cws_instrumentation_internal* @DataDog/agent-security - -# Internal kubernetes deploy -internal_kubernetes_deploy* @DataDog/container-integrations - # Deploy packages deploy_packages* @DataDog/agent-build-and-releases publish_winget* @DataDog/windows-agent diff --git a/.gitlab/internal_image_deploy/internal_image_deploy.yml b/.gitlab/internal_image_deploy/internal_image_deploy.yml deleted file mode 100644 index c15ab6ab7ad86..0000000000000 --- a/.gitlab/internal_image_deploy/internal_image_deploy.yml +++ /dev/null @@ -1,138 +0,0 @@ ---- -# internal_image_deploy stage -# Contains jobs to trigger a pipeline in the images repo to create internal Agent images. - -docker_trigger_internal: - stage: internal_image_deploy - rules: !reference [.on_deploy_internal_or_manual] - needs: - - job: docker_build_agent6_jmx - artifacts: false - - job: docker_build_agent6_jmx_arm64 - artifacts: false - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v9 - IMAGE_NAME: datadog-agent - RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-jmx - BUILD_TAG: ${CI_COMMIT_REF_SLUG}-jmx - TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx - TMPL_SRC_REPO: ci/datadog-agent/agent - RELEASE_STAGING: "true" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_SCHEDULER_TOKEN_SSM_NAME) - - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - - | - if [ "$BUCKET_BRANCH" = "nightly" ]; then - RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}" - TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly" - fi - - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master - --variable IMAGE_VERSION - --variable IMAGE_NAME - --variable RELEASE_TAG - --variable BUILD_TAG - --variable TMPL_SRC_IMAGE - --variable TMPL_SRC_REPO - --variable RELEASE_STAGING - --variable RELEASE_PROD - --variable DYNAMIC_BUILD_RENDER_RULES - --variable APPS - --variable BAZEL_TARGET - --variable DDR - --variable DDR_WORKFLOW_ID - --variable TARGET_ENV - --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS" - -docker_trigger_cluster_agent_internal: - stage: internal_image_deploy - rules: !reference [.on_deploy] - needs: - - job: docker_build_cluster_agent_amd64 - artifacts: false - - job: docker_build_cluster_agent_arm64 - artifacts: false - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v4 - IMAGE_NAME: datadog-cluster-agent - RELEASE_TAG: ${CI_COMMIT_REF_SLUG} - BUILD_TAG: ${CI_COMMIT_REF_SLUG} - TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - TMPL_SRC_REPO: ci/datadog-agent/cluster-agent - RELEASE_STAGING: "true" - RELEASE_PROD: "true" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_SCHEDULER_TOKEN_SSM_NAME) - - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - - | - if [ "$BUCKET_BRANCH" = "nightly" ]; then - RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}" - TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly" - fi - - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master - --variable IMAGE_VERSION - --variable IMAGE_NAME - --variable RELEASE_TAG - --variable BUILD_TAG - --variable TMPL_SRC_IMAGE - --variable TMPL_SRC_REPO - --variable RELEASE_STAGING - --variable RELEASE_PROD - --variable DYNAMIC_BUILD_RENDER_RULES - --variable APPS - --variable BAZEL_TARGET - --variable DDR - --variable DDR_WORKFLOW_ID - --variable TARGET_ENV - --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS" - -docker_trigger_cws_instrumentation_internal: - stage: internal_image_deploy - rules: !reference [.on_deploy] - needs: - - job: docker_build_cws_instrumentation_amd64 - artifacts: false - - job: docker_build_cws_instrumentation_arm64 - artifacts: false - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v1 - IMAGE_NAME: datadog-cws-instrumentation - RELEASE_TAG: ${CI_COMMIT_REF_SLUG} - BUILD_TAG: ${CI_COMMIT_REF_SLUG} - TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - TMPL_SRC_REPO: ci/datadog-agent/cws-instrumentation - RELEASE_STAGING: "true" - RELEASE_PROD: "true" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_SCHEDULER_TOKEN_SSM_NAME) - - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master - --variable IMAGE_VERSION - --variable IMAGE_NAME - --variable RELEASE_TAG - --variable BUILD_TAG - --variable TMPL_SRC_IMAGE - --variable TMPL_SRC_REPO - --variable RELEASE_STAGING - --variable RELEASE_PROD - --variable DYNAMIC_BUILD_RENDER_RULES - --variable APPS - --variable BAZEL_TARGET - --variable DDR - --variable DDR_WORKFLOW_ID - --variable TARGET_ENV - --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS" diff --git a/.gitlab/internal_kubernetes_deploy/include.yml b/.gitlab/internal_kubernetes_deploy/include.yml deleted file mode 100644 index 4161ee65e6f04..0000000000000 --- a/.gitlab/internal_kubernetes_deploy/include.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# internal_kubernetes_deploy stage -# Contains jobs to trigger a pipeline in our k8s-datadog-agent-ops repo - -include: - - .gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml - - .gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml \ No newline at end of file diff --git a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml deleted file mode 100644 index afe1053993cb6..0000000000000 --- a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -# internal_kubernetes_deploy stage -# Contains jobs to trigger a pipeline in our k8s-datadog-agent-ops repo - -include: - - https://gitlab-templates.ddbuild.io/slack-notifier/v3-sdm/template.yml - -internal_kubernetes_deploy_experimental: - stage: internal_kubernetes_deploy - rules: - - if: $FORCE_K8S_DEPLOYMENT == "true" - when: always - - if: $CI_COMMIT_BRANCH != "main" - when: never - - if: $DDR != "true" - when: never - - !reference [.on_deploy] - needs: - - job: docker_trigger_internal - artifacts: false - - job: docker_trigger_cluster_agent_internal - artifacts: false - - job: k8s-e2e-main # Currently only require container Argo workflow - artifacts: false - optional: true - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - OPTION_AUTOMATIC_ROLLOUT: "true" - OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=all-staging ${CI_COMMIT_REF_SLUG}-jmx-${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - SKIP_PLAN_CHECK: "true" - EXPLICIT_WORKFLOWS: "//workflows:beta_builds.agents_nightly.publish" - BUNDLE_VERSION_OVERRIDE: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_SCHEDULER_TOKEN_SSM_NAME) - - "inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main - --variable OPTION_AUTOMATIC_ROLLOUT - --variable EXPLICIT_WORKFLOWS - --variable OPTION_PRE_SCRIPT - --variable SKIP_PLAN_CHECK - --variable APPS,BAZEL_TARGET - --variable DDR - --variable DDR_WORKFLOW_ID - --variable TARGET_ENV - --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS - --variable BUNDLE_VERSION_OVERRIDE" - -notify-slack: - stage: internal_kubernetes_deploy - extends: .slack-notifier-base - rules: - - if: $FORCE_K8S_DEPLOYMENT == "true" - when: always - - if: $CI_COMMIT_BRANCH != "main" - when: never - - if: $DDR != "true" - when: never - - !reference [.on_deploy] - tags: ["arch:amd64"] - needs: ["internal_kubernetes_deploy_experimental"] - script: - - export SDM_JWT=$(vault read -field=token identity/oidc/token/sdm) - - python3 -m pip install -r tasks/libs/requirements-notifications.txt - - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} diff --git a/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml deleted file mode 100644 index ad381a66e1388..0000000000000 --- a/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -# internal_kubernetes_deploy stage -# Contains jobs to trigger a pipeline in our k8s-datadog-agent-ops repo to deploy release candidate build - -rc_kubernetes_deploy: - stage: internal_kubernetes_deploy - rules: - - if: $RC_K8S_DEPLOYMENTS == "true" - needs: - - job: docker_trigger_internal - artifacts: false - - job: docker_trigger_cluster_agent_internal - artifacts: false - - job: k8s-e2e-main # Currently only require container Argo workflow - artifacts: false - optional: true - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - OPTION_AUTOMATIC_ROLLOUT: "true" - OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=all-staging ${CI_COMMIT_REF_SLUG}-jmx ${CI_COMMIT_REF_SLUG}" - SKIP_PLAN_CHECK: "true" - EXPLICIT_WORKFLOWS: "//workflows:deploy_rc.agents_rc" - script: - - source /root/.bashrc - - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_SCHEDULER_TOKEN_SSM_NAME) - - "inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main - --no-follow - --variable OPTION_AUTOMATIC_ROLLOUT - --variable EXPLICIT_WORKFLOWS - --variable OPTION_PRE_SCRIPT - --variable SKIP_PLAN_CHECK - --variable APPS - --variable BAZEL_TARGET - --variable DDR - --variable DDR_WORKFLOW_ID - --variable TARGET_ENV - --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS" diff --git a/.gitlab/post_rc_build/post_rc_tasks.yml b/.gitlab/post_rc_build/post_rc_tasks.yml index ece24c2dc7a0b..9780d16a5e9af 100644 --- a/.gitlab/post_rc_build/post_rc_tasks.yml +++ b/.gitlab/post_rc_build/post_rc_tasks.yml @@ -5,9 +5,7 @@ update_rc_build_links: stage: post_rc_build rules: - if: $RC_BUILD == "true" - needs: - - job: docker_trigger_internal - artifacts: false + dependencies: [] image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] script: diff --git a/tasks/release.py b/tasks/release.py index 628da4bc0bc88..5aea72ad088cd 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -1605,7 +1605,7 @@ def _check_commit_in_repo(repo_name, branch, release_json_field): @task -def update_build_links(_ctx, new_version): +def update_build_links(_, new_version): """ Updates Agent release candidates build links on https://datadoghq.atlassian.net/wiki/spaces/agent/pages/2889876360/Build+links @@ -1619,7 +1619,7 @@ def update_build_links(_ctx, new_version): from atlassian import Confluence from atlassian.confluence import ApiError - BUILD_LINKS_PAGE_ID = 2889876360 + BUILD_LINKS_PAGE_ID = 4463690547 match = RC_VERSION_RE.match(new_version) if not match: