Kacper murzyn/rc automatic deployments (#20929) (#21010)

* Kacper murzyn/rc automatic deployments (#20929)

* Add flag to build-rc for staging deployments

* Deployment job added for test

* Test parameters

* Filename typo

* Filename typo 2

* Remove needs for test purpose

* Remove rules for test purpose

* Test workflow trigger

* Change to retrigger the pipeline

* Pipeline trigger

* Pipeline trigger

* Pipeline trigger

* Image references fix

* Test with new image tag

* Trigger pipeline

* Trigger pipeline

* Trigger pipeline

* Trigger pipeline

* Clean up from test values

* Updates after review

* Remove wrong rule

.gitlab-ci.yml

@@ -47,6 +47,7 @@ include:
   - /.gitlab/kitchen_common/testing.yml
   - /.gitlab/benchmarks/benchmarks.yml
   - /.gitlab/benchmarks/macrobenchmarks.yml
+  - /.gitlab/rc_kubernetes_deploy.yml
 
 default:
   retry:

.gitlab/rc_kubernetes_deploy.yml

@@ -0,0 +1,29 @@
+---
+# internal_kubernetes_deploy stage
+# Contains jobs to trigger a pipeline in our k8s-datadog-agent-ops repo to deploy release candidate build
+
+rc_kubernetes_deploy:
+  stage: internal_kubernetes_deploy
+  rules:
+  - if: $RC_K8S_DEPLOYMENTS == "true"
+    when: always
+  needs:
+  - job: docker_trigger_internal
+    artifacts: false
+  - job: docker_trigger_cluster_agent_internal
+    artifacts: false
+  - job: k8s-e2e-main # Currently only require container Argo workflow
+    artifacts: false
+    optional: true
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  variables:
+    OPTION_AUTOMATIC_ROLLOUT: "true"
+    OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=all-staging ${CI_COMMIT_REF_SLUG}-jmx ${CI_COMMIT_REF_SLUG}"
+    SKIP_PLAN_CHECK: "true"
+    EXPLICIT_WORKFLOWS: "//workflows:deploy_rc.agents_rc"
+  script:
+    - source /root/.bashrc
+    - set +x
+    - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
+    - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,EXPLICIT_WORKFLOWS,OPTION_PRE_SCRIPT,SKIP_PLAN_CHECK" --no-follow

tasks/libs/pipeline_tools.py

@@ -88,6 +88,7 @@ def trigger_agent_pipeline(
     deploy=False,
     all_builds=False,
     kitchen_tests=False,
+    rc_k8s_deployments=False,
 ):
     """
     Trigger a pipeline on the datadog-agent repositories. Multiple options are available:
@@ -123,6 +124,9 @@ def trigger_agent_pipeline(
     if branch is not None:
         args["BUCKET_BRANCH"] = branch
 
+    if rc_k8s_deployments:
+        args["RC_K8S_DEPLOYMENTS"] = "true"
+
     print(
         "Creating pipeline for datadog-agent on branch/tag {} with args:\n{}".format(  # noqa: FS002
             ref, "\n".join(f"  - {k}: {args[k]}" for k in args)

tasks/pipeline.py

@@ -211,6 +211,7 @@ def run(
     deploy=False,
     all_builds=True,
     kitchen_tests=True,
+    rc_k8s_deployments=False,
 ):
     """
     Run a pipeline on the given git ref (--git-ref <git ref>), or on the current branch if --here is given.
@@ -311,6 +312,7 @@ def run(
             deploy=deploy,
             all_builds=all_builds,
             kitchen_tests=kitchen_tests,
+            rc_k8s_deployments=rc_k8s_deployments,
         )
     except FilteredOutException:
         print(color_message(f"ERROR: pipeline does not match any workflow rule. Rules:\n{workflow_rules()}", "red"))

tasks/release.py

@@ -1132,11 +1132,13 @@ def create_rc(ctx, major_versions="6,7", patch_version=False, upstream="origin")
 
 
 @task
-def build_rc(ctx, major_versions="6,7", patch_version=False):
+def build_rc(ctx, major_versions="6,7", patch_version=False, k8s_deployments=False):
     """
     To be done after the PR created by release.create-rc is merged, with the same options
     as release.create-rc.
 
+    k8s_deployments - when set to True the child pipeline deploying to subset of k8s staging clusters will be triggered.
+
     Tags the new RC versions on the current commit, and creates the build pipeline for these
     new tags.
     """
@@ -1206,6 +1208,7 @@ def build_rc(ctx, major_versions="6,7", patch_version=False):
         major_versions=major_versions,
         repo_branch="beta",
         deploy=True,
+        rc_k8s_deployments=k8s_deployments,
     )