Merge branch 'main' into aws-billing-support-dynamic-group-by-types

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -3,63 +3,113 @@
 env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  MODULE: "kubernetes"
+
+  # Other deps
+  ASDF_KIND_VERSION: "0.20.0"
 
 steps:
-  - label: "Checks"
-    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
-    notify:
-      - github_commit_status:
-          context: "Deploy K8S/Checks"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+  - group: "Deploy/K8S"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
+
+    steps:
+    - label: "Checks"
+      command: |
+        set -euo pipefail
+        make -C deploy/kubernetes all
+        make check-no-changes
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s checks"
 
-  - label: "K8S Test/K8S version: v1.29.0"
-    key: "k8s-test-129"
-    env:
-      K8S_VERSION: "v1.29.0"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.29.0"
+      key: "k8s-test-129"
+      env:
+        K8S_VERSION: "v1.29.0"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.29.0"
 
-  - label: "K8S Test/K8S version: v1.28.0"
-    key: "k8s-test-128"
-    env:
-      K8S_VERSION: "v1.28.0"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.28.0"
+      key: "k8s-test-128"
+      env:
+        K8S_VERSION: "v1.28.0"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.28.0"
 
-  - label: "K8S Test/K8S version: v1.27.3"
-    key: "k8s-test-1273"
-    env:
-      K8S_VERSION: "v1.27.3"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.27.3"
+      key: "k8s-test-1273"
+      env:
+        K8S_VERSION: "v1.27.3"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.27.3"
 
-  - label: "K8S Test/K8S version: v1.26.6"
-    key: "k8s-test-1266"
-    env:
-      K8S_VERSION: "v1.26.6"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.26.6"
+      key: "k8s-test-1266"
+      env:
+        K8S_VERSION: "v1.26.6"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.26.6"

.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh

@@ -4,11 +4,15 @@ set -euo pipefail
 
 source .buildkite/env-scripts/util.sh
 
+export KUBECONFIG="${WORKSPACE}/kubecfg"
+export BIN="${WORKSPACE}/bin"
+add_bin_path
+
 echo "--- Installing kind & kubectl"
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh
+asdf plugin add kind
+asdf install kind $ASDF_KIND_VERSION
 
-echo "--- Setting up kind"
+echo "~~~ Setting up kind"
 max_retries=3
 timeout=5
 retries=0

.buildkite/env-scripts/env.sh

@@ -12,7 +12,6 @@ WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
-TMP_FOLDER="tmp.${REPO}"
 SNAPSHOT="true"
 PYTEST_ADDOPTS=""
 OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
@@ -38,7 +37,6 @@ export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
-export TMP_FOLDER
 export SNAPSHOT
 export PYTEST_ADDOPTS
 export OSS_MODULE_PATTERN