Merge branch 'main' into statsd

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -194,7 +194,7 @@ steps:
               context: "auditbeat: Linux arm64 Unit Tests"
 
       - label: ":mac: Auditbeat macOS x86_64 Unit Tests"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -214,7 +214,7 @@ steps:
               context: "auditbeat: macOS x86_64 Unit Tests"
 
       - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh

.ci/jobs/docker-compose.yml → .buildkite/deploy/docker/docker-compose.yml

@@ -6,10 +6,10 @@ services:
     image: busybox
     depends_on:
       localstack: { condition: service_healthy }
-      
+
   localstack:
     container_name: "${localstack_integration_test_container}"
-    image: localstack/localstack:2.1.0 # Latest stable release
+    image: localstack/localstack:3.1.0 # Latest stable release
     ports:
       - "127.0.0.1:4566:4566"            # LocalStack Gateway
     environment:

.buildkite/filebeat/filebeat-pipeline.yml

@@ -133,7 +133,7 @@ steps:
     steps:
       - label: ":mac: Filebeat macOS x86_64 Unit Tests"
         key: "macos-unit-tests-extended"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -154,7 +154,7 @@ steps:
 
       - label: ":mac: Filebeat macOS arm64 Unit Tests"
         key: "macos-arm64-unit-tests-extended"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -46,7 +46,7 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Ubuntu Unit Tests"
+              context: "heartbeat: Ubuntu Unit Tests"
 
       - label: ":rhel: Heartbeat Rhel9 Unit Tests"
         command: |
@@ -64,9 +64,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Rhel9 Unit Tests"
+              context: "heartbeat: Rhel9 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-2016 Unit Tests"
+      - label: ":windows: Win-2016 Unit Tests"
         key: "windows-2016"
         command: |
           Set-Location -Path heartbeat
@@ -84,7 +84,7 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2016 Unit Tests"
+              context: "heartbeat: Win-2016 Unit Tests"
 
       - label: ":windows: Heartbeat Win-2022 Unit Test"
         key: "windows-2022"
@@ -104,10 +104,19 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2022 Unit Tests"
+              context: "heartbeat: Win-2022 Unit Tests"
 
-      - label: ":ubuntu: Heartbeat Go Integration Tests"
+      - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
         command: |
+          set -euo pipefail
+          echo "~~~ Installing @elastic/synthetics"
+          npm install -g @elastic/synthetics
+
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet heartbeat
+
+          echo "~~~ Running tests"
           cd heartbeat
           mage goIntegTest
         retry:
@@ -122,10 +131,16 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Go Integration Tests"
+              context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Heartbeat Python Integration Tests"
+      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
         command: |
+          set -euo pipefail
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet heartbeat
+
+          echo "~~~ Running tests"
           cd heartbeat
           mage pythonIntegTest
         retry:
@@ -140,15 +155,16 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Python Integration Tests"
+              context: "heartbeat: Python Integration Tests"
 
-  - group: "Heartbeat ARM Tests"
-    key: "heartbeat-extended-tests-arm"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+  - group: "Heartbeat Extended Tests"
+    key: "heartbeat-extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
 
     steps:
-      - label: ":linux: Heartbeat ARM64 Unit Tests"
+      - label: ":linux: Heartbeat arm64 Unit Tests"
         key: "arm-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
           cd heartbeat
           mage build unitTest
@@ -162,14 +178,11 @@ steps:
         artifact_paths: "heartbeat/build/*.xml"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Ubuntu ARM64 Unit Tests"
+              context: "heartbeat: Linux arm64 Unit Tests"
 
-  - group: "Heartbeat Extended Testing MacOS"
-    key: "heartbeat-extended-tests-macos"
-    if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-    steps:
-      - label: ":mac: Heartbeat MacOS Unit Tests"
+      - label: ":mac: macOS x86_64 Unit Tests"
         key: "macos-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -186,10 +199,11 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: MacOS Unit Tests"
+              context: "heartbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: Heartbeat MacOS ARM Unit Tests"
+      - label: ":mac: macOS arm64 Unit Tests"
         key: "macos-arm-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -206,14 +220,14 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: MacOS ARM Unit Tests"
+              context: "heartbeat: macOS arm64 Unit Tests"
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
 
     steps:
-      - label: ":windows: Heartbeat Win-2019 Unit Tests"
+      - label: ":windows: Win-2019 Unit Tests"
         key: "heartbeat-win-extended-2019"
         command: |
           Set-Location -Path heartbeat
@@ -231,9 +245,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2019 Unit Tests"
+              context: "heartbeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-11 Unit Tests"
+      - label: ":windows: Win-11 Unit Tests"
         key: "heartbeat-windows-extended-11"
         command: |
           Set-Location -Path heartbeat
@@ -251,9 +265,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-11 Unit Tests"
+              context: "heartbeat: Win-11 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-10 Unit Tests"
+      - label: ":windows: Win-10 Unit Tests"
         key: "heartbeat-windows-extended-10"
         command: |
           Set-Location -Path heartbeat
@@ -271,17 +285,20 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-10 Unit Tests"
+              context: "heartbeat: Win-10 Unit Tests"
 
   - wait: ~
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on: "heartbeat-mandatory-tests"
 
   - group: "Heartbeat Packaging"
     key: "heartbeat-packaging"
     steps:
-      - label: ":ubuntu: Heartbeat Packaging Linux X86"
-        key: "heartbeat-package-linux-x86"
+      - label: ":ubuntu: Heartbeat Packaging Linux"
+        key: "heartbeat-package-linux"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
@@ -298,9 +315,9 @@ steps:
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Packaging Ubuntu x86_64"
+              context: "heartbeat: Packaging Linux"
 
-      - label: ":linux: Heartbeat Packaging Linux ARM"
+      - label: ":linux: Heartbeat Packaging Linux arm64"
         key: "heartbeat-package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
@@ -319,4 +336,4 @@ steps:
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Packaging Ubuntu ARM"
+              context: "heartbeat: Packaging Linux arm64"

.buildkite/metricbeat/pipeline.yml

@@ -244,7 +244,7 @@ steps:
 
   - group: "Metricbeat Extended MacOS Tests"
     key: "metricbeat-extended-macos-tests"
-    if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
     steps:
       - label: ":mac: MacOS x64_64 Unit Tests"
         key: "extended-macos-x64-64-unit-tests"

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -175,7 +175,7 @@ steps:
     steps:
       - label: ":mac: MacOS x86_64 Unit Tests"
         key: "macos-x86-64-unit-tests-extended"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh

.buildkite/pipeline.yml

@@ -250,7 +250,7 @@ steps:
           watch:
             - path:
                 - x-pack/osquerybeat/**
-                - .buildkite/x-pack/pipeline.xpack.metricbeat.yml
+                - .buildkite/x-pack/pipeline.xpack.osquerybeat.yml
                 - .buildkite/scripts/**
                 - .buildkite/hooks/**
                 # x-pack
@@ -279,7 +279,7 @@ steps:
       commit: "${BUILDKITE_COMMIT}"
       branch: "${BUILDKITE_BRANCH}"
 
-  - label: "Trigger Xpack/Winlogbeat"
+  - label: "Trigger x-pack/winlogbeat"
     if: build.pull_request.id != null
     plugins:
       - monorepo-diff#v1.0.1:
@@ -306,7 +306,7 @@ steps:
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
-  - label: "Triggering Build for Xpack/Winlogbeat"
+  - label: "Triggering Build for x-pack/winlogbeat"
     if: build.pull_request.id == null
     trigger: "beats-xpack-winlogbeat"
     build: