diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 1782626d2211..41aa771809ca 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -3,7 +3,6 @@ name: "beats-auditbeat"
 
 env:
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
-
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
@@ -30,8 +29,12 @@ steps:
     steps:
       - label: ":ubuntu: Auditbeat Unit Tests"
         command: |
+          set -euo pipefail
           cd auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,8 +48,12 @@ steps:
 
       - label: ":rhel: Auditbeat Unit Tests"
         command: |
+          set -euo pipefail
           cd auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -62,6 +69,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -79,6 +89,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -97,80 +110,147 @@ steps:
           make -C auditbeat crosscompile
         env:
           GOX_FLAGS: "-arch amd64"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
-              context: "auditbeat: Cross compile"
+              context: "auditbeat: Crosscompile"
+
+  - group: "Auditbeat Extended Tests"
+    key: "auditbeat-extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" ||  build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/
 
-  - group: "Auditbeat Linux arm64 Tests"
-    key: "auditbeat-extended-tests-linux-arm64"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
     steps:
-      - label: ":linux: Auditbeat Linux arm64 Unit Tests"
+      - label: ":linux: Auditbeat Ubuntu Integration Tests"
+        key: "auditbeat-extended-integ-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/
+        command: |
+          set -euo pipefail
+          cd auditbeat
+          mage build integTest
+        retry:
+          automatic:
+           - limit: 3
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Linux Integration Tests"
+
+      - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests"
+        key: "auditbeat-extended-arm64-integ-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/
+        command: |
+          set -euo pipefail
+          cd auditbeat
+          mage build integTest
+        retry:
+          automatic:
+           - limit: 3
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Linux arm64 Integration Tests"
+
+      - label: ":linux: Auditbeat Ubuntu arm64 Unit Tests"
         key: "auditbeat-extended-arm64-unit-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
           set -euo pipefail
           cd auditbeat
-          mage unitTest
+          mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Linux arm64 tests"
+              context: "auditbeat: Linux arm64 Unit Tests"
 
-  - group: "Auditbeat MacOS Extended"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-    steps:
       - label: ":mac: Auditbeat macOS x86_64 Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
-          mage unitTest
+          mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "auditbeat: Extended MacOS x86_64 Unit Tests"
+              context: "auditbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: Auditbeat macOS arm64 Unit Tests"
+      - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
-          mage unitTest
+          mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "auditbeat: Extended MacOS arm64 Unit Tests"
+              context: "auditbeat: macOS arm64 Unit Tests"
 
   - group: "Auditbeat Windows Extended Testing"
     key: "auditbeat-extended-tests-win"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
       - label: ":windows: Auditbeat Win-2019 Unit Tests"
         key: "auditbeat-extended-win-2019-unit-tests"
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Extended Windows 2019 Unit Tests"
@@ -180,13 +260,18 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Extended Windows 10 Unit Tests"
@@ -196,13 +281,18 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Extended Windows 11 Unit Tests"
@@ -223,6 +313,7 @@ steps:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
           SNAPSHOT: true
         command: |
+          set -euo pipefail
           cd auditbeat
           mage package
         agents:
@@ -240,6 +331,7 @@ steps:
           PACKAGES: "docker"
           SNAPSHOT: true
         command: |
+          set -euo pipefail
           cd auditbeat
           mage package
         agents:
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index a14cbdd2d7ca..630056fee892 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd filebeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -60,6 +66,9 @@ steps:
         command: |
           cd filebeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -76,6 +85,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -94,6 +106,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -118,6 +133,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -136,6 +154,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -152,6 +173,9 @@ steps:
         command: |
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -172,6 +196,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -190,6 +217,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -208,6 +238,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 707645a62899..f604beac3a25 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -61,6 +67,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -78,6 +87,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -94,6 +106,9 @@ steps:
         command:  |
           cd heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -109,6 +124,9 @@ steps:
         command: |
           cd heartbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -130,6 +148,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -151,6 +172,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -168,6 +192,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -188,6 +215,9 @@ steps:
         command:  |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -205,6 +235,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -222,6 +255,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index bd2dd411350a..e1b5749c5de2 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   source .buildkite/env-scripts/env.sh
 
   # Images with prefix "platform-ingest-beats-*" has Go pre-setup.
diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
index bc7f0752e53b..d1c56ae86dab 100644
--- a/.buildkite/libbeat/pipeline.libbeat.yml
+++ b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -21,6 +21,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -38,6 +41,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -55,6 +61,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -72,6 +81,9 @@ steps:
           set -euo pipefail
           cd libbeat
           make crosscompile
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -89,6 +101,9 @@ steps:
           set -euo pipefail
           cd libbeat
           make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -108,6 +123,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
index a3c531eaac1e..eb78fd838490 100644
--- a/.buildkite/metricbeat/pipeline.yml
+++ b/.buildkite/metricbeat/pipeline.yml
@@ -32,6 +32,9 @@ steps:
       - label: ":linux: Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: "cd metricbeat && mage build unitTest"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -66,6 +69,9 @@ steps:
           # uncomment line below (which causes skipping of module kubeconfig) when https://github.com/elastic/beats/issues/38874 has been resolved.
           # export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat && mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -96,6 +102,9 @@ steps:
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat && mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -110,6 +119,9 @@ steps:
       - label: ":negative_squared_cross_mark: Cross compile"
         key: "mandatory-cross-compile"
         command: "make -C metricbeat crosscompile"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -126,6 +138,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -144,6 +159,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -166,6 +184,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -184,6 +205,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -202,6 +226,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -225,6 +252,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -242,6 +272,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml
index 25b7ff92ad04..15d37f186655 100644
--- a/.buildkite/packetbeat/pipeline.packetbeat.yml
+++ b/.buildkite/packetbeat/pipeline.packetbeat.yml
@@ -28,6 +28,9 @@ steps:
         command: |
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -42,6 +45,9 @@ steps:
       - label: ":linux: Ubuntu ARM Unit Tests"
         key: "linux-arm64-unit-tests"
         command: "cd packetbeat && mage build unitTest"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -57,6 +63,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -79,6 +88,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -96,6 +108,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -113,6 +128,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -131,6 +149,9 @@ steps:
           Set-Location -Path packetbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -155,6 +176,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
diff --git a/.buildkite/pipeline-scheduler.yml b/.buildkite/pipeline-scheduler.yml
new file mode 100644
index 000000000000..3f9b628bc63a
--- /dev/null
+++ b/.buildkite/pipeline-scheduler.yml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+# this intermediate pipeline is required because we can't specify a custom agent (k8s image) yet
+# in catalog-info: https://github.com/elastic/ci/blob/71e83d340e3b93ab43fcf16a7a70ac33bdeec6e9/terrazzo/terrazzo/constructs/buildkite/pipelines.py#L787-L842
+
+steps:
+  - label: ":pipeline: Generate trigger steps for $PIPELINES_TO_TRIGGER"
+    command: |
+      set -eo pipefail
+      .buildkite/pipeline-scheduler.py >steps.yml
+      echo "~~~ Printing pipeline steps"
+      yq . steps.yml
+      echo "~~~ Uploading steps"
+      buildkite-agent pipeline upload steps.yml
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.1"
+      useCustomGlobalHooks: true
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 6e6b0919abae..51f88d4ed768 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -69,7 +69,8 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
 esac
 
 check_and_set_beat_vars() {
-  if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then
+  local BEATS_PROJECT_NAME=${BEATS_PROJECT_NAME:=""}
+  if [[ "${BEATS_PROJECT_NAME:=""}" == *"x-pack/"* ]]; then
     BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/}              #remove -
     BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_}      #replace / to _
     BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-}      #replace / to - for labels
diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml
index 88e43129c85b..5d61fbb3a049 100644
--- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml
+++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml
@@ -24,6 +24,9 @@ steps:
       - label: ":ubuntu: Winlogbeat Crossccompile"
         key: "mandatory-cross-compile"
         command: "make -C winlogbeat crosscompile"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -40,6 +43,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -58,6 +64,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -81,6 +90,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -99,6 +111,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -117,6 +132,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
index 89343c739c4e..2b183087834b 100644
--- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
@@ -36,6 +36,9 @@ steps:
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/auditbeat
           mage update build test
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -52,6 +55,9 @@ steps:
         command: |
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -68,6 +74,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -86,6 +95,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -108,6 +120,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -126,6 +141,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -144,6 +162,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -167,6 +188,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -183,6 +207,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -201,6 +228,9 @@ steps:
         command: |
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
index 76291321ab49..9496ef561f4a 100644
--- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
@@ -28,6 +28,9 @@ steps:
       - label: ":ubuntu: Xpack/Dockerlogbeat Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: "cd x-pack/dockerlogbeat && mage build unitTest"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -44,6 +47,9 @@ steps:
         command: "cd x-pack/dockerlogbeat && mage goIntegTest"
         env:
           MODULE: $MODULE
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index 21dd147d8c10..cf159b2f7dad 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd x-pack/filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -50,6 +53,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -70,6 +76,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -86,6 +95,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -104,6 +116,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -122,6 +137,9 @@ steps:
         command: |
           cd x-pack/filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -142,6 +160,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -160,6 +181,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -178,6 +202,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -200,6 +227,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/filebeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -217,6 +247,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/filebeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
index 25344342bd38..f2befa0e3b0b 100644
--- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
@@ -39,6 +39,9 @@ steps:
           echo "~~~ Running tests"
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -59,6 +62,9 @@ steps:
           echo "~~~ Running tests"
           cd x-pack/heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -76,6 +82,9 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -99,6 +108,9 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -117,6 +129,9 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -135,6 +150,9 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -153,6 +171,9 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -178,6 +199,9 @@ steps:
           installNodeJsDependencies
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -196,6 +220,9 @@ steps:
           installNodeJsDependencies
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
index 841d609e8984..21aa1e403493 100644
--- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
@@ -20,6 +20,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -36,6 +39,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -52,6 +58,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -72,6 +81,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index ca3e6123fdad..916319c6dcd6 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd x-pack/metricbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -50,6 +53,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -70,6 +76,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -86,6 +95,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -104,6 +116,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -126,6 +141,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -144,6 +162,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -162,6 +183,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -185,6 +209,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -202,6 +229,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
index 1b0fb40cee7d..b82c92c453d7 100644
--- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd x-pack/osquerybeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -46,6 +49,9 @@ steps:
         command: |
           cd x-pack/osquerybeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -62,6 +68,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "mandatory-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -85,6 +94,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -106,6 +118,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -128,6 +143,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/osquerybeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
index 48f088b4f389..fae77c7dc48e 100644
--- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
@@ -25,6 +25,9 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -42,6 +45,9 @@ steps:
           cd x-pack/packetbeat
           mage build unitTest
         if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -58,6 +64,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -80,6 +89,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -98,6 +110,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -122,6 +137,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
index c41dad93e442..6905c36d9ac9 100644
--- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
@@ -30,6 +30,9 @@ steps:
           mage build unitTest
         env:
           MODULE: $MODULE
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -48,6 +51,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -89,6 +95,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -107,6 +116,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -125,6 +137,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 34f14b0e16e5..a09b2dada040 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -3,6 +3,11 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.17.21]]
+=== Beats version 7.17.21
+https://github.com/elastic/beats/compare/v7.17.20\...v7.17.21[View commits]
+
+
 [[release-notes-7.17.20]]
 === Beats version 7.17.20
 https://github.com/elastic/beats/compare/v7.17.19\...v7.17.20[View commits]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 719edead7bc8..8e114c2c222a 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -35,6 +35,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Auditbeat*
 
 
+- Prevent scenario of losing children-related file events in a directory for recursive fsnotify backend of auditbeat file integrity module {pull}39133[39133]
+
+
 *Filebeat*
 
 - Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709]
@@ -122,6 +125,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 
 
+
+
+
 
 
 
diff --git a/auditbeat/module/file_integrity/monitor/monitor_test.go b/auditbeat/module/file_integrity/monitor/monitor_test.go
index 3842948ce0ba..2b907f74d824 100644
--- a/auditbeat/module/file_integrity/monitor/monitor_test.go
+++ b/auditbeat/module/file_integrity/monitor/monitor_test.go
@@ -177,6 +177,11 @@ func TestRecursiveSubdirPermissions(t *testing.T) {
 		t.Skip("Skipping permissions test on Windows")
 	}
 
+	if os.Getuid() == 0 {
+		t.Skip("skipping as root can access every file and thus this unittest will fail")
+		return
+	}
+
 	// Create dir to be watched
 
 	dir, err := ioutil.TempDir("", "monitor")
@@ -242,7 +247,7 @@ func TestRecursiveSubdirPermissions(t *testing.T) {
 	for {
 		// No event is received
 		ev, err := readTimeout(t, watcher)
-		if err == errReadTimeout {
+		if errors.Is(err, errReadTimeout) {
 			break
 		}
 		assertNoError(t, err)
diff --git a/auditbeat/module/file_integrity/monitor/recursive.go b/auditbeat/module/file_integrity/monitor/recursive.go
index 14cc99379d54..cf3957363b54 100644
--- a/auditbeat/module/file_integrity/monitor/recursive.go
+++ b/auditbeat/module/file_integrity/monitor/recursive.go
@@ -18,6 +18,7 @@
 package monitor
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -84,37 +85,63 @@ func (watcher *recursiveWatcher) ErrorChannel() <-chan error {
 	return watcher.inner.Errors
 }
 
+func (watcher *recursiveWatcher) watchFile(path string, info os.FileInfo) error {
+	var err error
+	if info == nil {
+		info, err = os.Lstat(path)
+		if err != nil {
+			return err
+		}
+	}
+
+	if info.IsDir() {
+		if err = watcher.tree.AddDir(path); err != nil {
+			return err
+		}
+
+		if err = watcher.inner.Add(path); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	return watcher.tree.AddFile(path)
+}
+
 func (watcher *recursiveWatcher) addRecursive(path string) error {
 	if watcher.isExcludedPath(path) {
 		return nil
 	}
 
+	if err := watcher.watchFile(path, nil); err != nil {
+		return fmt.Errorf("failed adding watcher to '%s': %w", path, err)
+	}
+
 	var errs multierror.Errors
-	err := filepath.Walk(path, func(path string, info os.FileInfo, fnErr error) error {
-		if watcher.isExcludedPath(path) {
+	err := filepath.Walk(path, func(walkPath string, info os.FileInfo, fnErr error) error {
+		if walkPath == path {
+			return nil
+		}
+
+		if watcher.isExcludedPath(walkPath) {
 			return nil
 		}
 
 		if fnErr != nil {
-			errs = append(errs, errors.Wrapf(fnErr, "error walking path '%s'", path))
+			errs = append(errs, fmt.Errorf("error walking path '%s': %w", walkPath, fnErr))
 			// If FileInfo is not nil, the directory entry can be processed
 			// even if there was some error
 			if info == nil {
 				return nil
 			}
 		}
-		var err error
-		if info.IsDir() {
-			if err = watcher.tree.AddDir(path); err == nil {
-				if err = watcher.inner.Add(path); err != nil {
-					errs = append(errs, errors.Wrapf(err, "failed adding watcher to '%s'", path))
-					return nil
-				}
-			}
-		} else {
-			err = watcher.tree.AddFile(path)
+
+		if err := watcher.watchFile(walkPath, info); err != nil {
+			errs = append(errs, fmt.Errorf("failed adding watcher to '%s': %w", walkPath, err))
 		}
-		return err
+
+		return nil
 	})
 	watcher.log.Debugw("Added recursive watch", "path", path)
 
diff --git a/catalog-info.yaml b/catalog-info.yaml
index c37a179a4f6a..0e79d2ad8484 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -39,10 +39,11 @@ spec:
       name: beats
       description: "Beats Main pipeline"
     spec:
-      branch_configuration: "main 7.* 8.* v7.* v8.*"
+      branch_configuration: "main 7.17 8.*"
       pipeline_file: ".buildkite/pipeline.yml"
       provider_settings:
         build_pull_request_forks: false
+        build_pull_request_labels_changed: true # automatically re trigger build if GH labels change
         build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         build_tags: true
         filter_enabled: true
@@ -60,7 +61,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -84,7 +85,7 @@ spec:
       name: beats-metricbeat
       description: "Beats Metricbeat pipeline"
     spec:
-      branch_configuration: "main 7.* 8.* v7.* v8.*"
+      branch_configuration: "main 7.17 8.*"
       pipeline_file: ".buildkite/metricbeat/pipeline.yml"
       maximum_timeout_in_minutes: 120
       provider_settings:
@@ -100,17 +101,14 @@ spec:
       cancel_intermediate_builds_branch_filter: "!main !7.* !8.*"
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: "!main !7.* !8.*"
-      # TODO uncomment this environment variable when pipeline definition is updated
-      # env:
-      #   ELASTIC_PR_COMMENTS_ENABLED: 'true'
       env:
-        # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved
+        # TODO set to truue once https://github.com/elastic/ingest-dev/issues/3001 has been resolved
         ELASTIC_PR_COMMENTS_ENABLED: "false"
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -134,8 +132,8 @@ spec:
       name: filebeat
       description: "Filebeat pipeline"
     spec:
+      branch_configuration: "main 7.17 8.*"
       pipeline_file: ".buildkite/filebeat/filebeat-pipeline.yml"
-      branch_configuration: "main 7.* 8.* v7.* v8.*"
       maximum_timeout_in_minutes: 120
       provider_settings:
         trigger_mode: none # don't trigger jobs from github activity
@@ -151,13 +149,13 @@ spec:
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: "!main !7.* !8.*"
       env:
-        # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved
+        # TODO set to truue once https://github.com/elastic/ingest-dev/issues/3001 has been resolved
         ELASTIC_PR_COMMENTS_ENABLED: "false"
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -181,7 +179,7 @@ spec:
       name: auditbeat
       description: "Auditbeat pipeline"
     spec:
-      #      branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks
+      #      branch_configuration: "main 7.17 8.*" TODO: temporarily commented to build PRs from forks
       pipeline_file: ".buildkite/auditbeat/auditbeat-pipeline.yml"
       #      maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready
       provider_settings:
@@ -198,13 +196,13 @@ spec:
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: "!main !7.* !8.*"
       env:
-         # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved
-         ELASTIC_PR_COMMENTS_ENABLED: "false"
+        # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved
+        ELASTIC_PR_COMMENTS_ENABLED: "false"
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -251,7 +249,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -275,7 +273,7 @@ spec:
       name: deploy-k8s
       description: "Deploy K8S pipeline"
     spec:
-      #      branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks
+      #      branch_configuration: "main 7.17 8.*" TODO: temporarily commented to build PRs from forks
       pipeline_file: ".buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml"
       #      maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready
       provider_settings:
@@ -298,7 +296,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -345,7 +343,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -392,7 +390,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -413,10 +411,10 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: xpack-elastic-agent
+      name: beats-xpack-elastic-agent
       description: "Beats xpack elastic agent pipeline"
     spec:
-      # branch_configuration: "7.17"
+      branch_configuration: "7.17"
       pipeline_file: ".buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml"
       provider_settings:
         trigger_mode: none # don't trigger jobs from github activity
@@ -438,7 +436,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -485,7 +483,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -532,7 +530,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -579,7 +577,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -626,7 +624,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -673,7 +671,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -708,7 +706,7 @@ spec:
         release-eng:
           access_level: BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -790,7 +788,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -837,7 +835,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -884,7 +882,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -931,7 +929,7 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
 
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
@@ -978,4 +976,176 @@ spec:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
-          access_level: READ_ONLY
+          access_level: BUILD_AND_READ
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: beats-xpack-agentbeat-package
+  description: Buildkite pipeline for packaging and publishing agentbeat
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/beats-xpack-agentbeat-package
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: beats-xpack-agentbeat-package
+      description: Buildkite pipeline for packaging and publishing agentbeat
+    spec:
+      repository: elastic/beats
+      pipeline_file: ".buildkite/x-pack/agentbeat/pipeline.xpack.agentbeat.package.yml"
+      # todo release branched must be 8.14+
+      branch_configuration: "main 8.14"
+      cancel_intermediate_builds: false
+      skip_intermediate_builds: false
+      provider_settings:
+        trigger_mode: code
+        build_pull_requests: false
+        build_branches: true
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        release-eng:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: beats-packaging-pipeline
+  description: Buildkite pipeline for packaging and publishing to DRA
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/beats-packaging-pipeline
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: beats-packaging-pipeline
+      description: Pipeline for Beats packaging and publishing DRA artifacts
+    spec:
+      repository: elastic/beats
+      pipeline_file: ".buildkite/packaging.pipeline.yml"
+      branch_configuration: "main 8.*"
+      # TODO enable after packaging backports for release branches
+      # branch_configuration: "main 8.* 7.17"
+      cancel_intermediate_builds: false
+      skip_intermediate_builds: false
+      maximum_timeout_in_minutes: 90
+      provider_settings:
+        build_branches: true
+        build_pull_request_forks: false
+        build_pull_requests: false
+        build_tags: false
+        filter_condition: >-
+          build.branch =~ /^[0-9]+\.[0-9]+$$/ || build.branch == "main"
+        filter_enabled: true
+        trigger_mode: code
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#ingest-notifications'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+        SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true'        
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        release-eng:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: beats-ironbank-validation
+  description: Buildkite pipeline for validating the Ironbank docker context
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/beats-ironbank-validation
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: beats-ironbank-validation
+      description: Buildkite pipeline for validating the Ironbank docker context
+    spec:
+      repository: elastic/beats
+      pipeline_file: ".buildkite/ironbank-validation.yml"
+      branch_configuration: "main 8.* 7.17"
+      cancel_intermediate_builds: false
+      skip_intermediate_builds: false
+      provider_settings:
+        trigger_mode: none
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        release-eng:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: beats-pipeline-scheduler
+  description: 'Scheduled runs of various Beats pipelines per release branch'
+  links:
+    - title: 'Scheduled runs of Beats pipelines per release branch'
+      url: https://buildkite.com/elastic/logstash-pipeline-scheduler
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: beats-pipeline-scheduler
+      description: ':alarm_clock: Scheduled runs of various Beats pipelines per release branch'
+    spec:
+      repository: elastic/beats
+      pipeline_file: ".buildkite/pipeline-scheduler.yml"
+      maximum_timeout_in_minutes: 240
+      schedules:
+        Daily run of Iron Bank validation:
+          branch: main
+          cronline: 30 02 * * *
+          message: Daily trigger of Iron Bank validation Pipeline per branch
+          env:
+            PIPELINES_TO_TRIGGER: 'beats-ironbank-validation'
+      skip_intermediate_builds: true
+      provider_settings:
+        trigger_mode: none
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#ingest-notifications'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        release-eng:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
diff --git a/deploy/kubernetes/auditbeat-kubernetes.yaml b/deploy/kubernetes/auditbeat-kubernetes.yaml
index 269285ebce6a..369e10c0ce93 100644
--- a/deploy/kubernetes/auditbeat-kubernetes.yaml
+++ b/deploy/kubernetes/auditbeat-kubernetes.yaml
@@ -109,7 +109,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: auditbeat
-        image: docker.elastic.co/beats/auditbeat:7.17.20
+        image: docker.elastic.co/beats/auditbeat:7.17.21
         args: [
           "-c", "/etc/auditbeat.yml",
           "-e",
diff --git a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml
index 545dd81005b1..a0f337989bf7 100644
--- a/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml
+++ b/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml
@@ -22,7 +22,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: elastic-agent
-          image: docker.elastic.co/beats/elastic-agent:7.17.20
+          image: docker.elastic.co/beats/elastic-agent:7.17.21
           env:
             - name: FLEET_ENROLL
               value: "1"
diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
index 8dc3c576024d..d0dc16594510 100644
--- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
+++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
@@ -471,7 +471,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: elastic-agent
-          image: docker.elastic.co/beats/elastic-agent:7.17.20
+          image: docker.elastic.co/beats/elastic-agent:7.17.21
           args: [
             "-c", "/etc/agent.yml",
             "-e",
diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml
index accf2bcead6e..3961dae3581e 100644
--- a/deploy/kubernetes/filebeat-kubernetes.yaml
+++ b/deploy/kubernetes/filebeat-kubernetes.yaml
@@ -64,7 +64,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: filebeat
-        image: docker.elastic.co/beats/filebeat:7.17.20
+        image: docker.elastic.co/beats/filebeat:7.17.21
         args: [
           "-c", "/etc/filebeat.yml",
           "-e",
diff --git a/deploy/kubernetes/heartbeat-kubernetes.yaml b/deploy/kubernetes/heartbeat-kubernetes.yaml
index 1cda6d3e1c76..3597abf284b1 100644
--- a/deploy/kubernetes/heartbeat-kubernetes.yaml
+++ b/deploy/kubernetes/heartbeat-kubernetes.yaml
@@ -74,7 +74,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: heartbeat
-        image: docker.elastic.co/beats/heartbeat:7.17.20
+        image: docker.elastic.co/beats/heartbeat:7.17.21
         args: [
           "-c", "/etc/heartbeat.yml",
           "-e",
diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml
index 60c544ab6fb8..b8c5ff35e38b 100644
--- a/deploy/kubernetes/metricbeat-kubernetes.yaml
+++ b/deploy/kubernetes/metricbeat-kubernetes.yaml
@@ -151,7 +151,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: metricbeat
-        image: docker.elastic.co/beats/metricbeat:7.17.20
+        image: docker.elastic.co/beats/metricbeat:7.17.21
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",
diff --git a/filebeat/docs/inputs/input-filestream-file-options.asciidoc b/filebeat/docs/inputs/input-filestream-file-options.asciidoc
index 3a70ca4c6ca8..f8c7a7987369 100644
--- a/filebeat/docs/inputs/input-filestream-file-options.asciidoc
+++ b/filebeat/docs/inputs/input-filestream-file-options.asciidoc
@@ -498,6 +498,30 @@ less than or equal to `prospector.scanner.check_interval`
 If `backoff.max` needs to be higher, it is recommended to close the file handler
 instead and let {beatname_uc} pick up the file again.
 
+[float]
+[id="{beatname_lc}-input-{type}-harvester-limit"]
+===== `harvester_limit`
+
+The `harvester_limit` option limits the number of harvesters that are started in
+parallel for one input. This directly relates to the maximum number of file
+handlers that are opened. The default for `harvester_limit` is 0, which means
+there is no limit. This configuration is useful if the number of files to be
+harvested exceeds the open file handler limit of the operating system.
+
+Setting a limit on the number of harvesters means that potentially not all files
+are opened in parallel. Therefore we recommended that you use this option in
+combination with the `close.on_state_change.*` options to make sure
+harvesters are stopped more often so that new files can be picked up.
+
+Currently if a new harvester can be started again, the harvester is picked
+randomly. This means it's possible that the harvester for a file that was just
+closed and then updated again might be started instead of the harvester for a
+file that hasn't been harvested for a longer period of time.
+
+This configuration option applies per input. You can use this option to
+indirectly set higher priorities on certain inputs by assigning a higher
+limit of harvesters.
+
 [float]
 ===== `file_identity`
 
diff --git a/filebeat/docs/inputs/input-filestream.asciidoc b/filebeat/docs/inputs/input-filestream.asciidoc
index e8dbaa054b5b..ccf51ed81fda 100644
--- a/filebeat/docs/inputs/input-filestream.asciidoc
+++ b/filebeat/docs/inputs/input-filestream.asciidoc
@@ -11,8 +11,9 @@ Use the `filestream` input to read lines from active log files. It is the
 new, improved alternative to the `log` input. It comes with various improvements
 to the existing input:
 
-1. Checking of `close_*` options happens out of band. Thus, if an output is blocked,
-{beatname_uc} can close the reader and avoid keeping too many files open.
+1. Checking of `close.on_state_change.*` options happens out of
+band. Thus, if an output is blocked, {beatname_uc} can close the
+reader and avoid keeping too many files open.
 
 2. Detailed metrics are available for all files that match the `paths` configuration
 regardless of the `harvester_limit`. This way, you can keep track of all files,
diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc
index 279f9903f247..05a88bf045c2 100644
--- a/libbeat/docs/release.asciidoc
+++ b/libbeat/docs/release.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-7.17.21>>
 * <<release-notes-7.17.20>>
 * <<release-notes-7.17.19>>
 * <<release-notes-7.17.18>>
diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc
index c80deb8efe1f..7525f6848d1d 100644
--- a/libbeat/docs/version.asciidoc
+++ b/libbeat/docs/version.asciidoc
@@ -1,4 +1,4 @@
-:stack-version: 7.17.20
+:stack-version: 7.17.21
 :doc-branch: 7.17
 :go-version: 1.21.9
 :release-state: unreleased
diff --git a/libbeat/version/version.go b/libbeat/version/version.go
index 8f70960cfd5d..2f5e1cb435f5 100644
--- a/libbeat/version/version.go
+++ b/libbeat/version/version.go
@@ -18,4 +18,4 @@
 // Code generated by dev-tools/set_version
 package version
 
-const defaultBeatVersion = "7.17.21"
+const defaultBeatVersion = "7.17.22"
diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml
index 456683750f88..c31c98d11c4f 100644
--- a/testing/environments/snapshot.yml
+++ b/testing/environments/snapshot.yml
@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21-b7eaea35-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21-86bc1e1e-SNAPSHOT
     healthcheck:
       test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"]
       retries: 300
@@ -22,7 +22,7 @@ services:
     - "ingest.geoip.downloader.enabled=false"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.17.21-b7eaea35-SNAPSHOT
+    image: docker.elastic.co/logstash/logstash:7.17.21-86bc1e1e-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 600
@@ -32,7 +32,7 @@ services:
       - ./docker/logstash/pki:/etc/pki:ro
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.17.21-b7eaea35-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:7.17.21-86bc1e1e-SNAPSHOT
     healthcheck:
       test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"]
       retries: 600