[8.13](backport #38605) [CI] Bk migration. Audit/filebeat: Bash chang…

…esets revision (#38715) * [CI] Bk migration. Audit/filebeat: Bash changesets revision (#38605) * [CI] Bk migration. Bash changesets revision * Added required UI triggers * Added required UI triggers * Added required UI triggers * Added required UI triggers * Added required UI triggers * filebeat * Cleanup * Cleanup * Notification contexts (cherry picked from commit 052efac) * renamed was ubuntu agent * updated pipeline.py * disabled main dynamic pipeline generator * cleanup --------- Co-authored-by: Pavel Zorin <[email protected]> Co-authored-by: Olga Naidjonoka <[email protected]>
elastic · Apr 4, 2024 · ef48eb4 · ef48eb4
1 parent fd74bf8
commit ef48eb4
Show file tree

Hide file tree

Showing 9 changed files with 535 additions and 276 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -1,145 +1,68 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: "beats-auditbeat"
 
 env:
-  BEATS_PROJECT_NAME: "auditbeat"
-  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
-  IMAGE_WIN_2016: "family/core-windows-2016"
-  IMAGE_WIN_2019: "family/core-windows-2019"
-  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+  IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
+  IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
+  IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
+  IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  BEATS_PROJECT_NAME: "auditbeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
 steps:
-  - group: "Auditbeat Mandatory Testing"
-    key: "mandatory-tests"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-
-    steps:
-      - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat: linux/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
-
-      - label: ":rhel: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat: rhel/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_RHEL9}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
-
-      - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat: windows/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "{{matrix.image}}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_WIN_2016}"
-              - "${IMAGE_WIN_2022}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
-
-      - label: ":linux: Crosscompile"
-        command:
-          - ".buildkite/auditbeat/scripts/crosscompile.sh"
-        env:
-          GOX_FLAGS: "-arch amd64"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat: Crosscompile"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-
-  - group: "Extended Testing"
-    key: "extended-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for extended support"
-
-    steps:
-      - label: ":linux: ARM64 Unit Tests"
-        key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat/Extended: Unit Tests ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "auditbeat/build/*.xml"
-
-      - label: ":mac: MacOS Unit Tests"
-        key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat/Extended: MacOS Unit Tests"
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "auditbeat/build/*.xml"
-
-  - group: "Windows Extended Testing"
-    key: "extended-tests-win"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
-
-    steps:
-      - label: ":windows: Win 2019 Unit Tests"
-        key: "win-extended-2019"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat/Extended: Win-2019 Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
+  - input: "Input Parameters"
+    key: "run_auditbeat"
+    fields:
+      - select: "auditbeat - run_auditbeat"
+        key: "run_auditbeat"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+      - select: "auditbeat - run_auditbeat_macos_tests"
+        key: "run_auditbeat_macos_tests"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+      - select: "auditbeat - run_auditbeat_arm_tests"
+        key: "run_auditbeat_arm_tests"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+      - select: "auditbeat - run_auditbeat_win_tests"
+        key: "run_auditbeat_win_tests"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+    if: "build.source == 'ui'"
 
-  - group: "Packaging"
-    key: "packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
 
-    steps:
-      - label: Package pipeline
-        commands: ".buildkite/scripts/packaging/package-step.sh"
+  - label: ":linux: Load dynamic auditbeat pipeline"
+    key: "auditbeat-pipeline"
+    command: ".buildkite/scripts/generate_auditbeat_pipeline.sh"
+    notify:
+      - github_commit_status:
+          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -1,145 +1,67 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: "beats-filebeat"
 
 env:
-  BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
-  IMAGE_WIN_2016: "family/core-windows-2016"
-  IMAGE_WIN_2019: "family/core-windows-2019"
-  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+  IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
+  IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
+  IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
+  IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  BEATS_PROJECT_NAME: "filebeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
 steps:
-  - group: "Filebeat Mandatory Testing"
-    key: "mandatory-tests"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-
-    steps:
-      - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: linux/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Go Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-gotests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Go Integration Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Python Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-pytests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Python Integration Tests"
-        agents:
-          provider: gcp
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: windows/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "{{matrix.image}}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_WIN_2016}"
-              - "${IMAGE_WIN_2022}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-  - group: "Extended Testing"
-    key: "extended-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
-
-    steps:
-      - label: ":linux: ARM64 Unit Tests"
-        key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Unit Tests ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "filebeat/build/*.xml"
-
-      - label: ":mac: MacOS Unit Tests"
-        key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: MacOS Unit Tests"
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "filebeat/build/*.xml"
-
-  - group: "Windows Extended Testing"
-    key: "extended-tests-win"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
-
-    steps:
-      - label: ":windows: Win 2019 Unit Tests"
-        key: "win-extended-2019"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Win-2019 Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
+  - input: "Input Parameters"
+    key: "run_filebeat"
+    fields:
+      - select: "filebeat - run_filebeat"
+        key: "run_filebeat"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+      - select: "filebeat - run_filebeat_macos_tests"
+        key: "run_filebeat_macos_tests"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+      - select: "filebeat - run_filebeat_arm_tests"
+        key: "run_filebeat_arm_tests"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+      - select: "filebeat - run_filebeat_win_tests"
+        key: "run_filebeat_win_tests"
+        options:
+          - label: "True"
+            value: "true"
+          - label: "False"
+            value: "false"
+        default: "false"
+    if: "build.source == 'ui'"
 
-  - group: "Packaging"
-    key: "packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
 
-    steps:
-      - label: Package pipeline
-        commands: ".buildkite/scripts/packaging/package-step.sh"
+  - label: ":linux: Load dynamic filebeat pipeline"
+    key: "filebeat-pipeline"
+    command: ".buildkite/scripts/generate_filebeat_pipeline.sh"
+    notify:
+      - github_commit_status:
+          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"