diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index b0bd67b036a..5d5d3c35433 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -26,6 +26,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "Heartbeat Check/Update" + key: "heartbeat-check-update" + + steps: + - label: "Run check/update" + command: | + make -C heartbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "hearbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "heartbeat-check-update" + - group: "Heartbeat Mandatory Testing" key: "heartbeat-mandatory-tests" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 86c0a59a704..1b59e5140d3 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -28,6 +28,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Metricbeat Check/Update" + key: "metricbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C metricbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "metricbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "metricbeat-check-update" + - group: "Metricbeat Mandatory Tests" key: "metricbeat-mandatory-tests" steps: @@ -209,7 +235,7 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: EmacOS x86_64 Unit Tests" + context: "metricbeat: macOS x86_64 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index eef5fc15380..c4ff374fdab 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -4,18 +4,18 @@ name: "beats-packetbeat" env: AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" - IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" - IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -53,8 +53,9 @@ steps: - group: "packetbeat Mandatory Tests" key: "packetbeat-mandatory-tests" + steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: Packetbeat Linux x86_64 Unit Tests" command: | cd packetbeat mage build unitTest @@ -70,11 +71,13 @@ steps: - "packetbeat/build/*.json" notify: - github_commit_status: - context: "packetbeat: Ubuntu Unit Tests" + context: "packetbeat: Linux x86_64 Unit Tests" - - label: ":linux: Ubuntu ARM Unit Tests" + - label: ":ubuntu: Packetbeat Linux arm64 Unit Tests" key: "linux-arm64-unit-tests" - command: "cd packetbeat && mage build unitTest" + command: | + cd packetbeat + mage build unitTest retry: automatic: - limit: 3 @@ -87,9 +90,9 @@ steps: - "packetbeat/build/*.json" notify: - github_commit_status: - context: "packetbeat: Ubuntu ARM Unit Tests" + context: "packetbeat: Linux arm64 Unit Tests" - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: Packetbeat Win-2019 Unit Tests" command: | Set-Location -Path packetbeat mage build unitTest @@ -107,33 +110,14 @@ steps: - "packetbeat/build/*.json" notify: - github_commit_status: - context: "packetbeat: Windows 2019 Unit Tests" + context: "packetbeat: Win-2019 Unit Tests" - - group: "Extended Windows Tests" + - group: "Packetbeat Extended Windows Tests" key: "packetbeat-extended-windows-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - steps: - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path packetbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "packetbeat/build/*.xml" - - "packetbeat/build/*.json" - notify: - - github_commit_status: - context: "packetbeat: Extended Windows 2016 Unit Tests" - - label: ":windows: Windows 2022 Unit Tests" + steps: + - label: ":windows: Packetbeat Win-2016 Unit Tests" command: | Set-Location -Path packetbeat mage build unitTest @@ -151,9 +135,9 @@ steps: - "packetbeat/build/*.json" notify: - github_commit_status: - context: "packetbeat: Extended Windows 2022 Unit Tests" + context: "packetbeat: Win-2016 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: Packetbeat Win 10 Unit Tests" command: | Set-Location -Path packetbeat mage build unitTest @@ -171,33 +155,13 @@ steps: - "packetbeat/build/*.json" notify: - github_commit_status: - context: "packetbeat: Extended Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path packetbeat - mage build unitTest - key: "extended-win-11-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "packetbeat/build/*.xml" - - "packetbeat/build/*.json" - notify: - - github_commit_status: - context: "packetbeat: Extended Windows 11 Unit Tests" + context: "packetbeat: Win 10 Unit Tests" - - group: "Extended Tests" + - group: "Packetbeat Extended Tests" key: "extended-tests" + steps: - - label: ":mac: MacOS x86_64 Unit Tests" + - label: ":mac: Packetbeat macOS x86_64 Unit Tests" key: "macos-x86-64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -216,46 +180,7 @@ steps: - "packetbeat/build/*.json" notify: - github_commit_status: - context: "packetbeat: Extended MacOS x86_64 Unit Tests" - - - label: ":mac: MacOS arm64 Unit Tests" - key: "macos-arm64-unit-tests-extended" - if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd packetbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "packetbeat/build/*.xml" - - "packetbeat/build/*.json" - notify: - - github_commit_status: - context: "packetbeat: Extended MacOS arm64 Unit Tests" - - - label: ":linux: Ubuntu ARM Unit Tests" - key: "linux-arm64-unit-tests-extended" - command: "cd packetbeat && mage build unitTest" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "packetbeat/build/*.xml" - - "packetbeat/build/*.json" - notify: - - github_commit_status: - context: "packetbeat: Extended Ubuntu ARM Unit Tests" + context: "packetbeat: macOS x86_64 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -267,7 +192,7 @@ steps: - group: "Packetbeat Packaging" key: "packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":ubuntu: Packetbeat Packaging Linux" key: "packaging-linux" command: | cd packetbeat @@ -288,7 +213,7 @@ steps: - github_commit_status: context: "packetbeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":ubuntu: Packetbeat Packaging arm64" key: "packaging-arm" command: | cd packetbeat @@ -306,4 +231,4 @@ steps: PACKAGES: "docker" notify: - github_commit_status: - context: "packetbeat: Packaging Linux ARM" + context: "packetbeat: Packaging Linux arm64" diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 7e79d2d3164..1ab80edc585 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -15,7 +15,7 @@ definePattern() { } defineExclusions() { - exclude="^$beatPath\/module\/(.*(?=1.2.0,<=3.1.0 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 70a3494d436..554b9df0b60 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 0a5646a9d75..b6517fea093 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index f3cab807dfd..f443a51dd3a 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.21.10 AS build-env +FROM golang:1.21.11 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 9dea6777c57..6301d44f4e2 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.21.10 +FROM golang:1.21.11 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 912cd87cc43..22c50eeb430 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 RUN \ apt-get update \ diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index bfbfe0ea7d1..f75d48577ee 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.22-b8cfff19-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.23-dc0072d1-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -22,7 +22,7 @@ services: - "ingest.geoip.downloader.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.17.22-b8cfff19-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.17.23-dc0072d1-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -32,7 +32,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.17.22-b8cfff19-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.17.23-dc0072d1-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 3b3a785a0ca..d1c4d5af3cd 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 RUN \ apt-get update \ diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 2901fa41c12..e09893cca6a 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 RUN \ apt-get update \ diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index 5023acb7b46..e7fd7774af8 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.21.10 AS build-env +FROM golang:1.21.11 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build .