Merge branch 'main' into metricbeat-process-multierr

.buildkite/aws-tests-pipeline.yml

@@ -0,0 +1,64 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
+  #  TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476
+  TEST_TAGS: "aws"
+  MODULE: "aws"
+
+  # Other deps
+  ASDF_TERRAFORM_VERSION: 1.0.2
+  ASDF_MAGE_VERSION: 1.15.0
+
+  # Unit tests
+  RACE_DETECTOR: "true"
+  TEST_COVERAGE: "true"
+
+steps:
+  - label: ":ubuntu: x-pack/filebeat: AWS Tests"
+    key: "x-pack-filebeat-extended-cloud-test"
+    skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467"
+    # Related issue: https://github.com/elastic/ingest-dev/issues/3467
+    env:
+      MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/initCloudEnv.sh
+      echo "~~~ Running tests"
+      cd x-pack/filebeat
+      mage build test goIntegTest
+    agents:
+      provider: "aws"
+      imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+      instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+    artifact_paths:
+      - "x-pack/filebeat/build/*.xml"
+      - "x-pack/filebeat/build/*.json"
+    notify:
+      - github_commit_status:
+          context: "x-pack/filebeat: AWS Tests"
+
+  - label: ":ubuntu: x-pack/metricbeat: AWS Tests"
+    key: "x-pack-metricbeat-extended-cloud-test"
+    env:
+      MODULE_DIR: "x-pack/metricbeat/module/aws"
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/initCloudEnv.sh
+      echo "~~~ Running tests"
+      cd x-pack/metricbeat
+      mage build test goIntegTest
+    agents:
+      provider: "aws"
+      imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+      instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+    artifact_paths:
+      - "x-pack/metricbeat/build/*.xml"
+      - "x-pack/metricbeat/build/*.json"
+    notify:
+      - github_commit_status:
+          context: "x-pack/metricbeat: AWS Tests"
+
+

.github/dependabot.yml

@@ -35,18 +35,23 @@ updates:
       - dependency-name: github.com/elastic/go-seccomp-bpf
       - dependency-name: github.com/elastic/toutoumomoma
       - dependency-name: github.com/elastic/ebpfevents
+      # Team:Team:obs-ds-hosted-services, Team:obs-infraobs-integrations
+      # Azure SDK dependencies
+      - dependency-name: github.com/Azure/azure-sdk-for-go/*
+      - dependency-name: github.com/Azure/azure-event-hubs-go/*
+      - dependency-name: github.com/Azure/azure-storage-blob-go/*
+      - dependency-name: github.com/Azure/go-autorest/*
+      # GCP SDK dependencies
+      - dependency-name: cloud.google.com/go/*
     groups:
       # Cloud providers' SDK dependencies
       azure-sdks:
         patterns:
-          - "github.com/Azure/azure-sdk-for-go/*"
-          - "github.com/Azure/azure-event-hubs-go/*"
+          - "github.com/Azure/azure-*-go"
           - "github.com/Azure/go-autorest/*"
-          - "github.com/Azure/azure-storage-blob-go/*"
       gcp-sdks:
         patterns:
           - "cloud.google.com/go/*"
-          - "google.golang.org/*"
     ignore:
       # Skip github.com/elastic/mito because it requires documentation updates.
       - dependency-name: github.com/elastic/mito

CHANGELOG.asciidoc

@@ -142,6 +142,7 @@ https://github.com/elastic/beats/compare/v8.13.4\...v8.14.0[View commits]
 - Fix concurrency/error handling bugs in the AWS S3 input that could drop data and prevent ingestion of large buckets. {pull}39131[39131]
 - Fix EntraID query handling. {issue}39419[39419] {pull}39420[39420]
 - Expand ID patterns in request trace logger for HTTP Endpoint. {pull}39656[39656]
+- Fix awscloudwarch input: set startTime to `0` for the first iteration of retrieving log events from CloudWatch. {pull}40079[40079]
 
 *Heartbeat*
 

CHANGELOG.next.asciidoc

@@ -44,8 +44,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Update Salesforce module to use new Salesforce input. {pull}37509[37509]
 - Tag events that come from a filestream in "take over" mode. {pull}39828[39828]
 - Fix high IO and handling of a corrupted registry log file. {pull}35893[35893]
+- Enable file ingestion to report detailed status to Elastic Agent {pull}40075[40075]
 - Filebeat, when running with Elastic-Agent, reports status for Filestream input. {pull}40121[40121]
 - Implement Elastic Agent status and health reporting for Winlog Filebeat input. {pull}40163[40163]
+- Fix filestream's registry GC: registry entries will never be removed if clean_inactive is set to "-1". {pull}40258[40258]
 
 *Heartbeat*
 
@@ -55,6 +57,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553]
 - Add support of Graphite series 1.1.0+ tagging extension for statsd module. {pull}39619[39619]
 - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation
+- Add support for Kibana status metricset in v8 format {pull}40275[40275]
 
 *Osquerybeat*
 
@@ -178,6 +181,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix behavior of pagetypeinfo metrics {pull}39985[39985]
 - Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787]
 - Set GCP metrics config period to the default (60s) when the value is below the minimum allowed period. {issue}30434[30434] {pull}40020[40020]
+- Add GCP 'instance_id' resource label in ECS cloud fields. {issue}40033[40033] {pull}40062[40062]
 - Fix missing metrics from CloudWatch when include_linked_accounts set to false. {issue}40071[40071] {pull}40135[40135]
 - Update beat module with apm-server monitoring metrics fields {pull}40127[40127]
 
@@ -319,6 +323,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 - Added status to monitor run log report.
 - Upgrade node to latest LTS v18.20.3. {pull}40038[40038]
+- Add journey duration to synthetics browser events. {pull}40230[40230]
 
 *Metricbeat*