Merge branch 'main' of https://github.com/aliabbas-elastic/integrations…

… into aws_benchmark_vpcflow
elastic · Mar 25, 2024 · 11c5ab9 · 11c5ab9
2 parents a4fec6f + 01a436f
commit 11c5ab9
Show file tree

Hide file tree

Showing 956 changed files with 76,154 additions and 448 deletions.
diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh
@@ -164,15 +164,15 @@ if branchExist "$BACKPORT_BRANCH_NAME"; then
 fi
 
 # backport branch does not exist, running checks and create branch
-echo "Check the entered version and PACKAGE_VERSION are equal"
-version="$(cat packages/${PACKAGE_NAME}/manifest.yml | yq -r .version)"
+version="$(git show "${BASE_COMMIT}":"packages/${PACKAGE_NAME}/manifest.yml" | yq -r .version)"
+echo "Check if version from ${BASE_COMMIT} (${version}) matches with version from input step ${PACKAGE_VERSION}"
 if [[ "${version}" != "${PACKAGE_VERSION}" ]]; then
   buildkite-agent annotate "Unexpected version found in packages/${PACKAGE_NAME}/manifest.yml" --style "error"
   exit 1
 fi
 
 echo "Check that this changeset is the one creating the version $PACKAGE_NAME"
-if ! git show -p ${BASE_COMMIT} packages/${PACKAGE_NAME}/manifest.yml | grep -E "^\+version: ${PACKAGE_VERSION}" ; then
+if ! git show -p ${BASE_COMMIT} packages/${PACKAGE_NAME}/manifest.yml | grep -E "^\+version: \"{0,1}${PACKAGE_VERSION}" ; then
   buildkite-agent annotate "This changeset does not creates the version ${PACKAGE_VERSION}" --style "error"
   exit 1
 fi

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -110,6 +110,7 @@
 /packages/cel @elastic/security-service-integrations
 /packages/ceph @elastic/obs-infraobs-integrations
 /packages/checkpoint @elastic/sec-deployment-and-devices
+/packages/cisa_kevs @elastic/security-service-integrations
 /packages/cisco_aironet @elastic/sec-deployment-and-devices
 /packages/cisco_asa @elastic/sec-deployment-and-devices
 /packages/cisco_duo @elastic/security-service-integrations
@@ -221,6 +222,7 @@
 /packages/lmd @elastic/ml-ui @elastic/sec-applied-ml
 /packages/log @elastic/elastic-agent-data-plane
 /packages/logstash @elastic/stack-monitoring
+/packages/lumos @elastic/security-service-integrations
 /packages/lyve_cloud @elastic/security-service-integrations
 /packages/m365_defender @elastic/security-service-integrations
 /packages/mattermost @elastic/security-service-integrations

diff --git a/docs/generic_guidelines.md b/docs/generic_guidelines.md
@@ -18,6 +18,21 @@ Starting with ECS 1.6, ECS is going to start using Basic types for some fields.
 
 All fields produced by an integration must be mapped by `fields.yml`. This guarantees that their index mapping is correct, and Kibana has enough info to deal with all fields.
 
+##### Field limits
+
+By default, data streams will have a `total_fields.limit` setting of 1000. Besides defined custom fields, this also includes dynamically generated ECS fields. If your data stream is expected to eventually house more than 1000 fields, set an explicit limit in the `manifest.yml` of the data stream:
+```yaml
+elasticsearch:
+  index_template:
+    settings:
+      index:
+        mapping:
+          total_fields:
+            limit: 5000
+```
+
+Note: For backwards compatibility, the limit is automatically bumped to 10000 fields if there are more than 500 fields explicitly defined for a data stream, however newly created integrations should not rely on this behavior but instead assume a fixed limit of 1000 fields.
+
 ##### Specify metric types and units
 
 As part of the field definition, there are two settings that add metadata which will help Kibana graphing it:

diff --git a/go.mod b/go.mod
@@ -187,7 +187,7 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect
 	google.golang.org/grpc v1.61.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.14.2 // indirect

diff --git a/go.sum b/go.sum
@@ -677,8 +677,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.13.1"
+  changes:
+    - description: Update latency parameter description
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9346
 - version: "2.13.0"
   changes:
     - description: Add Amazon MSK integration

diff --git a/packages/aws/data_stream/apigateway_metrics/manifest.yml b/packages/aws/data_stream/apigateway_metrics/manifest.yml
@@ -25,7 +25,7 @@ streams:
         required: false
         show_user: true
       - name: latency
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `5m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
         type: text
         title: Latency
         multi: false

diff --git a/packages/aws/data_stream/ebs/manifest.yml b/packages/aws/data_stream/ebs/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/ec2_metrics/manifest.yml b/packages/aws/data_stream/ec2_metrics/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/ecs_metrics/manifest.yml b/packages/aws/data_stream/ecs_metrics/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/elb_metrics/manifest.yml b/packages/aws/data_stream/elb_metrics/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/emr_metrics/manifest.yml b/packages/aws/data_stream/emr_metrics/manifest.yml
@@ -31,7 +31,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/firewall_metrics/manifest.yml b/packages/aws/data_stream/firewall_metrics/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/kafka_metrics/manifest.yml b/packages/aws/data_stream/kafka_metrics/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/kinesis/manifest.yml b/packages/aws/data_stream/kinesis/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/lambda/manifest.yml b/packages/aws/data_stream/lambda/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/natgateway/manifest.yml b/packages/aws/data_stream/natgateway/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/rds/manifest.yml b/packages/aws/data_stream/rds/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/redshift/manifest.yml b/packages/aws/data_stream/redshift/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/s3_daily_storage/manifest.yml b/packages/aws/data_stream/s3_daily_storage/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/s3_request/manifest.yml b/packages/aws/data_stream/s3_request/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/s3_storage_lens/manifest.yml b/packages/aws/data_stream/s3_storage_lens/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/sns/manifest.yml b/packages/aws/data_stream/sns/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/sqs/manifest.yml b/packages/aws/data_stream/sqs/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/transitgateway/manifest.yml b/packages/aws/data_stream/transitgateway/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/usage/manifest.yml b/packages/aws/data_stream/usage/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: include_linked_accounts
         type: bool
         title: Include Linked Accounts

diff --git a/packages/aws/data_stream/vpn/manifest.yml b/packages/aws/data_stream/vpn/manifest.yml
@@ -30,7 +30,7 @@ streams:
         multi: false
         required: false
         show_user: false
-        description: To address latency issues between certain AWS services and CloudWatch, specify a latency parameter to adjust the collection start time and end time in Metricbeat such as `15m`.
+        description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.13.0
+version: 2.13.1
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories: