Merge branch 'master' into archive-system-tests

* master: (61 commits) Add disk queue unit tests based on the queuetest package [Heartbeat] redact authorization headers from logger (elastic#26892) Expose custom process metrics (elastic#26912) [gcp/billing] always quote table name identifier (elastic#26870) Add Beats central management removal to BCs (elastic#26400) Add custom suffix to identifiers in filestream input when needed (elastic#26669) Update asa-ftd-pipeline.yml (elastic#26265) Use common host parser in vsphere module (elastic#26904) [automation] Update go release version 1.16.6 (elastic#26860) Skip flaky test: filestream and harvester group (elastic#26728) [Filebeat] Remove alias fields from Suricata and Traefik module mappings (elastic#26627) docs: apm-server.auth (elastic#26831) [Automation] Update elastic stack version to 8.0.0-2f008f4a for testing (elastic#26881) Clarify the scope of start/end multiline example (elastic#26786) [Heartbeat]: update Node.js version for synthetics (elastic#26867) [fix][httpjson] Fix incorrect key for template data (elastic#26848) [httpjson] Add value_type parameter to httpjson transforms (elastic#26847) [Heartbeat]: capture error from journey/end events (elastic#26781) [Winlogbeat] Fixes for wineventlog experimental api (elastic#26826) Set agent.id to Fleet Agent ID for each metric/log monitoring input (elastic#26776) ...
mdelapenya · Jul 19, 2021 · 78d203f · 78d203f
2 parents 7aa8e53 + c99fde1
commit 78d203f
Show file tree

Hide file tree

Showing 231 changed files with 55,797 additions and 3,535 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.13", "7.12" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.14", "7.13", "7.12" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -138,7 +138,11 @@ pipeline {
                     'linux/amd64',
                     'linux/386',
                     'linux/arm64',
-                    'linux/armv7',
+                    // armv7 packaging isn't working, and we don't currently
+                    // need it for release. Do not re-enable it without
+                    // confirming it is fixed, you will break the packaging
+                    // pipeline!
+                    //'linux/armv7',
                     // The platforms above are disabled temporarly as crossbuild images are
                     // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
                     //'linux/ppc64le',

diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py
@@ -5,8 +5,8 @@
 
 if __name__ == "__main__":
 
-    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
-    print("|-------|--------|----------|---------|------------|------|")
+    print("| Beat  | Stage  | Category | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|----------|---------|------------|------|")
     for root, dirs, files in os.walk("."):
         dirs.sort()
         for file in files:
@@ -18,6 +18,9 @@
                     withModule = False
                     platforms = [doc["platform"]]
                     when = "mandatory"
+                    category = 'default'
+                    if "stage" in doc["stages"][stage]:
+                        category = doc["stages"][stage]["stage"]
                     if "make" in doc["stages"][stage]:
                         command = doc["stages"][stage]["make"].replace("\n", " ")
                     if "mage" in doc["stages"][stage]:
@@ -33,5 +36,5 @@
                     if "when" in doc["stages"][stage]:
                         if "not_changeset_full_match" not in doc["stages"][stage]["when"]:
                             when = "optional"
-                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
-                        module, stage, command, withModule, platforms, when))
+                    print("| {} | {} | `{}` | `{}` | {} | `{}` | {} |".format(
+                        module, stage, category, command, withModule, platforms, when))
diff --git a/.ci/scripts/install-go.bat b/.ci/scripts/install-go.bat
@@ -0,0 +1,57 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+curl --version >nul 2>&1 && (
+    echo found curl
+) || (
+    choco install curl -y --no-progress --skipdownloadcache
+)
+
+mkdir %WORKSPACE%\bin
+
+IF EXIST "%PROGRAMFILES(X86)%" (
+    REM Force the gvm installation.
+    SET GVM_BIN=gvm.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
+    IF ERRORLEVEL 1 (
+        REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
+        exit /b 1
+    )
+) ELSE (
+    REM Windows 7 workers got a broken gvm installation.
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
+    IF ERRORLEVEL 1 (
+        REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
+        exit /b 1
+    )
+)
+
+SET GVM_BIN=gvm.exe
+WHERE /q %GVM_BIN%
+%GVM_BIN% version
+
+REM Install the given go version
+%GVM_BIN% --debug install %GO_VERSION%
+
+REM Configure the given go version
+FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
+
+go env
+IF ERRORLEVEL 1 (
+    REM go is not configured correctly.
+    rmdir %WORKSPACE%\.gvm /s /q
+    exit /b 1
+)
+
+where mage
+mage -version
+IF ERRORLEVEL 1 (
+    go get github.com/magefile/mage
+    IF ERRORLEVEL 1 (
+        exit /b 1
+    )
+)
diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat
@@ -1,6 +1,3 @@
-set GOPATH=%WORKSPACE%
-set MAGEFILE_CACHE=%WORKSPACE%\.magefile
-
 REM Configure GCC for either 32 or 64 bits
 set MINGW_ARCH=64
 IF NOT EXIST "%PROGRAMFILES(X86)%" (
@@ -14,51 +11,6 @@ curl --version >nul 2>&1 && (
     choco install curl -y --no-progress --skipdownloadcache
 )
 
-mkdir %WORKSPACE%\bin
-
-IF EXIST "%PROGRAMFILES(X86)%" (
-    REM Force the gvm installation.
-    SET GVM_BIN=gvm.exe
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
-    IF ERRORLEVEL 1 (
-        REM gvm installation has failed.
-        del bin\gvm.exe /s /f /q
-        exit /b 1
-    )
-) ELSE (
-    REM Windows 7 workers got a broken gvm installation.
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
-    IF ERRORLEVEL 1 (
-        REM gvm installation has failed.
-        del bin\gvm.exe /s /f /q
-        exit /b 1
-    )
-)
-
-SET GVM_BIN=gvm.exe
-WHERE /q %GVM_BIN%
-%GVM_BIN% version
-
-REM Install the given go version
-%GVM_BIN% --debug install %GO_VERSION%
-
-REM Configure the given go version
-FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
-
-go env
-IF ERRORLEVEL 1 (
-    REM go is not configured correctly.
-    rmdir %WORKSPACE%\.gvm /s /q
-    exit /b 1
-)
-
-go get github.com/magefile/mage
-where mage
-mage -version
-IF ERRORLEVEL 1 (
-    exit /b 1
-)
-
 REM Set the USERPROFILE to the previous location to fix issues with chocolatey in windows 2019
 SET PREVIOUS_USERPROFILE=%USERPROFILE%
 SET USERPROFILE=%OLD_USERPROFILE%

diff --git a/.ci/scripts/install-tools.sh b/.ci/scripts/install-tools.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-.ci/scripts/install-go.sh
 .ci/scripts/install-docker-compose.sh
 .ci/scripts/install-terraform.sh
-make mage
diff --git a/.ci/scripts/kind-setup.sh b/.ci/scripts/kind-setup.sh
@@ -1,5 +1,23 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-kind create cluster --image kindest/node:${K8S_VERSION}
+kind create cluster --image kindest/node:${K8S_VERSION} --config - <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    scheduler:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10251"
+        secure-port: "10259"
+    controllerManager:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10252"
+        secure-port: "10257"
+EOF
 kubectl cluster-info
diff --git a/.go-version b/.go-version
@@ -1 +1 @@
-1.16.5
+1.16.6
diff --git a/.mergify.yml b/.mergify.yml
@@ -15,7 +15,7 @@ pull_request_rules:
   - name: backport patches to 7.x branch
     conditions:
       - merged
-      - label=backport-v7.14.0
+      - label=backport-v7.15.0
     actions:
       backport:
         assignees:
@@ -103,6 +103,51 @@ pull_request_rules:
       - and:
         - label=automation
         - head~=^update-go-version
-        - files~=^.go-version$
+        - files~=^\.go-version$
+    actions:
+      delete_head_branch:
+  - name: backport patches to 7.14 branch
+    conditions:
+      - merged
+      - label=backport-v7.14.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "7.14"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: automatic approval for mergify pull requests with changes in bump-rules
+    conditions:
+      - author=mergify[bot]
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^\.mergify\.yml$
+      - head~=^add-backport-next.*
+    actions:
+      review:
+        type: APPROVE
+        message: Automatically approving mergify
+  - name: automatic squash and merge with success checks and the files matching the regex ^.mergify.yml is modified.
+    conditions:
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^\.mergify\.yml$
+      - head~=^add-backport-next.*
+    actions:
+      merge:
+        method: squash
+        strict: smart+fasttrack
+  - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^add-backport-next.*
+        - files~=^\.mergify\.yml$
     actions:
       delete_head_branch:
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,19 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.13.3]]
+=== Beats version 7.13.3
+https://github.com/elastic/beats/compare/v7.13.2...v7.13.3[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
+- Clone value when copy fields in processors to avoid crash. {issue}19206[19206] {pull}20500[20500]
+- Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
+- Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
+
 [[release-notes-7.13.2]]
 === Beats version 7.13.2
 https://github.com/elastic/beats/compare/v7.13.1...v7.13.2[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove id_field_data {pull}25239[25239]
 - Removed beats central management {pull}25696[25696], {issue}23908[23908]
 - MacOSX minimum supported version set to 10.14 {issue}24193{24193}
+- Add daemonset.name in pods controlled by DaemonSets {pull}26808[26808], {issue}25816[25816]
 
 *Auditbeat*
 
@@ -105,6 +106,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
 - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
 - Change source field for `event.action` in `fortinet.firewall` module to `fortinet.firewall.action` instead of `fortinet.firewall.eventtype`. {pull}24816[24816]
+- threatintel module: Changed the type of `threatintel.indicator.first_seen` from `keyword` to `date`. {pull}26765[26765]
+- Remove all alias fields pointing to ECS fields from modules. This affects the Suricata and Traefik modules. {issue}10535[10535] {pull}26627[26627]
 
 *Heartbeat*
 - Add support for screenshot blocks and use newer synthetics flags that only works in newer synthetics betas. {pull}25808[25808]
@@ -134,6 +137,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove xpack enabled flag on ES, Logstash, Beats and Kibana {pull}24427[24427]
 - Adjust host fields to adopt new names from 1.9.0 ECS. {pull}24312[24312]
 - Add replicas.ready field to state_statefulset in Kubernetes module{pull}26088[26088]
+- Added `statsd.mappings` configuration for Statsd module {pull}26220[26220]
+- Added Airflow lightweight module {pull}26220[26220]
+- Add state_job metricset to Kubernetes module{pull}26479[26479]
+- Recover service.address field in vsphere module {issue}26902[26902] {pull}26904[26904]
 
 *Packetbeat*
 
@@ -238,6 +245,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Omit full index template from errors that occur while loading the template. {pull}25743[25743]
 - In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively.
 - Fix encoding errors when using the disk queue on nested data with multi-byte characters {pull}26484[26484]
+- Fix `add_process_metadata` processor complaining about valid pid fields not being valid integers. {pull}26829[26829] {issue}26830[26830]
 
 *Auditbeat*
 
@@ -276,10 +284,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
-- Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
-- Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
-- Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
 - Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
+- Fix Elasticsearch compatibility for modules that use `copy_from` in `set` processors. {issue}26629[26629]
+- Change type of max_bytes in all configs to be cfgtype.ByteSize {pull}26699[26699]
 
 *Filebeat*
 
@@ -348,7 +355,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix mapping exception in the `googlecloud/audit` dataset pipeline. {issue}18465[18465] {pull}20465[20465]
 - Fix `cisco` asa and ftd parsing of messages 106102 and 106103. {pull}20469[20469]
 - Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
-- Clone value when copy fields in processors to avoid crash. {issue}19206[19206] {pull}20500[20500]
 - Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696]
 - Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
 - Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
@@ -390,6 +396,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
 - Removed incorrect `http.request.referrer` field from `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Fix `threatintel.indicator.url.full` not being populated. {issue}26351[26351] {pull}26508[26508]
+- Fix Elasticsearch compatibility for modules that use `type: ip` with `convert` processors. {issue}26629[26629] {pull}26676[26676]
+- Fix Elasticsearch compatibility for modules that use the `network_direction` processor. {issue}26629[26629] {pull}26676[26676]
+- Fix Elasticsearch compatibility for modules that use the `registered_domain` processor. {issue}26629[26629] {pull}26676[26676]
+- Fix Suricata metadata fields breaking visualizations, moved out of flattened datatype. {pull}26710[26710]
+- Fix `httpjson` template data key for `url.params`. {pull}26848[26848]
+- Cisco asa/ftd: Fix reversed usage of observer ingress and egress interfaces. {pull}26265[26265]
 
 *Heartbeat*
 
@@ -499,7 +511,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 - Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
 - Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412]
+- Fix memory leak in SQL module when database is not available. {issue}25840[25840] {pull}26607[26607]
 - Fix aws metric tags with resourcegroupstaggingapi paginator.  {issue}26385[26385] {pull}26443[26443]
+- Fix quoting in GCP billing table name  {issue}26855[26855] {pull}26870[26870]
 
 *Packetbeat*
 
@@ -513,6 +527,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236]
 - Protect against accessing an undefined variable in Security module. {pull}22937[22937]
 - Fix related.ip field in renameCommonAuthFields {pull}24892[24892]
+- Fix an issue with message template caching in the `wineventlog-experimental` API implementation. {pull}26826[26826]
 
 *Functionbeat*
 
@@ -844,6 +859,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
 - Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
 - Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524]
+- Add new template functions and `value_type` parameter to `httpjson` transforms. {pull}26847[26847]
 
 *Heartbeat*