Merge remote-tracking branch 'upstream/master' into add_dep_pod_owner

.ci/packaging.groovy

@@ -339,14 +339,14 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
   if (isPR()) {
     def version = "pr-${env.CHANGE_ID}"
-    parameters.push(booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true))
     parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
     parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
   }

.ci/scripts/generate_build_table.py

@@ -31,6 +31,7 @@
                     if "withModule" in doc["stages"][stage]:
                         withModule = doc["stages"][stage]["withModule"]
                     if "when" in doc["stages"][stage]:
-                        when = f"optional"
+                        if "not_changeset_full_match" not in doc["stages"][stage]["when"]:
+                            when = "optional"
                     print("| {} | {} | `{}` | {} | `{}` | {} |".format(
                         module, stage, command, withModule, platforms, when))

.go-version

@@ -1 +1 @@
-1.14.12
+1.15.7

CHANGELOG-developer.next.asciidoc

@@ -106,3 +106,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827]
 - Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]
 - Add support for customized monitoring API. {pull}22605[22605]
+- Update Go version to 1.15.7. {pull}22495[22495]

CHANGELOG.next.asciidoc

@@ -100,6 +100,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571]
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
+- Rename `s3` input to `aws-s3` input. {pull}23469[23469]
 
 *Heartbeat*
 - Adds negative body match. {pull}20728[20728]
@@ -229,6 +230,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix typo in config docs {pull}23185[23185]
 - Fix `nested` subfield handling in generated Elasticsearch templates. {issue}23178[23178] {pull}23183[23183]
 - Fix CPU usage metrics on VMs with dynamic CPU config {pull}23154[23154]
+- Add FAQ entry for madvdontneed variable {pull}23429[23429]
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
 - Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
 
@@ -268,8 +270,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix various processing errors in the Suricata module. {pull}23236[23236]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
+- CheckPoint Firewall module: Change event.severity JSON data type to a number because the field mapping is a `long`. {pull}23424[23424]
+- Cisco IOS: Change icmp.type/code and igmp.type JSON data types to strings because the fields mappings are `keyword`. {pull}23424[23424]
+- CrowdStrike Falcon: Change JSON field types to match the field mappings. {pull}23424[23424]
+- Fortinet Firewall: Drop `fortinet.firewall.assignip` when the value is "N/A". {pull}23424[23424]
+- Juniper SRX: Change JSON field types to match the field mappings. {pull}23424[23424]
+- Suricata EVE: Convert `suricata.eve.flow_id` to string because the field is a keyword in the mapping. {pull}23424[23424]
+- Zeek DNS: Ignore failures in data type conversions. And change `dns.id` JSON field to a string to match its `keyword` mapping. {pull}23424[23424]
 - Change the `event.created` in Netflow events to be the time the event was created by Filebeat
   to be consistent with ECS. {pull}23094[23094]
+- Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
 
 *Filebeat*
 
@@ -484,6 +494,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
+- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
+- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
 
 *Packetbeat*
 
@@ -500,6 +512,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+*Elastic Logging Plugin*
+- Fix out of date CLI flags on docs. {pull}23628[23628]
+
 
 ==== Added
 
@@ -579,6 +594,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Honor kube event resysncs to handle missed watch events {pull}22668[22668]
 - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324]
 - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330]
+- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539]
+- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595]
+- Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
+- Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629]
 
 *Auditbeat*
 
@@ -805,6 +824,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added support for first_event context in filebeat httpjson input {pull}23437[23437]
 - Add parsing of tcp flags to AWS vpcflow fileset {issue}228020[22820] {pull}23157[23157]
 - Added `alternative_host` option to google pubsub input {pull}23215[23215]
+- Added username parsing from Cisco ASA message 302013. {pull}21196[21196]
 - Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478]
 - Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521]
 - Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521]
@@ -971,6 +991,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add additional event categorization for security and sysmon modules. {pull}22988[22988]
 - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
 - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
+- Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684]
 
 *Elastic Log Driver*
 
@@ -980,6 +1001,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Selecting `full` in `ssl.verification_mode` option will not treat CommonName field in x509 certificates as
+  a hostname when Subject Alternative Name is not present from v8.0.
+  Please update your certificates so it contains at least one DNSName instead of relying on CommonName in the new
+  major version of Beats.
+
 *Filebeat*
 
 - The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed.

NOTICE.txt

@@ -2216,11 +2216,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/sarama
-Version: v1.19.1-0.20200629123429-0e7b69039eec
+Version: v1.19.1-0.20210120173147-5c8cb347d877
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20200629123429-0e7b69039eec/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20210120173147-5c8cb347d877/LICENSE:
 
 Copyright (c) 2013 Shopify
 
@@ -13708,11 +13708,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/prometheus@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/rcrowley/go-metrics
-Version: v0.0.0-20190826022208-cac0b30c2563
+Version: v0.0.0-20200313005456-10cdbea86bc0
 Licence type (autodetected): BSD-2-Clause-FreeBSD
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/rcrowley/[email protected]20190826022208-cac0b30c2563/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/rcrowley/[email protected]20200313005456-10cdbea86bc0/LICENSE:
 
 Copyright 2012 Richard Crowley. All rights reserved.
 
@@ -16079,11 +16079,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/crypto
-Version: v0.0.0-20200622213623-75b288015ac9
+Version: v0.0.0-20200820211705-5c72a883971a
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200622213623-75b288015ac9/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200820211705-5c72a883971a/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -16153,11 +16153,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
-Version: v0.0.0-20200707034311-ab3426394381
+Version: v0.0.0-20200904194848-62affa334b73
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200707034311-ab3426394381/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200904194848-62affa334b73/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -24968,6 +24968,39 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/creack/pty
+Version: v1.1.9
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/creack/[email protected]/LICENSE:
+
+Copyright (c) 2011 Keith Rarick
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall
+be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/cucumber/godog
 Version: v0.8.1
@@ -27077,11 +27110,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/frankban/quicktest
-Version: v1.7.2
+Version: v1.10.2
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.10.2/LICENSE:
 
 MIT License
 
@@ -35040,11 +35073,11 @@ match.go, match_test.go:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/klauspost/compress
-Version: v1.9.8
+Version: v1.11.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.9.8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.11.0/LICENSE:
 
 Copyright (c) 2012 The Go Authors. All rights reserved.
 Copyright (c) 2019 Klaus Post. All rights reserved.
@@ -35119,11 +35152,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 --------------------------------------------------------------------------------
 Dependency : github.com/kr/pretty
-Version: v0.2.0
+Version: v0.2.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/kr/[email protected].0/License:
+Contents of probable licence file $GOMODCACHE/github.com/kr/[email protected].1/License:
 
 Copyright 2012 Keith Rarick
 
@@ -35181,11 +35214,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/kr/text
-Version: v0.1.0
+Version: v0.2.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.1.0/License:
+Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.2.0/License:
 
 Copyright 2012 Keith Rarick
 
@@ -36662,6 +36695,35 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/niemeyer/pretty
+Version: v0.0.0-20200227124842-a10e7caefd8e
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/niemeyer/[email protected]/License:
+
+Copyright 2012 Keith Rarick
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/onsi/ginkgo
 Version: v1.11.0
@@ -38049,11 +38111,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/pierrec/lz4
-Version: v2.4.1+incompatible
+Version: v2.5.2+incompatible
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.4.1+incompatible/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.5.2+incompatible/LICENSE:
 
 Copyright (c) 2015, Pierre Curto
 All rights reserved.
@@ -41807,11 +41869,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/xerrors
-Version: v0.0.0-20191204190536-9bdfabe68543
+Version: v0.0.0-20200804184101-5ec99f83aff1
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20191204190536-9bdfabe68543/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200804184101-5ec99f83aff1/LICENSE:
 
 Copyright (c) 2019 The Go Authors. All rights reserved.
 
@@ -42122,11 +42184,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : gopkg.in/check.v1
-Version: v1.0.0-20190902080502-41f04d3bba15
+Version: v1.0.0-20200902074654-038fdea0a05b
 Licence type (autodetected): BSD-2-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/gopkg.in/[email protected]20190902080502-41f04d3bba15/LICENSE:
+Contents of probable licence file $GOMODCACHE/gopkg.in/[email protected]20200902074654-038fdea0a05b/LICENSE:
 
 Gocheck - A rich testing framework for Go
 
@@ -43145,11 +43207,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : gopkg.in/yaml.v3
-Version: v3.0.0-20200313102051-9f266ea9e77c
+Version: v3.0.0-20200615113413-eeeca48fe776
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/gopkg.in/[email protected]20200313102051-9f266ea9e77c/LICENSE:
+Contents of probable licence file $GOMODCACHE/gopkg.in/[email protected]20200615113413-eeeca48fe776/LICENSE:
 
 
 This project is covered by two different licenses: MIT and Apache.

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.12
+FROM golang:1.15.7
 
 RUN \
     apt-get update \