Merge branch 'master' into C0FFEEC0FFEE-patch-1

.ci/packaging.groovy

@@ -339,14 +339,14 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
   if (isPR()) {
     def version = "pr-${env.CHANGE_ID}"
-    parameters.push(booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true))
     parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
     parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
   }

.ci/schedule-weekly.groovy

@@ -20,8 +20,8 @@ pipeline {
   stages {
     stage('Nighly beats builds') {
       steps {
-        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)])
-        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)])
+        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
       }
     }
   }

.ci/scripts/install-tools.bat

@@ -79,4 +79,4 @@ gcc --version
 where gcc
 
 REM Reset the USERPROFILE
-SET USERPROFILE=%PREVIOUS_USERPROFILE%
+SET USERPROFILE=%PREVIOUS_USERPROFILE%

.go-version

@@ -1 +1 @@
-1.14.12
+1.15.7

CHANGELOG-developer.next.asciidoc

@@ -106,3 +106,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827]
 - Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]
 - Add support for customized monitoring API. {pull}22605[22605]
+- Update Go version to 1.15.7. {pull}22495[22495]

CHANGELOG.asciidoc

@@ -3,6 +3,22 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.10.2]]
+=== Beats version 7.10.2
+https://github.com/elastic/beats/compare/v7.10.1\...v7.10.2[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Add JSON body check for SQS message. {pull}21727[21727]
+- Fix cisco umbrella module config by adding input variable. {pull}22892[22892]
+- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
+
+*Metricbeat*
+
+- Change `vsphere.datastore.capacity.used.pct` value to betweeen 0 and 1. {pull}23148[23148]
+
 [[release-notes-7.10.1]]
 === Beats version 7.10.1
 https://github.com/elastic/beats/compare/v7.10.0\...v7.10.1[View commits]

CHANGELOG.next.asciidoc

@@ -100,6 +100,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571]
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
+- Rename `s3` input to `aws-s3` input. {pull}23469[23469]
 
 *Heartbeat*
 - Adds negative body match. {pull}20728[20728]
@@ -230,6 +231,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `nested` subfield handling in generated Elasticsearch templates. {issue}23178[23178] {pull}23183[23183]
 - Fix CPU usage metrics on VMs with dynamic CPU config {pull}23154[23154]
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
+- Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
 
 *Auditbeat*
 
@@ -267,6 +269,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix various processing errors in the Suricata module. {pull}23236[23236]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
+- Change the `event.created` in Netflow events to be the time the event was created by Filebeat
+  to be consistent with ECS. {pull}23094[23094]
+- Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
 
 *Filebeat*
 
@@ -363,6 +368,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for organization and custom prefix in AWS/CloudTrail fileset. {issue}23109[23109] {pull}23126[23126]
 - Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204]
 - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273]
+- Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534]
 
 *Heartbeat*
 
@@ -479,9 +485,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove io.time from windows {pull}22237[22237]
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
+- Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
+- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 
 *Packetbeat*
 
+- Fix SIP parser logic related to line length check. {pull}23411[23411]
+
 
 *Winlogbeat*
 
@@ -493,6 +503,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+*Elastic Logging Plugin*
+- Fix out of date CLI flags on docs. {pull}23628[23628]
+
 
 ==== Added
 
@@ -572,6 +585,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Honor kube event resysncs to handle missed watch events {pull}22668[22668]
 - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324]
 - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330]
+- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539]
+- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595]
+- Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
+- Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629]
 
 *Auditbeat*
 
@@ -796,8 +813,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Google Workspace module and mark Gsuite module as deprecated {pull}22950[22950]
 - Mark m365 defender, defender atp, okta and google workspace modules as GA {pull}23113[23113]
 - Added support for first_event context in filebeat httpjson input {pull}23437[23437]
+- Add parsing of tcp flags to AWS vpcflow fileset {issue}228020[22820] {pull}23157[23157]
 - Added `alternative_host` option to google pubsub input {pull}23215[23215]
 - Added username parsing from Cisco ASA message 302013. {pull}21196[21196]
+- Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478]
+- Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521]
+- Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521]
 
 *Heartbeat*
 
@@ -961,6 +982,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add additional event categorization for security and sysmon modules. {pull}22988[22988]
 - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
 - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
+- Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684]
 
 *Elastic Log Driver*
 
@@ -970,6 +992,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Selecting `full` in `ssl.verification_mode` option will not treat CommonName field in x509 certificates as
+  a hostname when Subject Alternative Name is not present from v8.0.
+  Please update your certificates so it contains at least one DNSName instead of relying on CommonName in the new
+  major version of Beats.
+
 *Filebeat*
 
 - The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed.

Jenkinsfile

@@ -80,6 +80,7 @@ pipeline {
             whenTrue(env.ONLY_DOCS == 'false') {
               cmd(label: "make check-python", script: "make check-python")
               cmd(label: "make check-go", script: "make check-go")
+              cmd(label: "make notice", script: "make notice")
               cmd(label: "Check for changes", script: "make check-no-changes")
             }
           }