Merge branch 'master' of github.com:elastic/beats

CHANGELOG.next.asciidoc

@@ -251,6 +251,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
 - Clone value when copy fields in processors to avoid crash. {issue}19206[19206] {pull}20500[20500]
 - Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696]
+- Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
+- Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
 
 *Heartbeat*
 
@@ -326,6 +328,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add required option for `metrics` in app_insights. {pull}20406[20406]
 - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
 - Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
+- Add fallback for PdhExpandWildCardPathW failing in perfmon metricset. {issue}20139[20139] {pull}20630[20630]
 - Fix resource tags in aws cloudwatch metricset {issue}20326[20326]  {pull}20385[20385]
 - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
 - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
@@ -397,6 +400,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
 - Add leader election for Kubernetes autodiscover. {pull}20281[20281]
 - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
+- Added support for more message types for Cisco ASA and FTD. {pull}20565[20565]
 - Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464]
 
 *Auditbeat*
@@ -655,6 +659,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
 - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
 - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
+- Request prometheus endpoints to be gzipped by default {pull}20766[20766]
 
 *Packetbeat*
 
@@ -667,6 +672,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 - Add basic ECS categorization and `cloud` fields. {pull}19174[19174]
+- Add support for parallelization factor for kinesis. {pull}20727[20727]
 
 *Winlogbeat*
 

filebeat/docs/fields.asciidoc

@@ -20677,6 +20677,116 @@ type: keyword
 The assigned DAP records
 
 
+type: keyword
+
+--
+
+*`cisco.asa.command_line_arguments`*::
++
+--
+The command line arguments logged by the local audit log
+
+
+type: keyword
+
+--
+
+*`cisco.asa.assigned_ip`*::
++
+--
+The IP address assigned to a VPN client successfully connecting
+
+
+type: ip
+
+--
+
+*`cisco.asa.privilege.old`*::
++
+--
+When a users privilege is changed this is the old value
+
+
+type: keyword
+
+--
+
+*`cisco.asa.privilege.new`*::
++
+--
+When a users privilege is changed this is the new value
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.object`*::
++
+--
+The related object for burst warnings
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.id`*::
++
+--
+The related rate ID for burst warnings
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.current_rate`*::
++
+--
+The current burst rate seen
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.configured_rate`*::
++
+--
+The current configured burst rate
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.avg_rate`*::
++
+--
+The current average burst rate seen
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.configured_avg_rate`*::
++
+--
+The current configured average burst rate allowed
+
+
+type: keyword
+
+--
+
+*`cisco.asa.burst.cumulative_count`*::
++
+--
+The total count of burst rate hits since the object was created or cleared
+
+
 type: keyword
 
 --

filebeat/module/auditd/log/ingest/pipeline.yml

@@ -141,24 +141,44 @@ processors:
     value: event
 - set:
     if: "ctx.auditd.log?.record_type == 'USER_AUTH'"
-    field: event.type
+    field: event.category
     value: authentication
 - set:
-    if: "ctx.auditd.log?.record_type == 'KERN_MODULE'"
+    if: "ctx.auditd.log?.record_type == 'USER_AUTH'"
     field: event.type
+    value: info
+- set:
+    if: "ctx.auditd.log?.record_type == 'KERN_MODULE'"
+    field: event.category
     value: driver
 - set:
-    if: "ctx.auditd.log?.record_type == 'SOFTWARE_UPDATE'"
+    if: "ctx.auditd.log?.record_type == 'KERN_MODULE'"
     field: event.type
+    value: info
+- set:
+    if: "ctx.auditd.log?.record_type == 'SOFTWARE_UPDATE'"
+    field: event.category
     value: package
 - set:
-    if: "ctx.auditd.log?.record_type == 'SYSTEM_BOOT' || ctx.auditd.log?.record_type == 'SYSTEM_SHUTDOWN'"
+    if: "ctx.auditd.log?.record_type == 'SOFTWARE_UPDATE'"
     field: event.type
+    value: info
+- set:
+    if: "ctx.auditd.log?.record_type == 'SYSTEM_BOOT' || ctx.auditd.log?.record_type == 'SYSTEM_SHUTDOWN'"
+    field: event.category
     value: host
 - set:
-    if: "ctx.auditd.log?.record_type == 'SYSCALL' && ctx.auditd.log?.syscall == 'execve'"
+    if: "ctx.auditd.log?.record_type == 'SYSTEM_BOOT' || ctx.auditd.log?.record_type == 'SYSTEM_SHUTDOWN'"
     field: event.type
+    value: info
+- set:
+    if: "ctx.auditd.log?.record_type == 'SYSCALL' && ctx.auditd.log?.syscall == 'execve'"
+    field: event.category
     value: process
+- set:
+    if: "ctx.auditd.log?.record_type == 'SYSCALL' && ctx.auditd.log?.syscall == 'execve'"
+    field: event.type
+    value: info
 - set:
     if: "ctx.auditd.log?.record_type == 'VIRT_CONTROL' || ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID'"
     field: event.category

filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json

@@ -89,11 +89,12 @@
         "auditd.log.ses": "4294967295",
         "auditd.log.subj": "system_u:system_r:init_t:s0",
         "event.action": "system_boot",
+        "event.category": "host",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "host",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 862,

filebeat/module/auditd/log/test/audit-rhel6.log-expected.json

@@ -212,11 +212,12 @@
         "auditd.log.sequence": 19623789,
         "auditd.log.ses": "6793",
         "event.action": "user_auth",
+        "event.category": "authentication",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "authentication",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1926,
@@ -234,11 +235,12 @@
         "auditd.log.sequence": 19623807,
         "auditd.log.ses": "12286",
         "event.action": "user_auth",
+        "event.category": "authentication",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "authentication",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 2122,

filebeat/module/auditd/log/test/audit-rhel7.log-expected.json

@@ -45,11 +45,12 @@
         "auditd.log.ses": "4294967295",
         "auditd.log.subj": "system_u:system_r:init_t:s0",
         "event.action": "system_boot",
+        "event.category": "host",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "host",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 419,

filebeat/module/auditd/log/test/test.log-expected.json

@@ -167,11 +167,12 @@
         "auditd.log.sw": "gcc-4.8.5-39.el7.x86_64",
         "auditd.log.sw_type": "rpm",
         "event.action": "software_update",
+        "event.category": "package",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "package",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1893,
@@ -188,11 +189,12 @@
         "auditd.log.ses": "4294967295",
         "auditd.log.subj": "system_u:system_r:init_t:s0",
         "event.action": "system_boot",
+        "event.category": "host",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "host",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 2196,
@@ -210,11 +212,12 @@
         "auditd.log.ses": "4294967295",
         "auditd.log.subj": "system_u:system_r:init_t:s0",
         "event.action": "system_shutdown",
+        "event.category": "host",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
-        "event.type": "host",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 2438,
@@ -254,10 +257,11 @@
         "auditd.log.syscall": "execve",
         "auditd.log.tty": "pts0",
         "event.action": "syscall",
+        "event.category": "process",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
-        "event.type": "process",
+        "event.type": "info",
         "fileset.name": "log",
         "host.architecture": "x86_64",
         "input.type": "log",
@@ -283,10 +287,11 @@
         "auditd.log.name": "mymodule",
         "auditd.log.sequence": 579397,
         "event.action": "kern_module",
+        "event.category": "driver",
         "event.dataset": "auditd.log",
         "event.kind": "event",
         "event.module": "auditd",
-        "event.type": "driver",
+        "event.type": "info",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 3153,

filebeat/registrar/migrate.go

@@ -31,7 +31,6 @@ import (
 	helper "github.com/elastic/beats/v7/libbeat/common/file"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/paths"
-	"github.com/elastic/beats/v7/libbeat/statestore"
 	"github.com/elastic/beats/v7/libbeat/statestore/backend/memlog"
 )
 
@@ -214,17 +213,15 @@ func (m *Migrator) updateToVersion1(regHome string) error {
 	registryBackend, err := memlog.New(logp.NewLogger("migration"), memlog.Settings{
 		Root:               m.dataPath,
 		FileMode:           m.permissions,
-		Checkpoint:         func(_ uint64) bool { return true },
+		Checkpoint:         func(sz uint64) bool { return false },
 		IgnoreVersionCheck: true,
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create new registry backend")
 	}
+	defer registryBackend.Close()
 
-	reg := statestore.NewRegistry(registryBackend)
-	defer reg.Close()
-
-	store, err := reg.Get("filebeat")
+	store, err := registryBackend.Access("filebeat")
 	if err != nil {
 		return errors.Wrap(err, "failed to open filebeat registry store")
 	}
@@ -234,6 +231,13 @@ func (m *Migrator) updateToVersion1(regHome string) error {
 		return errors.Wrap(err, "failed to migrate registry states")
 	}
 
+	if checkpointer, ok := store.(interface{ Checkpoint() error }); ok {
+		err := checkpointer.Checkpoint()
+		if err != nil {
+			return fmt.Errorf("failed to fsync filebeat storage state: %w", err)
+		}
+	}
+
 	if err := os.Remove(origDataFile); err != nil {
 		return errors.Wrapf(err, "migration complete but failed to remove original data file: %v", origDataFile)
 	}