Merge branch 'master' into kubernetes-module-metadata

CHANGELOG.asciidoc

@@ -48,6 +48,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - RabbitMQ node metricset only collects metrics of the instance it connects to, `node.collect: cluster` can be used to collect all nodes as before. {issue}6556[6556] {pull}6971[6971]
 - Change http/server metricset to put events by default under http.server and prefix config options with server.. {pull}7100[7100]
 - Disable dedotting in docker module configuration. This will change the out-of-the-box behaviour, but not the one of already configured instances. {pull}7485[7485]
+- Fix typo in etcd/self metricset fields from *.bandwithrate to *.bandwidthrate. {pull}7456[7456]
 
 *Packetbeat*
 
@@ -77,22 +78,23 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - When we fail to build a Kubernetes' indexer or matcher we produce a warning but we don't add them to the execution. {pull}7466[7466]
 - Fix default value for logging.files.keepfiles. It was being set to 0 and now
   it's set to the documented value of 7. {issue}7494[7494]
+- Retain compatibility with older Docker server versions. {issue}7542[7542]
 
 *Auditbeat*
 
 - Add hex decoding for the name field in audit path records. {pull}6687[6687]
 - Fixed a deadlock in the file_integrity module under Windows. {issue}6864[6864]
 - Fixed parsing of AppArmor audit messages. {pull}6978[6978]
 - Allow `auditbeat setup` to run without requiring elevated privileges for the audit client. {issue}7111[7111]
-- Fix goroutine leak that occured when the auditd module was stopped. {pull}7163[7163]
+- Fix goroutine leak that occurred when the auditd module was stopped. {pull}7163[7163]
 
 *Filebeat*
 
 - Fix panic when log prospector configuration fails to load. {issue}6800[6800]
 - Fix memory leak in log prospector when files cannot be read. {issue}6797[6797]
 - Add raw JSON to message field when JSON parsing fails. {issue}6516[6516]
 - Commit registry writes to stable storage to avoid corrupt registry files. {issue}6792[6792]
-- Fix a data race between stopping and starting of the harverters. {issue}#6879[6879]
+- Fix a data race between stopping and starting of the harvesters. {issue}#6879[6879]
 - Fix a parsing issue in the syslog input for RFC3339 timestamp and time with nanoseconds. {pull}7046[7046]
 - Comply with PostgreSQL database name format {pull}7198[7198]
 - Fix an issue with an overflowing wait group when using the TCP input. {issue}7202[7202]
@@ -152,7 +154,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add `has_fields` conditional to filter events based on the existence of all the given fields. {issue}6285[6285] {pull}6653[6653]
 - Add support for spooling to disk to the beats event publishing pipeline. {pull}6581[6581]
 - Added logging of system info at Beat startup. {issue}5946[5946]
-- Do not log errors if X-Pack Monitoring is enabled but Elastisearch X-Pack is not. {pull}6627[6627]
+- Do not log errors if X-Pack Monitoring is enabled but Elasticsearch X-Pack is not. {pull}6627[6627]
 - Add rename processor. {pull}6292[6292]
 - Add IP-addresses and MAC-addresses to add_host_metadata. {pull}6878[6878]
 - Added a seccomp (secure computing) filter on Linux that whitelists the
@@ -197,7 +199,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add Logstash module support for main log and the slow log, support the plain text or structured JSON format {pull}5481[5481]
 - Add stream filtering when using `docker` prospector. {pull}6057[6057]
 - Add support for CRI logs format. {issue}5630[5630]
-- Add json.ignore_decoding_error config to not log json decoding erors. {issue}6547[6547]
+- Add json.ignore_decoding_error config to not log json decoding errors. {issue}6547[6547]
 - Make registry file permission configurable. {pull}6455[6455]
 - Add MongoDB module. {pull}6283[6238]
 - Add Ingest pipeline loading to setup. {pull}6814[6814]
@@ -208,7 +210,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Support MySQL 5.7.19 by mysql/slowlog {pull}6969[6969]
 - Correctly join partial log lines when using `docker` input. {pull}6967[6967]
 - Add support for TLS with client authentication to the TCP input {pull}7056[7056]
-- Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
+- Converted part of pipeline from treafik/access metricSet to dissect to improve efficiency. {pull}7209[7209]
 - Add GC fileset to the Elasticsearch module. {pull}7305[7305]
 - Add Audit log fileset to the Elasticsearch module. {pull}7365[7365]
 - Add Slow log fileset to the Elasticsearch module. {pull}7473[7473]
@@ -298,6 +300,8 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Collect accumulated docker network metrics and mark old ones as deprecated. {pull}7253[7253]
 - Add TLS support to MongoDB module. {pull}7401[7401]
 - Added Traefik module with health metricset. {pull}7413[7413]
+- Add Elasticsearch ml_job metricsets. {pull}7196[7196]
+- Add support for bearer token files to HTTP helper. {pull}7527[7527]
 
 *Packetbeat*
 
@@ -408,7 +412,7 @@ https://github.com/elastic/beats/compare/v6.1.3...v6.2.0[View commits]
 *Auditbeat*
 
 - Fixed an issue where the proctitle value was being truncated. {pull}6080[6080]
-- Fixed an issue where values were incorrectly interpretted as hex data. {pull}6080[6080]
+- Fixed an issue where values were incorrectly interpreted as hex data. {pull}6080[6080]
 - Fixed parsing of the `key` value when multiple keys are present. {pull}6080[6080]
 - Fix possible resource leak if file_integrity module is used with config
   reloading on Windows or Linux. {pull}6198[6198]
@@ -447,7 +451,7 @@ https://github.com/elastic/beats/compare/v6.1.3...v6.2.0[View commits]
 - Use structured logging for the metrics that are periodically logged via the
   `logging.metrics` feature. {pull}5915[5915]
 - Improve Elasticsearch output metrics to count number of dropped and duplicate (if event ID is given) events. {pull}5811[5811]
-- Add the abilility for the add_docker_metadata process to enrich based on process ID. {pull}6100[6100]
+- Add the ability for the add_docker_metadata process to enrich based on process ID. {pull}6100[6100]
 - The `add_docker_metadata` and `add_kubernetes_metadata` processors are now GA, instead of Beta. {pull}6105[6105]
 - Update go-ucfg library to support top level key reference and cyclic key reference for the
   keystore {pull}6098[6098]
@@ -473,7 +477,7 @@ https://github.com/elastic/beats/compare/v6.1.3...v6.2.0[View commits]
 - Update the MySQL dashboard to use the Time Series Visual Builder. {pull}5996[5996]
 - Add experimental uwsgi module. {pull}6006[6006]
 - Docker and Kubernetes modules are now GA, instead of Beta. {pull}6105[6105]
-- Support haproxy stats gathering using http (additionaly to tcp socket). {pull}5819[5819]
+- Support haproxy stats gathering using http (additionally to tcp socket). {pull}5819[5819]
 - Support to optionally 'de dot' keys in http/json metricset to prevent collisions. {pull}5957[5957]
 
 *Packetbeat*
@@ -576,7 +580,7 @@ https://github.com/elastic/beats/compare/v6.0.1...v6.1.0[View commits]
 
 *Affecting all Beats*
 
-- Support dashboard loading without Elasticseach {pull}5653[5653]
+- Support dashboard loading without Elasticsearch {pull}5653[5653]
 - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051]
 - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051]
 - Fix reloader error message to only print on actual error {pull}5066[5066]
@@ -817,7 +821,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...v6.0.0-beta2[View commit
 
 - Added missing mongodb configuration file to the `modules.d` folder. {pull}4870[4870]
 - Fix wrong MySQL CRUD queries timelion visualization {pull}4857[4857]
-- Add new metrics to CPU metricsset {pull}4969[4969]
+- Add new metrics to CPU metricset {pull}4969[4969]
 
 *Packetbeat*
 
@@ -1502,7 +1506,7 @@ https://github.com/elastic/beats/compare/v5.2.2...v5.3.0[View commits]
 - Add Filebeat modules for system, apache2, mysql, and nginx. {issue}3159[3159]
 - Add the `pipeline` config option at the prospector level, for configuring the Ingest Node pipeline ID. {pull}3433[3433]
 - Update regular expressions used for matching file names or lines (multiline, include/exclude functionality) to new matchers improving performance of simple string matches. {pull}3469[3469]
-- The `symlinks` and `harverster_limit` settings are now GA, instead of experimental. {pull}3525[3525]
+- The `symlinks` and `harvester_limit` settings are now GA, instead of experimental. {pull}3525[3525]
 - close_timeout is also applied when the output is blocking. {pull}3511[3511]
 - Improve handling of different path variants on Windows. {pull}3781[3781]
 - Add multiline.flush_pattern option, for specifying the 'end' of a multiline pattern {pull}4019[4019]
@@ -1777,7 +1781,7 @@ The list below covers the changes between 5.0.0-rc1 and 5.0.0 GA only.
 
 - Fix high CPU usage on macOS when encountering processes with long command lines. {issue}2747[2747]
 - Fix high value of `system.memory.actual.free` and `system.memory.actual.used`. {issue}2653[2653]
-- Change several `OpenProcess` calls on Windows to request the lowest possible access provilege.  {issue}1897[1897]
+- Change several `OpenProcess` calls on Windows to request the lowest possible access privilege.  {issue}1897[1897]
 - Fix system.memory.actual.free high value on Windows. {issue}2653[2653]
 
 *Filebeat*
@@ -2035,7 +2039,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...v5.0.0-alpha4[View comm
 
 *Affecting all Beats*
 
-- The topology_expire option of the Elasticserach output was removed. {pull}1907[1907]
+- The topology_expire option of the Elasticsearch output was removed. {pull}1907[1907]
 
 *Filebeat*
 
@@ -2532,7 +2536,7 @@ https://github.com/elastic/beats/compare/v1.0.1...v1.1.0[View commits]
 - Add multiline support for combining multiple related lines into one event. {issue}461[461]
 - Add `exclude_lines` and `include_lines` options for regexp based line filtering. {pull}430[430]
 - Add `exclude_files` configuration option. {pull}563[563]
-- Add experimental option to enable filebeat publisher pipeline to operate asynchonrously {pull}782[782]
+- Add experimental option to enable filebeat publisher pipeline to operate asynchronously {pull}782[782]
 
 *Winlogbeat*
 
@@ -2573,7 +2577,7 @@ https://github.com/elastic/beats/compare/1.0.0-rc2...1.0.0[Check 1.0.0 diff]
 *Affecting all Beats*
 
 - Fix random panic on shutdown by calling shutdown handler only once. elastic/filebeat#204
-- Fix credentials are not send when pinging an elasticsearch host. elastic/fileabeat#287
+- Fix credentials are not send when pinging an elasticsearch host. elastic/filebeat#287
 
 *Filebeat*
 

Vagrantfile

@@ -11,7 +11,7 @@
 # Usage and Features:
 #   - Two users exist: Administrator and Vagrant. Both have the password: vagrant
 #   - Use 'vagrant ssh' to open a Windows command prompt.
-#   - Use 'vagrant rdp' to open a Windows Remote Deskop session. Mac users must
+#   - Use 'vagrant rdp' to open a Windows Remote Desktop session. Mac users must
 #     install the Microsoft Remote Desktop Client from the App Store.
 #   - There is a desktop shortcut labeled "Beats Shell" that opens a command prompt
 #     to C:\Gopath\src\github.com\elastic\beats where the code is mounted.
@@ -54,9 +54,9 @@ $Shortcut.WorkingDirectory = "C:\\Gopath\\src\\github.com\\elastic\\beats"
 $Shortcut.Save()
 
 echo "Disable automatic updates"
-$AUSettigns = (New-Object -com "Microsoft.Update.AutoUpdate").Settings
-$AUSettigns.NotificationLevel = 1
-$AUSettigns.Save()
+$AUSettings = (New-Object -com "Microsoft.Update.AutoUpdate").Settings
+$AUSettings.NotificationLevel = 1
+$AUSettings.Save()
 SCRIPT
 
 # Provisioning for Unix/Linux

auditbeat/auditbeat.reference.yml

@@ -38,6 +38,8 @@ auditbeat.modules:
   rate_limit: 0
   include_raw_message: false
   include_warnings: false
+  # Load audit rules from separate files. Same format as audit.rules(7).
+  audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ]
   audit_rules: |
     ## Define audit rules here.
     ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these

auditbeat/auditbeat.yml

@@ -11,6 +11,8 @@
 auditbeat.modules:
 
 - module: auditd
+  # Load audit rules from separate files. Same format as audit.rules(7).
+  audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ]
   audit_rules: |
     ## Define audit rules here.
     ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these

auditbeat/docs/modules/auditd.asciidoc

@@ -141,6 +141,11 @@ embedded in the string using `#` as a prefix. The format for rules is the same
 used by the Linux `auditctl` utility. {beatname_uc} supports adding file watches
 (`-w`) and syscall rules (`-a` or `-A`).
 
+*`audit_rule_files`*:: A list of files to load audit rules from. This files are
+loaded after the rules declared in `audit_rules` are loaded. Wildcards are
+supported and will expand in lexicographical order. The format is the same as
+that of the `audit_rules` field.
+
 [float]
 === Audit rules
 
@@ -193,6 +198,8 @@ is an example configuration:
 ----
 auditbeat.modules:
 - module: auditd
+  # Load audit rules from separate files. Same format as audit.rules(7).
+  audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ]
   audit_rules: |
     ## Define audit rules here.
     ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these

auditbeat/docs/setting-up-running.asciidoc

@@ -4,7 +4,7 @@
 // that is unique to each beat.
 /////
 
-[[seting-up-and-running]]
+[[setting-up-and-running]]
 == Setting up and running {beatname_uc}
 
 Before reading this section, see the

auditbeat/magefile.go

@@ -135,6 +135,9 @@ func makeConfigTemplates() error {
 
 // customizePackaging modifies the package specs to use templated config files
 // instead of the defaults.
+//
+// Customizations specific to Auditbeat:
+// - Include audit.rules.d directory in packages.
 func customizePackaging() {
 	var (
 		shortConfig = mage.PackageFile{
@@ -149,6 +152,26 @@ func customizePackaging() {
 		}
 	)
 
+	archiveRulesDir := "audit.rules.d"
+	linuxPkgRulesDir := "/etc/{{.BeatName}}/audit.rules.d"
+	rulesSrcDir := "module/auditd/_meta/audit.rules.d"
+	sampleRules := mage.PackageFile{
+		Mode:   0644,
+		Source: rulesSrcDir,
+		Dep: func(spec mage.PackageSpec) error {
+			if spec.OS == "linux" {
+				params := map[string]interface{}{
+					"ArchBits": archBits,
+				}
+				rulesFile := spec.MustExpand(rulesSrcDir+"/sample-rules-linux-{{call .ArchBits .GOARCH}}bit.conf", params)
+				if err := mage.Copy(rulesFile, spec.MustExpand("{{.PackageDir}}/audit.rules.d/sample-rules.conf.disabled")); err != nil {
+					return errors.Wrap(err, "failed to copy sample rules")
+				}
+			}
+			return nil
+		},
+	}
+
 	for _, args := range mage.Packages {
 		pkgType := args.Types[0]
 		switch pkgType {
@@ -161,6 +184,13 @@ func customizePackaging() {
 		default:
 			panic(errors.Errorf("unhandled package type: %v", pkgType))
 		}
+		if args.OS == "linux" {
+			rulesDest := archiveRulesDir
+			if pkgType != mage.TarGz {
+				rulesDest = linuxPkgRulesDir
+			}
+			args.Spec.Files[rulesDest] = sampleRules
+		}
 	}
 }
 

auditbeat/module/auditd/_meta/audit.rules.d/sample-rules-linux-32bit.conf

@@ -0,0 +1,14 @@
+## Executions.
+-a always,exit -F arch=b32 -S execve,execveat -k exec
+
+## External access (warning: these can be expensive to audit).
+-a always,exit -F arch=b32 -S accept,bind,connect -F key=external-access
+
+## Identity changes.
+-w /etc/group -p wa -k identity
+-w /etc/passwd -p wa -k identity
+-w /etc/gshadow -p wa -k identity
+
+## Unauthorized access attempts.
+-a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access
+-a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access

auditbeat/module/auditd/_meta/audit.rules.d/sample-rules-linux-64bit.conf

@@ -0,0 +1,20 @@
+## If you are on a 64 bit platform, everything should be running
+## in 64 bit mode. This rule will detect any use of the 32 bit syscalls
+## because this might be a sign of someone exploiting a hole in the 32
+## bit API.
+-a always,exit -F arch=b32 -S all -F key=32bit-abi
+
+## Executions.
+-a always,exit -F arch=b64 -S execve,execveat -k exec
+
+## External access (warning: these can be expensive to audit).
+-a always,exit -F arch=b64 -S accept,bind,connect -F key=external-access
+
+## Identity changes.
+-w /etc/group -p wa -k identity
+-w /etc/passwd -p wa -k identity
+-w /etc/gshadow -p wa -k identity
+
+## Unauthorized access attempts.
+-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access
+-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access

auditbeat/module/auditd/_meta/config.yml.tmpl

@@ -12,6 +12,8 @@
   include_raw_message: false
   include_warnings: false
   {{ end -}}
+  # Load audit rules from separate files. Same format as audit.rules(7).
+  audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ]
   audit_rules: |
     ## Define audit rules here.
     ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these

auditbeat/module/auditd/_meta/docs.asciidoc

@@ -136,6 +136,11 @@ embedded in the string using `#` as a prefix. The format for rules is the same
 used by the Linux `auditctl` utility. {beatname_uc} supports adding file watches
 (`-w`) and syscall rules (`-a` or `-A`).
 
+*`audit_rule_files`*:: A list of files to load audit rules from. This files are
+loaded after the rules declared in `audit_rules` are loaded. Wildcards are
+supported and will expand in lexicographical order. The format is the same as
+that of the `audit_rules` field.
+
 [float]
 === Audit rules
 

auditbeat/module/auditd/audit_linux.go

@@ -202,10 +202,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 }
 
 func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error {
-	rules, err := ms.config.rules()
-	if err != nil {
-		return errors.Wrap(err, "failed to add rules")
-	}
+	rules := ms.config.rules()
 
 	if len(rules) == 0 {
 		ms.log.Info("No audit_rules were specified.")
@@ -793,7 +790,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) {
 		}
 		return c.SocketType, nil
 	}
-	rules, _ := c.rules()
+	rules := c.rules()
 
 	isLocked := status.Enabled == auditLocked
 	hasMulticast := hasMulticastSupport()