Merge pull request elastic#6615 from tsg/automatic_merge_from_master_…

…to_6.x_branch

Automatic merge from master to 6.x branch

CHANGELOG.asciidoc

@@ -24,6 +24,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - The default value for pipelining is reduced to 2 to avoid high memory in the Logstash beats input. {pull}6250[6250]
 - Add logging when monitoring cannot connect to Elasticsearch. {pull}6365[6365]
 - Rename beat.cpu.*.time metrics to beat.cpu.*.time.ms. {pull}6449[6449]
+- Mark `system.syslog.message` and `system.auth.message` as `text` instead of `keyword`. {pull}6589[6589]
 
 *Auditbeat*
 
@@ -65,6 +66,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add filtering option by exact device names in system.diskio. `diskio.include_devices`. {pull}6085[6085]
 - Fix dealing with new process status codes in Linux kernel 4.14+. {pull}6306[6306]
 - Add config option for windows/perfmon metricset to ignore non existent counters. {pull}6432[6432]
+- Support apache status pages for versions older than 2.4.16. {pull}6450[6450]
 
 *Packetbeat*
 
@@ -143,6 +145,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix namespace disambiguation in Kubernetes state_* metricsets. {issue}6281[6281]
 - Fix Kubernetes overview dashboard views for non default time ranges. {issue}6395{6395}
 - Fix Windows perfmon metricset so that it sends metrics when an error occurs. {pull}6542[6542]
+- Fix Kubernetes calculated fields store. {pull}6564{6564}
 
 *Packetbeat*
 
@@ -152,6 +155,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix corruption when parsing repeated headers in an HTTP request or response. {pull}6325[6325]
 - Fix panic when parsing partial AMQP messages. {pull}6384[6384]
 - Fix out of bounds access to slice in MongoDB parser. {pull}6256[6256]
+- Fix sniffer hanging on exit under Linux. {pull}6535[6535]
 
 *Winlogbeat*
 
@@ -194,6 +198,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Experimental feature setup.template.append_fields added. {pull}6024[6024]
 - Add appender support to autodiscover {pull}6469[6469]
 - Add add_host_metadata processor {pull}5968[5968]
+- Retry configuration to load dashboards if Kibana is not reachable when the beat starts. {pull}6560[6560]
 
 *Auditbeat*
 
@@ -251,6 +256,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Making the jolokia/jmx module GA. {pull}6143[6143]
 - Making the MongoDB module GA. {pull}6554[6554]
 - Allow to disable labels `dedot` in Docker module, in favor of a safe way to keep dots. {pull}6490[6490]
+- Add experimental module to collect metrics from munin nodes. {pull}6517[6517]
 
 *Packetbeat*
 

NOTICE.txt

@@ -3107,7 +3107,7 @@ OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: github.com/tsg/gopacket
-Revision: 8e703b9968693c15f25cabb6ba8be4370cf431d0
+Revision: f289b3ea3e41a01b2822be9caf5f40c01fdda05c
 License type (autodetected): BSD-3-Clause
 ./vendor/github.com/tsg/gopacket/LICENSE:
 --------------------------------------------------------------------
@@ -3326,7 +3326,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/net
-Revision: e90d6d0afc4c315a0d87a568ae68577cc15149a0
+Version: release-branch.go1.9
+Revision: 44b7c21cbf19450f38b337eb6b6fe4f6496fb5b3
 License type (autodetected): BSD-3-Clause
 ./vendor/golang.org/x/net/LICENSE:
 --------------------------------------------------------------------

auditbeat/auditbeat.reference.yml

@@ -744,6 +744,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

auditbeat/docs/fields.asciidoc

@@ -2563,23 +2563,23 @@ Host architecture (e.g. x86_64, arm, ppc, mips).
 [float]
 === `host.os.platform`
 
-type: object
+type: keyword
 
 OS platform (e.g. centos, ubuntu, windows).
 
 
 [float]
 === `host.os.version`
 
-type: object
+type: keyword
 
 OS version.
 
 
 [float]
 === `host.os.family`
 
-type: object
+type: keyword
 
 OS family (e.g. redhat, debian, freebsd, windows).
 

filebeat/docs/fields.asciidoc

@@ -683,23 +683,23 @@ Host architecture (e.g. x86_64, arm, ppc, mips).
 [float]
 === `host.os.platform`
 
-type: object
+type: keyword
 
 OS platform (e.g. centos, ubuntu, windows).
 
 
 [float]
 === `host.os.version`
 
-type: object
+type: keyword
 
 OS version.
 
 
 [float]
 === `host.os.family`
 
-type: object
+type: keyword
 
 OS family (e.g. redhat, debian, freebsd, windows).
 
@@ -2352,6 +2352,8 @@ The PID of the process that sent the auth message.
 [float]
 === `system.auth.message`
 
+type: text
+
 The message in the log line.
 
 
@@ -2590,6 +2592,8 @@ The PID of the process that sent the syslog message.
 [float]
 === `system.syslog.message`
 
+type: text
+
 The message in the log line.
 
 

filebeat/filebeat.reference.yml

@@ -1223,6 +1223,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

filebeat/module/system/auth/_meta/fields.yml

@@ -17,6 +17,7 @@
       description: >
         The PID of the process that sent the auth message.
     - name: message
+      type: text
       description: >
         The message in the log line.
     - name: user

filebeat/module/system/syslog/_meta/fields.yml

@@ -16,5 +16,6 @@
       description: >
         The PID of the process that sent the syslog message.
     - name: message
+      type: text
       description: >
         The message in the log line.

heartbeat/docs/fields.asciidoc

@@ -332,23 +332,23 @@ Host architecture (e.g. x86_64, arm, ppc, mips).
 [float]
 === `host.os.platform`
 
-type: object
+type: keyword
 
 OS platform (e.g. centos, ubuntu, windows).
 
 
 [float]
 === `host.os.version`
 
-type: object
+type: keyword
 
 OS version.
 
 
 [float]
 === `host.os.family`
 
-type: object
+type: keyword
 
 OS family (e.g. redhat, debian, freebsd, windows).
 

heartbeat/heartbeat.reference.yml

@@ -853,6 +853,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

libbeat/_meta/config.reference.yml

@@ -639,6 +639,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

libbeat/autodiscover/appenders/registry.go

@@ -23,7 +23,7 @@ func init() {
 	p.MustRegisterLoader(pluginKey, func(ifc interface{}) error {
 		app, ok := ifc.(appenderPlugin)
 		if !ok {
-			return errors.New("plugin does not match processor plugin type")
+			return errors.New("plugin does not match appender plugin type")
 		}
 
 		return autodiscover.Registry.AddAppender(app.name, app.appender)

libbeat/autodiscover/builder.go

@@ -63,7 +63,7 @@ func (r *registry) BuildBuilder(c *common.Config) (Builder, error) {
 
 	builder := r.GetBuilder(config.Type)
 	if builder == nil {
-		return nil, fmt.Errorf("Unknown autodiscover builder %s", config.Type)
+		return nil, fmt.Errorf("unknown autodiscover builder %s", config.Type)
 	}
 
 	return builder(c)

libbeat/autodiscover/builder/helper.go

@@ -77,30 +77,39 @@ func IsNoOp(hints common.MapStr, key string) bool {
 // GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats.
 func GenerateHints(annotations common.MapStr, container, prefix string) common.MapStr {
 	hints := common.MapStr{}
-	plen := len(prefix)
-
-	for key, value := range annotations {
-		// Filter out all annotations which start with the prefix
-		if strings.Index(key, prefix) == 0 {
-			subKey := key[plen:]
-			// Split an annotation by /. Ex co.elastic.metrics/module would split to ["metrics", "module"]
-			// part[0] would give the type of config and part[1] would give the config entry
-			parts := strings.Split(subKey, "/")
-			if len(parts) == 0 || parts[0] == "" {
-				continue
-			}
-			// tc stands for type and container
-			// Split part[0] to get the builder type and the container if it exists
-			tc := strings.Split(parts[0], ".")
-			k := fmt.Sprintf("%s.%s", tc[0], parts[1])
-			if len(tc) == 2 && container != "" && tc[1] == container {
-				// Container specific properties always carry higher preference.
-				// Overwrite properties even if they exist.
-				hints.Put(k, value)
-			} else {
-				// Only insert the config if it doesn't already exist
-				if _, err := hints.GetValue(k); err != nil {
-					hints.Put(k, value)
+	if rawEntries, err := annotations.GetValue(prefix); err == nil {
+		if entries, ok := rawEntries.(common.MapStr); ok {
+			for key, rawValue := range entries {
+				// If there are top level hints like co.elastic.logs/ then just add the values after the /
+				// Only consider namespaced annotations
+				parts := strings.Split(key, "/")
+				if len(parts) == 2 {
+					hintKey := fmt.Sprintf("%s.%s", parts[0], parts[1])
+					// Insert only if there is no entry already. container level annotations take
+					// higher priority.
+					if _, err := hints.GetValue(hintKey); err != nil {
+						hints.Put(hintKey, rawValue)
+					}
+				} else if container != "" {
+					// Only consider annotations that are of type common.MapStr as we are looking for
+					// container level nesting
+					builderHints, ok := rawValue.(common.MapStr)
+					if !ok {
+						continue
+					}
+
+					// Check for <containerName>/ prefix
+					for hintKey, rawVal := range builderHints {
+						if strings.HasPrefix(hintKey, container) {
+							// Split the key to get part[1] to be the hint
+							parts := strings.Split(hintKey, "/")
+							if len(parts) == 2 {
+								// key will be the hint type
+								hintKey := fmt.Sprintf("%s.%s", key, parts[1])
+								hints.Put(hintKey, rawVal)
+							}
+						}
+					}
 				}
 			}
 		}

libbeat/autodiscover/builder/helper_test.go

@@ -10,12 +10,12 @@ import (
 
 func TestGenerateHints(t *testing.T) {
 	tests := []struct {
-		annotations common.MapStr
+		annotations map[string]string
 		result      common.MapStr
 	}{
 		// Empty annotations should return empty hints
 		{
-			annotations: common.MapStr{},
+			annotations: map[string]string{},
 			result:      common.MapStr{},
 		},
 
@@ -25,11 +25,12 @@ func TestGenerateHints(t *testing.T) {
 		// not.to.include must not be part of hints
 		// period is annotated at both container and pod level. Container level value must be in hints
 		{
-			annotations: common.MapStr{
+			annotations: map[string]string{
 				"co.elastic.logs/multiline.pattern": "^test",
 				"co.elastic.metrics/module":         "prometheus",
 				"co.elastic.metrics/period":         "10s",
 				"co.elastic.metrics.foobar/period":  "15s",
+				"co.elastic.metrics.foobar1/period": "15s",
 				"not.to.include":                    "true",
 			},
 			result: common.MapStr{
@@ -47,6 +48,10 @@ func TestGenerateHints(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		assert.Equal(t, GenerateHints(test.annotations, "foobar", "co.elastic."), test.result)
+		annMap := common.MapStr{}
+		for k, v := range test.annotations {
+			annMap.Put(k, v)
+		}
+		assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic"), test.result)
 	}
 }

libbeat/autodiscover/builder/plugin.go

@@ -23,7 +23,7 @@ func init() {
 	p.MustRegisterLoader(pluginKey, func(ifc interface{}) error {
 		b, ok := ifc.(builderPlugin)
 		if !ok {
-			return errors.New("plugin does not match processor plugin type")
+			return errors.New("plugin does not match builder plugin type")
 		}
 
 		return autodiscover.Registry.AddBuilder(b.name, b.builder)