Cherry-pick elastic#7051 to 6.3: Add host.name in the events (elastic…

…#7068)

* Add host.name in the events (elastic#7051)

As a solution for elastic#7050, we're adding a `host.name` field to
all events. This is duplicate information from `beat.name`,
but is used to avoid the mapping conflict and to slowly
introduce the "host as an object" approach.

To remove the duplication, you can remove `beat.name` like this:

    processors:
      - drop_fields.fields: ["beat.name"]

Closes elastic#7050.

(cherry picked from commit 1889f9c)

* changelog cleanup

CHANGELOG.asciidoc

@@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Rename beat.cpu.*.time metrics to beat.cpu.*.time.ms. {pull}6449[6449]
 - Mark `system.syslog.message` and `system.auth.message` as `text` instead of `keyword`. {pull}6589[6589]
 - Allow override of dynamic template `match_mapping_type` for fields with object_type. {pull}6691[6691]
+- Add `host.name` field to all events, to avoid mapping conflicts. This could be breaking Logstash configs if you rely on the `host` field being a string. {pull}7051[7051]
 
 *Auditbeat*
 

dev-tools/jenkins_ci.ps1

@@ -36,13 +36,14 @@ exec { go get -u github.com/jstemmer/go-junit-report }
 echo "Building $env:beat"
 exec { go build } "Build FAILURE"
 
+# always build the libbeat fields
+cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml
+cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml
+cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml
+
 if ($env:beat -eq "metricbeat") {
     cp .\_meta\fields.common.yml .\_meta\fields.generated.yml
     python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml
-} elseif ($env:beat -eq "libbeat") {
-    cp .\_meta\fields.common.yml .\_meta\fields.generated.yml
-    cat processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath .\_meta\fields.generated.yml
-    cp .\_meta\fields.generated.yml .\fields.yml
 }
 
 echo "Unit testing $env:beat"

libbeat/publisher/pipeline/module.go

@@ -50,10 +50,15 @@ func Load(
 		Processors:    processors,
 		Annotations: Annotations{
 			Event: config.EventMetadata,
-			Beat: common.MapStr{
-				"name":     name,
-				"hostname": beatInfo.Hostname,
-				"version":  beatInfo.Version,
+			Builtin: common.MapStr{
+				"beat": common.MapStr{
+					"name":     name,
+					"hostname": beatInfo.Hostname,
+					"version":  beatInfo.Version,
+				},
+				"host": common.MapStr{
+					"name": name,
+				},
 			},
 		},
 	}

libbeat/publisher/pipeline/pipeline.go

@@ -62,9 +62,9 @@ type pipelineProcessors struct {
 	// The pipeline its processor settings for
 	// constructing the clients complete processor
 	// pipeline on connect.
-	beatsMeta common.MapStr
-	fields    common.MapStr
-	tags      []string
+	builtinMeta common.MapStr
+	fields      common.MapStr
+	tags        []string
 
 	processors beat.Processor
 
@@ -91,8 +91,8 @@ type Settings struct {
 // processors, so all processors configured with the pipeline or client will see
 // the same/complete event.
 type Annotations struct {
-	Beat  common.MapStr
-	Event common.EventMetadata
+	Event   common.EventMetadata
+	Builtin common.MapStr
 }
 
 // WaitCloseMode enumerates the possible behaviors of WaitClose in a pipeline.
@@ -401,8 +401,8 @@ func makePipelineProcessors(
 		p.processors = tmp
 	}
 
-	if meta := annotations.Beat; meta != nil {
-		p.beatsMeta = common.MapStr{"beat": meta}
+	if meta := annotations.Builtin; meta != nil {
+		p.builtinMeta = meta
 	}
 
 	if em := annotations.Event; len(em.Fields) > 0 {

libbeat/publisher/pipeline/processor.go

@@ -99,8 +99,8 @@ func newProcessorPipeline(
 	// setup 5: client processor list
 	processors.add(localProcessors)
 
-	// setup 6: add beats metadata
-	if meta := global.beatsMeta; len(meta) > 0 {
+	// setup 6: add beats and host metadata
+	if meta := global.builtinMeta; len(meta) > 0 {
 		processors.add(makeAddFieldsProcessor("beatsMeta", meta, needsCopy))
 	}
 

metricbeat/tests/system/metricbeat.py

@@ -7,7 +7,7 @@
 from beat.beat import TestCase
 
 COMMON_FIELDS = ["@timestamp", "beat", "metricset.name", "metricset.host",
-                 "metricset.module", "metricset.rtt"]
+                 "metricset.module", "metricset.rtt", "host.name"]
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 

metricbeat/tests/system/test_processors.py

@@ -35,7 +35,7 @@ def test_drop_fields(self):
         print(evt.keys())
         self.assertItemsEqual(self.de_dot([
             'beat', '@timestamp', 'system', 'metricset.module',
-            'metricset.rtt', 'metricset.name'
+            'metricset.rtt', 'metricset.name', 'host'
         ]), evt.keys())
         cpu = evt["system"]["cpu"]
         print(cpu.keys())