Set agent.name to hostname by default (#18000)

Since ECS does not define agent.hostname it will be removed in a future release. In order to always have a field available to identify the agent by name we will set the agent.name to hostname unless the user has provided a custom name. Relates #16377
elastic · May 2, 2020 · 3711ee6 · 3711ee6
1 parent f39d869
commit 3711ee6
Show file tree

Hide file tree

Showing 24 changed files with 64 additions and 27 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -210,6 +210,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191]
+- Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
 
 *Auditbeat*
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -2458,7 +2458,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -3112,7 +3112,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/tests/system/test_fields.py b/filebeat/tests/system/test_fields.py
@@ -63,7 +63,7 @@ def test_custom_fields_under_root(self):
     def test_beat_fields(self):
         """
         Checks that it's possible to set a custom shipper name. Also
-        tests that beat.hostname  has values.
+        tests that agent.hostname has values.
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/test.log",
@@ -80,5 +80,6 @@ def test_beat_fields(self):
         output = self.read_output()
         doc = output[0]
         assert doc["host.name"] == "testShipperName"
+        assert doc["agent.name"] == "testShipperName"
         assert doc["agent.hostname"] == socket.gethostname()
         assert "fields" not in doc
diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -214,7 +214,7 @@ def _test_expected_events(self, test_file, objects):
 
 def clean_keys(obj):
     # These keys are host dependent
-    host_keys = ["host.name", "agent.hostname", "agent.type", "agent.ephemeral_id", "agent.id"]
+    host_keys = ["host.name", "agent.name", "agent.hostname", "agent.type", "agent.ephemeral_id", "agent.id"]
     # The create timestamps area always new
     time_keys = ["event.created"]
     # source path and agent.version can be different for each run

diff --git a/heartbeat/cmd/root.go b/heartbeat/cmd/root.go
@@ -41,7 +41,7 @@ var RootCmd *cmd.BeatsRootCmd
 func init() {
 	settings := instance.Settings{
 		Name:          Name,
-		Processing:    processing.MakeDefaultSupport(true, processing.WithECS, processing.WithBeatMeta("agent")),
+		Processing:    processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()),
 		HasDashboards: false,
 	}
 	RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings)

diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -40,7 +40,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc
@@ -33,7 +33,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/_meta/fields.common.yml b/libbeat/_meta/fields.common.yml
@@ -6,7 +6,9 @@
   fields:
     - name: agent.hostname
       type: keyword
-      description: Hostname of the agent.
+      description: >
+        Deprecated - use agent.name or agent.id to identify an agent.
+        Hostname of the agent.
 
     - name: beat.timezone
       type: alias

diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go
@@ -76,14 +76,14 @@ type builtinModifier func(beat.Info) common.MapStr
 // MakeDefaultBeatSupport automatically adds the `ecs.version`, `host.name` and `agent.X` fields
 // to each event.
 func MakeDefaultBeatSupport(normalize bool) SupportFactory {
-	return MakeDefaultSupport(normalize, WithECS, WithHost, WithBeatMeta("agent"))
+	return MakeDefaultSupport(normalize, WithECS, WithHost, WithAgentMeta())
 }
 
 // MakeDefaultObserverSupport creates a new SupportFactory based on NewDefaultSupport.
 // MakeDefaultObserverSupport automatically adds the `ecs.version` and `observer.X` fields
 // to each event.
 func MakeDefaultObserverSupport(normalize bool) SupportFactory {
-	return MakeDefaultSupport(normalize, WithECS, WithBeatMeta("observer"))
+	return MakeDefaultSupport(normalize, WithECS, WithObserverMeta())
 }
 
 // MakeDefaultSupport creates a new SupportFactory for use with the publisher pipeline.
@@ -139,21 +139,42 @@ var WithHost modifier = builtinModifier(func(info beat.Info) common.MapStr {
 	}
 })
 
-// WithBeatMeta adds beat meta information as builtin fields to a processing pipeline.
-// The `key` parameter defines the field to be used.
-func WithBeatMeta(key string) modifier {
+// WithAgentMeta adds agent meta information as builtin fields to a processing
+// pipeline.
+func WithAgentMeta() modifier {
 	return builtinModifier(func(info beat.Info) common.MapStr {
 		metadata := common.MapStr{
-			"type":         info.Beat,
 			"ephemeral_id": info.EphemeralID.String(),
-			"hostname":     info.Hostname,
 			"id":           info.ID.String(),
+			"name":         info.Hostname,
+			"type":         info.Beat,
+			"version":      info.Version,
+			// hostname is deprecated. To be removed for 8.0. It's not in ECS.
+			// See https://github.com/elastic/beats/issues/16377.
+			"hostname": info.Hostname,
+		}
+		if info.Name != "" {
+			metadata["name"] = info.Name
+		}
+		return common.MapStr{"agent": metadata}
+	})
+}
+
+// WithObserverMeta adds beat meta information as builtin fields to a processing
+// pipeline.
+func WithObserverMeta() modifier {
+	return builtinModifier(func(info beat.Info) common.MapStr {
+		metadata := common.MapStr{
+			"type":         info.Beat,                 // Per ECS this is not a valid type value.
+			"ephemeral_id": info.EphemeralID.String(), // Not in ECS.
+			"hostname":     info.Hostname,
+			"id":           info.ID.String(), // Not in ECS.
 			"version":      info.Version,
 		}
 		if info.Name != info.Hostname {
 			metadata.Put("name", info.Name)
 		}
-		return common.MapStr{key: metadata}
+		return common.MapStr{"observer": metadata}
 	})
 }
 

diff --git a/libbeat/publisher/processing/default_test.go b/libbeat/publisher/processing/default_test.go
@@ -184,6 +184,7 @@ func TestProcessorsConfigs(t *testing.T) {
 				"agent": common.MapStr{
 					"ephemeral_id": "123e4567-e89b-12d3-a456-426655440000",
 					"hostname":     "test.host.name",
+					"name":         "test.host.name",
 					"id":           "123e4567-e89b-12d3-a456-426655440001",
 					"type":         "test",
 					"version":      "0.1",

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -4631,7 +4631,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc
@@ -452,7 +452,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/packetbeat/include/fields.go b/packetbeat/include/fields.go
diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go
@@ -36,6 +36,8 @@ import (
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
 	// mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
+	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/notests"
 	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/test"

diff --git a/packetbeat/tests/system/test_0099_golden_files.py b/packetbeat/tests/system/test_0099_golden_files.py
@@ -66,6 +66,7 @@ def clean_keys(obj):
         "agent.ephemeral_id",
         "agent.hostname",
         "agent.id",
+        "agent.name",
         "agent.type",
         "agent.version",
         "ecs.version",

diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc
@@ -36,7 +36,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/winlogbeat/include/fields.go b/winlogbeat/include/fields.go
diff --git a/x-pack/functionbeat/docs/fields.asciidoc b/x-pack/functionbeat/docs/fields.asciidoc
@@ -33,7 +33,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

diff --git a/x-pack/functionbeat/include/fields.go b/x-pack/functionbeat/include/fields.go