Merge remote-tracking branch 'origin/7.x' into backport_24468_7.x

elastic · Mar 11, 2021 · e8ff8aa · e8ff8aa
2 parents 1740863 + 8b43fcc
commit e8ff8aa
Show file tree

Hide file tree

Showing 31 changed files with 437 additions and 504 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -19,6 +19,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938]
 - Make error message about locked data path actionable. {pull}18667[18667]
 - Fix panic with inline SSL when the certificate or key were small than 256 bytes. {pull}23820[23820]
+- Use alias to report container image in k8s metadata. {pull}24380[24380]
 
 *Auditbeat*
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -11968,7 +11968,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -11981,7 +11981,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -86567,7 +86567,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -86580,7 +86580,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

diff --git a/filebeat/docs/inputs/input-common-file-options.asciidoc b/filebeat/docs/inputs/input-common-file-options.asciidoc
@@ -126,7 +126,7 @@ is renamed. This happens, for example, when rotating files. By default, the
 harvester stays open and keeps reading the file because the file handler does
 not depend on the file name. If the `close_renamed` option is enabled and the
 file is renamed or moved in such a way that it's no longer matched by the file
-patterns specified for the , the file will not be picked up again.
+patterns specified for the path, the file will not be picked up again.
 {beatname_uc} will not finish reading the file.
 
 Do not use this option when `path` based `file_identity` is configured. It does

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -9550,7 +9550,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -9563,7 +9563,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc
@@ -9895,7 +9895,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -9908,7 +9908,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go
@@ -349,18 +349,14 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 		// so it works also on `stop` if containers have been already deleted.
 		eventID := fmt.Sprintf("%s.%s", pod.GetObjectMeta().GetUID(), c.Name)
 
-		meta := p.metagen.Generate(
-			pod,
-			metadata.WithFields("container.name", c.Name),
-			metadata.WithFields("container.image", c.Image),
-		)
+		meta := p.metagen.Generate(pod, metadata.WithFields("container.name", c.Name))
 
 		cmeta := common.MapStr{
-			"id": cid,
+			"id":      cid,
+			"runtime": runtimes[c.Name],
 			"image": common.MapStr{
 				"name": c.Image,
 			},
-			"runtime": runtimes[c.Name],
 		}
 
 		// Information that can be used in discovering a workload
@@ -387,6 +383,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 				"host":       host,
 				"port":       0,
 				"kubernetes": kubemeta,
+				//Actual metadata that will enrich the event
 				"meta": common.MapStr{
 					"kubernetes": meta,
 					"container":  cmeta,

diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go
@@ -451,10 +451,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -576,10 +574,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -621,10 +617,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -792,10 +786,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -895,15 +887,13 @@ func TestEmitEvent(t *testing.T) {
 					"meta": common.MapStr{
 						"kubernetes": common.MapStr{
 							"namespace": "default",
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
-							},
 							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -1003,15 +993,13 @@ func TestEmitEvent(t *testing.T) {
 					"meta": common.MapStr{
 						"kubernetes": common.MapStr{
 							"namespace": "default",
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
-							},
 							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -1120,10 +1108,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -1234,10 +1220,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -1380,10 +1364,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat",
 							},
 						},
 						"container": common.MapStr{
@@ -1426,10 +1408,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat-init",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat-init",
 							},
 						},
 						"container": common.MapStr{
@@ -1472,10 +1452,8 @@ func TestEmitEvent(t *testing.T) {
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
-							},
-							"container": common.MapStr{
-								"name":  "filebeat-ephemeral",
-								"image": "elastic/filebeat:6.3.0",
+							}, "container": common.MapStr{
+								"name": "filebeat-ephemeral",
 							},
 						},
 						"container": common.MapStr{

diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go
@@ -62,7 +62,8 @@ const defaultLevel = InfoLevel
 // Beat is supposed to be run within.
 func DefaultConfig(environment Environment) Config {
 	return Config{
-		Level: defaultLevel,
+		Level:   defaultLevel,
+		ToFiles: true,
 		Files: FileConfig{
 			MaxSize:         10 * 1024 * 1024,
 			MaxBackups:      7,

diff --git a/libbeat/processors/add_kubernetes_metadata/_meta/fields.yml b/libbeat/processors/add_kubernetes_metadata/_meta/fields.yml
@@ -72,9 +72,10 @@
         - name: container.name
           type: keyword
           description: >
-            Kubernetes container name
+            Kubernetes container name (different than the name from the runtime)
 
         - name: container.image
-          type: keyword
+          type: alias
+          path: container.image.name
           description: >
             Kubernetes container image
diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go
@@ -271,8 +271,10 @@ func (k *kubernetesAnnotator) Run(event *beat.Event) (*beat.Event, error) {
 	}
 
 	kubeMeta := metadata.Clone()
+	// remove container meta from kubernetes.container.*
 	kubeMeta.Delete("container.id")
 	kubeMeta.Delete("container.runtime")
+	kubeMeta.Delete("container.image")
 	event.Fields.DeepUpdate(common.MapStr{
 		"kubernetes": kubeMeta,
 	})

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -27491,7 +27491,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -27504,7 +27504,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

diff --git a/metricbeat/module/kubernetes/state_container/_meta/data.json b/metricbeat/module/kubernetes/state_container/_meta/data.json
@@ -13,14 +13,13 @@
       "app": "playground"
     },
     "container": {
+      "name": "ubuntu",
       "cpu": {
         "request": {
           "nanocores": 200000000
         }
-      },      
-      "image": "ubuntu:latest",
+      },
       "id": "docker://5f8ce416d10ab0b28ce5c7d521de2264aa03ff4d001e1194076f6a02a330139f",
-      "name": "ubuntu",
       "status": {
         "ready": true,
         "restarts": 0,
@@ -62,11 +61,14 @@
   },
   "container": {
     "runtime": "docker",
-    "id": "5f8ce416d10ab0b28ce5c7d521de2264aa03ff4d001e1194076f6a02a330139f"
+    "id": "5f8ce416d10ab0b28ce5c7d521de2264aa03ff4d001e1194076f6a02a330139f",
+    "image": {
+      "name": "ubuntu:latest"
+    },
   },
   "event": {
     "dataset": "kubernetes.container",
     "module": "kubernetes",
     "duration": 33750820
   }
-}
+}