Merge remote-tracking branch 'upstream/master' into hostfs_cgroups_li…

…bbeat

CHANGELOG.asciidoc

@@ -3,6 +3,26 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.11.2]]
+=== Beats version 7.11.2
+https://github.com/elastic/beats/compare/v7.11.1...v7.11.2[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
+
+*Filebeat*
+
+- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
+- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
+- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
+
+*Metricbeat*
+
+- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
+
 [[release-notes-7.11.1]]
 === Beats version 7.11.1
 https://github.com/elastic/beats/compare/v7.11.0...v7.11.1[View commits]

CHANGELOG.next.asciidoc

@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update to ECS 1.7.0. {pull}22571[22571]
 - Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
 - Fix panic with inline SSL when the certificate or key were small than 256 bytes. {pull}23820[23820]
+- Use alias to report container image in k8s metadata. {pull}24380[24380]
 
 *Auditbeat*
 
@@ -103,7 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
 - Rename `s3` input to `aws-s3` input. {pull}23469[23469]
-- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
+- Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295]
 
 *Heartbeat*
 - Adds negative body match. {pull}20728[20728]
@@ -391,13 +392,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
 - Use rfc6587 framing for fortinet firewall and clientendpoint filesets when transferring over tcp. {pull}23837[23837]
 - Fix httpjson input logging so it doesn't conflict with ECS. {pull}23972[23972]
-- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
 - Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709]
 - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
 - Fix Netflow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
-- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
 - in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336]
+- Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
 
 *Heartbeat*
 
@@ -515,11 +515,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
-- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
 - Fix ec2 metricset fields.yml and the integration test {pull}23726[23726]
 - Unskip s3_request integration test. {pull}23887[23887]
 - Add system.hostfs configuration option for system module. {pull}23831[23831]
 - Fix GCP not able to request Cloudfunctions metrics if a region filter was set {pull}24218[24218]
+- Fix type of `uwsgi.status.worker.rss` type. {pull}24468[24468]
 
 *Packetbeat*
 
@@ -1057,6 +1057,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
 - Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684]
 - Add new ECS 1.8 improvements. {pull}23563[23563]
+- Remove deprecated eventlogging api that was used for Windows XP/2003 and associated unused code. {pull}24463[24463]
 
 *Elastic Log Driver*
 

auditbeat/docs/fields.asciidoc

@@ -11970,7 +11970,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -11983,7 +11983,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

filebeat/docs/fields.asciidoc

@@ -86569,7 +86569,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -86582,7 +86582,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

filebeat/input/filestream/environment_test.go

@@ -105,17 +105,69 @@ func (e *inputTestingEnvironment) mustWriteLinesToFile(filename string, lines []
 	}
 }
 
+func (e *inputTestingEnvironment) mustAppendLinesToFile(filename string, lines []byte) {
+	path := e.abspath(filename)
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		e.t.Fatalf("failed to open file '%s': %+v", path, err)
+	}
+	defer f.Close()
+
+	_, err = f.Write(lines)
+	if err != nil {
+		e.t.Fatalf("append lines to file '%s': %+v", path, err)
+	}
+}
+
 func (e *inputTestingEnvironment) mustRenameFile(oldname, newname string) {
 	err := os.Rename(e.abspath(oldname), e.abspath(newname))
 	if err != nil {
 		e.t.Fatalf("failed to rename file '%s': %+v", oldname, err)
 	}
 }
 
+func (e *inputTestingEnvironment) mustRemoveFile(filename string) {
+	path := e.abspath(filename)
+	err := os.Remove(path)
+	if err != nil {
+		e.t.Fatalf("failed to rename file '%s': %+v", path, err)
+	}
+}
+
+func (e *inputTestingEnvironment) mustSymlink(filename, symlinkname string) {
+	err := os.Symlink(e.abspath(filename), e.abspath(symlinkname))
+	if err != nil {
+		e.t.Fatalf("failed to create symlink to file '%s': %+v", filename, err)
+	}
+}
+
+func (e *inputTestingEnvironment) mustTruncateFile(filename string, size int64) {
+	path := e.abspath(filename)
+	err := os.Truncate(path, size)
+	if err != nil {
+		e.t.Fatalf("failed to truncate file '%s': %+v", path, err)
+	}
+}
+
 func (e *inputTestingEnvironment) abspath(filename string) string {
 	return filepath.Join(e.workingDir, filename)
 }
 
+func (e *inputTestingEnvironment) requireRegistryEntryCount(expectedCount int) {
+	inputStore, _ := e.stateStore.Access()
+
+	actual := 0
+	err := inputStore.Each(func(_ string, _ statestore.ValueDecoder) (bool, error) {
+		actual += 1
+		return true, nil
+	})
+	if err != nil {
+		e.t.Fatalf("error while iterating through registry: %+v", err)
+	}
+
+	require.Equal(e.t, actual, expectedCount)
+}
+
 // requireOffsetInRegistry checks if the expected offset is set for a file.
 func (e *inputTestingEnvironment) requireOffsetInRegistry(filename string, expectedOffset int) {
 	filepath := e.abspath(filename)
@@ -131,6 +183,32 @@ func (e *inputTestingEnvironment) requireOffsetInRegistry(filename string, expec
 	require.Equal(e.t, expectedOffset, entry.Cursor.Offset)
 }
 
+func (e *inputTestingEnvironment) requireNoEntryInRegistry(filename string) {
+	filepath := e.abspath(filename)
+	fi, err := os.Stat(filepath)
+	if err != nil {
+		e.t.Fatalf("cannot stat file when cheking for offset: %+v", err)
+	}
+
+	inputStore, _ := e.stateStore.Access()
+
+	identifier, _ := newINodeDeviceIdentifier(nil)
+	src := identifier.GetSource(loginp.FSEvent{Info: fi, Op: loginp.OpCreate, NewPath: filepath})
+
+	var entry registryEntry
+	err = inputStore.Get(src.Name(), &entry)
+	if err == nil {
+		e.t.Fatalf("key is not expected to be present '%s'", src.Name())
+	}
+}
+
+// requireOffsetInRegistry checks if the expected offset is set for a file.
+func (e *inputTestingEnvironment) requireOffsetInRegistryByID(key string, expectedOffset int) {
+	entry := e.getRegistryState(key)
+
+	require.Equal(e.t, expectedOffset, entry.Cursor.Offset)
+}
+
 func (e *inputTestingEnvironment) getRegistryState(key string) registryEntry {
 	inputStore, _ := e.stateStore.Access()
 
@@ -153,10 +231,58 @@ func (e *inputTestingEnvironment) waitUntilEventCount(count int) {
 		if sum == count {
 			return
 		}
+		if count < sum {
+			e.t.Fatalf("too many events; expected: %d, actual: %d", count, sum)
+		}
 		time.Sleep(10 * time.Millisecond)
 	}
 }
 
+// waitUntilHarvesterIsDone detects Harvester stop by checking if the last client has been closed
+// as when a Harvester stops the client is closed.
+func (e *inputTestingEnvironment) waitUntilHarvesterIsDone() {
+	for !e.pipeline.clients[len(e.pipeline.clients)-1].closed {
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+// requireEventReceived requires that the list of messages has made it into the output.
+func (e *inputTestingEnvironment) requireEventsReceived(events []string) {
+	foundEvents := make([]bool, len(events))
+	checkedEventCount := 0
+	for _, c := range e.pipeline.clients {
+		for _, evt := range c.GetEvents() {
+			if len(events) == checkedEventCount {
+				e.t.Fatalf("not enough expected elements")
+			}
+			message := evt.Fields["message"].(string)
+			if message == events[checkedEventCount] {
+				foundEvents[checkedEventCount] = true
+			}
+			checkedEventCount += 1
+		}
+	}
+
+	var missingEvents []string
+	for i, found := range foundEvents {
+		if !found {
+			missingEvents = append(missingEvents, events[i])
+		}
+	}
+
+	require.Equal(e.t, 0, len(missingEvents), "following events are missing: %+v", missingEvents)
+}
+
+func (e *inputTestingEnvironment) getOutputMessages() []string {
+	messages := make([]string, 0)
+	for _, c := range e.pipeline.clients {
+		for _, evt := range c.GetEvents() {
+			messages = append(messages, evt.Fields["message"].(string))
+		}
+	}
+	return messages
+}
+
 type testInputStore struct {
 	registry *statestore.Registry
 }

heartbeat/docs/fields.asciidoc

@@ -9552,7 +9552,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -9565,7 +9565,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

journalbeat/docs/fields.asciidoc

@@ -9897,7 +9897,7 @@ type: keyword
 *`kubernetes.container.name`*::
 +
 --
-Kubernetes container name
+Kubernetes container name (different than the name from the runtime)
 
 
 type: keyword
@@ -9910,7 +9910,9 @@ type: keyword
 Kubernetes container image
 
 
-type: keyword
+type: alias
+
+alias to: container.image.name
 
 --
 

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -349,18 +349,14 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 		// so it works also on `stop` if containers have been already deleted.
 		eventID := fmt.Sprintf("%s.%s", pod.GetObjectMeta().GetUID(), c.Name)
 
-		meta := p.metagen.Generate(
-			pod,
-			metadata.WithFields("container.name", c.Name),
-			metadata.WithFields("container.image", c.Image),
-		)
+		meta := p.metagen.Generate(pod, metadata.WithFields("container.name", c.Name))
 
 		cmeta := common.MapStr{
-			"id": cid,
+			"id":      cid,
+			"runtime": runtimes[c.Name],
 			"image": common.MapStr{
 				"name": c.Image,
 			},
-			"runtime": runtimes[c.Name],
 		}
 
 		// Information that can be used in discovering a workload
@@ -387,6 +383,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 				"host":       host,
 				"port":       0,
 				"kubernetes": kubemeta,
+				//Actual metadata that will enrich the event
 				"meta": common.MapStr{
 					"kubernetes": meta,
 					"container":  cmeta,