Update beats framework to 5554677

Makefile

@@ -16,7 +16,8 @@ NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%S')
 GOBUILD_FLAGS=-i -ldflags "-s -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
 MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 
-.DEFAULT_GOAL := ${BEAT_NAME}
+# overwrite some beats targets cleanly
+.OVER := original-
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/libbeat/scripts/Makefile

NOTICE.txt

@@ -245,7 +245,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 96e059d2c28214536905d8691af0505106ede3ea
+Revision: 5554677dc86b78f48b1ff2a214c661f5f78cbfe1
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/dev-tools/ecs-migration.yml

@@ -463,7 +463,26 @@
 - from: system.auth.hostname
   to: host.hostname
   alias: true
-  copy_to: false
+  beat: filebeat
+
+- from: system.auth.message
+  to: message
+  alias: true
+  beat: filebeat
+
+- from: system.auth.program
+  to: process.name
+  alias: true
+  beat: filebeat
+
+- from: system.auth.timestamp
+  to: '@timestamp'
+  alias: true
+  beat: filebeat
+
+- from: system.auth.user
+  to: user.name
+  alias: true
   beat: filebeat
 
 - from: system.auth.pid
@@ -476,12 +495,22 @@
   alias: true
   beat: filebeat
 
+- from: system.auth.groupadd.name
+  to: group.name
+  alias: true
+  beat: filebeat
+
+- from: system.auth.useradd.gid
+  to: group.id
+  alias: true
+  beat: filebeat
+
 - from: system.auth.useradd.uid
   to: user.id
   alias: true
   beat: filebeat
 
-- from: system.auth.useradd.user
+- from: system.auth.useradd.name
   to: user.name
   alias: true
   beat: filebeat
@@ -682,11 +711,6 @@
 
 ## Elasticsearch module
 
-- from: elasticsearch.audit.event_type
-  to: event.type
-  alias: true
-  beat: filebeat
-
 - from: elasticsearch.audit.origin_address
   to: source.ip
   alias: true
@@ -1151,6 +1175,11 @@
 
 ## NGINX module
 
+- from: nginx.access.remote_ip
+  to: source.address
+  alias: true
+  beat: filebeat
+
 - from: nginx.access.user_name
   to: user.name
   alias: true

_beats/libbeat/scripts/Makefile

@@ -101,6 +101,7 @@ include $(ES_BEATS)/dev-tools/make/mage.mk
 
 ### BUILDING ###
 
+.DEFAULT_GOAL := ${BEAT_NAME}
 
 ${BEAT_NAME}: $(GOFILES_ALL) ## @build build the beat application
 	go build $(GOBUILD_FLAGS)
@@ -128,15 +129,15 @@ check: check-headers python-env prepare-tests ## @build Checks project and sourc
 	@${FIND} -wholename "*tests/system/test_*.py" -perm ${PERM_EXEC} -exec false {} + || (echo "Python test files shouldn't be executable, otherwise nose doesn't find them" && false)
 	@${FIND} -name "*.yml" -perm ${PERM_EXEC} -exec false {} + || (echo "YAML files should not be executable" && false)
 
-.PHONY: check-headers
-check-headers:
+.PHONY: $(.OVER)check-headers
+$(.OVER)check-headers:
 ifndef CHECK_HEADERS_DISABLED
 	@go get -u github.com/elastic/go-licenser
 	@go-licenser -d -license ${LICENSE}
 endif
 
-.PHONY: add-headers
-add-headers:
+.PHONY: $(.OVER)add-headers
+$(.OVER)add-headers:
 ifndef CHECK_HEADERS_DISABLED
 	@go get github.com/elastic/go-licenser
 	@go-licenser -license ${LICENSE}
@@ -381,8 +382,8 @@ docs-preview:  ## @build Preview the documents for the beat in the browser
 ES_URL?=http://localhost:9200
 KIBANA_URL?=http://localhost:5601
 
-.PHONY: import-dashboards
-import-dashboards: update ${BEAT_NAME}
+.PHONY: $(.OVER)import-dashboards
+$(.OVER)import-dashboards: update ${BEAT_NAME}
 	${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME} setup -E setup.dashboards.directory=${PWD}/_meta/kibana.generated -E setup.kibana.host=${KIBANA_URL} --dashboards
 
 ### CONTAINER ENVIRONMENT ####

_beats/libbeat/tests/system/beat/beat.py

@@ -408,6 +408,30 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False):
 
         return counter
 
+    def log_contains_countmap(self, pattern, capture_group, logfile=None):
+        """
+        Returns a map of the number of appearances of each captured group in the log file
+        """
+        counts = {}
+
+        if logfile is None:
+            logfile = self.beat_name + ".log"
+
+        try:
+            with open(os.path.join(self.working_dir, logfile), "r") as f:
+                for line in f:
+                    res = pattern.search(line)
+                    if res is not None:
+                        capt = res.group(capture_group)
+                        if capt in counts:
+                            counts[capt] += 1
+                        else:
+                            counts[capt] = 1
+        except IOError:
+            pass
+
+        return counts
+
     def output_lines(self, output_file=None):
         """ Count number of lines in a file."""
         if output_file is None:

_beats/testing/environments/docker/kafka/Dockerfile

@@ -4,14 +4,14 @@ ENV KAFKA_HOME /kafka
 # The advertised host is kafka. This means it will not work if container is started locally and connected from localhost to it
 ENV KAFKA_ADVERTISED_HOST kafka
 ENV KAFKA_LOGS_DIR="/kafka-logs"
-ENV KAFKA_VERSION 2.1.0
+ENV KAFKA_VERSION 2.1.1
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
 
 RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat
 
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \
-    "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+    "http://mirror.easyname.ch/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
     tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
 
 ADD run.sh /run.sh