Merge remote-tracking branch 'upstream/master' into feature/arm-aws-test

* upstream/master: [CI] bump gvm version and use the binary (elastic#24571) [Filebeat] Add Dashboards to Threat Intel Module (elastic#24488) Fix default scope in add_nomad_metadata (elastic#24559) [Heartbeat] Produce error rather than panic on missing source (elastic#24404) [Auditbeat] btmp offset check (elastic#24515) Clarify that the Tomcat module is for ingesting access logs (elastic#24543) [Ingest Manager] Move logging defaults to agent (elastic#24535)
v1v · Mar 17, 2021 · 5b0169f · 5b0169f
2 parents 58cf144 + 34e5c09
commit 5b0169f
Show file tree

Hide file tree

Showing 31 changed files with 11,749 additions and 26 deletions.
diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh
@@ -33,9 +33,9 @@ fi
 echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}"
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-${OS}-${GVM_ARCH_SUFFIX}"
 chmod +x "${GVM_CMD}"
 
-gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
+${GVM_CMD} "${GO_VERSION}" |cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
 
-eval $(gvm ${GO_VERSION})
+eval "$("${GVM_CMD}" "${GO_VERSION}")"
diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat
@@ -17,14 +17,14 @@ mkdir %WORKSPACE%\bin
 IF EXIST "%PROGRAMFILES(X86)%" (
     REM Force the gvm installation.
     SET GVM_BIN=gvm.exe
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.4/gvm-windows-amd64.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
     IF ERRORLEVEL 1 (
         REM gvm installation has failed.
         exit /b 1
     )
 ) ELSE (
     REM Windows 7 workers got a broken gvm installation.
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.4/gvm-windows-386.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
     IF ERRORLEVEL 1 (
         REM gvm installation has failed.
         exit /b 1

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -264,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
 - Note incompatibility of system/socket on ARM. {pull}23381[23381]
 - system/login: Fixed offset reset on inode reuse. {pull}24414[24414]
+- system/login: Add additional offset check for utmp files. {pull}24515[24515]
 
 *Filebeat*
 
@@ -398,6 +399,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Netflow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
 - in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336]
 - Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
+- Fix default `scope` in `add_nomad_metadata`. {issue}24559[24559]
 
 *Heartbeat*
 

diff --git a/Vagrantfile b/Vagrantfile
@@ -44,7 +44,7 @@ if (-Not (Test-Path $gopath_beats)) {
 if (-Not (Get-Command "gvm" -ErrorAction SilentlyContinue)) {
     echo "Installing gvm to manage go version"
     [Net.ServicePointManager]::SecurityProtocol = "tls12"
-    Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe
+    Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe
     C:\\Windows\\System32\\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression
     go version
 
@@ -119,7 +119,7 @@ def linuxGvmProvision(arch="amd64")
   return <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-linux-#{arch}
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-linux-#{arch}
   chmod +x ~/bin/gvm
   ~/bin/gvm #{GO_VERSION}
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile

diff --git a/filebeat/docs/images/filebeat-threatintel-abuse-malware.png b/filebeat/docs/images/filebeat-threatintel-abuse-malware.png
diff --git a/filebeat/docs/images/filebeat-threatintel-abuse-url.png b/filebeat/docs/images/filebeat-threatintel-abuse-url.png
diff --git a/filebeat/docs/images/filebeat-threatintel-alienvault-otx.png b/filebeat/docs/images/filebeat-threatintel-alienvault-otx.png
diff --git a/filebeat/docs/images/filebeat-threatintel-anomali-limo.png b/filebeat/docs/images/filebeat-threatintel-anomali-limo.png
diff --git a/filebeat/docs/images/filebeat-threatintel-misp.png b/filebeat/docs/images/filebeat-threatintel-misp.png
diff --git a/filebeat/docs/images/filebeat-threatintel-overview.png b/filebeat/docs/images/filebeat-threatintel-overview.png
diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc
@@ -6,7 +6,7 @@ This file is generated! See scripts/docs_collector.py
 [role="xpack"]
 
 :modulename: threatintel
-:has-dashboards: false
+:has-dashboards: true
 
 == Threat Intel module
 beta[]
@@ -341,6 +341,49 @@ Anomali Threat Intel is mapped to the following ECS fields.
 
 `anomali.pattern` is mapped to the appropriate field dependent on attribute type.
 
+:has-dashboards!:
+
+[float]
+=== Dashboards
+
+This module comes with dashboards for the threat information feeds.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-overview.png[]
+
+[float]
+Overview of the information provided, and the health of, the Threat Intel module.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-abuse-malware.png[]
+
+[float]
+Overview of the information provided by the Abuse.ch Malware feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-abuse-url.png[]
+
+[float]
+Overview of the information provided by the Abuse.ch URL feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-alienvault-otx.png[]
+
+[float]
+Overview of the information provided by the AlienVault OTX feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-anomali-limo.png[]
+
+[float]
+Overview of the information provided by the Anomali Limo feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-misp.png[]
+
+[float]
+Overview of the information provided by the MSIP feed.
+
 :modulename!:
 
 

diff --git a/filebeat/docs/modules/tomcat.asciidoc b/filebeat/docs/modules/tomcat.asciidoc
@@ -12,7 +12,7 @@ This file is generated! See scripts/docs_collector.py
 
 experimental[]
 
-This is a module for receiving Apache Tomcat logs over Syslog or a file.
+This is a module for receiving Apache Tomcat access logs over Syslog or a file.
 
 include::../include/gs-link.asciidoc[]
 

diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go
@@ -62,8 +62,7 @@ const defaultLevel = InfoLevel
 // Beat is supposed to be run within.
 func DefaultConfig(environment Environment) Config {
 	return Config{
-		Level:   defaultLevel,
-		ToFiles: true,
+		Level: defaultLevel,
 		Files: FileConfig{
 			MaxSize:         10 * 1024 * 1024,
 			MaxBackups:      7,

diff --git a/x-pack/auditbeat/module/system/login/utmp.go b/x-pack/auditbeat/module/system/login/utmp.go
@@ -181,14 +181,14 @@ func (r *UtmpFileReader) readNewInFile(loginRecordC chan<- LoginRecord, errorC c
 
 	size := utmpFile.Size
 	oldSize := savedUtmpFile.Size
-	if size < oldSize {
+	if size < oldSize || utmpFile.Offset > size {
 		// UTMP files are append-only and so this is weird. It might be a sign of
 		// a highly unlikely inode reuse - or of something more nefarious.
 		// Setting isKnownFile to false so we read the whole file from the beginning.
 		isKnownFile = false
 
-		r.log.Warnf("Unexpectedly, the file %v is smaller than before (new: %v, old: %v) - reading whole file.",
-			utmpFile.Path, size, oldSize)
+		r.log.Warnf("saved size or offset illogical (new=%+v, saved=%+v) - reading whole file.",
+			utmpFile, savedUtmpFile)
 	}
 
 	if !isKnownFile && size == 0 {
@@ -221,7 +221,7 @@ func (r *UtmpFileReader) readNewInFile(loginRecordC chan<- LoginRecord, errorC c
 
 		// This will be the usual case, but we do not want to seek with the stored offset
 		// if the saved size is smaller than the current one.
-		if size >= oldSize {
+		if size >= oldSize && utmpFile.Offset <= size {
 			_, err = f.Seek(utmpFile.Offset, 0)
 			if err != nil {
 				errorC <- errors.Wrapf(err, "error setting offset %d for file %v", utmpFile.Offset, utmpFile.Path)
@@ -230,7 +230,7 @@ func (r *UtmpFileReader) readNewInFile(loginRecordC chan<- LoginRecord, errorC c
 
 		// If the saved size is smaller than the current one, or the previous Seek failed,
 		// we retry one more time, this time resetting to the beginning of the file.
-		if size < oldSize || err != nil {
+		if size < oldSize || utmpFile.Offset > size || err != nil {
 			_, err = f.Seek(0, 0)
 			if err != nil {
 				errorC <- errors.Wrapf(err, "error setting offset 0 for file %v", utmpFile.Path)

diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go
@@ -55,17 +55,19 @@ func new(name string, cfg *Config) (*Logger, error) {
 		return nil, err
 	}
 
+	var outputs []zapcore.Core
 	if cfg.ToFiles {
 		internal, err := makeInternalFileOutput(cfg)
 		if err != nil {
 			return nil, err
 		}
 
-		if err := configure.LoggingWithOutputs("", commonCfg, internal); err != nil {
-			return nil, fmt.Errorf("error initializing logging: %v", err)
-		}
+		outputs = append(outputs, internal)
 	}
 
+	if err := configure.LoggingWithOutputs("", commonCfg, outputs...); err != nil {
+		return nil, fmt.Errorf("error initializing logging: %v", err)
+	}
 	return logp.NewLogger(name), nil
 }
 
@@ -92,6 +94,7 @@ func DefaultLoggingConfig() *Config {
 	cfg := logp.DefaultConfig(logp.DefaultEnvironment)
 	cfg.Beat = agentName
 	cfg.Level = logp.InfoLevel
+	cfg.ToFiles = true
 	cfg.Files.Path = paths.Logs()
 	cfg.Files.Name = fmt.Sprintf("%s.log", agentName)
 

diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc
@@ -1,7 +1,7 @@
 [role="xpack"]
 
 :modulename: threatintel
-:has-dashboards: false
+:has-dashboards: true
 
 == Threat Intel module
 beta[]
@@ -336,4 +336,47 @@ Anomali Threat Intel is mapped to the following ECS fields.
 
 `anomali.pattern` is mapped to the appropriate field dependent on attribute type.
 
+:has-dashboards!:
+
+[float]
+=== Dashboards
+
+This module comes with dashboards for the threat information feeds.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-overview.png[]
+
+[float]
+Overview of the information provided, and the health of, the Threat Intel module.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-abuse-malware.png[]
+
+[float]
+Overview of the information provided by the Abuse.ch Malware feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-abuse-url.png[]
+
+[float]
+Overview of the information provided by the Abuse.ch URL feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-alienvault-otx.png[]
+
+[float]
+Overview of the information provided by the AlienVault OTX feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-anomali-limo.png[]
+
+[float]
+Overview of the information provided by the Anomali Limo feed.
+
+[role="screenshot"]
+image::./images/filebeat-threatintel-misp.png[]
+
+[float]
+Overview of the information provided by the MSIP feed.
+
 :modulename!: