Introduce CLI flag -environment

CHANGELOG.next.asciidoc

@@ -39,8 +39,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
+- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
 - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568]
 - Fix missing output in dockerlogbeat {pull}15719[15719]
+- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442]
 
 *Auditbeat*
 

dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl

@@ -2,23 +2,25 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-	<key>Label</key>
+    <key>Label</key>
         <string>{{.identifier}}</string>
-	<key>ProgramArguments</key>
-	<array>
-            <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
-            <string>-c</string>
-            <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
-            <string>--path.home</string>
-            <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
-            <string>--path.config</string>
-            <string>/etc/{{.BeatName}}</string>
-            <string>--path.data</string>
-            <string>/var/lib/{{.BeatName}}</string>
-            <string>--path.logs</string>
-            <string>/var/log/{{.BeatName}}</string>
-        </array>
-	<key>RunAtLoad</key>
-        <true/>
+    <key>ProgramArguments</key>
+    <array>
+        <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
+        <string>-environment</string>
+        <string>macOS_service</string>
+        <string>-c</string>
+        <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
+        <string>--path.home</string>
+        <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
+        <string>--path.config</string>
+        <string>/etc/{{.BeatName}}</string>
+        <string>--path.data</string>
+        <string>/var/lib/{{.BeatName}}</string>
+        <string>--path.logs</string>
+        <string>/var/log/{{.BeatName}}</string>
+    </array>
+    <key>RunAtLoad</key>
+    <true/>
 </dict>
 </plist>

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -50,4 +50,4 @@ EXPOSE {{ $port }}
 
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
-CMD ["-e"]
+CMD ["-environment", "container"]

dev-tools/packaging/templates/linux/systemd.unit.tmpl

@@ -9,10 +9,10 @@ After=network-online.target
 User={{ .BeatUser }}
 Group={{ .BeatUser }}
 {{- end }}
-Environment="BEAT_LOG_OPTS=-e"
+Environment="BEAT_LOG_OPTS="
 Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml"
 Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}"
-ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
+ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} -environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
 Restart=always
 
 [Install]

dev-tools/packaging/templates/windows/install-service.ps1.tmpl

@@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path
 # Create the new service.
 New-Service -name {{.BeatName}} `
   -displayName {{.BeatName | title}} `
-  -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
+  -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
 
 # Attempt to set the service to delayed start using sc config.
 Try {

libbeat/cmd/root.go

@@ -88,6 +88,7 @@ func GenRootCmdWithSettings(beatCreator beat.Creator, settings instance.Settings
 	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d"))
 	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v"))
 	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("e"))
+	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("environment"))
 	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config"))
 	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data"))
 	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs"))

libbeat/docs/command-reference.asciidoc

@@ -910,6 +910,13 @@ messages.
 *`-e, --e`*::
 Logs to stderr and disables syslog/file output.
 
+*`-environment`*::
+For logging purposes, specifies the environment that {beatname_uc} is running in.
+This setting is used to select a default log output when no log output is configured.
+Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`.
+If `systemd` or `container` is specified, {beatname_uc} will log to stdout and stderr
+by default.
+
 *`--path.config`*::
 Sets the path for configuration files. See the <<directory-layout>> section for
 details.

libbeat/docs/shared-systemd.asciidoc

@@ -54,10 +54,6 @@ Logs are stored by default in journald. To view the Logs, use `journalctl`:
 journalctl -u {beatname_lc}.service
 ------------------------------------------------
 
-NOTE: The unit file included in the packages sets the `-e` flag by default.
-This flag makes {beatname_uc} log to stderr and disables other log outputs.
-Systemd stores all output sent to stderr in journald.
-
 [float]
 === Customize systemd unit for {beatname_uc}
 
@@ -67,7 +63,7 @@ override to change the default options.
 [cols="<h,<,<m",options="header",]
 |=======================================
 | Variable | Description | Default value
-| BEAT_LOG_OPTS | Log options | `-e`
+| BEAT_LOG_OPTS | Log options |
 | BEAT_CONFIG_OPTS | Flags for configuration file path | +-c /etc/{beatname_lc}/{beatname_lc}.yml+
 | BEAT_PATH_OPTS | Other paths | +-path.home /usr/share/{beatname_lc} -path.config /etc/{beatname_lc} -path.data /var/lib/{beatname_lc} -path.logs /var/log/{beatname_lc}+
 |=======================================
@@ -82,16 +78,7 @@ would override `BEAT_LOG_OPTS` to enable debug for Elasticsearch output.
 ["source", "systemd", subs="attributes"]
 ------------------------------------------------
 [Service]
-Environment="BEAT_LOG_OPTS=-e -d elasticsearch"
-------------------------------------------------
-
-To change the logging output from the {beatname_uc} configuration file, empty
-the environment variable. For example:
-
-["source", "systemd", subs="attributes"]
-------------------------------------------------
-[Service]
-Environment="BEAT_LOG_OPTS="
+Environment="BEAT_LOG_OPTS=-d elasticsearch"
 ------------------------------------------------
 
 To apply your changes, reload the systemd configuration and restart
@@ -109,4 +96,4 @@ include drop-in unit files. If you need to add a drop-in manually, use
 
 ifdef::apm-server[]
 include::./../config-ownership.asciidoc[]
-endif::apm-server[]
+endif::apm-server[]

libbeat/logp/config.go

@@ -17,7 +17,9 @@
 
 package logp
 
-import "time"
+import (
+	"time"
+)
 
 // Config contains the configuration options for the logger. To create a Config
 // from a common.Config use logp/config.Build.
@@ -52,20 +54,39 @@ type FileConfig struct {
 	RedirectStderr  bool          `config:"redirect_stderr"`
 }
 
-var defaultConfig = Config{
-	Level:   InfoLevel,
-	ToFiles: true,
-	Files: FileConfig{
-		MaxSize:         10 * 1024 * 1024,
-		MaxBackups:      7,
-		Permissions:     0600,
-		Interval:        0,
-		RotateOnStartup: true,
-	},
-	addCaller: true,
+// DefaultConfig returns the default config options for a given environment the
+// Beat is supposed to be run within.
+func DefaultConfig(environment Environment) Config {
+	switch environment {
+	case SystemdEnvironment, ContainerEnvironment:
+		return defaultToStderrConfig()
+
+	case MacOSServiceEnvironment, WindowsServiceEnvironment:
+		fallthrough
+	default:
+		return defaultToFileConfig()
+	}
+}
+
+func defaultToStderrConfig() Config {
+	return Config{
+		Level:     InfoLevel,
+		ToStderr:  true,
+		addCaller: true,
+	}
 }
 
-// DefaultConfig returns the default config options.
-func DefaultConfig() Config {
-	return defaultConfig
+func defaultToFileConfig() Config {
+	return Config{
+		Level:   InfoLevel,
+		ToFiles: true,
+		Files: FileConfig{
+			MaxSize:         10 * 1024 * 1024,
+			MaxBackups:      7,
+			Permissions:     0600,
+			Interval:        0,
+			RotateOnStartup: true,
+		},
+		addCaller: true,
+	}
 }

libbeat/logp/configure/logging.go

@@ -19,6 +19,7 @@ package configure
 
 import (
 	"flag"
+	"fmt"
 	"strings"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -30,18 +31,22 @@ var (
 	verbose        bool
 	toStderr       bool
 	debugSelectors []string
+	environment    logp.Environment
 )
 
+type environmentVar logp.Environment
+
 func init() {
 	flag.BoolVar(&verbose, "v", false, "Log at INFO level")
 	flag.BoolVar(&toStderr, "e", false, "Log to stderr and disable syslog/file output")
 	common.StringArrVarFlag(nil, &debugSelectors, "d", "Enable certain debug selectors")
+	flag.Var((*environmentVar)(&environment), "environment", "set environment the Beat is run in")
 }
 
 // Logging builds a logp.Config based on the given common.Config and the specified
 // CLI flags.
 func Logging(beatName string, cfg *common.Config) error {
-	config := logp.DefaultConfig()
+	config := logp.DefaultConfig(environment)
 	config.Beat = beatName
 	if cfg != nil {
 		if err := cfg.Unpack(&config); err != nil {
@@ -69,3 +74,17 @@ func applyFlags(cfg *logp.Config) {
 		cfg.Level = logp.DebugLevel
 	}
 }
+
+func (v *environmentVar) Set(in string) error {
+	env := logp.ParseEnvironment(in)
+	if env == logp.InvalidEnvironment {
+		return fmt.Errorf("'%v' is not supported", in)
+	}
+
+	*(*logp.Environment)(v) = env
+	return nil
+}
+
+func (v *environmentVar) String() string {
+	return (*logp.Environment)(v).String()
+}

libbeat/logp/environment.go

@@ -0,0 +1,82 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package logp
+
+import "strings"
+
+// Environment indicates the environment the logger is supped to be run in.
+// The default logger configuration may be different for different environments.
+type Environment int
+
+const (
+	// DefaultEnvironment is used if the environment the process runs in is not known.
+	DefaultEnvironment Environment = iota
+
+	// SystemdEnvironment indicates that the process is started and managed by systemd.
+	SystemdEnvironment
+
+	// ContainerEnvironment indicates that the process is running within a container (docker, k8s, rkt, ...).
+	ContainerEnvironment
+
+	// MacOSServiceEnvironment indicates that the process is running as a daemon on macOS (e.g. managed via launchctl).
+	MacOSServiceEnvironment
+
+	// WindowsServiceEnvironment indicates the the process is run as a windows service.
+	WindowsServiceEnvironment
+
+	// InvalidEnvironment indicates that the environment name given is unknown or invalid.
+	InvalidEnvironment
+)
+
+// String returns the string representation the configured environment
+func (v Environment) String() string {
+	switch v {
+	case DefaultEnvironment:
+		return "default"
+	case SystemdEnvironment:
+		return "systemd"
+	case ContainerEnvironment:
+		return "container"
+	case MacOSServiceEnvironment:
+		return "macOS_service"
+	case WindowsServiceEnvironment:
+		return "windows_service"
+	default:
+		return "<invalid>"
+	}
+}
+
+// ParseEnvironment returns the environment type by name.
+// The parse is case insensitive.
+// InvalidEnvironment is returned if the environment type is unknown.
+func ParseEnvironment(in string) Environment {
+	switch strings.ToLower(in) {
+	case "default":
+		return DefaultEnvironment
+	case "systemd":
+		return SystemdEnvironment
+	case "container":
+		return ContainerEnvironment
+	case "macos_service":
+		return MacOSServiceEnvironment
+	case "windows_service":
+		return WindowsServiceEnvironment
+	default:
+		return InvalidEnvironment
+	}
+}