Merge remote-tracking branch 'upstream/master' into fix/ci-tools-inst…

…allation

* upstream/master:
  Check expand_event_list_from_field when json in map[string]interface{} format (elastic#20370)
  [docs] Remove deprecated security roles (elastic#20162)
  Modify doc in app_insights metricset (elastic#20185)
  [Elastic Agent] Add skeleton for client/server for agent control protocol (elastic#20163)
  Auditbeat: Allow multiple instances by grouping kprobes by PID (elastic#20325)
  [Filebeat][Fortinet] Remove pre populated event.timezone (elastic#20273)
  Add an explicit system test for processes on unix systems (elastic#20320)

CHANGELOG.next.asciidoc

@@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - auditd: Fix spelling of anomaly in `event.category`.
 - auditd: Fix typo in `event.action` of `removed-user-role-from`. {pull}19300[19300]
 - auditd: Fix typo in `event.action` of `used-suspicious-link`. {pull}19300[19300]
+- system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325]
 
 *Filebeat*
 
@@ -225,9 +226,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984]
 - Fix auditd module syscall table for ppc64 and ppc64le. {pull}20052[20052]
 - Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552]
-- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962]
+- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] {pull}20370[20370]
 - Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138]
 - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
+- Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273]
 
 *Heartbeat*
 
@@ -298,6 +300,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
+- Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
 
 *Packetbeat*
 

filebeat/docs/fields.asciidoc

@@ -45379,7 +45379,7 @@ Roles to which the principal belongs
 
 type: keyword
 
-example: ['kibana_user', 'beats_admin']
+example: ['kibana_admin', 'beats_admin']
 
 --
 

filebeat/module/elasticsearch/audit/_meta/fields.yml

@@ -24,7 +24,7 @@
       type: keyword
     - name: user.roles
       description: "Roles to which the principal belongs"
-      example: [ "kibana_user", "beats_admin" ]
+      example: [ "kibana_admin", "beats_admin" ]
       type: keyword
     - name: action
       description: "The name of the action that was executed"

libbeat/docs/security/users.asciidoc

@@ -169,7 +169,7 @@ users who need to monitor {beatname_uc}:
 |====
 |Role | Purpose
 
-|`kibana_user`
+|`kibana_admin`
 |Use {kib}
 
 |`monitoring_user`
@@ -231,7 +231,7 @@ endif::serverless[]
 Users who publish events to {es} need to create and write to {beatname_uc}
 indices. To minimize the privileges required by the writer role, use the
 <<privileges-to-setup-beats,setup role>> to pre-load dependencies. This section
-assumes that you've pre-loaded dependencies.
+assumes that you've run the setup.
 
 ifndef::no_ilm[]
 When using ILM, turn off the ILM setup check in the {beatname_uc} config file before

libbeat/docs/tab-widgets/set-connection.asciidoc

@@ -58,8 +58,8 @@ include the scheme and port: `http://mykibanahost:5601/path`.
 <2> The `username` and `password` settings for {kib} are optional. If you don't
 specify credentials for {kib}, {beatname_uc} uses the `username` and `password`
 specified for the {es} output.
-<3> To use the pre-built Kibana dashboards, this user must have the
-`kibana_user` {ref}/built-in-roles.html[built-in role] or equivalent
-privileges.
+<3> To use the pre-built {kib} dashboards, this user must be authorized to
+view dashboards or have the
+`kibana_admin` {ref}/built-in-roles.html[built-in role].
 // end::self-managed[]
 

metricbeat/module/system/test_system.py

@@ -392,41 +392,85 @@ def test_process(self):
         self.assertGreater(len(output), 0)
 
         found_cmdline = False
-        found_env = False
+        for evt in output:
+            process = evt["system"]["process"]
+            found_cmdline |= "cmdline" in process
+
+            # Remove 'env' prior to checking documented fields because its keys are dynamic.
+            process.pop("env", None)
+            self.assert_fields_are_documented(evt)
+
+            # Remove optional keys.
+            process.pop("cgroup", None)
+            process.pop("fd", None)
+            process.pop("cmdline", None)
+
+            self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys())
+
+        self.assertTrue(found_cmdline, "cmdline not found in any process events")
+
+    @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os")
+    def test_process_unix(self):
+        """
+        Test system/process output for fields specific of unix systems.
+        """
+        import getpass
+
+        self.render_config_template(
+            modules=[{
+                "name": "system",
+                "metricsets": ["process"],
+                "period": "5s",
+                "extras": {
+                    "process.env.whitelist": ["PATH"],
+                    "process.include_cpu_ticks": True,
+
+                    # Remove 'percpu' prior to checking documented fields because its keys are dynamic.
+                    "process.include_per_cpu": False,
+                },
+            }],
+            # Some info is only guaranteed in processes with permissions, check
+            # only on own processes.
+            processors=[{
+                "drop_event": {
+                    "when": "not.equals.user.name: " + getpass.getuser(),
+                },
+            }],
+        )
+        proc = self.start_beat()
+        self.wait_until(lambda: self.output_lines() > 0)
+        proc.check_kill_and_wait()
+        self.assert_no_logged_warnings()
+
+        output = self.read_output_json()
+        self.assertGreater(len(output), 0)
+
         found_fd = False
+        found_env = False
         found_cwd = not sys.platform.startswith("linux")
         for evt in output:
+            found_cwd |= "working_directory" in evt["process"]
+
             process = evt["system"]["process"]
+            found_fd |= "fd" in process
+            found_env |= "env" in process
 
             # Remove 'env' prior to checking documented fields because its keys are dynamic.
             env = process.pop("env", None)
-            if env is not None:
-                found_env = True
-
             self.assert_fields_are_documented(evt)
 
             # Remove optional keys.
             process.pop("cgroup", None)
-            cmdline = process.pop("cmdline", None)
-            if cmdline is not None:
-                found_cmdline = True
-            fd = process.pop("fd", None)
-            if fd is not None:
-                found_fd = True
-            cwd = process.pop("cwd", None)
-            if cwd is not None:
-                found_cwd = True
+            process.pop("cmdline", None)
+            process.pop("fd", None)
 
             self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys())
 
-        self.assertTrue(found_cmdline, "cmdline not found in any process events")
-
-        if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"):
+        if not sys.platform.startswith("darwin"):
             self.assertTrue(found_fd, "fd not found in any process events")
 
-        if sys.platform.startswith("linux") or sys.platform.startswith("freebsd")\
-                or sys.platform.startswith("darwin"):
-            self.assertTrue(found_env, "env not found in any process events")
+        self.assertTrue(found_env, "env not found in any process events")
+        self.assertTrue(found_cwd, "working_directory not found in any process events")
 
     @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os")
     def test_process_metricbeat(self):

x-pack/auditbeat/module/system/socket/socket_linux.go

@@ -9,8 +9,11 @@ package socket
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -36,17 +39,18 @@ import (
 )
 
 const (
-	moduleName     = "system"
-	metricsetName  = "socket"
-	fullName       = moduleName + "/" + metricsetName
-	namespace      = "system.audit.socket"
-	detailSelector = metricsetName + "detailed"
-	auditbeatGroup = "auditbeat"
+	moduleName      = "system"
+	metricsetName   = "socket"
+	fullName        = moduleName + "/" + metricsetName
+	namespace       = "system.audit.socket"
+	detailSelector  = metricsetName + "detailed"
+	groupNamePrefix = "auditbeat_"
 	// Magic value to detect clock-sync events generated by the metricset.
 	clockSyncMagic uint64 = 0x42DEADBEEFABCDEF
 )
 
 var (
+	groupName     = fmt.Sprintf("%s%d", groupNamePrefix, os.Getpid())
 	kernelVersion string
 	eventCount    uint64
 )
@@ -290,7 +294,7 @@ func (m *MetricSet) Setup() (err error) {
 		extra = WithFilterPort(22)
 	}
 	m.installer = newProbeInstaller(traceFS,
-		WithGroup(auditbeatGroup),
+		WithGroup(groupName),
 		WithTemplates(m.templateVars),
 		extra)
 	defer func() {
@@ -300,10 +304,18 @@ func (m *MetricSet) Setup() (err error) {
 	}()
 
 	//
-	// remove existing KProbes from Auditbeat
+	// remove dangling KProbes from terminated Auditbeat processes.
+	// Not a fatal error if they can't be removed.
 	//
-	if err = m.installer.UninstallIf(isOwnProbe); err != nil {
-		return errors.Wrap(err, "unable to delete existing KProbes. Is Auditbeat already running?")
+	if err = m.installer.UninstallIf(isDeadAuditbeat); err != nil {
+		m.log.Debugf("Removing existing probes from terminated instances: %+v", err)
+	}
+
+	//
+	// remove existing Auditbeat KProbes that match the current PID.
+	//
+	if err = m.installer.UninstallIf(isThisAuditbeat); err != nil {
+		return errors.Wrapf(err, "unable to delete existing KProbes for group %s", groupName)
 	}
 
 	//
@@ -409,7 +421,7 @@ func (m *MetricSet) Cleanup() {
 		}
 	}
 	if m.installer != nil {
-		if err := m.installer.UninstallIf(isOwnProbe); err != nil {
+		if err := m.installer.UninstallIf(isThisAuditbeat); err != nil {
 			m.log.Warnf("Failed to remove KProbes on exit: %v", err)
 		}
 	}
@@ -468,8 +480,28 @@ func triggerClockSync() {
 	unix.Uname(&buf)
 }
 
-func isOwnProbe(probe tracing.Probe) bool {
-	return probe.Group == auditbeatGroup
+func isRunningAuditbeat(pid int) bool {
+	path := fmt.Sprintf("/proc/%d/exe", pid)
+	exePath, err := os.Readlink(path)
+	if err != nil {
+		// Not a running process
+		return false
+	}
+	exeName := filepath.Base(exePath)
+	return strings.HasPrefix(exeName, "auditbeat")
+}
+
+func isDeadAuditbeat(probe tracing.Probe) bool {
+	if strings.HasPrefix(probe.Group, groupNamePrefix) && probe.Group != groupName {
+		if pid, err := strconv.Atoi(probe.Group[len(groupNamePrefix):]); err == nil && !isRunningAuditbeat(pid) {
+			return true
+		}
+	}
+	return false
+}
+
+func isThisAuditbeat(probe tracing.Probe) bool {
+	return probe.Group == groupName
 }
 
 type mountPoint struct {

x-pack/elastic-agent/pkg/agent/control/addr.go

@@ -0,0 +1,20 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build !windows
+
+package control
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+)
+
+// Address returns the address to connect to Elastic Agent daemon.
+func Address() string {
+	data := paths.Data()
+	return fmt.Sprintf("unix://%s", filepath.Join(data, "agent.sock"))
+}

x-pack/elastic-agent/pkg/agent/control/addr_windows.go

@@ -0,0 +1,22 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build windows
+
+package control
+
+import (
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+)
+
+// Address returns the address to connect to Elastic Agent daemon.
+func Address() string {
+	data = paths.Data()
+	// entire string cannot be longer than 256 characters, this forces the
+	// length to always be 87 characters (but unique per data path)
+	return fmt.Sprintf(`\\.\pipe\elastic-agent-%s`, sha256.Sum256(data))
+}