Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0

* upstream/master:
  Check expand_event_list_from_field when json in map[string]interface{} format (elastic#20370)
  [docs] Remove deprecated security roles (elastic#20162)
  Modify doc in app_insights metricset (elastic#20185)

CHANGELOG.next.asciidoc

@@ -226,7 +226,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984]
 - Fix auditd module syscall table for ppc64 and ppc64le. {pull}20052[20052]
 - Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552]
-- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962]
+- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] {pull}20370[20370]
 - Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138]
 - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
 - Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273]
@@ -300,6 +300,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
+- Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
 
 *Packetbeat*
 

filebeat/docs/fields.asciidoc

@@ -45379,7 +45379,7 @@ Roles to which the principal belongs
 
 type: keyword
 
-example: ['kibana_user', 'beats_admin']
+example: ['kibana_admin', 'beats_admin']
 
 --
 

filebeat/module/elasticsearch/audit/_meta/fields.yml

@@ -24,7 +24,7 @@
       type: keyword
     - name: user.roles
       description: "Roles to which the principal belongs"
-      example: [ "kibana_user", "beats_admin" ]
+      example: [ "kibana_admin", "beats_admin" ]
       type: keyword
     - name: action
       description: "The name of the action that was executed"

libbeat/docs/security/users.asciidoc

@@ -169,7 +169,7 @@ users who need to monitor {beatname_uc}:
 |====
 |Role | Purpose
 
-|`kibana_user`
+|`kibana_admin`
 |Use {kib}
 
 |`monitoring_user`
@@ -231,7 +231,7 @@ endif::serverless[]
 Users who publish events to {es} need to create and write to {beatname_uc}
 indices. To minimize the privileges required by the writer role, use the
 <<privileges-to-setup-beats,setup role>> to pre-load dependencies. This section
-assumes that you've pre-loaded dependencies.
+assumes that you've run the setup.
 
 ifndef::no_ilm[]
 When using ILM, turn off the ILM setup check in the {beatname_uc} config file before

libbeat/docs/tab-widgets/set-connection.asciidoc

@@ -58,8 +58,8 @@ include the scheme and port: `http://mykibanahost:5601/path`.
 <2> The `username` and `password` settings for {kib} are optional. If you don't
 specify credentials for {kib}, {beatname_uc} uses the `username` and `password`
 specified for the {es} output.
-<3> To use the pre-built Kibana dashboards, this user must have the
-`kibana_user` {ref}/built-in-roles.html[built-in role] or equivalent
-privileges.
+<3> To use the pre-built {kib} dashboards, this user must be authorized to
+view dashboards or have the
+`kibana_admin` {ref}/built-in-roles.html[built-in role].
 // end::self-managed[]
 

x-pack/filebeat/input/s3/input.go

@@ -526,10 +526,11 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3
 			return nil
 		}
 
-		offset, err = p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx)
+		offsetNew, err := p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx)
 		if err != nil {
 			return err
 		}
+		offset = offsetNew
 	}
 }
 
@@ -554,6 +555,27 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash
 			return offset, nil
 		}
 	case map[string]interface{}:
+		if p.config.ExpandEventListFromField != "" {
+			textValues, ok := f[p.config.ExpandEventListFromField]
+			if !ok {
+				err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField)
+				p.logger.Error(err)
+				return offset, err
+			}
+
+			valuesConverted := textValues.([]interface{})
+			for _, textValue := range valuesConverted {
+				offsetNew, err := p.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx)
+				if err != nil {
+					err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name)
+					p.logger.Error(err)
+					return offset, err
+				}
+				offset = offsetNew
+			}
+			return offset, nil
+		}
+
 		offset, err := p.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx)
 		if err != nil {
 			err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name)

x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc

@@ -45,17 +45,15 @@ This value is only valid when segment is specified.
 `filter`:: (_string_) An expression used to filter the results.
 This value should be a valid OData filter expression where the keys of each clause should be applicable dimensions for the metric you are retrieving.
 
-Users can select the options to retrieve all metrics from a specific namespace using the following:
+Example configuration:
 
 ["source","yaml"]
 ----
- metrics:
- - id: ["*"]
-   timespan: "Microsoft.Storage/storageAccounts"
+metrics:
+ - id: ["requests/count", "requests/failed"]
+   segment: "request/name"
+   aggregation: ["sum"]
 ----
 
 
 
-A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or  a multiply of it.
-
-