diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0c333a35561..9a18671f36a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -258,6 +258,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552] - Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984] +- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] {pull}20370[20370] - Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138] - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245] - Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273] diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 15f9384b7cf..65984dace45 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -526,10 +526,11 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3 return nil } - offset, err = p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) + offsetNew, err := p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) if err != nil { return err } + offset = offsetNew } } @@ -554,6 +555,27 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash return offset, nil } case map[string]interface{}: + if p.config.ExpandEventListFromField != "" { + textValues, ok := f[p.config.ExpandEventListFromField] + if !ok { + err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + p.logger.Error(err) + return offset, err + } + + valuesConverted := textValues.([]interface{}) + for _, textValue := range valuesConverted { + offsetNew, err := p.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) + p.logger.Error(err) + return offset, err + } + offset = offsetNew + } + return offset, nil + } + offset, err := p.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx) if err != nil { err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name)