diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 95d2bbbcbde..6c681a43add 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -158,6 +158,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847] - Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834] - Fixed typo in log message. {pull}17897[17897] +- Unescape file name from SQS message. {pull}18370[18370] *Heartbeat* diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 48da3296363..e4f6f792686 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -13,6 +13,7 @@ import ( "encoding/json" "fmt" "io" + "net/url" "strings" "sync" "time" @@ -360,10 +361,16 @@ func handleSQSMessage(m sqs.Message) ([]s3Info, error) { var s3Infos []s3Info for _, record := range msg.Records { if record.EventSource == "aws:s3" && strings.HasPrefix(record.EventName, "ObjectCreated:") { + // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" + filename, err := url.QueryUnescape(record.S3.object.Key) + if err != nil { + return nil, errors.Wrapf(err, "url.QueryUnescape failed") + } + s3Infos = append(s3Infos, s3Info{ region: record.AwsRegion, name: record.S3.bucket.Name, - key: record.S3.object.Key, + key: filename, arn: record.S3.bucket.Arn, }) } else { @@ -381,6 +388,7 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC defer s3Ctx.done() for _, info := range s3Infos { + p.logger.Debugf("Processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) err := p.createEventsFromS3Info(svc, info, s3Ctx) if err != nil { err = errors.Wrapf(err, "createEventsFromS3Info failed for %v", info.key) diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/input_test.go index b4ad597c146..4d5ca16acf8 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/input_test.go @@ -89,6 +89,30 @@ func TestHandleMessage(t *testing.T) { }, }, }, + { + "sqs message with event source aws:s3, event name ObjectCreated:Put and encoded filename", + sqs.Message{ + Body: awssdk.String("{\"Records\":[{\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-southeast-1\",\"eventTime\":\"2019-06-21T16:16:54.629Z\",\"eventName\":\"ObjectCreated:Put\",\"s3\":{\"configurationId\":\"object-created-event\",\"bucket\":{\"name\":\"test-s3-ks-2\",\"arn\":\"arn:aws:s3:::test-s3-ks-2\"},\"object\":{\"key\":\"year%3D2020/month%3D05/test1.txt\"}}}]}"), + }, + []s3Info{ + { + name: "test-s3-ks-2", + key: "year=2020/month=05/test1.txt", + }, + }, + }, + { + "sqs message with event source aws:s3, event name ObjectCreated:Put and gzip filename", + sqs.Message{ + Body: awssdk.String("{\"Records\":[{\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-southeast-1\",\"eventTime\":\"2019-06-21T16:16:54.629Z\",\"eventName\":\"ObjectCreated:Put\",\"s3\":{\"configurationId\":\"object-created-event\",\"bucket\":{\"name\":\"test-s3-ks-2\",\"arn\":\"arn:aws:s3:::test-s3-ks-2\"},\"object\":{\"key\":\"428152502467_CloudTrail_us-east-2_20191219T1655Z_WXCas1PVnOaTpABD.json.gz\"}}}]}"), + }, + []s3Info{ + { + name: "test-s3-ks-2", + key: "428152502467_CloudTrail_us-east-2_20191219T1655Z_WXCas1PVnOaTpABD.json.gz", + }, + }, + }, } for _, c := range casesPositive {