[Filebeat] Fix s3 access key grok pattern (elastic#13811)

* Fix s3 access key grok pattern
* Improve error handling when a file doesn't exist or not readable

x-pack/filebeat/input/s3/input.go

@@ -362,10 +362,13 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC
 		objectHash := s3ObjectHash(s3Info)
 
 		// read from s3 object
-		reader, err := newS3BucketReader(svc, s3Info, p.context)
+		reader, err := p.newS3BucketReader(svc, s3Info)
 		if err != nil {
 			return errors.Wrap(err, "newS3BucketReader failed")
 		}
+		if reader == nil {
+			continue
+		}
 
 		offset := 0
 		for {
@@ -405,17 +408,24 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC
 	return nil
 }
 
-func newS3BucketReader(svc s3iface.ClientAPI, s3Info s3Info, context *channelContext) (*bufio.Reader, error) {
+func (p *s3Input) newS3BucketReader(svc s3iface.ClientAPI, s3Info s3Info) (*bufio.Reader, error) {
 	s3GetObjectInput := &s3.GetObjectInput{
 		Bucket: awssdk.String(s3Info.name),
 		Key:    awssdk.String(s3Info.key),
 	}
 	req := svc.GetObjectRequest(s3GetObjectInput)
 
-	resp, err := req.Send(context)
+	resp, err := req.Send(p.context)
 	if err != nil {
-		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled {
-			return nil, nil
+		if awsErr, ok := err.(awserr.Error); ok {
+			if awsErr.Code() == awssdk.ErrCodeRequestCanceled {
+				return nil, nil
+			}
+
+			if awsErr.Code() == "NoSuchKey" {
+				p.logger.Warn("Cannot find s3 file with key ", s3Info.key)
+				return nil, nil
+			}
 		}
 		return nil, errors.Wrapf(err, "s3 get object request failed %v", s3Info.key)
 	}

x-pack/filebeat/input/s3/input_test.go

@@ -121,7 +121,8 @@ func TestHandleMessage(t *testing.T) {
 }
 
 func TestNewS3BucketReader(t *testing.T) {
-	reader, err := newS3BucketReader(mockSvc, info, &channelContext{})
+	p := &s3Input{context: &channelContext{}}
+	reader, err := p.newS3BucketReader(mockSvc, info)
 	assert.NoError(t, err)
 	for i := 0; i < 3; i++ {
 		switch i {
@@ -142,12 +143,14 @@ func TestNewS3BucketReader(t *testing.T) {
 }
 
 func TestNewS3BucketReaderErr(t *testing.T) {
-	reader, err := newS3BucketReader(mockSvcErr, info, &channelContext{})
+	p := &s3Input{context: &channelContext{}}
+	reader, err := p.newS3BucketReader(mockSvcErr, info)
 	assert.Error(t, err, "s3 get object response body is empty")
 	assert.Nil(t, reader)
 }
 
 func TestCreateEvent(t *testing.T) {
+	p := &s3Input{context: &channelContext{}}
 	errC := make(chan error)
 	s3Context := &s3Context{
 		refs: 1,
@@ -163,7 +166,7 @@ func TestCreateEvent(t *testing.T) {
 	}
 	s3ObjectHash := s3ObjectHash(s3Info)
 
-	reader, err := newS3BucketReader(mockSvc, s3Info, &channelContext{})
+	reader, err := p.newS3BucketReader(mockSvc, s3Info)
 	assert.NoError(t, err)
 	var events []beat.Event
 	for {

x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml

@@ -17,7 +17,7 @@ processors:
         S3REQUESTER: "[a-zA-Z0-9\\/_\\.\\-%:@]+"
         S3REQUESTID: "[a-zA-Z0-9]+"
         S3OPERATION: "%{WORD}.%{WORD}.%{WORD}"
-        S3KEY: "[a-zA-Z0-9\\/_\\.\\-%]+"
+        S3KEY: "[a-zA-Z0-9\\/_\\.\\-%+]+"
         S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+"
         S3VERSION: "[a-zA-Z0-9.]+"
 

x-pack/filebeat/module/aws/s3access/test/s3_server_access.log

@@ -3,3 +3,4 @@
 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2
 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2
 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/[email protected] 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2
+36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/[email protected] 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2

x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json

@@ -131,5 +131,29 @@
         "input.type": "log",
         "log.offset": 2875,
         "service.type": "aws"
+    },
+    {
+        "@timestamp": "2019-09-19T17:06:39.000Z",
+        "aws.s3access.authentication_type": "AuthHeader",
+        "aws.s3access.bucket": "test-s3-ks",
+        "aws.s3access.bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2",
+        "aws.s3access.cipher_suite": "ECDHE-RSA-AES128-SHA",
+        "aws.s3access.host_header": "s3-ap-southeast-1.amazonaws.com",
+        "aws.s3access.host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=",
+        "aws.s3access.http_status": 204,
+        "aws.s3access.key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png",
+        "aws.s3access.object_size": 57138,
+        "aws.s3access.operation": "BATCH.DELETE.OBJECT",
+        "aws.s3access.remote_ip": "174.29.206.152",
+        "aws.s3access.request_id": "6CE38F1312D32BDD",
+        "aws.s3access.requester": "arn:aws:iam::123456:user/[email protected]",
+        "aws.s3access.signature_version": "SigV4",
+        "aws.s3access.tls_version": "TLSv1.2",
+        "event.dataset": "aws.s3access",
+        "event.module": "aws",
+        "fileset.name": "s3access",
+        "input.type": "log",
+        "log.offset": 3280,
+        "service.type": "aws"
     }
 ]