From 8865e3fa7f52823eb2a9669a4e560ef5a7011f6f Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 26 Sep 2019 15:14:42 -0600 Subject: [PATCH] [Filebeat] Fix s3 access key grok pattern (#13811) * Fix s3 access key grok pattern * Improve error handling when a file doesn't exist or not readable --- x-pack/filebeat/input/s3/input.go | 20 ++++++++++++---- x-pack/filebeat/input/s3/input_test.go | 9 ++++--- .../module/aws/s3access/ingest/pipeline.yml | 2 +- .../aws/s3access/test/s3_server_access.log | 1 + .../test/s3_server_access.log-expected.json | 24 +++++++++++++++++++ 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 3c08ec88082b..51f9ee49eb94 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -362,10 +362,13 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC objectHash := s3ObjectHash(s3Info) // read from s3 object - reader, err := newS3BucketReader(svc, s3Info, p.context) + reader, err := p.newS3BucketReader(svc, s3Info) if err != nil { return errors.Wrap(err, "newS3BucketReader failed") } + if reader == nil { + continue + } offset := 0 for { @@ -405,17 +408,24 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC return nil } -func newS3BucketReader(svc s3iface.ClientAPI, s3Info s3Info, context *channelContext) (*bufio.Reader, error) { +func (p *s3Input) newS3BucketReader(svc s3iface.ClientAPI, s3Info s3Info) (*bufio.Reader, error) { s3GetObjectInput := &s3.GetObjectInput{ Bucket: awssdk.String(s3Info.name), Key: awssdk.String(s3Info.key), } req := svc.GetObjectRequest(s3GetObjectInput) - resp, err := req.Send(context) + resp, err := req.Send(p.context) if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { - return nil, nil + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == awssdk.ErrCodeRequestCanceled { + return nil, nil + } + + if awsErr.Code() == "NoSuchKey" { + p.logger.Warn("Cannot find s3 file with key ", s3Info.key) + return nil, nil + } } return nil, errors.Wrapf(err, "s3 get object request failed %v", s3Info.key) } diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/input_test.go index 80532a2851ee..23d7c4d1da27 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/input_test.go @@ -121,7 +121,8 @@ func TestHandleMessage(t *testing.T) { } func TestNewS3BucketReader(t *testing.T) { - reader, err := newS3BucketReader(mockSvc, info, &channelContext{}) + p := &s3Input{context: &channelContext{}} + reader, err := p.newS3BucketReader(mockSvc, info) assert.NoError(t, err) for i := 0; i < 3; i++ { switch i { @@ -142,12 +143,14 @@ func TestNewS3BucketReader(t *testing.T) { } func TestNewS3BucketReaderErr(t *testing.T) { - reader, err := newS3BucketReader(mockSvcErr, info, &channelContext{}) + p := &s3Input{context: &channelContext{}} + reader, err := p.newS3BucketReader(mockSvcErr, info) assert.Error(t, err, "s3 get object response body is empty") assert.Nil(t, reader) } func TestCreateEvent(t *testing.T) { + p := &s3Input{context: &channelContext{}} errC := make(chan error) s3Context := &s3Context{ refs: 1, @@ -163,7 +166,7 @@ func TestCreateEvent(t *testing.T) { } s3ObjectHash := s3ObjectHash(s3Info) - reader, err := newS3BucketReader(mockSvc, s3Info, &channelContext{}) + reader, err := p.newS3BucketReader(mockSvc, s3Info) assert.NoError(t, err) var events []beat.Event for { diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index 9c1d0c5a48ee..8090da4d465d 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -17,7 +17,7 @@ processors: S3REQUESTER: "[a-zA-Z0-9\\/_\\.\\-%:@]+" S3REQUESTID: "[a-zA-Z0-9]+" S3OPERATION: "%{WORD}.%{WORD}.%{WORD}" - S3KEY: "[a-zA-Z0-9\\/_\\.\\-%]+" + S3KEY: "[a-zA-Z0-9\\/_\\.\\-%+]+" S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+" S3VERSION: "[a-zA-Z0-9.]+" diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log index b8848397fcb2..f96091a76798 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log @@ -3,3 +3,4 @@ 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2 diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json index af44251d597a..a9ce57e1f411 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json @@ -131,5 +131,29 @@ "input.type": "log", "log.offset": 2875, "service.type": "aws" + }, + { + "@timestamp": "2019-09-19T17:06:39.000Z", + "aws.s3access.authentication_type": "AuthHeader", + "aws.s3access.bucket": "test-s3-ks", + "aws.s3access.bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "aws.s3access.cipher_suite": "ECDHE-RSA-AES128-SHA", + "aws.s3access.host_header": "s3-ap-southeast-1.amazonaws.com", + "aws.s3access.host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=", + "aws.s3access.http_status": 204, + "aws.s3access.key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png", + "aws.s3access.object_size": 57138, + "aws.s3access.operation": "BATCH.DELETE.OBJECT", + "aws.s3access.remote_ip": "174.29.206.152", + "aws.s3access.request_id": "6CE38F1312D32BDD", + "aws.s3access.requester": "arn:aws:iam::123456:user/test@elastic.co", + "aws.s3access.signature_version": "SigV4", + "aws.s3access.tls_version": "TLSv1.2", + "event.dataset": "aws.s3access", + "event.module": "aws", + "fileset.name": "s3access", + "input.type": "log", + "log.offset": 3280, + "service.type": "aws" } ] \ No newline at end of file