Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Filebeat] Fix s3 access key grok pattern #13811

Merged
merged 2 commits into from
Sep 26, 2019
Merged

[Filebeat] Fix s3 access key grok pattern #13811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
57b9bbd
Fix s3 access key grok pattern
kaiyan-sheng Sep 26, 2019
f6a5ffb
Improve error handling when a file doesn't exist or not readable
kaiyan-sheng Sep 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions x-pack/filebeat/input/s3/input.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,10 +362,13 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC
objectHash := s3ObjectHash(s3Info)

// read from s3 object
reader, err := newS3BucketReader(svc, s3Info, p.context)
reader, err := p.newS3BucketReader(svc, s3Info)
if err != nil {
return errors.Wrap(err, "newS3BucketReader failed")
}
if reader == nil {
continue
}

offset := 0
for {
Expand Down Expand Up @@ -405,17 +408,24 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC
return nil
}

func newS3BucketReader(svc s3iface.ClientAPI, s3Info s3Info, context *channelContext) (*bufio.Reader, error) {
func (p *s3Input) newS3BucketReader(svc s3iface.ClientAPI, s3Info s3Info) (*bufio.Reader, error) {
s3GetObjectInput := &s3.GetObjectInput{
Bucket: awssdk.String(s3Info.name),
Key: awssdk.String(s3Info.key),
}
req := svc.GetObjectRequest(s3GetObjectInput)

resp, err := req.Send(context)
resp, err := req.Send(p.context)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled {
return nil, nil
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == awssdk.ErrCodeRequestCanceled {
return nil, nil
}

if awsErr.Code() == "NoSuchKey" {
p.logger.Warn("Cannot find s3 file with key ", s3Info.key)
return nil, nil
}
}
return nil, errors.Wrapf(err, "s3 get object request failed %v", s3Info.key)
}
Expand Down
9 changes: 6 additions & 3 deletions x-pack/filebeat/input/s3/input_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ func TestHandleMessage(t *testing.T) {
}

func TestNewS3BucketReader(t *testing.T) {
reader, err := newS3BucketReader(mockSvc, info, &channelContext{})
p := &s3Input{context: &channelContext{}}
reader, err := p.newS3BucketReader(mockSvc, info)
assert.NoError(t, err)
for i := 0; i < 3; i++ {
switch i {
Expand All @@ -142,12 +143,14 @@ func TestNewS3BucketReader(t *testing.T) {
}

func TestNewS3BucketReaderErr(t *testing.T) {
reader, err := newS3BucketReader(mockSvcErr, info, &channelContext{})
p := &s3Input{context: &channelContext{}}
reader, err := p.newS3BucketReader(mockSvcErr, info)
assert.Error(t, err, "s3 get object response body is empty")
assert.Nil(t, reader)
}

func TestCreateEvent(t *testing.T) {
p := &s3Input{context: &channelContext{}}
errC := make(chan error)
s3Context := &s3Context{
refs: 1,
Expand All @@ -163,7 +166,7 @@ func TestCreateEvent(t *testing.T) {
}
s3ObjectHash := s3ObjectHash(s3Info)

reader, err := newS3BucketReader(mockSvc, s3Info, &channelContext{})
reader, err := p.newS3BucketReader(mockSvc, s3Info)
assert.NoError(t, err)
var events []beat.Event
for {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ processors:
S3REQUESTER: "[a-zA-Z0-9\\/_\\.\\-%:@]+"
S3REQUESTID: "[a-zA-Z0-9]+"
S3OPERATION: "%{WORD}.%{WORD}.%{WORD}"
S3KEY: "[a-zA-Z0-9\\/_\\.\\-%]+"
S3KEY: "[a-zA-Z0-9\\/_\\.\\-%+]+"
S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+"
S3VERSION: "[a-zA-Z0-9.]+"

Expand Down
1 change: 1 addition & 0 deletions x-pack/filebeat/module/aws/s3access/test/s3_server_access.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2
36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2
36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/[email protected] 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2
36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/[email protected] 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2
24 changes: 24 additions & 0 deletions x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,5 +131,29 @@
"input.type": "log",
"log.offset": 2875,
"service.type": "aws"
},
{
"@timestamp": "2019-09-19T17:06:39.000Z",
"aws.s3access.authentication_type": "AuthHeader",
"aws.s3access.bucket": "test-s3-ks",
"aws.s3access.bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2",
"aws.s3access.cipher_suite": "ECDHE-RSA-AES128-SHA",
"aws.s3access.host_header": "s3-ap-southeast-1.amazonaws.com",
"aws.s3access.host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=",
"aws.s3access.http_status": 204,
"aws.s3access.key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png",
"aws.s3access.object_size": 57138,
"aws.s3access.operation": "BATCH.DELETE.OBJECT",
"aws.s3access.remote_ip": "174.29.206.152",
"aws.s3access.request_id": "6CE38F1312D32BDD",
"aws.s3access.requester": "arn:aws:iam::123456:user/[email protected]",
"aws.s3access.signature_version": "SigV4",
"aws.s3access.tls_version": "TLSv1.2",
"event.dataset": "aws.s3access",
"event.module": "aws",
"fileset.name": "s3access",
"input.type": "log",
"log.offset": 3280,
"service.type": "aws"
}
]