Skip to content

Commit

Permalink
Remove url.QueryUnescape()
Browse files Browse the repository at this point in the history
We introduced [^1] the `url.QueryUnescape()` function to unescape
object keys from S3 notification in SQS messages.

However, the object keys in the S3 list object responses do not
require [^2] unescape.

We must remove the unescape to avoid unintended changes to the S3
object key.

[^1]: elastic#18370
[^2]: elastic#38012 (comment)
  • Loading branch information
zmoog committed Feb 23, 2024
1 parent db704dc commit 62e9992
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 10 deletions.
12 changes: 2 additions & 10 deletions x-pack/filebeat/input/awss3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"context"
"errors"
"fmt"
"net/url"
"sync"
"time"

Expand Down Expand Up @@ -208,14 +207,7 @@ func (p *s3Poller) GetS3Objects(ctx context.Context, s3ObjectPayloadChan chan<-
// Metrics
p.metrics.s3ObjectsListedTotal.Add(uint64(totListedObjects))
for _, object := range page.Contents {
// Unescape s3 key name. For example, convert "%3D" back to "=".
filename, err := url.QueryUnescape(*object.Key)
if err != nil {
p.log.Errorw("Error when unescaping object key, skipping.", "error", err, "s3_object", *object.Key)
continue
}

state := newState(bucketName, filename, *object.ETag, p.listPrefix, *object.LastModified)
state := newState(bucketName, *object.Key, *object.ETag, p.listPrefix, *object.LastModified)
if p.states.MustSkip(state, p.store) {
p.log.Debugw("skipping state.", "state", state)
continue
Expand All @@ -240,7 +232,7 @@ func (p *s3Poller) GetS3Objects(ctx context.Context, s3ObjectPayloadChan chan<-
s3ObjectHandler: s3Processor,
s3ObjectInfo: s3ObjectInfo{
name: bucketName,
key: filename,
key: *object.Key,
etag: *object.ETag,
lastModified: *object.LastModified,
listingID: listingID.String(),
Expand Down
9 changes: 9 additions & 0 deletions x-pack/filebeat/input/awss3/s3_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ func TestS3Poller(t *testing.T) {
Key: aws.String("key5"),
LastModified: aws.Time(time.Now()),
},
{
ETag: aws.String("etag6"),
Key: aws.String("2024-02-08T08:35:00+00:02.json.gz"),
LastModified: aws.Time(time.Now()),
},
},
}, nil
})
Expand Down Expand Up @@ -124,6 +129,10 @@ func TestS3Poller(t *testing.T) {
GetObject(gomock.Any(), gomock.Eq(bucket), gomock.Eq("key5")).
Return(nil, errFakeConnectivityFailure)

mockAPI.EXPECT().
GetObject(gomock.Any(), gomock.Eq(bucket), gomock.Eq("2024-02-08T08:35:00+00:02.json.gz")).
Return(nil, errFakeConnectivityFailure)

s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockAPI, nil, backupConfig{}, numberOfWorkers)
receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, mockPublisher, s3ObjProc, newStates(inputCtx), store, bucket, "key", "region", "provider", numberOfWorkers, pollInterval)
require.Error(t, context.DeadlineExceeded, receiver.Poll(ctx))
Expand Down

0 comments on commit 62e9992

Please sign in to comment.