From 27e565f861b9973ca95fd5ead6e9803d60bbb95a Mon Sep 17 00:00:00 2001 From: Havrileck Alexandre Date: Fri, 14 Jul 2023 17:40:02 +0200 Subject: [PATCH] feat: Allow to disable listing files and folders Related to #370 --- conf/config-example.yaml | 3 + docs/configuration/example.md | 3 + docs/configuration/structure.md | 17 +- pkg/s3-proxy/bucket/requestContext.go | 14 ++ pkg/s3-proxy/bucket/requestContext_test.go | 33 ++++ pkg/s3-proxy/config/config.go | 1 + .../server/server_integration_test.go | 177 ++++++++++++++++++ 7 files changed, 240 insertions(+), 8 deletions(-) diff --git a/conf/config-example.yaml b/conf/config-example.yaml index 2b571cb8..790ff5d3 100644 --- a/conf/config-example.yaml +++ b/conf/config-example.yaml @@ -359,6 +359,9 @@ targets: # redirectToSignedUrl: true # # Signed URL expiration time # signedUrlExpiration: 15m + # # Disable listing + # # Note: This will return an empty list or you should change the folder list template (in general or in this target) + # disableListing: false # # Webhooks # webhooks: [] # # Action for PUT requests on target diff --git a/docs/configuration/example.md b/docs/configuration/example.md index b80d41ad..5db4a510 100644 --- a/docs/configuration/example.md +++ b/docs/configuration/example.md @@ -369,6 +369,9 @@ targets: # redirectToSignedUrl: true # # Signed URL expiration time # signedUrlExpiration: 15m + # # Disable listing + # # Note: This will return an empty list or you should change the folder list template (in general or in this target) + # disableListing: false # # Webhooks # webhooks: [] # # Action for PUT requests on target diff --git a/docs/configuration/structure.md b/docs/configuration/structure.md index 774973b5..bdf4ce85 100644 --- a/docs/configuration/structure.md +++ b/docs/configuration/structure.md @@ -237,14 +237,15 @@ See more information [here](../feature-guide/key-rewrite.md). ## GetActionConfigConfiguration -| Key | Type | Required | Default | Description | -| ---------------------------------------- | ----------------------------------------------- | -------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| redirectWithTrailingSlashForNotFoundFile | Boolean | No | `false` | This option allow to do a redirect with a trailing slash when a GET request on a file (not a folder) encountered a 404 not found. | -| indexDocument | String | No | `""` | The index document name. If this document is found, get it instead of list folder. Example: `index.html` | -| streamedFileHeaders | Map[String]String | No | `nil`  |  Headers containing templates that will be added to streamed files in this target. Key corresponds to header and value to the template. If templated value is empty, the header won't be added to answer. More information [here](../feature-guide/templates.md#stream-file-case). | -| redirectToSignedUrl | Boolean | No | `false`  | Instead of streaming the file through S3-Proxy application, it will redirect to a S3 signed URL to perform the actual download. | -| signedUrlExpiration | String | No | `15m` | This will allow to set an expiration time on generated signed URL. | -| webhooks | [[WebhookConfiguration](#webhookconfiguration)] | No | `nil` | Webhooks configuration list to call when a GET request is performed | +| Key | Type | Required | Default | Description | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| redirectWithTrailingSlashForNotFoundFile | Boolean | No | `false` | This option allow to do a redirect with a trailing slash when a GET request on a file (not a folder) encountered a 404 not found. | +| indexDocument | String | No | `""` | The index document name. If this document is found, get it instead of list folder. Example: `index.html` | +| streamedFileHeaders | Map[String]String | No | `nil`  |  Headers containing templates that will be added to streamed files in this target. Key corresponds to header and value to the template. If templated value is empty, the header won't be added to answer. More information [here](../feature-guide/templates.md#stream-file-case). | +| redirectToSignedUrl | Boolean | No | `false`  | Instead of streaming the file through S3-Proxy application, it will redirect to a S3 signed URL to perform the actual download. | +| signedUrlExpiration | String | No | `15m` | This will allow to set an expiration time on generated signed URL. | +| disableListing | That will disable the listing action. That will display an empty list or you should change the folder list template (general or per target). | No | `false` | +| webhooks | [[WebhookConfiguration](#webhookconfiguration)] | No | `nil` | Webhooks configuration list to call when a GET request is performed | ## PutActionConfiguration diff --git a/pkg/s3-proxy/bucket/requestContext.go b/pkg/s3-proxy/bucket/requestContext.go index dc06b671..b9122e37 100644 --- a/pkg/s3-proxy/bucket/requestContext.go +++ b/pkg/s3-proxy/bucket/requestContext.go @@ -270,6 +270,20 @@ func (rctx *requestContext) manageGetFolder(ctx context.Context, key string, inp } } + // Check if list folders is disabled + if rctx.targetCfg.Actions != nil && rctx.targetCfg.Actions.GET != nil && + rctx.targetCfg.Actions.GET.Config != nil && + rctx.targetCfg.Actions.GET.Config.DisableListing { + // Answer directly + resHan.FoldersFilesList( + rctx.LoadFileContent, + make([]*responsehandler.Entry, 0), + ) + + // Stop + return + } + // Directory listing case s3Entries, info, err := rctx.s3ClientManager. GetClientForTarget(rctx.targetCfg.Name). diff --git a/pkg/s3-proxy/bucket/requestContext_test.go b/pkg/s3-proxy/bucket/requestContext_test.go index f0521f58..c876fd2a 100644 --- a/pkg/s3-proxy/bucket/requestContext_test.go +++ b/pkg/s3-proxy/bucket/requestContext_test.go @@ -2144,6 +2144,39 @@ func Test_requestContext_Get(t *testing.T) { times: 1, }, }, + { + name: "list bucket should not be done when disable listing is enabled", + fields: fields{ + targetCfg: &config.TargetConfig{ + Name: "target", + Bucket: &config.BucketConfig{ + Name: "bucket1", + Prefix: "/", + }, + KeyRewriteList: []*config.TargetKeyRewriteConfig{{ + SourceRegex: regexp.MustCompile(`^/folder/index\.html$`), + Target: "/fake/fake.html", + TargetType: config.RegexTargetKeyRewriteTargetType, + }}, + Actions: &config.ActionsConfig{GET: &config.GetActionConfig{ + Config: &config.GetActionConfigConfig{ + DisableListing: true, + }, + }}, + }, + mountPath: "/mount", + }, + args: args{ + input: &GetInput{RequestPath: "/folder/"}, + }, + s3ClientListFilesAndDirectoriesMockResult: s3ClientListFilesAndDirectoriesMockResult{ + times: 0, + }, + responseHandlerFoldersFilesListMockResult: responseHandlerFoldersFilesListMockResult{ + times: 1, + input2: []*responsehandler.Entry{}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/s3-proxy/config/config.go b/pkg/s3-proxy/config/config.go index b6aa87a7..f7c84648 100644 --- a/pkg/s3-proxy/config/config.go +++ b/pkg/s3-proxy/config/config.go @@ -424,6 +424,7 @@ type GetActionConfigConfig struct { SignedURLExpiration time.Duration RedirectWithTrailingSlashForNotFoundFile bool `mapstructure:"redirectWithTrailingSlashForNotFoundFile"` RedirectToSignedURL bool `mapstructure:"redirectToSignedUrl"` + DisableListing bool `mapstructure:"disableListing"` } // WebhookConfig Webhook configuration. diff --git a/pkg/s3-proxy/server/server_integration_test.go b/pkg/s3-proxy/server/server_integration_test.go index ebf7631b..a47e3416 100644 --- a/pkg/s3-proxy/server/server_integration_test.go +++ b/pkg/s3-proxy/server/server_integration_test.go @@ -3504,6 +3504,183 @@ func TestPublicRouter(t *testing.T) {

Not Found /mount/folder1/

+`, + expectedHeaders: map[string]string{ + "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", + "Content-Type": "text/html; charset=utf-8", + }, + }, + { + name: "GET a folder list with disable listing enabled", + args: args{ + cfg: &config.Config{ + Server: svrCfg, + ListTargets: &config.ListTargetsConfig{}, + Tracing: tracingConfig, + Templates: testsDefaultGeneralTemplateConfig, + Targets: map[string]*config.TargetConfig{ + "target1": { + Name: "target1", + Bucket: &config.BucketConfig{ + Name: bucket, + Region: region, + S3Endpoint: s3server.URL, + Credentials: &config.BucketCredentialConfig{ + AccessKey: &config.CredentialConfig{Value: accessKey}, + SecretKey: &config.CredentialConfig{Value: secretAccessKey}, + }, + DisableSSL: true, + }, + Mount: &config.MountConfig{ + Path: []string{"/mount/"}, + }, + Actions: &config.ActionsConfig{ + GET: &config.GetActionConfig{ + Enabled: true, + Config: &config.GetActionConfigConfig{DisableListing: true}, + }, + }, + }, + }, + }, + }, + inputMethod: "GET", + inputURL: "http://localhost/mount/folder1/", + expectedCode: 200, + expectedBody: ` + + +

Index of /mount/folder1/

+ + + + + + + + + + + + + + + +
EntrySizeLast modified
.. - -
+ +`, + expectedHeaders: map[string]string{ + "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", + "Content-Type": "text/html; charset=utf-8", + }, + }, + { + name: "GET a folder list with disable listing enabled, another status code and another content (general templates)", + args: args{ + cfg: &config.Config{ + Server: svrCfg, + ListTargets: &config.ListTargetsConfig{}, + Tracing: tracingConfig, + Templates: &config.TemplateConfig{ + Helpers: testsDefaultHelpersTemplateConfig, + FolderList: testsDefaultNotFoundErrorTemplateConfig, + TargetList: testsDefaultTargetListTemplateConfig, + BadRequestError: testsDefaultBadRequestErrorTemplateConfig, + NotFoundError: testsDefaultNotFoundErrorTemplateConfig, + InternalServerError: testsDefaultInternalServerErrorTemplateConfig, + UnauthorizedError: testsDefaultUnauthorizedErrorTemplateConfig, + ForbiddenError: testsDefaultForbiddenErrorTemplateConfig, + }, + Targets: map[string]*config.TargetConfig{ + "target1": { + Name: "target1", + Bucket: &config.BucketConfig{ + Name: bucket, + Region: region, + S3Endpoint: s3server.URL, + Credentials: &config.BucketCredentialConfig{ + AccessKey: &config.CredentialConfig{Value: accessKey}, + SecretKey: &config.CredentialConfig{Value: secretAccessKey}, + }, + DisableSSL: true, + }, + Mount: &config.MountConfig{ + Path: []string{"/mount/"}, + }, + Actions: &config.ActionsConfig{ + GET: &config.GetActionConfig{ + Enabled: true, + Config: &config.GetActionConfigConfig{DisableListing: true}, + }, + }, + }, + }, + }, + }, + inputMethod: "GET", + inputURL: "http://localhost/mount/folder1/", + expectedCode: 404, + expectedBody: ` + + +

Not Found /mount/folder1/

+ +`, + expectedHeaders: map[string]string{ + "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", + "Content-Type": "text/html; charset=utf-8", + }, + }, + { + name: "GET a folder list with disable listing enabled, another status code and another content (target override)", + args: args{ + cfg: &config.Config{ + Server: svrCfg, + ListTargets: &config.ListTargetsConfig{}, + Tracing: tracingConfig, + Templates: testsDefaultGeneralTemplateConfig, + Targets: map[string]*config.TargetConfig{ + "target1": { + Name: "target1", + Bucket: &config.BucketConfig{ + Name: bucket, + Region: region, + S3Endpoint: s3server.URL, + Credentials: &config.BucketCredentialConfig{ + AccessKey: &config.CredentialConfig{Value: accessKey}, + SecretKey: &config.CredentialConfig{Value: secretAccessKey}, + }, + DisableSSL: true, + }, + Mount: &config.MountConfig{ + Path: []string{"/mount/"}, + }, + Actions: &config.ActionsConfig{ + GET: &config.GetActionConfig{ + Enabled: true, + }, + }, + Templates: &config.TargetTemplateConfig{ + FolderList: &config.TargetTemplateConfigItem{ + Path: "../../../templates/not-found-error.tpl", + Headers: map[string]string{ + "Content-Type": "{{ template \"main.headers.contentType\" . }}", + }, + Status: "404", + }, + }, + }, + }, + }, + }, + inputMethod: "GET", + inputURL: "http://localhost/mount/folder1/", + expectedCode: 404, + expectedBody: ` + + +

Not Found /mount/folder1/

+ `, expectedHeaders: map[string]string{ "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0",