From 0c705ccaeb903672a6c7414724d79ecf9d827c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9ctor=20Veiga=20Ortiz?= Date: Fri, 12 Apr 2024 11:28:56 -0700 Subject: [PATCH] feature: add support for S3 Express #225 --- .../00-check-for-required-env.sh | 3 +- common/etc/nginx/include/s3gateway.js | 4 +- common/etc/nginx/nginx.conf | 1 + .../templates/default_s3express.conf.template | 319 ++++++++++++++++++ docs/getting_started.md | 61 ++-- .../20-envsubst-on-templates.sh | 7 + .../upstreams_s3express.conf.template | 13 + settings.example | 1 + standalone_ubuntu_oss_install.sh | 6 +- test.sh | 4 + test/docker-compose.yaml | 1 + 11 files changed, 387 insertions(+), 33 deletions(-) create mode 100644 common/etc/nginx/templates/default_s3express.conf.template create mode 100644 plus/etc/nginx/templates/upstreams_s3express.conf.template diff --git a/common/docker-entrypoint.d/00-check-for-required-env.sh b/common/docker-entrypoint.d/00-check-for-required-env.sh index 604214d2..6c1df3ac 100755 --- a/common/docker-entrypoint.d/00-check-for-required-env.sh +++ b/common/docker-entrypoint.d/00-check-for-required-env.sh @@ -22,7 +22,7 @@ set -e failed=0 -required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO" +required=("S3_SERVICE" "S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO" "S3_REGION" "S3_STYLE" "ALLOW_DIRECTORY_LIST" "AWS_SIGS_VERSION" "CORS_ENABLED") @@ -123,6 +123,7 @@ fi echo "S3 Backend Environment" echo "Access Key ID: ${AWS_ACCESS_KEY_ID}" +echo "Service: ${S3_SERVICE}" echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}" echo "Region: ${S3_REGION}" echo "Addressing Style: ${S3_STYLE}" diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 7a497cf8..5acab72b 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -32,6 +32,7 @@ import awssig2 from "./awssig2.js"; import awssig4 from "./awssig4.js"; import utils from "./utils.js"; +_requireEnvVars('S3_SERVICE'); _requireEnvVars('S3_BUCKET_NAME'); _requireEnvVars('S3_SERVER'); _requireEnvVars('S3_SERVER_PROTO'); @@ -84,9 +85,10 @@ const INDEX_PAGE = "index.html"; /** * Constant defining the service requests are being signed for. + * can be 's3' or 's3express'. * @type {string} */ -const SERVICE = 's3'; +const SERVICE = process.env['S3_SERVICE']; /** * Transform the headers returned from S3 such that there isn't information diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index cd938089..1aa6de74 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -12,6 +12,7 @@ load_module modules/ngx_http_xslt_filter_module.so; env AWS_ACCESS_KEY_ID; env AWS_SECRET_ACCESS_KEY; env AWS_SESSION_TOKEN; +env S3_SERVICE; env S3_BUCKET_NAME; env S3_SERVER; env S3_SERVER_PORT; diff --git a/common/etc/nginx/templates/default_s3express.conf.template b/common/etc/nginx/templates/default_s3express.conf.template new file mode 100644 index 00000000..2fcb4a21 --- /dev/null +++ b/common/etc/nginx/templates/default_s3express.conf.template @@ -0,0 +1,319 @@ +js_import /etc/nginx/include/awscredentials.js; +js_import /etc/nginx/include/s3gateway.js; + + +# We include only the variables needed for the authentication signatures that +# we plan to use. +include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_js_vars.conf; + +# Extracts only the path from the requested URI. This strips out all query +# parameters and anchors in order to prevent extraneous data from being sent +# to S3. +map $request_uri $uri_full_path { + "~^(?P.*?)(\?.*)*$" $path; +} + +# Remove/replace a portion of request URL (if configured) +map $uri_full_path $uri_path { + "~^$STRIP_LEADING_DIRECTORY_PATH(.*)" $PREFIX_LEADING_DIRECTORY_PATH$1; + default $PREFIX_LEADING_DIRECTORY_PATH$uri_full_path; +} + +map $S3_STYLE $s3_host_hdr { + virtual "${S3_BUCKET_NAME}.${S3_SERVER}"; + path "${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}"; + default "${S3_BUCKET_NAME}.${S3_SERVER}"; +} + +js_var $indexIsEmpty true; +js_var $forIndexPage true; +# This creates the HTTP authentication header to be sent to S3 +js_set $s3auth s3gateway.s3auth; +js_set $awsSessionToken awscredentials.sessionToken; +js_set $s3uri s3gateway.s3uri; + +server { + include /etc/nginx/conf.d/gateway/server_variables.conf; + + # Don't display the NGINX version number because we don't want to reveal + # information that could be used to find an exploit. + server_tokens off; + + # Uncomment this for a HTTP header that will let you know the cache status + # of an object. + # add_header X-Cache-Status $upstream_cache_status; + + # Proxy caching configuration. Customize this for your needs. + proxy_cache s3_cache; + proxy_cache_valid 200 302 ${PROXY_CACHE_VALID_OK}; + proxy_cache_valid 404 ${PROXY_CACHE_VALID_NOTFOUND}; + proxy_cache_valid 403 ${PROXY_CACHE_VALID_FORBIDDEN}; + proxy_cache_methods GET HEAD; + # When this is enabled a HEAD request to NGINX will result in a GET + # request upstream. Unfortunately, proxy_cache_convert_head has to be + # disabled because there is no way for the signatures generation code to + # get access to the metadata in the GET request that is sent upstream. + proxy_cache_convert_head off; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504; + proxy_cache_key "$request_method$host$uri"; + + # If you need to support proxying range request, refer to this article: + # https://www.nginx.com/blog/smart-efficient-byte-range-caching-nginx/ + + # Do not proxy the S3 SOAP API. The S3 API has a less-documented feature + # where the object name "soap" is used for the SOAP API. We don't allow + # access to it. + location /soap { + return 404; + } + + location /health { + return 200; + } + + location / { + # This value is templated in based on the value of $CORS_ENABLED. When + # CORS is enabled, acceptable methods are GET, HEAD, and OPTIONS. + # Otherwise, they are GET and HEAD. + limit_except ${LIMIT_METHODS_TO} {} + + # CORS is implemented by returning the appropriate headers as part of + # the response to an OPTIONS request. If you want to customize the + # CORS response, the cors.conf.template file can be overwritten and + # extended to meet your needs. + include /etc/nginx/conf.d/gateway/cors.conf; + + auth_request /aws/credentials/retrieve; + + # Redirect to the proper location based on the client request - either + # @s3, @s3PreListing or @error405. + + js_content s3gateway.redirectToS3; + } + + location /aws/credentials/retrieve { + internal; + js_content awscredentials.fetchCredentials; + + include /etc/nginx/conf.d/gateway/js_fetch_trusted_certificate.conf; + } + + # This is the primary location that proxies the request to s3 + # See the included s3_location_common.conf file for all logic + location @s3 { + include /etc/nginx/conf.d/gateway/s3_location_common.conf; + } + + # Same as the primary location above but handling and caching + # byte range requests efficiently + location @s3_sliced { + proxy_cache s3_cache_slices; + proxy_cache_valid 200 302 206 ${PROXY_CACHE_VALID_OK}; + proxy_cache_key "$request_method$host$uri$slice_range"; + + slice ${PROXY_CACHE_SLICE_SIZE}; + proxy_set_header Range $slice_range; + include /etc/nginx/conf.d/gateway/s3_location_common.conf; + } + + location @s3PreListing { + # We include only the headers needed for the authentication signatures that + # we plan to use. + include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf; + + # The CORS configuration needs to be imported in several places in order for + # it to be applied within different contexts. + include /etc/nginx/conf.d/gateway/cors.conf; + + # Don't allow any headers from the client - we don't want them messing + # with S3 at all. + proxy_pass_request_headers off; + + # Enable passing of the server name through TLS Server Name Indication extension. + proxy_ssl_server_name on; + proxy_ssl_name ${S3_BUCKET_NAME}.${S3_SERVER}; + + # Set the Authorization header to the AWS Signatures credentials + proxy_set_header Authorization $s3auth; + proxy_set_header X-Amz-Security-Token $awsSessionToken; + + # We set the host as the bucket name to inform the S3 API of the bucket + proxy_set_header Host $s3_host_hdr; + + # Use keep alive connections in order to improve performance + proxy_http_version 1.1; + proxy_set_header Connection ''; + + # We strip off all of the AWS specific headers from the server so that + # there is nothing identifying the object as having originated in an + # object store. + js_header_filter s3gateway.editHeaders; + + # Apply XSL transformation to the XML returned from S3 directory listing + # results such that we can output an HTML directory contents list. + xslt_stylesheet /etc/nginx/include/listing.xsl; + xslt_string_param rootPath '${DIRECTORY_LISTING_PATH_PREFIX}'; + xslt_types application/xml; + + # We apply an output filter to the XML input received from S3 before it + # is passed to XSLT in order to determine if the resource is not a valid + # S3 directory. If it isn't a valid directory, we do a dirty hack to + # corrupt the contents of the XML causing the XSLT to fail and thus + # nginx to return a 404 to the client. If you don't care about empty + # directory listings for invalid directories, remove this. + js_body_filter s3gateway.filterListResponse; + + # Catch all errors from S3 and sanitize them so that the user can't + # gain intelligence about the S3 bucket being proxied. + proxy_intercept_errors on; + + # Comment out this line to receive the error messages returned by S3 + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + + js_content s3gateway.loadContent; + include /etc/nginx/conf.d/gateway/s3listing_location.conf; + } + + location @s3Directory { + # We include only the headers needed for the authentication signatures that + # we plan to use. + include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf; + + # Necessary for determining the correct URI to construct. + set $forIndexPage false; + + # The CORS configuration needs to be imported in several places in order for + # it to be applied within different contexts. + include /etc/nginx/conf.d/gateway/cors.conf; + + # Don't allow any headers from the client - we don't want them messing + # with S3 at all. + proxy_pass_request_headers off; + + # Enable passing of the server name through TLS Server Name Indication extension. + proxy_ssl_server_name on; + proxy_ssl_name ${S3_BUCKET_NAME}.${S3_SERVER}; + + # Set the Authorization header to the AWS Signatures credentials + proxy_set_header Authorization $s3auth; + proxy_set_header X-Amz-Security-Token $awsSessionToken; + + # We set the host as the bucket name to inform the S3 API of the bucket + proxy_set_header Host $s3_host_hdr; + + # Use keep alive connections in order to improve performance + proxy_http_version 1.1; + proxy_set_header Connection ''; + + # We strip off all of the AWS specific headers from the server so that + # there is nothing identifying the object as having originated in an + # object store. + js_header_filter s3gateway.editHeaders; + + # Apply XSL transformation to the XML returned from S3 directory listing + # results such that we can output an HTML directory contents list. + xslt_stylesheet /etc/nginx/include/listing.xsl; + xslt_string_param rootPath '${DIRECTORY_LISTING_PATH_PREFIX}'; + xslt_types application/xml; + + # We apply an output filter to the XML input received from S3 before it + # is passed to XSLT in order to determine if the resource is not a valid + # S3 directory. If it isn't a valid directory, we do a dirty hack to + # corrupt the contents of the XML causing the XSLT to fail and thus + # nginx to return a 404 to the client. If you don't care about empty + # directory listings for invalid directories, remove this. + js_body_filter s3gateway.filterListResponse; + + # Catch all errors from S3 and sanitize them so that the user can't + # gain intelligence about the S3 bucket being proxied. + proxy_intercept_errors on; + + # Comment out this line to receive the error messages returned by S3 + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + + proxy_pass ${S3_SERVER_PROTO}://storage_urls$s3Uri; + include /etc/nginx/conf.d/gateway/s3listing_location.conf; + } + + location ~ /index.html$ { + # Configuration for handling locations ending with /index.html + + # Necessary for determining the correct URI to construct. + set $forIndexPage true; + + # We include only the headers needed for the authentication signatures that + # we plan to use. + include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf; + + # The CORS configuration needs to be imported in several places in order for + # it to be applied within different contexts. + include /etc/nginx/conf.d/gateway/cors.conf; + + # Don't allow any headers from the client - we don't want them messing + # with S3 at all. + proxy_pass_request_headers off; + + # Enable passing of the server name through TLS Server Name Indication extension. + proxy_ssl_server_name on; + proxy_ssl_name ${S3_BUCKET_NAME}.${S3_SERVER}; + + # Set the Authorization header to the AWS Signatures credentials + proxy_set_header Authorization $s3auth; + proxy_set_header X-Amz-Security-Token $awsSessionToken; + + # We set the host as the bucket name to inform the S3 API of the bucket + proxy_set_header Host $s3_host_hdr; + + # Use keep alive connections in order to improve performance + proxy_http_version 1.1; + proxy_set_header Connection ''; + + # We strip off all of the AWS specific headers from the server so that + # there is nothing identifying the object as having originated in an + # object store. + js_header_filter s3gateway.editHeaders; + + # Catch all errors from S3 and sanitize them so that the user can't + # gain intelligence about the S3 bucket being proxied. + proxy_intercept_errors on; + + # Comment out this line to receive the error messages returned by S3 + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + + proxy_pass ${S3_SERVER_PROTO}://storage_urls$s3uri; + include /etc/nginx/conf.d/gateway/s3listing_location.conf; + } + + location @error404 { + # The CORS configuration needs to be imported in several places in order for + # it to be applied within different contexts. + include /etc/nginx/conf.d/gateway/cors.conf; + + return 404; + } + + location @trailslashControl { + # Checks if requesting a folder without trailing slash, and return 302 + # appending a slash to it when using for static site hosting. + js_content s3gateway.trailslashControl; + } + + location @trailslash { + # 302 to request without slashes + # Adding a ? to the end of the replacement param in `rewrite` prevents it from + # appending the query string. + rewrite ^ $scheme://$http_host$uri/$is_args$query_string? redirect; + } + + # Provide a hint to the client on 405 errors of the acceptable request methods + error_page 405 @error405; + location @error405 { + add_header Allow "${LIMIT_METHODS_TO_CSV}" always; + return 405; + } + + include /etc/nginx/conf.d/gateway/s3_server.conf; +} diff --git a/docs/getting_started.md b/docs/getting_started.md index d3380817..0ada5346 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -13,36 +13,37 @@ The following environment variables are used to configure the gateway when running as a Container or as a Systemd service. -| Name | Required? | Allowed Values | Default | Description | -| ------------------------------------- | --------- | ---------------------------- | --------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ALLOW_DIRECTORY_LIST` | Yes | `true`, `false` | `false` | Flag enabling directory listing | -| `AWS_SIGS_VERSION` | Yes | 2, 4 | | AWS Signatures API version | -| `AWS_ACCESS_KEY_ID` | Yes | | | Access key | -| `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | -| `AWS_SESSION_TOKEN` | No | | | Session token. | -| `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | -| `S3_REGION` | Yes | | | Region associated with API | -| `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | -| `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | -| `S3_SERVER` | Yes | | | S3 host to connect to | -| `S3_STYLE` | Yes | `virtual`, `path`, `default` | `default` | The S3 host/path method.
  • `virtual` is the method that that uses DNS-style bucket+hostname:port. This is the `default` value.
  • `path` is a method that appends the bucket name as the first directory in the URI's path. This method is used by many S3 compatible services.

    See this [AWS blog article](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/) for further information. | -| `DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output | -| `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No | `true`, `false` | `false` | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. | -| `DIRECTORY_LISTING_PATH_PREFIX` | No | | | In `ALLOW_DIRECTORY_LIST=true` mode [adds defined prefix to links](#configuring-directory-listing) | -| `DNS_RESOLVERS` | No | | | DNS resolvers (separated by single spaces) to configure NGINX with | -| `PROXY_CACHE_MAX_SIZE` | No | | `10g` | Limits cache size | -| `PROXY_CACHE_INACTIVE` | No | | `60m` | Cached data that are not accessed during the time specified by the parameter get removed from the cache regardless of their freshness -| `PROXY_CACHE_SLICE_SIZE` | No | | `1m` | For requests with a `Range` header included, determines the size of the chunks in which the file is fetched. Values much smaller than the requests can lead to inefficiencies due to reading and writing many files. See [below for more details](#byte-range-requests-and-caching) | | -| `PROXY_CACHE_VALID_OK` | No | | `1h` | Sets caching time for response code 200 and 302 | -| `PROXY_CACHE_VALID_NOTFOUND` | No | | `1m` | Sets caching time for response code 404 | -| `PROXY_CACHE_VALID_FORBIDDEN` | No | | `30s` | Sets caching time for response code 403 | -| `PROVIDE_INDEX_PAGE` | No | `true`, `false` | `false` | Flag which returns the index page if there is one when requesting a directory. | -| `JS_TRUSTED_CERT_PATH` | No | | | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path | -| `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-` | -| `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | -| `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) | -| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). | -| `PREFIX_LEADING_DIRECTORY_PATH` | No | | | Prefix to prepend to all S3 object paths. Useful to serve only a subset of an S3 bucket. When used in combination with `STRIP_LEADING_DIRECTORY_PATH`, this allows the leading path to be replaced, rather than just removed. | +| Name | Required? | Allowed Values | Default | Description | +|---------------------------------------| --------- |------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ALLOW_DIRECTORY_LIST` | Yes | `true`, `false` | `false` | Flag enabling directory listing | +| `AWS_SIGS_VERSION` | Yes | 2, 4 | | AWS Signatures API version | +| `AWS_ACCESS_KEY_ID` | Yes | | | Access key | +| `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | +| `AWS_SESSION_TOKEN` | No | | | Session token. | +| `S3_SERVICE` | Yes | `s3`, `s3express` | `s3` | Name of S3 service | +| `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | +| `S3_REGION` | Yes | | | Region associated with API | +| `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | +| `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | +| `S3_SERVER` | Yes | | | S3 host to connect to | +| `S3_STYLE` | Yes | `virtual`, `path`, `default` | `default` | The S3 host/path method.
  • `virtual` is the method that that uses DNS-style bucket+hostname:port. This is the `default` value.
  • `path` is a method that appends the bucket name as the first directory in the URI's path. This method is used by many S3 compatible services.

    See this [AWS blog article](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/) for further information. | +| `DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output | +| `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No | `true`, `false` | `false` | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. | +| `DIRECTORY_LISTING_PATH_PREFIX` | No | | | In `ALLOW_DIRECTORY_LIST=true` mode [adds defined prefix to links](#configuring-directory-listing) | +| `DNS_RESOLVERS` | No | | | DNS resolvers (separated by single spaces) to configure NGINX with | +| `PROXY_CACHE_MAX_SIZE` | No | | `10g` | Limits cache size | +| `PROXY_CACHE_INACTIVE` | No | | `60m` | Cached data that are not accessed during the time specified by the parameter get removed from the cache regardless of their freshness +| `PROXY_CACHE_SLICE_SIZE` | No | | `1m` | For requests with a `Range` header included, determines the size of the chunks in which the file is fetched. Values much smaller than the requests can lead to inefficiencies due to reading and writing many files. See [below for more details](#byte-range-requests-and-caching) | | +| `PROXY_CACHE_VALID_OK` | No | | `1h` | Sets caching time for response code 200 and 302 | +| `PROXY_CACHE_VALID_NOTFOUND` | No | | `1m` | Sets caching time for response code 404 | +| `PROXY_CACHE_VALID_FORBIDDEN` | No | | `30s` | Sets caching time for response code 403 | +| `PROVIDE_INDEX_PAGE` | No | `true`, `false` | `false` | Flag which returns the index page if there is one when requesting a directory. | +| `JS_TRUSTED_CERT_PATH` | No | | | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path | +| `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-` | +| `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | +| `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) | +| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). | +| `PREFIX_LEADING_DIRECTORY_PATH` | No | | | Prefix to prepend to all S3 object paths. Useful to serve only a subset of an S3 bucket. When used in combination with `STRIP_LEADING_DIRECTORY_PATH`, this allows the leading path to be replaced, rather than just removed. | If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from diff --git a/plus/docker-entrypoint.d/20-envsubst-on-templates.sh b/plus/docker-entrypoint.d/20-envsubst-on-templates.sh index c57faed6..cfd72054 100644 --- a/plus/docker-entrypoint.d/20-envsubst-on-templates.sh +++ b/plus/docker-entrypoint.d/20-envsubst-on-templates.sh @@ -41,6 +41,13 @@ auto_envsubst() { echo >&3 "$ME: Running envsubst on $template to $output_path" envsubst "$defined_envs" < "$template" > "$output_path" done + + # Special logic for S3 Express + if [ "${S3_SERVICE}" == 's3express' ]; then + echo >&3 "$ME: Detected 's3express' configuration. Proceeding with configuration file updates." + mv /etc/nginx/conf.d/default_s3express.conf /etc/nginx/conf.d/default.conf + mv /etc/nginx/conf.d/upstreams_s3express.conf /etc/nginx/conf.d/upstreams.conf + fi } auto_envsubst diff --git a/plus/etc/nginx/templates/upstreams_s3express.conf.template b/plus/etc/nginx/templates/upstreams_s3express.conf.template new file mode 100644 index 00000000..4105d607 --- /dev/null +++ b/plus/etc/nginx/templates/upstreams_s3express.conf.template @@ -0,0 +1,13 @@ +# This configuration with NGINX Plus should dynamically reload S3 backends +# as they change in DNS. + +# Use NGINX's non-blocking DNS resolution +resolver ${DNS_RESOLVERS}; + +upstream storage_urls { + zone s3_backends 64k; + + # Be sure to specify the port in the S3_SERVER and be sure that port + # corresponds to the https/http in the proxy_pass directive. + server ${S3_BUCKET}.${S3_SERVER}:${S3_SERVER_PORT} resolve; +} diff --git a/settings.example b/settings.example index b2884a63..e93339fc 100644 --- a/settings.example +++ b/settings.example @@ -2,6 +2,7 @@ S3_BUCKET_NAME=my-bucket AWS_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ AWS_SECRET_ACCESS_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa AWS_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +S3_SERVICE=s3 S3_SERVER=s3.us-east-1.amazonaws.com S3_SERVER_PORT=443 S3_SERVER_PROTO=https diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh index ee173499..91c012a6 100644 --- a/standalone_ubuntu_oss_install.sh +++ b/standalone_ubuntu_oss_install.sh @@ -30,7 +30,7 @@ fi failed=0 -required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO" +required=("S3_SERVICE" "S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO" "S3_REGION" "S3_STYLE" "ALLOW_DIRECTORY_LIST" "AWS_SIGS_VERSION") if [ ! -z ${AWS_CONTAINER_CREDENTIALS_RELATIVE_URI+x} ]; then @@ -83,6 +83,7 @@ echo "Installing using github '${branch}' branch" echo "S3 Backend Environment" echo "Access Key ID: ${AWS_ACCESS_KEY_ID}" +echo "Service: ${S3_SERVICE}" echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}" echo "Region: ${S3_REGION}" echo "Addressing Style: ${S3_STYLE}" @@ -150,6 +151,8 @@ ALLOW_DIRECTORY_LIST=${ALLOW_DIRECTORY_LIST:-'false'} DIRECTORY_LISTING_PATH_PREFIX=${DIRECTORY_LISTING_PATH_PREFIX:-''} # AWS Authentication signature version (2=v2 authentication, 4=v4 authentication) AWS_SIGS_VERSION=${AWS_SIGS_VERSION} +# Name of S3 service - 's3' or 's3express' +S3_SERVICE=${S3_SERVICE:-'s3'} # Name of S3 bucket to proxy requests to S3_BUCKET_NAME=${S3_BUCKET_NAME} # Region associated with API @@ -331,6 +334,7 @@ EOF fi cat >> /etc/nginx/nginx.conf << 'EOF' +env S3_SERVICE; env S3_BUCKET_NAME; env S3_SERVER; env S3_SERVER_PORT; diff --git a/test.sh b/test.sh index a12fd28e..b661ead0 100755 --- a/test.sh +++ b/test.sh @@ -350,6 +350,7 @@ runUnitTestWithOutSessionToken() { -v "$(pwd)/test/unit:/var/tmp" \ --workdir /var/tmp \ -e "DEBUG=true" \ + -e "S3_SERVICE=s3" \ -e "S3_STYLE=virtual" \ -e "AWS_ACCESS_KEY_ID=unit_test" \ -e "AWS_SECRET_ACCESS_KEY=unit_test" \ @@ -369,6 +370,7 @@ runUnitTestWithOutSessionToken() { -v "$(pwd)/test/unit:/var/tmp" \ --workdir /var/tmp \ -e "DEBUG=true" \ + -e "S3_SERVICE=s3" \ -e "S3_STYLE=virtual" \ -e "AWS_ACCESS_KEY_ID=unit_test" \ -e "AWS_SECRET_ACCESS_KEY=unit_test" \ @@ -395,6 +397,7 @@ runUnitTestWithSessionToken() { -v "$(pwd)/test/unit:/var/tmp" \ --workdir /var/tmp \ -e "DEBUG=true" \ + -e "S3_SERVICE=s3" \ -e "S3_STYLE=virtual" \ -e "AWS_ACCESS_KEY_ID=unit_test" \ -e "AWS_SECRET_ACCESS_KEY=unit_test" \ @@ -415,6 +418,7 @@ runUnitTestWithSessionToken() { -v "$(pwd)/test/unit:/var/tmp" \ --workdir /var/tmp \ -e "DEBUG=true" \ + -e "S3_SERVICE=s3" \ -e "S3_STYLE=virtual" \ -e "AWS_ACCESS_KEY_ID=unit_test" \ -e "AWS_SECRET_ACCESS_KEY=unit_test" \ diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index 44c58763..b2658fe3 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -22,6 +22,7 @@ services: S3_SERVER_PROTO: "http" S3_REGION: "us-east-1" DEBUG: "true" + S3_SERVICE: "s3" S3_STYLE: "virtual" ALLOW_DIRECTORY_LIST: PROVIDE_INDEX_PAGE: