From aab7f9433c5037a7b269a1aa14821cb855e3b473 Mon Sep 17 00:00:00 2001 From: Hui Yu Date: Wed, 26 Apr 2023 14:07:11 +0800 Subject: [PATCH 1/2] Add debug tools --- Dockerfile.oss | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.oss b/Dockerfile.oss index 393a7130..28fb764b 100644 --- a/Dockerfile.oss +++ b/Dockerfile.oss @@ -30,6 +30,7 @@ RUN set -eux \ apt-get update; \ apt-get install --no-install-recommends --no-install-suggests --yes \ curl nginx-module-njs=${NGINX_VERSION}+${NJS_VERSION}-${PKG_RELEASE}; \ + apt-get install --yes vim procps net-tools iputils-ping; \ apt-get remove --purge --auto-remove --yes; \ rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list From bdbcd676f72f26822892aa3336e9916df2bdfed9 Mon Sep 17 00:00:00 2001 From: Hui Yu Date: Fri, 28 Apr 2023 11:27:41 +0800 Subject: [PATCH 2/2] Support PUT and DELETE operation --- Dockerfile.oss | 12 +++++------ common/docker-entrypoint.sh | 8 ++++---- common/etc/nginx/include/awssig4.js | 20 +++++++++++++++---- common/etc/nginx/include/s3gateway.js | 12 +++++------ .../etc/nginx/templates/default.conf.template | 8 ++++---- .../gateway/v4_headers.conf.template | 2 +- .../gateway/v4_js_vars.conf.template | 2 ++ 7 files changed, 38 insertions(+), 26 deletions(-) diff --git a/Dockerfile.oss b/Dockerfile.oss index 28fb764b..3947ef96 100644 --- a/Dockerfile.oss +++ b/Dockerfile.oss @@ -16,16 +16,10 @@ ENV CORS_ENABLED 0 # 4. Explicitly install the version of njs coded in the environment variable # above. -COPY common/etc /etc -COPY common/docker-entrypoint.sh /docker-entrypoint.sh -COPY common/docker-entrypoint.d /docker-entrypoint.d/ -COPY oss/etc /etc - RUN set -eux \ export DEBIAN_FRONTEND=noninteractive; \ mkdir -p /var/cache/nginx/s3_proxy; \ chown nginx:nginx /var/cache/nginx/s3_proxy; \ - chmod -R -v +x /docker-entrypoint.sh /docker-entrypoint.d/*.sh; \ echo "deb https://nginx.org/packages/mainline/debian/ $(echo $PKG_RELEASE | cut -f2 -d~) nginx" >> /etc/apt/sources.list.d/nginx.list; \ apt-get update; \ apt-get install --no-install-recommends --no-install-suggests --yes \ @@ -34,3 +28,9 @@ RUN set -eux \ apt-get remove --purge --auto-remove --yes; \ rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list +COPY common/docker-entrypoint.sh /docker-entrypoint.sh +COPY common/docker-entrypoint.d /docker-entrypoint.d/ +COPY oss/etc /etc +COPY common/etc /etc + +RUN chmod -R -v +x /docker-entrypoint.sh /docker-entrypoint.d/*.sh; \ \ No newline at end of file diff --git a/common/docker-entrypoint.sh b/common/docker-entrypoint.sh index 024d0aa7..f9e0cfa2 100644 --- a/common/docker-entrypoint.sh +++ b/common/docker-entrypoint.sh @@ -52,11 +52,11 @@ export CORS_ENABLED="$(parseBoolean "${CORS_ENABLED}")" # is not normally used as part of the gateway. The following variable # defines the set of acceptable headers. if [ "${CORS_ENABLED}" == "1" ]; then - export LIMIT_METHODS_TO="GET HEAD OPTIONS" - export LIMIT_METHODS_TO_CSV="GET, HEAD, OPTIONS" + export LIMIT_METHODS_TO="GET HEAD OPTIONS PUT DELETE" + export LIMIT_METHODS_TO_CSV="GET, HEAD, OPTIONS, PUT, DELETE" else - export LIMIT_METHODS_TO="GET HEAD" - export LIMIT_METHODS_TO_CSV="GET, HEAD" + export LIMIT_METHODS_TO="GET HEAD PUT DELETE" + export LIMIT_METHODS_TO_CSV="GET, HEAD, PUT, DELETE" fi if [ -z "${CORS_ALLOWED_ORIGIN+x}" ]; then diff --git a/common/etc/nginx/include/awssig4.js b/common/etc/nginx/include/awssig4.js index 6206796c..cd28f29a 100644 --- a/common/etc/nginx/include/awssig4.js +++ b/common/etc/nginx/include/awssig4.js @@ -48,8 +48,9 @@ const DEFAULT_SIGNED_HEADERS = 'host;x-amz-content-sha256;x-amz-date'; function signatureV4(r, timestamp, region, service, uri, queryParams, host, credentials) { const eightDigitDate = utils.getEightDigitDate(timestamp); const amzDatetime = utils.getAmzDatetime(timestamp, eightDigitDate); + const payloadHash = getPayloadHash(r.requestText); const canonicalRequest = _buildCanonicalRequest( - r.method, uri, queryParams, host, amzDatetime, credentials.sessionToken); + r.method, uri, queryParams, host, amzDatetime, credentials.sessionToken, payloadHash); const signature = _buildSignatureV4(r, amzDatetime, eightDigitDate, credentials, region, service, canonicalRequest); const authHeader = 'AWS4-HMAC-SHA256 Credential=' @@ -61,6 +62,16 @@ function signatureV4(r, timestamp, region, service, uri, queryParams, host, cred return authHeader; } +function getPayloadHash(payloadString) { + let payloadHash = EMPTY_PAYLOAD_HASH; + if (typeof payloadString !== 'undefined' && payloadString.length > 0) { + payloadHash = mod_hmac.createHash('sha256') + .update(payloadString) + .digest('hex'); + } + return payloadHash; +} + /** * Creates a canonical request that will later be signed * @@ -73,9 +84,9 @@ function signatureV4(r, timestamp, region, service, uri, queryParams, host, cred * @returns {string} string with concatenated request parameters * @private */ -function _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, sessionToken) { +function _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, sessionToken, payloadHash) { let canonicalHeaders = 'host:' + host + '\n' + - 'x-amz-content-sha256:' + EMPTY_PAYLOAD_HASH + '\n' + + 'x-amz-content-sha256:' + payloadHash + '\n' + 'x-amz-date:' + amzDatetime + '\n'; if (sessionToken) { @@ -87,7 +98,7 @@ function _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, ses canonicalRequest += queryParams + '\n'; canonicalRequest += canonicalHeaders + '\n'; canonicalRequest += _signedHeaders(sessionToken) + '\n'; - canonicalRequest += EMPTY_PAYLOAD_HASH; + canonicalRequest += payloadHash; return canonicalRequest; } @@ -253,6 +264,7 @@ function _splitCachedValues(cached) { export default { signatureV4, + getPayloadHash, // These functions do not need to be exposed, but they are exposed so that // unit tests can run against them. _buildCanonicalRequest, diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 6d694749..6c741a1e 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -155,6 +155,10 @@ function awsHeaderDate(r) { return utils.getAmzDatetime(NOW, utils.getEightDigitDate(NOW)); } +function awsContentSHA256(r) { + return awssig4.getPayloadHash(r.requestText); +} + /** * Creates an AWS authentication signature based on the global settings and * the passed request parameter. @@ -342,13 +346,6 @@ function _s3DirQueryParams(uriPath, method) { * @param r {Request} HTTP request object */ function redirectToS3(r) { - // This is a read-only S3 gateway, so we do not support any other methods - if (!(r.method === 'GET' || r.method === 'HEAD')) { - utils.debug_log(r, 'Invalid method requested: ' + r.method); - r.internalRedirect("@error405"); - return; - } - const uriPath = r.variables.uri_path; const isDirectoryListing = ALLOW_LISTING && _isDirectory(uriPath); @@ -704,6 +701,7 @@ async function _fetchWebIdentityCredentials(r) { export default { awsHeaderDate, + awsContentSHA256, fetchCredentials, s3date, s3auth, diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index cf4cd605..df6419d9 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -128,10 +128,10 @@ server { # Catch all errors from S3 and sanitize them so that the user can't # gain intelligence about the S3 bucket being proxied. - proxy_intercept_errors on; + #proxy_intercept_errors on; # Comment out this line to receive the error messages returned by S3 - error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + #error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; error_page 404 @trailslashControl; @@ -188,10 +188,10 @@ server { # Catch all errors from S3 and sanitize them so that the user can't # gain intelligence about the S3 bucket being proxied. - proxy_intercept_errors on; + proxy_intercept_errors off; # Comment out this line to receive the error messages returned by S3 - error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; + #error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404; proxy_pass ${S3_SERVER_PROTO}://storage_urls$s3Uri; include /etc/nginx/conf.d/gateway/s3listing_location.conf; diff --git a/common/etc/nginx/templates/gateway/v4_headers.conf.template b/common/etc/nginx/templates/gateway/v4_headers.conf.template index e6ca2daf..9bec2f59 100644 --- a/common/etc/nginx/templates/gateway/v4_headers.conf.template +++ b/common/etc/nginx/templates/gateway/v4_headers.conf.template @@ -4,4 +4,4 @@ proxy_set_header x-amz-date $awsDate; # All HTTP bodies are empty because we are only doing GET/HEAD requests, # so we can hardcode the body checksum. -proxy_set_header x-amz-content-sha256 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; +proxy_set_header x-amz-content-sha256 $awsContentSHA256; diff --git a/common/etc/nginx/templates/gateway/v4_js_vars.conf.template b/common/etc/nginx/templates/gateway/v4_js_vars.conf.template index b7baeee4..35364aff 100644 --- a/common/etc/nginx/templates/gateway/v4_js_vars.conf.template +++ b/common/etc/nginx/templates/gateway/v4_js_vars.conf.template @@ -2,3 +2,5 @@ # specifies the timestamp in which the signature was generated and is used with # the x-amz-date header. js_set $awsDate s3gateway.awsHeaderDate; + +js_set $awsContentSHA256 s3gateway.awsContentSHA256;