feat: Support AWS S3 Express One Zone buckets

.gitignore

@@ -346,4 +346,45 @@ test-settings.*
 s3-requests.http
 httpRequests/
 
-.bin/
+.bin/
+
+# Created by https://www.toptal.com/developers/gitignore/api/terraform
+# Edit at https://www.toptal.com/developers/gitignore?templates=terraform
+
+### Terraform ###
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+.tfplan
+# End of https://www.toptal.com/developers/gitignore/api/terraform

common/etc/nginx/include/s3gateway.js

@@ -86,7 +86,7 @@ const INDEX_PAGE = "index.html";
  * Constant defining the service requests are being signed for.
  * @type {string}
  */
-const SERVICE = 's3';
+const SERVICE = 's3express';
 
 /**
  * Transform the headers returned from S3 such that there isn't information

deployments/s3_express/.tool-versions

@@ -0,0 +1 @@
+terraform 1.8.1

deployments/s3_express/README.md

@@ -0,0 +1,45 @@
+# Purpose
+This Terraform script sets up an AWS S3 Express One Zone bucket for testing.
+
+## Usage
+Use environment variables to authenticate:
+
+```bash
+export AWS_ACCESS_KEY_ID="anaccesskey"
+export AWS_SECRET_ACCESS_KEY="asecretkey"
+export AWS_REGION="us-west-2"
+```
+
+Generate a plan:
+```bash
+terraform plan -out=plan.tfplan \
+>   -var="bucket_name=my-bucket-name--usw2-az1--x-s3" \
+>   -var="region=us-west-2" \
+>   -var="availability_zone_id=usw2-az1" \
+>   -var="[email protected]"
+```
+> [!NOTE] 
+> Note that AWS S3 Express One Zone is only available in [certain regions and availability zones](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html#s3-express-endpoints). If you get an error like this: `api error InvalidBucketName`.  If you have met the [naming rules](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html), this likely means you have chosen a bad region/availability zone combination.
+
+
+If you are comfortable with the plan, apply it:
+```
+terraform apply "plan.tfplan"
+```
+
+Then build the image (you can also use the latest release)
+```bash
+docker build --file Dockerfile.oss --tag nginx-s3-gateway:oss --tag nginx-s3-gateway .
+```
+
+Configure and run the image:
+
+```bash
+docker run --rm --env-file ./settings.s3express.example --publish 80:80 --name nginx-s3-gateway \
+    nginx-s3-gateway:oss
+```
+
+Confirm that it is working. The terraform script will prepopulate the bucket with a single test object
+```bash
+curl http://localhost:80/test.txt
+```

deployments/s3_express/main.tf

@@ -0,0 +1,51 @@
+provider "aws" {
+  region = var.region
+}
+
+resource "aws_s3_directory_bucket" "example" {
+  bucket = var.bucket_name
+  location {
+    name = var.availability_zone_id
+  }
+
+  force_destroy = true
+}
+
+data "aws_partition" "current" {}
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_policy_document" "example" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3express:*",
+    ]
+
+    resources = [
+      aws_s3_directory_bucket.example.arn,
+    ]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
+}
+
+resource "aws_s3_bucket_policy" "example" {
+  bucket = aws_s3_directory_bucket.example.bucket
+  policy = data.aws_iam_policy_document.example.json
+}
+
+# The filemd5() function is available in Terraform 0.11.12 and later
+# For Terraform 0.11.11 and earlier, use the md5() function and the file() function:
+# etag = "${md5(file("path/to/file"))}"
+# etag = filemd5("path/to/file")
+resource "aws_s3_object" "example" {
+  bucket = aws_s3_directory_bucket.example.bucket
+  key    = "test.txt"
+  source = "${path.root}/test_data/test.txt"
+}
+
+

deployments/s3_express/settings.s3express.example

@@ -0,0 +1,21 @@
+S3_BUCKET_NAME=my-bucket-name--usw2-az1--x-s3
+AWS_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ
+AWS_SECRET_ACCESS_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+AWS_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+S3_SERVER=s3express-usw2-az1.us-west-2.amazonaws.com
+S3_SERVER_PORT=443
+S3_SERVER_PROTO=https
+S3_REGION=us-west-2
+S3_STYLE=virtual
+DEBUG=true
+AWS_SIGS_VERSION=4
+ALLOW_DIRECTORY_LIST=false
+PROVIDE_INDEX_PAGE=false
+APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=false
+DIRECTORY_LISTING_PATH_PREFIX=""
+PROXY_CACHE_MAX_SIZE=10g
+PROXY_CACHE_SLICE_SIZE="1m"
+PROXY_CACHE_INACTIVE=60m
+PROXY_CACHE_VALID_OK=1h
+PROXY_CACHE_VALID_NOTFOUND=1m
+PROXY_CACHE_VALID_FORBIDDEN=30s

deployments/s3_express/test_data/test.txt

@@ -0,0 +1,2 @@
+Congratulations, friend. You are using Amazon S3 Express One Zone.
+🚂🚂🚂 Choo-choo~ 🚂🚂🚂

deployments/s3_express/variables.tf

@@ -0,0 +1,20 @@
+# Format for bucket name [bucket_name]--[azid]--x-s3
+variable "bucket_name" {
+  type    = string
+  default = "example--usw2-az2--x-s3"
+}
+
+variable "owner_email" {
+  type = string
+}
+
+variable "region" {
+  type    = string
+  default = "us-west-2"
+}
+
+# "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#az-ids"
+variable "availability_zone_id" {
+  type    = string
+  default = "usw2-az2"
+}

deployments/s3_express/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.45.0"
+    }
+  }
+}

docs/getting_started.md

@@ -124,6 +124,37 @@ Setting your slice size too small can have performance impacts since NGINX perfo
 
 You may make byte-range requests and normal requests for the same file and NGINX will automatically handle them differently.  The caches for file chunks and normal file requests are separate on disk.
 
+## Usage with AWS S3 Express One Zone
+The gateway may be used to proxy files in the AWS S3 Express One Zone product (also called Directory Buckets).
+
+To do so, be sure that `S3_STYLE` is set to `virtual`. Additionally, the `S3_SERVER` configuration must be set a combination of the bucket name and the [Zonal Endpoint](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html#s3-express-endpoints).
+
+### Directory Bucket Names
+See the [official documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html) for the most up to date rules on Directory Bucket naming.
+
+Directory Buckets must have names matching this format:
+```
+bucket-base-name--azid--x-s3
+```
+For example:
+```
+bucket-base-name--usw2-az1--x-s3
+```
+### Final Configuration
+The bucket name must be prepended to the zonal endpoint like this
+```
+bucket-base-name--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com
+```
+The above is the value that must be provided to the `S3_SERVER` variable.
+Additionally, the `S3_BUCKET_NAME` must be set to the full bucket name with the suffix:
+```
+bucket-base-name--usw2-az1--x-s3
+```
+Buckets created in the AWS UI don't require manual specification of a suffix but it must be included in the gateway configuration.
+
+### Trying it Out
+A sample Terraform script to provision a bucket is provided in `/deployments/s3_express`.
+
 ## Running as a Systemd Service
 
 An [install script](/standalone_ubuntu_oss_install.sh) for the gateway shows

oss/etc/nginx/templates/upstreams.conf.template

@@ -7,5 +7,5 @@ upstream storage_urls {
 
     # Be sure to specify the port in the S3_SERVER and be sure that port
     # corresponds to the https/http in the proxy_pass directive.
-    server ${S3_SERVER}:${S3_SERVER_PORT};
+    server ${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT};
 }

plus/etc/nginx/templates/upstreams.conf.template

@@ -9,5 +9,5 @@ upstream storage_urls {
 
     # Be sure to specify the port in the S3_SERVER and be sure that port
     # corresponds to the https/http in the proxy_pass directive.
-    server ${S3_SERVER}:${S3_SERVER_PORT} resolve;
+    server ${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT} resolve;
 }