diff --git a/.gitignore b/.gitignore index d2a86aba..f5e56d93 100644 --- a/.gitignore +++ b/.gitignore @@ -346,4 +346,45 @@ test-settings.* s3-requests.http httpRequests/ -.bin/ \ No newline at end of file +.bin/ + +# Created by https://www.toptal.com/developers/gitignore/api/terraform +# Edit at https://www.toptal.com/developers/gitignore?templates=terraform + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# End of https://www.toptal.com/developers/gitignore/api/terraform diff --git a/deployments/s3_express/.terraform.lock.hcl b/deployments/s3_express/.terraform.lock.hcl new file mode 100644 index 00000000..d6034f09 --- /dev/null +++ b/deployments/s3_express/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.45.0" + constraints = "5.45.0" + hashes = [ + "h1:8m3+C1VNevzU/8FsABoKp2rTOx3Ue7674INfhfk0TZY=", + "zh:1379bcf45aef3d486ee18b4f767bfecd40a0056510d26107f388be3d7994c368", + "zh:1615a6f5495acfb3a0cb72324587261dd4d72711a3cc51aff13167b14531501e", + "zh:18b69a0f33f8b1862fbd3f200756b7e83e087b73687085f2cf9c7da4c318e3e6", + "zh:2c5e7aecd197bc3d3b19290bad8cf4c390c2c6a77bb165da4e11f53f2dfe2e54", + "zh:3794da9bef97596e3bc60e12cdd915bda5ec2ed62cd1cd93723d58b4981905fe", + "zh:40a5e45ed91801f83db76dffd467dcf425ea2ca8642327cf01119601cb86021c", + "zh:4abfc3f53d0256a7d5d1fa5e931e4601b02db3d1da28f452341d3823d0518f1a", + "zh:4eb0e98078f79aeb06b5ff6115286dc2135d12a80287885698d04036425494a2", + "zh:75470efbadea4a8d783642497acaeec5077fc4a7f3df3340defeaa1c7de29bf7", + "zh:8861a0b4891d5fa2fa7142f236ae613cea966c45b5472e3915a4ac3abcbaf487", + "zh:8bf6f21cd9390b742ca0b4393fde92616ca9e6553fb75003a0999006ad233d35", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ad73008a044e75d337acda910fb54d8b81a366873c8a413fec1291034899a814", + "zh:bf261713b0b8bebfe8c199291365b87d9043849f28a2dc764bafdde73ae43693", + "zh:da3bafa1fd830be418dfcc730e85085fe67c0d415c066716f2ac350a2306f40a", + ] +} diff --git a/deployments/s3_express/.tool-versions b/deployments/s3_express/.tool-versions new file mode 100644 index 00000000..fad6fadc --- /dev/null +++ b/deployments/s3_express/.tool-versions @@ -0,0 +1 @@ +terraform 1.8.1 diff --git a/deployments/s3_express/README.md b/deployments/s3_express/README.md new file mode 100644 index 00000000..75242564 --- /dev/null +++ b/deployments/s3_express/README.md @@ -0,0 +1,45 @@ +# Purpose +This Terraform script sets up an AWS S3 Express One Zone bucket for testing. + +## Usage +Use environment variables to authenticate: + +```bash +export AWS_ACCESS_KEY_ID="anaccesskey" +export AWS_SECRET_ACCESS_KEY="asecretkey" +export AWS_REGION="us-west-2" +``` + +Generate a plan: +```bash +terraform plan -out=plan.tfplan \ +> -var="bucket_name=my-bucket-name--usw2-az1--x-s3" \ +> -var="region=us-west-2" \ +> -var="availability_zone_id=usw2-az1" \ +> -var="owner_email=my_email@foo.com" +``` +> [!NOTE] +> Note that AWS S3 Express One Zone is only available in [certain regions and availability zones](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html#s3-express-endpoints). If you get an error like this: `api error InvalidBucketName`. If you have met the [naming rules](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html), this likely means you have chosen a bad region/availability zone combination. + + +If you are comfortable with the plan, apply it: +``` +terraform apply "plan.tfplan" +``` + +Then build the image (you can also use the latest release) +```bash +docker build --file Dockerfile.oss --tag nginx-s3-gateway:oss --tag nginx-s3-gateway . +``` + +Configure and run the image: + +```bash +docker run --rm --env-file ./settings.s3express.example --publish 80:80 --name nginx-s3-gateway \ + nginx-s3-gateway:oss +``` + +Confirm that it is working. The terraform script will prepopulate the bucket with a single test object +```bash +curl http://localhost:80/test.txt +``` diff --git a/deployments/s3_express/main.tf b/deployments/s3_express/main.tf new file mode 100644 index 00000000..55edfd62 --- /dev/null +++ b/deployments/s3_express/main.tf @@ -0,0 +1,51 @@ +provider "aws" { + region = var.region +} + +resource "aws_s3_directory_bucket" "example" { + bucket = var.bucket_name + location { + name = var.availability_zone_id + } + + force_destroy = true +} + +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +data "aws_iam_policy_document" "example" { + statement { + effect = "Allow" + + actions = [ + "s3express:*", + ] + + resources = [ + aws_s3_directory_bucket.example.arn, + ] + + principals { + type = "AWS" + identifiers = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root"] + } + } +} + +resource "aws_s3_bucket_policy" "example" { + bucket = aws_s3_directory_bucket.example.bucket + policy = data.aws_iam_policy_document.example.json +} + +# The filemd5() function is available in Terraform 0.11.12 and later +# For Terraform 0.11.11 and earlier, use the md5() function and the file() function: +# etag = "${md5(file("path/to/file"))}" +# etag = filemd5("path/to/file") +resource "aws_s3_object" "example" { + bucket = aws_s3_directory_bucket.example.bucket + key = "test.txt" + source = "${path.root}/test_data/test.txt" +} + + diff --git a/deployments/s3_express/plan.tfplan b/deployments/s3_express/plan.tfplan new file mode 100644 index 00000000..294916db Binary files /dev/null and b/deployments/s3_express/plan.tfplan differ diff --git a/deployments/s3_express/settings.s3express.example b/deployments/s3_express/settings.s3express.example new file mode 100644 index 00000000..00630be2 --- /dev/null +++ b/deployments/s3_express/settings.s3express.example @@ -0,0 +1,21 @@ +S3_BUCKET_NAME=bucket_name=my-bucket-name--usw2-az1--x-s3 +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_SESSION_TOKEN= +S3_SERVER=bucket_name=my-bucket-name--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com +S3_SERVER_PORT=443 +S3_SERVER_PROTO=https +S3_REGION=us-west-2 +S3_STYLE=virtual +DEBUG=true +AWS_SIGS_VERSION=4 +ALLOW_DIRECTORY_LIST=false +PROVIDE_INDEX_PAGE=false +APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=false +DIRECTORY_LISTING_PATH_PREFIX="" +PROXY_CACHE_MAX_SIZE=10g +PROXY_CACHE_SLICE_SIZE="1m" +PROXY_CACHE_INACTIVE=60m +PROXY_CACHE_VALID_OK=1h +PROXY_CACHE_VALID_NOTFOUND=1m +PROXY_CACHE_VALID_FORBIDDEN=30s diff --git a/deployments/s3_express/test_data/test.txt b/deployments/s3_express/test_data/test.txt new file mode 100644 index 00000000..b0a9adc7 --- /dev/null +++ b/deployments/s3_express/test_data/test.txt @@ -0,0 +1,2 @@ +Congratulations, friend. You are using Amazon S3 Express One Zone. +🚂🚂🚂 Choo-choo~ 🚂🚂🚂 \ No newline at end of file diff --git a/deployments/s3_express/variables.tf b/deployments/s3_express/variables.tf new file mode 100644 index 00000000..689eef56 --- /dev/null +++ b/deployments/s3_express/variables.tf @@ -0,0 +1,20 @@ +# Format for bucket name [bucket_name]--[azid]--x-s3 +variable "bucket_name" { + type = string + default = "example--usw2-az2--x-s3" +} + +variable "owner_email" { + type = string +} + +variable "region" { + type = string + default = "us-west-2" +} + +# "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#az-ids" +variable "availability_zone_id" { + type = string + default = "usw2-az2" +} diff --git a/deployments/s3_express/versions.tf b/deployments/s3_express/versions.tf new file mode 100644 index 00000000..a1aaa0de --- /dev/null +++ b/deployments/s3_express/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.45.0" + } + } +} diff --git a/docs/getting_started.md b/docs/getting_started.md index d3380817..036b8da6 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -124,6 +124,37 @@ Setting your slice size too small can have performance impacts since NGINX perfo You may make byte-range requests and normal requests for the same file and NGINX will automatically handle them differently. The caches for file chunks and normal file requests are separate on disk. +## Usage with AWS S3 Express One Zone +The gateway may be used to proxy files in the AWS S3 Express One Zone product (also called Directory Buckets). + +To do so, be sure that `S3_STYLE` is set to `virtual`. Additionally, the `S3_SERVER` configuration must be set a combination of the bucket name and the [Zonal Endpoint](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html#s3-express-endpoints). + +### Directory Bucket Names +See the [official documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html) for the most up to date rules on Directory Bucket naming. + +Directory Buckets must have names matching this format: +``` +bucket-base-name--azid--x-s3 +``` +For example: +``` +bucket-base-name--usw2-az1--x-s3 +``` +### Final Configuration +The bucket name must be prepended to the zonal endpoint like this +``` +bucket-base-name--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com +``` +The above is the value that must be provided to the `S3_SERVER` variable. +Additionally, the `S3_BUCKET_NAME` must be set to the full bucket name with the suffix: +``` +bucket-base-name--usw2-az1--x-s3 +``` +Buckets created in the AWS UI don't require manual specification of a suffix but it must be included in the gateway configuration. + +### Trying it Out +A sample Terraform script to provision a bucket is provided in `/deployments/s3_express`. + ## Running as a Systemd Service An [install script](/standalone_ubuntu_oss_install.sh) for the gateway shows