Skip to content

Commit

Permalink
Add support for S3 Express One Zone (#3465)
Browse files Browse the repository at this point in the history
## Motivation and Context
Allows the Rust SDK to use [S3 Express One
Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/)

## Description
The PR adds the said S3-specific functionality to the Rust SDK. The code
changes have already been reviewed by previous sub PRs, but it's worth
going through them again as a whole:
- #3386
- #3388
- #3390
- #3432
- #3433
- #3459
- #3457
- #3462

In addition to the PRs above, commit eebe8af increases the canary
lambda's memory size to 512MB from 128MB (also makes it configurable
through a command line arg for `canary-runner`). By default, lambda's
allowed memory size is 128MB but with the addition of `canary-wasm` in
main, canary lambda's memory usage will be 152MB, causing the lambda to
be killed by a signal during runtime. The commit addresses that issue.

## Testing
- Unit tests in
[aws/rust-runtime/aws-inlineable/src/s3_express.rs](https://github.com/smithy-lang/smithy-rs/blob/7f8c28b7038372927ec6196eff88384452f908dd/aws/rust-runtime/aws-inlineable/src/s3_express.rs)
- Integration tests in
[aws/sdk/integration-tests/s3/tests/express.rs](https://github.com/smithy-lang/smithy-rs/blob/7f8c28b7038372927ec6196eff88384452f908dd/aws/sdk/integration-tests/s3/tests/express.rs)
- Canary in smithy-rs#3462

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: John DiSanti <[email protected]>
Co-authored-by: AWS SDK Rust Bot <[email protected]>
Co-authored-by: AWS SDK Rust Bot <[email protected]>
Co-authored-by: Zelda Hessler <[email protected]>
Co-authored-by: Russell Cohen <[email protected]>
  • Loading branch information
6 people authored Mar 11, 2024
1 parent a413b9b commit f3b332d
Show file tree
Hide file tree
Showing 37 changed files with 3,196 additions and 352 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.next.toml
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,9 @@ message = "Upgrade Smithy to 1.45."
references = ["smithy-rs#3470"]
meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "all" }
authors = ["jdisanti"]

[[aws-sdk-rust]]
message = "Add support for S3 Express One Zone. See [the user guide](https://github.com/awslabs/aws-sdk-rust/discussions/1091) for more details."
references = ["aws-sdk-rust#992", "smithy-rs#3465"]
meta = { "breaking" = false, "bug" = false, "tada" = true }
author = "ysaito1001"
8 changes: 8 additions & 0 deletions aws/rust-runtime/aws-inlineable/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ publish = false
repository = "https://github.com/smithy-lang/smithy-rs"

[dependencies]
# Used by lru, and this forces it to be a later version that avoids
# https://github.com/tkaitchuck/aHash/issues/200
# when built with `cargo update -Z minimal-versions`
ahash = "0.8.11"
aws-credential-types = { path = "../aws-credential-types" }
aws-runtime = { path = "../aws-runtime", features = ["http-02x"] }
aws-sigv4 = { path = "../aws-sigv4" }
Expand All @@ -22,10 +26,14 @@ aws-smithy-runtime = { path = "../../../rust-runtime/aws-smithy-runtime", featur
aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] }
aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["http-body-0-4-x"] }
bytes = "1"
fastrand = "2.0.0"
hex = "0.4.3"
http = "0.2.9"
http-body = "0.4.5"
hmac = "0.12"
lru = "0.12.2"
ring = "0.17.5"
sha2 = "0.10"
tokio = "1.23.1"
tracing = "0.1"

Expand Down
54 changes: 52 additions & 2 deletions aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,45 @@ impl Storable for RequestChecksumInterceptorState {
type Storer = StoreReplace<Self>;
}

type CustomDefaultFn = Box<
dyn Fn(Option<ChecksumAlgorithm>, &ConfigBag) -> Option<ChecksumAlgorithm>
+ Send
+ Sync
+ 'static,
>;

pub(crate) struct DefaultRequestChecksumOverride {
custom_default: CustomDefaultFn,
}
impl fmt::Debug for DefaultRequestChecksumOverride {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("DefaultRequestChecksumOverride").finish()
}
}
impl Storable for DefaultRequestChecksumOverride {
type Storer = StoreReplace<Self>;
}
impl DefaultRequestChecksumOverride {
pub(crate) fn new<F>(custom_default: F) -> Self
where
F: Fn(Option<ChecksumAlgorithm>, &ConfigBag) -> Option<ChecksumAlgorithm>
+ Send
+ Sync
+ 'static,
{
Self {
custom_default: Box::new(custom_default),
}
}
pub(crate) fn custom_default(
&self,
original: Option<ChecksumAlgorithm>,
config_bag: &ConfigBag,
) -> Option<ChecksumAlgorithm> {
(self.custom_default)(original, config_bag)
}
}

pub(crate) struct RequestChecksumInterceptor<AP> {
algorithm_provider: AP,
}
Expand Down Expand Up @@ -102,7 +141,7 @@ where
/// Calculate a checksum and modify the request to include the checksum as a header
/// (for in-memory request bodies) or a trailer (for streaming request bodies).
/// Streaming bodies must be sized or this will return an error.
fn modify_before_retry_loop(
fn modify_before_signing(
&self,
context: &mut BeforeTransmitInterceptorContextMut<'_>,
_runtime_components: &RuntimeComponents,
Expand All @@ -112,7 +151,8 @@ where
.load::<RequestChecksumInterceptorState>()
.expect("set in `read_before_serialization`");

if let Some(checksum_algorithm) = state.checksum_algorithm {
let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg);
if let Some(checksum_algorithm) = checksum_algorithm {
let request = context.request_mut();
add_checksum_for_request_body(request, checksum_algorithm, cfg)?;
}
Expand All @@ -121,6 +161,16 @@ where
}
}

fn incorporate_custom_default(
checksum: Option<ChecksumAlgorithm>,
cfg: &ConfigBag,
) -> Option<ChecksumAlgorithm> {
match cfg.load::<DefaultRequestChecksumOverride>() {
Some(checksum_override) => checksum_override.custom_default(checksum, cfg),
None => checksum,
}
}

fn add_checksum_for_request_body(
request: &mut HttpRequest,
checksum_algorithm: ChecksumAlgorithm,
Expand Down
5 changes: 5 additions & 0 deletions aws/rust-runtime/aws-inlineable/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ pub mod presigning;
/// Presigning interceptors
pub mod presigning_interceptors;

// This module uses module paths that assume the target crate to which it is copied, e.g.
// `crate::config::endpoint::Params`. If included into `aws-inlineable`, this module would
// fail to compile.
// pub mod s3_express;

/// Special logic for extracting request IDs from S3's responses.
#[allow(dead_code)]
pub mod s3_request_id;
Expand Down
Loading

0 comments on commit f3b332d

Please sign in to comment.