From 8ba9571d1e52d0bbfcfdc3ecba6af54268a72792 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Oct 2023 15:26:53 -0500 Subject: [PATCH 1/6] Avoid making `FsBuilder` aware of http-body version at interface level --- .../src/http_request_checksum.rs | 2 +- .../glacier/tests/custom-headers.rs | 6 +- .../integration-tests/s3/tests/checksums.rs | 2 +- .../customize/RequiredCustomizations.kt | 4 +- .../ServerRequiredCustomizations.kt | 4 +- .../src/types.rs | 4 +- rust-runtime/aws-smithy-types/Cargo.toml | 2 +- rust-runtime/aws-smithy-types/src/body.rs | 7 +- .../aws-smithy-types/src/byte_stream.rs | 104 +++++++++++++++--- .../src/byte_stream/bytestream_util.rs | 73 ++++-------- .../src/byte_stream/http_body_0_4_x.rs | 83 +------------- 11 files changed, 123 insertions(+), 168 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 239133414e..ef07f2c34b 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -269,7 +269,7 @@ mod tests { let crc32c_checksum = crc32c_checksum.finalize(); let mut request = HttpRequest::new( - ByteStream::read_with_body_0_4_from() + ByteStream::read_from() .path(&file) .buffer_size(1024) .build() diff --git a/aws/sdk/integration-tests/glacier/tests/custom-headers.rs b/aws/sdk/integration-tests/glacier/tests/custom-headers.rs index 46536dd2cd..52194567cc 100644 --- a/aws/sdk/integration-tests/glacier/tests/custom-headers.rs +++ b/aws/sdk/integration-tests/glacier/tests/custom-headers.rs @@ -21,11 +21,7 @@ async fn set_correct_headers() { let _resp = client .upload_archive() .vault_name("vault") - .body( - ByteStream::from_path_body_0_4("tests/test-file.txt") - .await - .unwrap(), - ) + .body(ByteStream::from_path("tests/test-file.txt").await.unwrap()) .send() .await; let req = handler.expect_request(); diff --git a/aws/sdk/integration-tests/s3/tests/checksums.rs b/aws/sdk/integration-tests/s3/tests/checksums.rs index 97155bfc09..37782182e0 100644 --- a/aws/sdk/integration-tests/s3/tests/checksums.rs +++ b/aws/sdk/integration-tests/s3/tests/checksums.rs @@ -177,7 +177,7 @@ async fn test_checksum_on_streaming_request<'a>( use std::io::Write; file.write_all(body).unwrap(); - let body = aws_sdk_s3::primitives::ByteStream::read_with_body_0_4_from() + let body = aws_sdk_s3::primitives::ByteStream::read_from() .path(file.path()) .buffer_size(1024) .build() diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt index 22ec93e842..0c2bc8ef93 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt @@ -71,12 +71,12 @@ class RequiredCustomizations : ClientCodegenDecorator { override fun extras(codegenContext: ClientCodegenContext, rustCrate: RustCrate) { val rc = codegenContext.runtimeConfig - // Add rt-tokio and http-body-0-4-x features for `ByteStream::from_path_0_4` + // Add rt-tokio feature for `ByteStream::from_path` rustCrate.mergeFeature( Feature( "rt-tokio", true, - listOf("aws-smithy-async/rt-tokio", "aws-smithy-types/rt-tokio", "aws-smithy-types/http-body-0-4-x"), + listOf("aws-smithy-async/rt-tokio", "aws-smithy-types/rt-tokio"), ), ) diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt index dde2e91bae..1117cd1d16 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt @@ -37,12 +37,12 @@ class ServerRequiredCustomizations : ServerCodegenDecorator { override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) { val rc = codegenContext.runtimeConfig - // Add rt-tokio and http-body-0-4-x features for `ByteStream::from_path_body_0_4` + // Add rt-tokio feature for `ByteStream::from_path` rustCrate.mergeFeature( Feature( "rt-tokio", true, - listOf("aws-smithy-types/rt-tokio", "aws-smithy-types/http-body-0-4-x"), + listOf("aws-smithy-types/rt-tokio"), ), ) diff --git a/rust-runtime/aws-smithy-http-server-python/src/types.rs b/rust-runtime/aws-smithy-http-server-python/src/types.rs index a475f62351..1af75dbd27 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/types.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/types.rs @@ -407,7 +407,7 @@ impl ByteStream { #[staticmethod] pub fn from_path_blocking(py: Python, path: String) -> PyResult> { let byte_stream = Handle::current().block_on(async { - aws_smithy_types::byte_stream::ByteStream::from_path_body_0_4(path) + aws_smithy_types::byte_stream::ByteStream::from_path(path) .await .map_err(|e| PyRuntimeError::new_err(e.to_string())) })?; @@ -423,7 +423,7 @@ impl ByteStream { #[staticmethod] pub fn from_path(py: Python, path: String) -> PyResult<&PyAny> { pyo3_asyncio::tokio::future_into_py(py, async move { - let byte_stream = aws_smithy_types::byte_stream::ByteStream::from_path_body_0_4(path) + let byte_stream = aws_smithy_types::byte_stream::ByteStream::from_path(path) .await .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; Ok(Self(Arc::new(Mutex::new(byte_stream)))) diff --git a/rust-runtime/aws-smithy-types/Cargo.toml b/rust-runtime/aws-smithy-types/Cargo.toml index 42f50191a9..a4e0d39fcd 100644 --- a/rust-runtime/aws-smithy-types/Cargo.toml +++ b/rust-runtime/aws-smithy-types/Cargo.toml @@ -14,7 +14,7 @@ repository = "https://github.com/awslabs/smithy-rs" byte-stream-poll-next = [] http-body-0-4-x = ["dep:http-body-0-4"] hyper-0-14-x = ["dep:hyper-0-14"] -rt-tokio = ["dep:tokio-util", "dep:tokio", "tokio?/rt", "tokio?/fs", "tokio?/io-util", "tokio-util?/io"] +rt-tokio = ["dep:http-body-0-4", "dep:tokio-util", "dep:tokio", "tokio?/rt", "tokio?/fs", "tokio?/io-util", "tokio-util?/io"] test-util = [] serde-serialize = [] serde-deserialize = [] diff --git a/rust-runtime/aws-smithy-types/src/body.rs b/rust-runtime/aws-smithy-types/src/body.rs index 043aa47df3..099711b087 100644 --- a/rust-runtime/aws-smithy-types/src/body.rs +++ b/rust-runtime/aws-smithy-types/src/body.rs @@ -95,10 +95,9 @@ impl SdkBody { /// _Note: This is probably not what you want_ /// /// All bodies constructed from in-memory data (`String`, `Vec`, `Bytes`, etc.) will be - /// retryable out of the box. If you want to read data from a file, you should turn on a feature - /// `http-body-0-4-x` and use `ByteStream::from_path_body_0_4`. - /// - /// This function is only necessary when you need to enable retries for your own streaming container. + /// retryable out of the box. If you want to read data from a file, you should use + /// [`ByteStream::from_path`](crate::byte_stream::ByteStream::from_path). This function + /// is only necessary when you need to enable retries for your own streaming container. pub fn retryable(f: impl Fn() -> SdkBody + Send + Sync + 'static) -> Self { let initial = f(); SdkBody { diff --git a/rust-runtime/aws-smithy-types/src/byte_stream.rs b/rust-runtime/aws-smithy-types/src/byte_stream.rs index 0db39f4bd1..1018518d5c 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream.rs @@ -78,10 +78,10 @@ //! //! ### Create a ByteStream from a file //! -//! _Note: This is only available with `rt-tokio` and `http-body-0-4-x` enabled._ +//! _Note: This is only available with `rt-tokio` enabled._ //! //! ```no_run -//! # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] +//! # #[cfg(feature = "rt-tokio")] //! # { //! use aws_smithy_types::byte_stream::ByteStream; //! use std::path::Path; @@ -90,7 +90,7 @@ //! } //! //! async fn bytestream_from_file() -> GetObjectInput { -//! let bytestream = ByteStream::from_path_body_0_4("docs/some-large-file.csv") +//! let bytestream = ByteStream::from_path("docs/some-large-file.csv") //! .await //! .expect("valid path"); //! GetObjectInput { body: bytestream } @@ -102,7 +102,7 @@ //! or the length of the file, use an `FsBuilder`. //! //! ```no_run -//! # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] +//! # #[cfg(feature = "rt-tokio")] //! # { //! use aws_smithy_types::byte_stream::{ByteStream, Length}; //! use std::path::Path; @@ -111,7 +111,7 @@ //! } //! //! async fn bytestream_from_file() -> GetObjectInput { -//! let bytestream = ByteStream::read_with_body_0_4_from().path("docs/some-large-file.csv") +//! let bytestream = ByteStream::read_from().path("docs/some-large-file.csv") //! .buffer_size(32_784) //! .length(Length::Exact(123_456)) //! .build() @@ -235,26 +235,18 @@ pin_project! { /// ``` /// /// 2. **From a file**: ByteStreams created from a path can be retried. A new file descriptor will be opened if a retry occurs. - /// - /// _Note: The `http-body-0-4-x` feature must be active to call `ByteStream::from_body_0_4`._ - /// /// ```no_run - /// #[cfg(all(feature = "tokio-rt", feature = "http-body-0-4-x"))] + /// #[cfg(feature = "tokio-rt")] /// # { /// use aws_smithy_types::byte_stream::ByteStream; - /// let stream = ByteStream::from_path_body_0_4("big_file.csv"); + /// let stream = ByteStream::from_path("big_file.csv"); /// # } /// ``` /// /// 3. **From an `SdkBody` directly**: For more advanced / custom use cases, a ByteStream can be created directly /// from an SdkBody. **When created from an SdkBody, care must be taken to ensure retriability.** An SdkBody is retryable /// when constructed from in-memory data or when using [`SdkBody::retryable`](crate::body::SdkBody::retryable). - /// - /// _Note: The `http-body-0-4-x` feature must be active to construct an `SdkBody` with `from_body_0_4`._ - /// /// ```no_run - /// # #[cfg(feature = "http-body-0-4-x")] - /// # { /// # use hyper_0_14 as hyper; /// use aws_smithy_types::byte_stream::ByteStream; /// use aws_smithy_types::body::SdkBody; @@ -265,7 +257,6 @@ pin_project! { /// tx.send_data(Bytes::from_static(b"hello world!")); /// tx.send_data(Bytes::from_static(b"hello again!")); /// // NOTE! You must ensure that `tx` is dropped to ensure that EOF is sent - /// # } /// ``` /// #[derive(Debug)] @@ -353,6 +344,87 @@ impl ByteStream { self.inner.collect().await.map_err(Error::streaming) } + /// Returns a [`FsBuilder`](crate::byte_stream::FsBuilder), allowing you to build a `ByteStream` with + /// full control over how the file is read (eg. specifying the length of the file or the size of the buffer used to read the file). + /// ```no_run + /// # #[cfg(feature = "rt-tokio")] + /// # { + /// use aws_smithy_types::byte_stream::{ByteStream, Length}; + /// + /// async fn bytestream_from_file() -> ByteStream { + /// let bytestream = ByteStream::read_from() + /// .path("docs/some-large-file.csv") + /// // Specify the size of the buffer used to read the file (in bytes, default is 4096) + /// .buffer_size(32_784) + /// // Specify the length of the file used (skips an additional call to retrieve the size) + /// .length(Length::Exact(123_456)) + /// .build() + /// .await + /// .expect("valid path"); + /// bytestream + /// } + /// # } + /// ``` + #[cfg(feature = "rt-tokio")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] + pub fn read_from() -> crate::byte_stream::FsBuilder { + crate::byte_stream::FsBuilder::new() + } + + /// Create a ByteStream that streams data from the filesystem + /// + /// This function creates a retryable ByteStream for a given `path`. The returned ByteStream + /// will provide a size hint when used as an HTTP body. If the request fails, the read will + /// begin again by reloading the file handle. + /// + /// ## Warning + /// The contents of the file MUST not change during retries. The length & checksum of the file + /// will be cached. If the contents of the file change, the operation will almost certainly fail. + /// + /// Furthermore, a partial write MAY seek in the file and resume from the previous location. + /// + /// Note: If you want more control, such as specifying the size of the buffer used to read the file + /// or the length of the file, use a [`FsBuilder`](crate::byte_stream::FsBuilder) as returned + /// from `ByteStream::read_from` + /// + /// # Examples + /// ```no_run + /// use aws_smithy_types::byte_stream::ByteStream; + /// use std::path::Path; + /// async fn make_bytestream() -> ByteStream { + /// ByteStream::from_path("docs/rows.csv").await.expect("file should be readable") + /// } + /// ``` + #[cfg(feature = "rt-tokio")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] + pub async fn from_path( + path: impl AsRef, + ) -> Result { + crate::byte_stream::FsBuilder::new() + .path(path) + .build() + .await + } + + /// Create a ByteStream from a file + /// + /// NOTE: This will NOT result in a retryable ByteStream. For a ByteStream that can be retried in the case of + /// upstream failures, use [`ByteStream::from_path`](ByteStream::from_path) + #[deprecated( + since = "0.40.0", + note = "Prefer the more extensible ByteStream::read_from() API" + )] + #[cfg(feature = "rt-tokio")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] + pub async fn from_file( + file: tokio::fs::File, + ) -> Result { + crate::byte_stream::FsBuilder::new() + .file(file) + .build() + .await + } + #[cfg(feature = "rt-tokio")] /// Convert this `ByteStream` into a struct that implements [`AsyncRead`](tokio::io::AsyncRead). /// diff --git a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs index 467733b002..1dfe87b1e3 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs @@ -8,7 +8,6 @@ use crate::byte_stream::{error::Error, error::ErrorKind, ByteStream}; use std::future::Future; use std::path::PathBuf; use std::pin::Pin; -use std::sync::Arc; use tokio::fs::File; use tokio::io::{self, AsyncReadExt, AsyncSeekExt}; use tokio_util::io::ReaderStream; @@ -56,11 +55,8 @@ impl PathBody { /// Builder for creating [`ByteStreams`](ByteStream) from a file/path, with full control over advanced options. /// -/// _Note: A cargo feature `http-body-0-4-x` should be active to call `ByteStream::read_with_body_0_4_from` in the following example._ -/// -/// Example usage: /// ```no_run -/// # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] +/// # #[cfg(feature = "rt-tokio")] /// # { /// use aws_smithy_types::byte_stream::{ByteStream, Length}; /// use std::path::Path; @@ -69,7 +65,7 @@ impl PathBody { /// } /// /// async fn bytestream_from_file() -> GetObjectInput { -/// let bytestream = ByteStream::read_with_body_0_4_from() +/// let bytestream = ByteStream::read_from() /// .path("docs/some-large-file.csv") /// // Specify the size of the buffer used to read the file (in bytes, default is 4096) /// .buffer_size(32_784) @@ -89,12 +85,6 @@ pub struct FsBuilder { length: Option, buffer_size: usize, offset: Option, - sdk_body_creator: SdkBodyCreator, -} - -enum SdkBodyCreator { - #[cfg(feature = "http-body-0-4-x")] - HttpBody04(Arc SdkBody + Send + Sync + 'static>), } /// The length (in bytes) to read. Determines whether or not a short read counts as an error. @@ -108,22 +98,16 @@ pub enum Length { } impl FsBuilder { - #[cfg(feature = "http-body-0-4-x")] /// Create a new [`FsBuilder`] (using a default read buffer of 4096 bytes). /// /// You must then call either [`file`](FsBuilder::file) or [`path`](FsBuilder::path) to specify what to read from. - /// - /// _Note: This is only available with `http-body-0-4-x` enabled._ - pub fn new_with_body_0_4() -> Self { + pub fn new() -> Self { Self { buffer_size: DEFAULT_BUFFER_SIZE, file: None, length: None, offset: None, path: None, - sdk_body_creator: SdkBodyCreator::HttpBody04(Arc::new(|body: PathBody| { - SdkBody::from_body_0_4(body) - })), } } @@ -203,19 +187,12 @@ impl FsBuilder { let body_loader = move || { // If an offset was provided, seeking will be handled in `PathBody::poll_data` each // time the file is loaded. - match &self.sdk_body_creator { - #[cfg(feature = "http-body-0-4-x")] - SdkBodyCreator::HttpBody04(f) => f(PathBody::from_path( - path.clone(), - length, - buffer_size, - self.offset, - )), - #[allow(unreachable_patterns)] - _ => unreachable!( - "`http-body-0-4-x` should've been enabled to create an `FsBuilder`" - ), - } + SdkBody::from_body_0_4(PathBody::from_path( + path.clone(), + length, + buffer_size, + self.offset, + )) }; Ok(ByteStream::new(SdkBody::retryable(body_loader))) @@ -225,14 +202,7 @@ impl FsBuilder { let _s = file.seek(io::SeekFrom::Start(offset)).await?; } - let body = match self.sdk_body_creator { - #[cfg(feature = "http-body-0-4-x")] - SdkBodyCreator::HttpBody04(f) => f(PathBody::from_file(file, length, buffer_size)), - #[allow(unreachable_patterns)] - _ => unreachable!( - "`http-body-0-4-x` should've been enabled to create an `FsBuilder`" - ), - }; + let body = SdkBody::from_body_0_4(PathBody::from_file(file, length, buffer_size)); Ok(ByteStream::new(body)) } else { @@ -256,7 +226,6 @@ enum State { Loaded(ReaderStream>), } -#[cfg(feature = "http-body-0-4-x")] impl http_body_0_4::Body for PathBody { type Data = bytes::Bytes; type Error = Box; @@ -343,7 +312,7 @@ mod test { .expect("file metadata is accessible") .len(); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) .buffer_size(16384) .length(Length::Exact(file_length)) @@ -382,7 +351,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // The file is longer than 1 byte, let's see if this is used to generate the size hint .length(Length::Exact(1)) @@ -406,7 +375,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // We're going to read line 0 only .length(Length::Exact(line_0.len() as u64)) @@ -430,7 +399,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - assert!(FsBuilder::new_with_body_0_4() + assert!(FsBuilder::new() .path(&file) // The file is 30 bytes so this is fine .length(Length::Exact(29)) @@ -438,7 +407,7 @@ mod test { .await .is_ok()); - assert!(FsBuilder::new_with_body_0_4() + assert!(FsBuilder::new() .path(&file) // The file is 30 bytes so this is fine .length(Length::Exact(30)) @@ -446,7 +415,7 @@ mod test { .await .is_ok()); - assert!(FsBuilder::new_with_body_0_4() + assert!(FsBuilder::new() .path(&file) // Larger than 30 bytes, this will cause an error .length(Length::Exact(31)) @@ -467,7 +436,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // We're going to skip the first line by using offset .offset(line_0.len() as u64) @@ -495,7 +464,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // We're going to skip line 0 by using offset .offset(line_0.len() as u64) @@ -524,7 +493,7 @@ mod test { file.flush().expect("flushing is OK"); assert_eq!( - FsBuilder::new_with_body_0_4() + FsBuilder::new() .path(&file) // We're going to skip all file contents by setting an offset // much larger than the file size @@ -549,7 +518,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) .length(Length::UpTo(9000)) .build() @@ -601,7 +570,7 @@ mod test { chunk_size }; - let byte_stream = FsBuilder::new_with_body_0_4() + let byte_stream = FsBuilder::new() .path(&file_path) .offset(i * chunk_size) .length(Length::Exact(length)) diff --git a/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs b/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs index 5edbaaba87..8bdae026f6 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs @@ -18,87 +18,6 @@ impl ByteStream { { ByteStream::new(SdkBody::from_body_0_4(body)) } - - /// Returns a [`FsBuilder`](crate::byte_stream::FsBuilder), allowing you to build a `ByteStream` with - /// full control over how the file is read (eg. specifying the length of the file or the size of the buffer used to read the file). - /// ```no_run - /// # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] - /// # { - /// use aws_smithy_types::byte_stream::{ByteStream, Length}; - /// - /// async fn bytestream_from_file() -> ByteStream { - /// let bytestream = ByteStream::read_with_body_0_4_from() - /// .path("docs/some-large-file.csv") - /// // Specify the size of the buffer used to read the file (in bytes, default is 4096) - /// .buffer_size(32_784) - /// // Specify the length of the file used (skips an additional call to retrieve the size) - /// .length(Length::Exact(123_456)) - /// .build() - /// .await - /// .expect("valid path"); - /// bytestream - /// } - /// # } - /// ``` - #[cfg(feature = "rt-tokio")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] - pub fn read_with_body_0_4_from() -> crate::byte_stream::FsBuilder { - crate::byte_stream::FsBuilder::new_with_body_0_4() - } - - /// Create a ByteStream that streams data from the filesystem - /// - /// This function creates a retryable ByteStream for a given `path`. The returned ByteStream - /// will provide a size hint when used as an HTTP body. If the request fails, the read will - /// begin again by reloading the file handle. - /// - /// ## Warning - /// The contents of the file MUST not change during retries. The length & checksum of the file - /// will be cached. If the contents of the file change, the operation will almost certainly fail. - /// - /// Furthermore, a partial write MAY seek in the file and resume from the previous location. - /// - /// Note: If you want more control, such as specifying the size of the buffer used to read the file - /// or the length of the file, use a [`FsBuilder`](crate::byte_stream::FsBuilder) as returned - /// from `ByteStream::read_with_body_0_4_from` - /// - /// # Examples - /// ```no_run - /// use aws_smithy_types::byte_stream::ByteStream; - /// use std::path::Path; - /// async fn make_bytestream() -> ByteStream { - /// ByteStream::from_path_body_0_4("docs/rows.csv").await.expect("file should be readable") - /// } - /// ``` - #[cfg(feature = "rt-tokio")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] - pub async fn from_path_body_0_4( - path: impl AsRef, - ) -> Result { - crate::byte_stream::FsBuilder::new_with_body_0_4() - .path(path) - .build() - .await - } - - /// Create a ByteStream from a file - /// - /// NOTE: This will NOT result in a retryable ByteStream. For a ByteStream that can be retried in the case of - /// upstream failures, use [`ByteStream::from_path_body_0_4`](ByteStream::from_path_body_0_4) - #[deprecated( - since = "0.40.0", - note = "Prefer the more extensible ByteStream::read_from() API" - )] - #[cfg(feature = "rt-tokio")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] - pub async fn from_file_body_0_4( - file: tokio::fs::File, - ) -> Result { - crate::byte_stream::FsBuilder::new_with_body_0_4() - .file(file) - .build() - .await - } } #[cfg(feature = "hyper-0-14-x")] @@ -141,7 +60,7 @@ mod tests { for i in 0..10000 { writeln!(file, "Brian was here. Briefly. {}", i)?; } - let body = ByteStream::from_path_body_0_4(&file).await?.into_inner(); + let body = ByteStream::from_path(&file).await?.into_inner(); // assert that a valid size hint is immediately ready assert_eq!(body.content_length(), Some(298890)); let mut body1 = body.try_clone().expect("retryable bodies are cloneable"); From 4f52e851f1eb415898cf5872139fdf5e4838c482 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Oct 2023 15:35:53 -0500 Subject: [PATCH 2/6] Update CHANGELOG.next.toml --- CHANGELOG.next.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index f74058dd8d..3ae4b10f27 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -489,7 +489,7 @@ author = "rcoh" [[smithy-rs]] message = "Publicly exposed types from `http-body` and `hyper` crates within `aws-smithy-types` are now feature-gated. See the [upgrade guidance](https://github.com/awslabs/smithy-rs/discussions/3089) for details." -references = ["smithy-rs#3033", "smithy-rs#3088"] +references = ["smithy-rs#3033", "smithy-rs#3088", "smithy-rs#3101"] meta = { "breaking" = true, "tada" = false, "bug" = false, "target" = "all" } author = "ysaito1001" From 6f910d9fe987298f11787f27e3b39db2ebabdbec Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Oct 2023 15:43:28 -0500 Subject: [PATCH 3/6] Fix clippy warning `new_without_default` --- .../aws-smithy-types/src/byte_stream/bytestream_util.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs index 1dfe87b1e3..39965c90c5 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs @@ -87,6 +87,12 @@ pub struct FsBuilder { offset: Option, } +impl Default for FsBuilder { + fn default() -> Self { + Self::new() + } +} + /// The length (in bytes) to read. Determines whether or not a short read counts as an error. #[allow(missing_debug_implementations)] pub enum Length { From baeef4bffddc7a7ddb214e97cc8a31e863b3f2e5 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Oct 2023 15:26:53 -0500 Subject: [PATCH 4/6] Avoid making `FsBuilder` aware of http-body version at interface level --- .../src/http_request_checksum.rs | 2 +- .../glacier/tests/custom-headers.rs | 6 +- .../integration-tests/s3/tests/checksums.rs | 2 +- .../customize/RequiredCustomizations.kt | 4 +- .../ServerRequiredCustomizations.kt | 4 +- .../src/types.rs | 4 +- rust-runtime/aws-smithy-types/Cargo.toml | 2 +- rust-runtime/aws-smithy-types/src/body.rs | 7 +- .../aws-smithy-types/src/byte_stream.rs | 104 +++++++++++++++--- .../src/byte_stream/bytestream_util.rs | 73 ++++-------- .../src/byte_stream/http_body_0_4_x.rs | 83 +------------- 11 files changed, 123 insertions(+), 168 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 239133414e..ef07f2c34b 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -269,7 +269,7 @@ mod tests { let crc32c_checksum = crc32c_checksum.finalize(); let mut request = HttpRequest::new( - ByteStream::read_with_body_0_4_from() + ByteStream::read_from() .path(&file) .buffer_size(1024) .build() diff --git a/aws/sdk/integration-tests/glacier/tests/custom-headers.rs b/aws/sdk/integration-tests/glacier/tests/custom-headers.rs index 46536dd2cd..52194567cc 100644 --- a/aws/sdk/integration-tests/glacier/tests/custom-headers.rs +++ b/aws/sdk/integration-tests/glacier/tests/custom-headers.rs @@ -21,11 +21,7 @@ async fn set_correct_headers() { let _resp = client .upload_archive() .vault_name("vault") - .body( - ByteStream::from_path_body_0_4("tests/test-file.txt") - .await - .unwrap(), - ) + .body(ByteStream::from_path("tests/test-file.txt").await.unwrap()) .send() .await; let req = handler.expect_request(); diff --git a/aws/sdk/integration-tests/s3/tests/checksums.rs b/aws/sdk/integration-tests/s3/tests/checksums.rs index 97155bfc09..37782182e0 100644 --- a/aws/sdk/integration-tests/s3/tests/checksums.rs +++ b/aws/sdk/integration-tests/s3/tests/checksums.rs @@ -177,7 +177,7 @@ async fn test_checksum_on_streaming_request<'a>( use std::io::Write; file.write_all(body).unwrap(); - let body = aws_sdk_s3::primitives::ByteStream::read_with_body_0_4_from() + let body = aws_sdk_s3::primitives::ByteStream::read_from() .path(file.path()) .buffer_size(1024) .build() diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt index 22ec93e842..0c2bc8ef93 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt @@ -71,12 +71,12 @@ class RequiredCustomizations : ClientCodegenDecorator { override fun extras(codegenContext: ClientCodegenContext, rustCrate: RustCrate) { val rc = codegenContext.runtimeConfig - // Add rt-tokio and http-body-0-4-x features for `ByteStream::from_path_0_4` + // Add rt-tokio feature for `ByteStream::from_path` rustCrate.mergeFeature( Feature( "rt-tokio", true, - listOf("aws-smithy-async/rt-tokio", "aws-smithy-types/rt-tokio", "aws-smithy-types/http-body-0-4-x"), + listOf("aws-smithy-async/rt-tokio", "aws-smithy-types/rt-tokio"), ), ) diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt index dde2e91bae..1117cd1d16 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/ServerRequiredCustomizations.kt @@ -37,12 +37,12 @@ class ServerRequiredCustomizations : ServerCodegenDecorator { override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) { val rc = codegenContext.runtimeConfig - // Add rt-tokio and http-body-0-4-x features for `ByteStream::from_path_body_0_4` + // Add rt-tokio feature for `ByteStream::from_path` rustCrate.mergeFeature( Feature( "rt-tokio", true, - listOf("aws-smithy-types/rt-tokio", "aws-smithy-types/http-body-0-4-x"), + listOf("aws-smithy-types/rt-tokio"), ), ) diff --git a/rust-runtime/aws-smithy-http-server-python/src/types.rs b/rust-runtime/aws-smithy-http-server-python/src/types.rs index a475f62351..1af75dbd27 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/types.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/types.rs @@ -407,7 +407,7 @@ impl ByteStream { #[staticmethod] pub fn from_path_blocking(py: Python, path: String) -> PyResult> { let byte_stream = Handle::current().block_on(async { - aws_smithy_types::byte_stream::ByteStream::from_path_body_0_4(path) + aws_smithy_types::byte_stream::ByteStream::from_path(path) .await .map_err(|e| PyRuntimeError::new_err(e.to_string())) })?; @@ -423,7 +423,7 @@ impl ByteStream { #[staticmethod] pub fn from_path(py: Python, path: String) -> PyResult<&PyAny> { pyo3_asyncio::tokio::future_into_py(py, async move { - let byte_stream = aws_smithy_types::byte_stream::ByteStream::from_path_body_0_4(path) + let byte_stream = aws_smithy_types::byte_stream::ByteStream::from_path(path) .await .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; Ok(Self(Arc::new(Mutex::new(byte_stream)))) diff --git a/rust-runtime/aws-smithy-types/Cargo.toml b/rust-runtime/aws-smithy-types/Cargo.toml index 42f50191a9..a4e0d39fcd 100644 --- a/rust-runtime/aws-smithy-types/Cargo.toml +++ b/rust-runtime/aws-smithy-types/Cargo.toml @@ -14,7 +14,7 @@ repository = "https://github.com/awslabs/smithy-rs" byte-stream-poll-next = [] http-body-0-4-x = ["dep:http-body-0-4"] hyper-0-14-x = ["dep:hyper-0-14"] -rt-tokio = ["dep:tokio-util", "dep:tokio", "tokio?/rt", "tokio?/fs", "tokio?/io-util", "tokio-util?/io"] +rt-tokio = ["dep:http-body-0-4", "dep:tokio-util", "dep:tokio", "tokio?/rt", "tokio?/fs", "tokio?/io-util", "tokio-util?/io"] test-util = [] serde-serialize = [] serde-deserialize = [] diff --git a/rust-runtime/aws-smithy-types/src/body.rs b/rust-runtime/aws-smithy-types/src/body.rs index 043aa47df3..099711b087 100644 --- a/rust-runtime/aws-smithy-types/src/body.rs +++ b/rust-runtime/aws-smithy-types/src/body.rs @@ -95,10 +95,9 @@ impl SdkBody { /// _Note: This is probably not what you want_ /// /// All bodies constructed from in-memory data (`String`, `Vec`, `Bytes`, etc.) will be - /// retryable out of the box. If you want to read data from a file, you should turn on a feature - /// `http-body-0-4-x` and use `ByteStream::from_path_body_0_4`. - /// - /// This function is only necessary when you need to enable retries for your own streaming container. + /// retryable out of the box. If you want to read data from a file, you should use + /// [`ByteStream::from_path`](crate::byte_stream::ByteStream::from_path). This function + /// is only necessary when you need to enable retries for your own streaming container. pub fn retryable(f: impl Fn() -> SdkBody + Send + Sync + 'static) -> Self { let initial = f(); SdkBody { diff --git a/rust-runtime/aws-smithy-types/src/byte_stream.rs b/rust-runtime/aws-smithy-types/src/byte_stream.rs index 0db39f4bd1..1018518d5c 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream.rs @@ -78,10 +78,10 @@ //! //! ### Create a ByteStream from a file //! -//! _Note: This is only available with `rt-tokio` and `http-body-0-4-x` enabled._ +//! _Note: This is only available with `rt-tokio` enabled._ //! //! ```no_run -//! # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] +//! # #[cfg(feature = "rt-tokio")] //! # { //! use aws_smithy_types::byte_stream::ByteStream; //! use std::path::Path; @@ -90,7 +90,7 @@ //! } //! //! async fn bytestream_from_file() -> GetObjectInput { -//! let bytestream = ByteStream::from_path_body_0_4("docs/some-large-file.csv") +//! let bytestream = ByteStream::from_path("docs/some-large-file.csv") //! .await //! .expect("valid path"); //! GetObjectInput { body: bytestream } @@ -102,7 +102,7 @@ //! or the length of the file, use an `FsBuilder`. //! //! ```no_run -//! # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] +//! # #[cfg(feature = "rt-tokio")] //! # { //! use aws_smithy_types::byte_stream::{ByteStream, Length}; //! use std::path::Path; @@ -111,7 +111,7 @@ //! } //! //! async fn bytestream_from_file() -> GetObjectInput { -//! let bytestream = ByteStream::read_with_body_0_4_from().path("docs/some-large-file.csv") +//! let bytestream = ByteStream::read_from().path("docs/some-large-file.csv") //! .buffer_size(32_784) //! .length(Length::Exact(123_456)) //! .build() @@ -235,26 +235,18 @@ pin_project! { /// ``` /// /// 2. **From a file**: ByteStreams created from a path can be retried. A new file descriptor will be opened if a retry occurs. - /// - /// _Note: The `http-body-0-4-x` feature must be active to call `ByteStream::from_body_0_4`._ - /// /// ```no_run - /// #[cfg(all(feature = "tokio-rt", feature = "http-body-0-4-x"))] + /// #[cfg(feature = "tokio-rt")] /// # { /// use aws_smithy_types::byte_stream::ByteStream; - /// let stream = ByteStream::from_path_body_0_4("big_file.csv"); + /// let stream = ByteStream::from_path("big_file.csv"); /// # } /// ``` /// /// 3. **From an `SdkBody` directly**: For more advanced / custom use cases, a ByteStream can be created directly /// from an SdkBody. **When created from an SdkBody, care must be taken to ensure retriability.** An SdkBody is retryable /// when constructed from in-memory data or when using [`SdkBody::retryable`](crate::body::SdkBody::retryable). - /// - /// _Note: The `http-body-0-4-x` feature must be active to construct an `SdkBody` with `from_body_0_4`._ - /// /// ```no_run - /// # #[cfg(feature = "http-body-0-4-x")] - /// # { /// # use hyper_0_14 as hyper; /// use aws_smithy_types::byte_stream::ByteStream; /// use aws_smithy_types::body::SdkBody; @@ -265,7 +257,6 @@ pin_project! { /// tx.send_data(Bytes::from_static(b"hello world!")); /// tx.send_data(Bytes::from_static(b"hello again!")); /// // NOTE! You must ensure that `tx` is dropped to ensure that EOF is sent - /// # } /// ``` /// #[derive(Debug)] @@ -353,6 +344,87 @@ impl ByteStream { self.inner.collect().await.map_err(Error::streaming) } + /// Returns a [`FsBuilder`](crate::byte_stream::FsBuilder), allowing you to build a `ByteStream` with + /// full control over how the file is read (eg. specifying the length of the file or the size of the buffer used to read the file). + /// ```no_run + /// # #[cfg(feature = "rt-tokio")] + /// # { + /// use aws_smithy_types::byte_stream::{ByteStream, Length}; + /// + /// async fn bytestream_from_file() -> ByteStream { + /// let bytestream = ByteStream::read_from() + /// .path("docs/some-large-file.csv") + /// // Specify the size of the buffer used to read the file (in bytes, default is 4096) + /// .buffer_size(32_784) + /// // Specify the length of the file used (skips an additional call to retrieve the size) + /// .length(Length::Exact(123_456)) + /// .build() + /// .await + /// .expect("valid path"); + /// bytestream + /// } + /// # } + /// ``` + #[cfg(feature = "rt-tokio")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] + pub fn read_from() -> crate::byte_stream::FsBuilder { + crate::byte_stream::FsBuilder::new() + } + + /// Create a ByteStream that streams data from the filesystem + /// + /// This function creates a retryable ByteStream for a given `path`. The returned ByteStream + /// will provide a size hint when used as an HTTP body. If the request fails, the read will + /// begin again by reloading the file handle. + /// + /// ## Warning + /// The contents of the file MUST not change during retries. The length & checksum of the file + /// will be cached. If the contents of the file change, the operation will almost certainly fail. + /// + /// Furthermore, a partial write MAY seek in the file and resume from the previous location. + /// + /// Note: If you want more control, such as specifying the size of the buffer used to read the file + /// or the length of the file, use a [`FsBuilder`](crate::byte_stream::FsBuilder) as returned + /// from `ByteStream::read_from` + /// + /// # Examples + /// ```no_run + /// use aws_smithy_types::byte_stream::ByteStream; + /// use std::path::Path; + /// async fn make_bytestream() -> ByteStream { + /// ByteStream::from_path("docs/rows.csv").await.expect("file should be readable") + /// } + /// ``` + #[cfg(feature = "rt-tokio")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] + pub async fn from_path( + path: impl AsRef, + ) -> Result { + crate::byte_stream::FsBuilder::new() + .path(path) + .build() + .await + } + + /// Create a ByteStream from a file + /// + /// NOTE: This will NOT result in a retryable ByteStream. For a ByteStream that can be retried in the case of + /// upstream failures, use [`ByteStream::from_path`](ByteStream::from_path) + #[deprecated( + since = "0.40.0", + note = "Prefer the more extensible ByteStream::read_from() API" + )] + #[cfg(feature = "rt-tokio")] + #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] + pub async fn from_file( + file: tokio::fs::File, + ) -> Result { + crate::byte_stream::FsBuilder::new() + .file(file) + .build() + .await + } + #[cfg(feature = "rt-tokio")] /// Convert this `ByteStream` into a struct that implements [`AsyncRead`](tokio::io::AsyncRead). /// diff --git a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs index 467733b002..1dfe87b1e3 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs @@ -8,7 +8,6 @@ use crate::byte_stream::{error::Error, error::ErrorKind, ByteStream}; use std::future::Future; use std::path::PathBuf; use std::pin::Pin; -use std::sync::Arc; use tokio::fs::File; use tokio::io::{self, AsyncReadExt, AsyncSeekExt}; use tokio_util::io::ReaderStream; @@ -56,11 +55,8 @@ impl PathBody { /// Builder for creating [`ByteStreams`](ByteStream) from a file/path, with full control over advanced options. /// -/// _Note: A cargo feature `http-body-0-4-x` should be active to call `ByteStream::read_with_body_0_4_from` in the following example._ -/// -/// Example usage: /// ```no_run -/// # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] +/// # #[cfg(feature = "rt-tokio")] /// # { /// use aws_smithy_types::byte_stream::{ByteStream, Length}; /// use std::path::Path; @@ -69,7 +65,7 @@ impl PathBody { /// } /// /// async fn bytestream_from_file() -> GetObjectInput { -/// let bytestream = ByteStream::read_with_body_0_4_from() +/// let bytestream = ByteStream::read_from() /// .path("docs/some-large-file.csv") /// // Specify the size of the buffer used to read the file (in bytes, default is 4096) /// .buffer_size(32_784) @@ -89,12 +85,6 @@ pub struct FsBuilder { length: Option, buffer_size: usize, offset: Option, - sdk_body_creator: SdkBodyCreator, -} - -enum SdkBodyCreator { - #[cfg(feature = "http-body-0-4-x")] - HttpBody04(Arc SdkBody + Send + Sync + 'static>), } /// The length (in bytes) to read. Determines whether or not a short read counts as an error. @@ -108,22 +98,16 @@ pub enum Length { } impl FsBuilder { - #[cfg(feature = "http-body-0-4-x")] /// Create a new [`FsBuilder`] (using a default read buffer of 4096 bytes). /// /// You must then call either [`file`](FsBuilder::file) or [`path`](FsBuilder::path) to specify what to read from. - /// - /// _Note: This is only available with `http-body-0-4-x` enabled._ - pub fn new_with_body_0_4() -> Self { + pub fn new() -> Self { Self { buffer_size: DEFAULT_BUFFER_SIZE, file: None, length: None, offset: None, path: None, - sdk_body_creator: SdkBodyCreator::HttpBody04(Arc::new(|body: PathBody| { - SdkBody::from_body_0_4(body) - })), } } @@ -203,19 +187,12 @@ impl FsBuilder { let body_loader = move || { // If an offset was provided, seeking will be handled in `PathBody::poll_data` each // time the file is loaded. - match &self.sdk_body_creator { - #[cfg(feature = "http-body-0-4-x")] - SdkBodyCreator::HttpBody04(f) => f(PathBody::from_path( - path.clone(), - length, - buffer_size, - self.offset, - )), - #[allow(unreachable_patterns)] - _ => unreachable!( - "`http-body-0-4-x` should've been enabled to create an `FsBuilder`" - ), - } + SdkBody::from_body_0_4(PathBody::from_path( + path.clone(), + length, + buffer_size, + self.offset, + )) }; Ok(ByteStream::new(SdkBody::retryable(body_loader))) @@ -225,14 +202,7 @@ impl FsBuilder { let _s = file.seek(io::SeekFrom::Start(offset)).await?; } - let body = match self.sdk_body_creator { - #[cfg(feature = "http-body-0-4-x")] - SdkBodyCreator::HttpBody04(f) => f(PathBody::from_file(file, length, buffer_size)), - #[allow(unreachable_patterns)] - _ => unreachable!( - "`http-body-0-4-x` should've been enabled to create an `FsBuilder`" - ), - }; + let body = SdkBody::from_body_0_4(PathBody::from_file(file, length, buffer_size)); Ok(ByteStream::new(body)) } else { @@ -256,7 +226,6 @@ enum State { Loaded(ReaderStream>), } -#[cfg(feature = "http-body-0-4-x")] impl http_body_0_4::Body for PathBody { type Data = bytes::Bytes; type Error = Box; @@ -343,7 +312,7 @@ mod test { .expect("file metadata is accessible") .len(); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) .buffer_size(16384) .length(Length::Exact(file_length)) @@ -382,7 +351,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // The file is longer than 1 byte, let's see if this is used to generate the size hint .length(Length::Exact(1)) @@ -406,7 +375,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // We're going to read line 0 only .length(Length::Exact(line_0.len() as u64)) @@ -430,7 +399,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - assert!(FsBuilder::new_with_body_0_4() + assert!(FsBuilder::new() .path(&file) // The file is 30 bytes so this is fine .length(Length::Exact(29)) @@ -438,7 +407,7 @@ mod test { .await .is_ok()); - assert!(FsBuilder::new_with_body_0_4() + assert!(FsBuilder::new() .path(&file) // The file is 30 bytes so this is fine .length(Length::Exact(30)) @@ -446,7 +415,7 @@ mod test { .await .is_ok()); - assert!(FsBuilder::new_with_body_0_4() + assert!(FsBuilder::new() .path(&file) // Larger than 30 bytes, this will cause an error .length(Length::Exact(31)) @@ -467,7 +436,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // We're going to skip the first line by using offset .offset(line_0.len() as u64) @@ -495,7 +464,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) // We're going to skip line 0 by using offset .offset(line_0.len() as u64) @@ -524,7 +493,7 @@ mod test { file.flush().expect("flushing is OK"); assert_eq!( - FsBuilder::new_with_body_0_4() + FsBuilder::new() .path(&file) // We're going to skip all file contents by setting an offset // much larger than the file size @@ -549,7 +518,7 @@ mod test { // Ensure that the file was written to file.flush().expect("flushing is OK"); - let body = FsBuilder::new_with_body_0_4() + let body = FsBuilder::new() .path(&file) .length(Length::UpTo(9000)) .build() @@ -601,7 +570,7 @@ mod test { chunk_size }; - let byte_stream = FsBuilder::new_with_body_0_4() + let byte_stream = FsBuilder::new() .path(&file_path) .offset(i * chunk_size) .length(Length::Exact(length)) diff --git a/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs b/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs index 5edbaaba87..8bdae026f6 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream/http_body_0_4_x.rs @@ -18,87 +18,6 @@ impl ByteStream { { ByteStream::new(SdkBody::from_body_0_4(body)) } - - /// Returns a [`FsBuilder`](crate::byte_stream::FsBuilder), allowing you to build a `ByteStream` with - /// full control over how the file is read (eg. specifying the length of the file or the size of the buffer used to read the file). - /// ```no_run - /// # #[cfg(all(feature = "rt-tokio", feature = "http-body-0-4-x"))] - /// # { - /// use aws_smithy_types::byte_stream::{ByteStream, Length}; - /// - /// async fn bytestream_from_file() -> ByteStream { - /// let bytestream = ByteStream::read_with_body_0_4_from() - /// .path("docs/some-large-file.csv") - /// // Specify the size of the buffer used to read the file (in bytes, default is 4096) - /// .buffer_size(32_784) - /// // Specify the length of the file used (skips an additional call to retrieve the size) - /// .length(Length::Exact(123_456)) - /// .build() - /// .await - /// .expect("valid path"); - /// bytestream - /// } - /// # } - /// ``` - #[cfg(feature = "rt-tokio")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] - pub fn read_with_body_0_4_from() -> crate::byte_stream::FsBuilder { - crate::byte_stream::FsBuilder::new_with_body_0_4() - } - - /// Create a ByteStream that streams data from the filesystem - /// - /// This function creates a retryable ByteStream for a given `path`. The returned ByteStream - /// will provide a size hint when used as an HTTP body. If the request fails, the read will - /// begin again by reloading the file handle. - /// - /// ## Warning - /// The contents of the file MUST not change during retries. The length & checksum of the file - /// will be cached. If the contents of the file change, the operation will almost certainly fail. - /// - /// Furthermore, a partial write MAY seek in the file and resume from the previous location. - /// - /// Note: If you want more control, such as specifying the size of the buffer used to read the file - /// or the length of the file, use a [`FsBuilder`](crate::byte_stream::FsBuilder) as returned - /// from `ByteStream::read_with_body_0_4_from` - /// - /// # Examples - /// ```no_run - /// use aws_smithy_types::byte_stream::ByteStream; - /// use std::path::Path; - /// async fn make_bytestream() -> ByteStream { - /// ByteStream::from_path_body_0_4("docs/rows.csv").await.expect("file should be readable") - /// } - /// ``` - #[cfg(feature = "rt-tokio")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] - pub async fn from_path_body_0_4( - path: impl AsRef, - ) -> Result { - crate::byte_stream::FsBuilder::new_with_body_0_4() - .path(path) - .build() - .await - } - - /// Create a ByteStream from a file - /// - /// NOTE: This will NOT result in a retryable ByteStream. For a ByteStream that can be retried in the case of - /// upstream failures, use [`ByteStream::from_path_body_0_4`](ByteStream::from_path_body_0_4) - #[deprecated( - since = "0.40.0", - note = "Prefer the more extensible ByteStream::read_from() API" - )] - #[cfg(feature = "rt-tokio")] - #[cfg_attr(docsrs, doc(cfg(feature = "rt-tokio")))] - pub async fn from_file_body_0_4( - file: tokio::fs::File, - ) -> Result { - crate::byte_stream::FsBuilder::new_with_body_0_4() - .file(file) - .build() - .await - } } #[cfg(feature = "hyper-0-14-x")] @@ -141,7 +60,7 @@ mod tests { for i in 0..10000 { writeln!(file, "Brian was here. Briefly. {}", i)?; } - let body = ByteStream::from_path_body_0_4(&file).await?.into_inner(); + let body = ByteStream::from_path(&file).await?.into_inner(); // assert that a valid size hint is immediately ready assert_eq!(body.content_length(), Some(298890)); let mut body1 = body.try_clone().expect("retryable bodies are cloneable"); From be9f8b2c810175c7212be10d714bfad14c20d45b Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Oct 2023 15:35:53 -0500 Subject: [PATCH 5/6] Update CHANGELOG.next.toml --- CHANGELOG.next.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 7f9f7ff212..29a71b5528 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -489,7 +489,7 @@ author = "rcoh" [[smithy-rs]] message = "Publicly exposed types from `http-body` and `hyper` crates within `aws-smithy-types` are now feature-gated. See the [upgrade guidance](https://github.com/awslabs/smithy-rs/discussions/3089) for details." -references = ["smithy-rs#3033", "smithy-rs#3088"] +references = ["smithy-rs#3033", "smithy-rs#3088", "smithy-rs#3101"] meta = { "breaking" = true, "tada" = false, "bug" = false, "target" = "all" } author = "ysaito1001" From 66dc4b01ecfdf36a0b39ef027986e57a0a3f0756 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Oct 2023 15:43:28 -0500 Subject: [PATCH 6/6] Fix clippy warning `new_without_default` --- .../aws-smithy-types/src/byte_stream/bytestream_util.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs index 1dfe87b1e3..39965c90c5 100644 --- a/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs +++ b/rust-runtime/aws-smithy-types/src/byte_stream/bytestream_util.rs @@ -87,6 +87,12 @@ pub struct FsBuilder { offset: Option, } +impl Default for FsBuilder { + fn default() -> Self { + Self::new() + } +} + /// The length (in bytes) to read. Determines whether or not a short read counts as an error. #[allow(missing_debug_implementations)] pub enum Length {