diff --git a/Cargo.lock b/Cargo.lock index 1dc58f1446..9d9abfed4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -566,6 +566,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct 0.6.1", +] + [[package]] name = "ctor" version = "0.1.22" @@ -742,6 +751,9 @@ dependencies = [ "errno", "futures", "glibc_version 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper", + "hyper-proxy", + "hyper-rustls 0.23.0", "lazy_static", "libc", "log", @@ -1196,6 +1208,31 @@ dependencies = [ "ahash", ] +[[package]] +name = "headers" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d" +dependencies = [ + "base64", + "bitflags", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha-1", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] + [[package]] name = "heck" version = "0.3.3" @@ -1312,6 +1349,42 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-proxy" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca815a891b24fdfb243fa3239c86154392b0953ee584aa1a2a1f66d20cbe75cc" +dependencies = [ + "bytes", + "futures", + "headers", + "http", + "hyper", + "hyper-rustls 0.22.1", + "rustls-native-certs 0.5.0", + "tokio", + "tokio-rustls 0.22.0", + "tower-service", + "webpki 0.21.4", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls 0.19.1", + "rustls-native-certs 0.5.0", + "tokio", + "tokio-rustls 0.22.0", + "webpki 0.21.4", +] + [[package]] name = "hyper-rustls" version = "0.23.0" @@ -1321,10 +1394,10 @@ dependencies = [ "http", "hyper", "log", - "rustls", - "rustls-native-certs", + "rustls 0.20.6", + "rustls-native-certs 0.6.2", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -2295,7 +2368,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.0", "hyper-tls", "ipnet", "js-sys", @@ -2305,14 +2378,14 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.20.6", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util", "tower-service", "url", @@ -2351,7 +2424,7 @@ dependencies = [ "futures", "http", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.0", "hyper-tls", "lazy_static", "log", @@ -2473,6 +2546,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct 0.6.1", + "webpki 0.21.4", +] + [[package]] name = "rustls" version = "0.20.6" @@ -2481,8 +2567,20 @@ checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls 0.19.1", + "schannel", + "security-framework", ] [[package]] @@ -2544,6 +2642,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sct" version = "0.7.0" @@ -2673,6 +2781,17 @@ dependencies = [ "syn", ] +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3005,15 +3124,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.1", + "tokio", + "webpki 0.21.4", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.6", "tokio", - "webpki", + "webpki 0.22.0", ] [[package]] @@ -3328,6 +3458,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki" version = "0.22.0" @@ -3344,7 +3484,7 @@ version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" dependencies = [ - "webpki", + "webpki 0.22.0", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index bb2ee166c5..4ce346e6f5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -44,6 +44,9 @@ rusoto_s3 = { version = "0.48", default-features = false, optional = true } rusoto_sts = { version = "0.48", default-features = false, optional = true } rusoto_dynamodb = { version = "0.48", default-features = false, optional = true } maplit = { version = "1", optional = true } +hyper = { version = "0.14.19", default-features = false, optional = true} +hyper-rustls = { version = "0.23.0", default-features = false, optional = true, features = ["http2", "rustls-native-certs", "tokio-runtime"] } +hyper-proxy = { version = "0.9.1", default-features = false, optional = true, features = ["rustls"] } # Glue rusoto_glue = { version = "0.48", default-features = false, optional = true } @@ -91,6 +94,9 @@ s3 = [ "rusoto_dynamodb/native-tls", "maplit", "dynamodb_lock/native-tls", + "hyper", + "hyper-rustls", + "hyper-proxy" ] s3-rustls = [ "rusoto_core/rustls", @@ -100,6 +106,9 @@ s3-rustls = [ "rusoto_dynamodb/rustls", "maplit", "dynamodb_lock/rustls", + "hyper", + "hyper-rustls", + "hyper-proxy" ] gcs = ["async-stream", "tame-gcs", "tame-oauth", "reqwest"] glue = ["s3", "rusoto_glue"] diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs index 9328464fcf..3abc6f410e 100644 --- a/rust/src/storage/mod.rs +++ b/rust/src/storage/mod.rs @@ -1,5 +1,7 @@ //! Object storage backend abstraction layer for Delta Table transaction logs and data +#[cfg(any(feature = "s3", feature = "s3-rustls"))] +use hyper::http::uri::InvalidUri; use std::fmt::Debug; use std::pin::Pin; @@ -408,6 +410,15 @@ pub enum StorageError { /// Uri error details when the URI is invalid. source: UriError, }, + + /// Error returned when the URI is invalid. + #[cfg(any(feature = "s3", feature = "s3-rustls"))] + #[error("Invalid URI parsing")] + ParsingUri { + #[from] + /// Uri error details when the URI parsing is invalid. + source: InvalidUri, + }, } impl StorageError { diff --git a/rust/src/storage/s3/mod.rs b/rust/src/storage/s3/mod.rs index 64e1a3e557..266840a4e6 100644 --- a/rust/src/storage/s3/mod.rs +++ b/rust/src/storage/s3/mod.rs @@ -6,6 +6,7 @@ use std::{fmt, pin::Pin}; use chrono::{DateTime, FixedOffset, Utc}; use futures::Stream; + use log::debug; use rusoto_core::{HttpClient, HttpConfig, Region, RusotoError}; use rusoto_credential::AutoRefreshingProvider; @@ -28,6 +29,12 @@ use uuid::Uuid; use dynamodb_lock::{LockClient, LockItem, DEFAULT_MAX_RETRY_ACQUIRE_LOCK_ATTEMPTS}; +use hyper::client::HttpConnector; +use hyper_proxy::{Intercept, Proxy, ProxyConnector}; +use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; + +use std::env; + /// Lock data which stores an attempt to rename `source` into `destination` #[derive(Clone, Debug, Serialize, Deserialize)] pub struct LockData { @@ -416,10 +423,32 @@ fn get_sts_assume_role_provider( Ok(AutoRefreshingProvider::new(provider)?) } -fn create_http_client(pool_idle_timeout: Duration) -> Result { +fn create_http_client( + pool_idle_timeout: Duration, +) -> Result>>, StorageError> { let mut config = HttpConfig::new(); config.pool_idle_timeout(pool_idle_timeout); - Ok(HttpClient::new_with_config(config)?) + let https_connector = HttpsConnectorBuilder::new() + .with_native_roots() + .https_or_http() + .enable_http2() + .build(); + match env::var("HTTPS_PROXY") { + Ok(proxy_uri) => { + let proxy = Proxy::new(Intercept::All, proxy_uri.parse()?); + let proxy_connector = ProxyConnector::from_proxy(https_connector, proxy)?; + Ok(HttpClient::>>::from_connector_with_config( + proxy_connector, + config, + )) + } + Err(_) => Ok( + HttpClient::>>::from_connector_with_config( + ProxyConnector::new(https_connector)?, + config, + ), + ), + } } fn get_web_identity_provider() -> Result, StorageError>