From ee431f47e248fbfd7d316ad04878cf9eb4f99ee5 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Thu, 30 Jun 2022 16:34:54 +0200 Subject: [PATCH 1/6] Implement S3 backend storage --- Cargo.lock | 231 +++++++++++++++++++++- wally-registry-backend/Cargo.toml | 2 + wally-registry-backend/src/main.rs | 25 ++- wally-registry-backend/src/storage/mod.rs | 3 + wally-registry-backend/src/storage/s3.rs | 61 ++++++ 5 files changed, 314 insertions(+), 8 deletions(-) create mode 100644 wally-registry-backend/src/storage/s3.rs diff --git a/Cargo.lock b/Cargo.lock index e6b22677..25225998 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,10 +159,19 @@ dependencies = [ "cc", "cfg-if 0.1.10", "constant_time_eq", - "crypto-mac", + "crypto-mac 0.8.0", "digest", ] +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "0.2.15" @@ -371,6 +380,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.2.1" @@ -489,6 +507,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "darling" version = "0.12.2" @@ -584,6 +612,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.5" @@ -591,7 +629,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" dependencies = [ "libc", - "redox_users", + "redox_users 0.3.5", + "winapi", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users 0.4.3", "winapi", ] @@ -979,6 +1028,16 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac 0.11.1", + "digest", +] + [[package]] name = "htmlescape" version = "0.3.1" @@ -1316,6 +1375,17 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer", + "digest", + "opaque-debug", +] + [[package]] name = "measure_time" version = "0.7.0" @@ -1487,6 +1557,12 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "opener" version = "0.5.0" @@ -1559,7 +1635,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.5", + "redox_syscall 0.2.13", "smallvec", "winapi", ] @@ -1838,9 +1914,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] @@ -1856,6 +1932,17 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.2", + "redox_syscall 0.2.13", + "thiserror", +] + [[package]] name = "ref-cast" version = "1.0.6" @@ -2063,6 +2150,88 @@ dependencies = [ "winapi", ] +[[package]] +name = "rusoto_core" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" +dependencies = [ + "async-trait", + "base64 0.13.0", + "bytes", + "crc32fast", + "futures", + "http", + "hyper", + "hyper-tls", + "lazy_static", + "log", + "rusoto_credential", + "rusoto_signature", + "rustc_version 0.4.0", + "serde", + "serde_json", + "tokio", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" +dependencies = [ + "async-trait", + "chrono", + "dirs-next", + "futures", + "hyper", + "serde", + "serde_json", + "shlex", + "tokio", + "zeroize", +] + +[[package]] +name = "rusoto_s3" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d" +dependencies = [ + "async-trait", + "bytes", + "futures", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" +dependencies = [ + "base64 0.13.0", + "bytes", + "chrono", + "digest", + "futures", + "hex", + "hmac", + "http", + "hyper", + "log", + "md-5", + "percent-encoding", + "pin-project-lite", + "rusoto_credential", + "rustc_version 0.4.0", + "serde", + "sha2", + "tokio", +] + [[package]] name = "rust-argon2" version = "0.8.3" @@ -2094,6 +2263,15 @@ dependencies = [ "semver 0.9.0", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.11", +] + [[package]] name = "rustls" version = "0.19.0" @@ -2202,6 +2380,12 @@ dependencies = [ "serde", ] +[[package]] +name = "semver" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d92beeab217753479be2f74e54187a6aed4c125ff0703a866c3147a02f0c6dd" + [[package]] name = "semver-parser" version = "0.7.0" @@ -2302,6 +2486,25 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "signal-hook" version = "0.3.10" @@ -2422,7 +2625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -2610,7 +2813,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall 0.2.13", "remove_dir_all", "winapi", ] @@ -3082,6 +3285,8 @@ dependencies = [ "reqwest", "rocket", "rocket_contrib", + "rusoto_core", + "rusoto_s3", "semver 0.11.0", "serde", "serde_json", @@ -3263,6 +3468,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + [[package]] name = "yaml-rust" version = "0.4.5" @@ -3278,6 +3489,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" +[[package]] +name = "zeroize" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20b578acffd8516a6c3f2a1bdefc1ec37e547bb4e0fb8b6b01a4cafc886b4442" + [[package]] name = "zip" version = "0.5.11" diff --git a/wally-registry-backend/Cargo.toml b/wally-registry-backend/Cargo.toml index e97ffe5d..195893f4 100644 --- a/wally-registry-backend/Cargo.toml +++ b/wally-registry-backend/Cargo.toml @@ -23,6 +23,8 @@ git2 = "0.13.17" reqwest = { version = "0.11.0", features = ["blocking", "json"] } rocket = { git = "https://github.com/SergioBenitez/Rocket", rev = "2893ce754d6535e0a752586e60d7e292343016c0" } rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket", rev = "2893ce754d6535e0a752586e60d7e292343016c0" } +rusoto_core = "0.48.0" +rusoto_s3 = "0.48.0" semver = "0.11.0" serde = { version = "1.0.120", features = ["derive"] } serde_json = "1.0.61" diff --git a/wally-registry-backend/src/main.rs b/wally-registry-backend/src/main.rs index fc77cfd6..edfeffb5 100644 --- a/wally-registry-backend/src/main.rs +++ b/wally-registry-backend/src/main.rs @@ -13,6 +13,7 @@ mod tests; use std::convert::TryInto; use std::io::{Cursor, Read, Seek}; use std::sync::RwLock; +use std::env; use anyhow::{format_err, Context}; use figment::{ @@ -46,7 +47,7 @@ use crate::auth::{ReadAccess, WriteAccess}; use crate::config::Config; use crate::error::{ApiErrorContext, ApiErrorStatus, Error}; use crate::search::SearchBackend; -use crate::storage::{GcsStorage, LocalStorage, StorageBackend, StorageOutput}; +use crate::storage::{S3Storage, GcsStorage, LocalStorage, StorageBackend, StorageOutput}; #[get("/")] fn root() -> Json { @@ -228,6 +229,7 @@ pub fn server(figment: Figment) -> rocket::Rocket { let storage_backend: Box = match config.storage { StorageMode::Local { path } => Box::new(LocalStorage::new(path)), StorageMode::Gcs { bucket } => Box::new(configure_gcs(bucket).unwrap()), + StorageMode::S3 { bucket } => Box::new(configure_s3(bucket).unwrap()), }; println!("Cloning package index repository..."); @@ -272,6 +274,27 @@ fn configure_gcs(bucket: String) -> anyhow::Result { Ok(GcsStorage::new(client)) } +fn configure_s3(bucket: String) -> anyhow::Result { + use rusoto_core::{ + request::HttpClient, + credential::ChainProvider, + Region + }; + + use rusoto_s3::S3Client; + + let client = S3Client::new_with( + HttpClient::new()?, + ChainProvider::new(), + Region::Custom { + name: env::var("AWS_REGION_NAME").unwrap_or("us-east-1".to_string()).to_owned(), + endpoint: env::var("AWS_REGION_ENDPOINT")?.to_owned(), + } + ); + + Ok(S3Storage::new(client, bucket)) +} + struct Cors; #[rocket::async_trait] diff --git a/wally-registry-backend/src/storage/mod.rs b/wally-registry-backend/src/storage/mod.rs index 589375c3..28cf44fe 100644 --- a/wally-registry-backend/src/storage/mod.rs +++ b/wally-registry-backend/src/storage/mod.rs @@ -1,5 +1,6 @@ mod gcs; mod local; +mod s3; use std::path::PathBuf; @@ -10,12 +11,14 @@ use tokio::io::AsyncRead; pub use gcs::GcsStorage; pub use local::LocalStorage; +pub use s3::S3Storage; #[derive(Debug, Deserialize, Serialize)] #[serde(tag = "type", rename_all = "kebab-case")] pub enum StorageMode { Local { path: Option }, Gcs { bucket: String }, + S3 { bucket: String }, } pub type StorageOutput = Box; diff --git a/wally-registry-backend/src/storage/s3.rs b/wally-registry-backend/src/storage/s3.rs new file mode 100644 index 00000000..66133590 --- /dev/null +++ b/wally-registry-backend/src/storage/s3.rs @@ -0,0 +1,61 @@ +use std::io::Cursor; + +use async_trait::async_trait; +use futures::TryStreamExt; +use libwally::package_id::PackageId; +use tokio::sync::Mutex; + +use rusoto_s3::{GetObjectRequest, PutObjectRequest, S3Client, S3}; + +use super::{StorageBackend, StorageOutput}; + +pub struct S3Storage { + client: Mutex, + bucket: String, +} + +impl S3Storage { + pub fn new(client: S3Client, bucket: String) -> Self { + Self { + client: Mutex::new(client), + bucket, + } + } +} + +#[async_trait] +impl StorageBackend for S3Storage { + async fn read(&self, key: &PackageId) -> anyhow::Result { + let name = key.to_string(); + let client = self.client.lock().await; + + let result = client + .get_object(GetObjectRequest { + bucket: self.bucket.to_owned(), + key: name.to_owned(), + ..Default::default() + }) + .await?; + + let stream = result.body.unwrap(); + let data = stream.map_ok(|chunk| chunk.to_vec()).try_concat().await?; + Ok(Box::new(Cursor::new(data))) + } + + async fn write(&self, id: &PackageId, contents: &[u8]) -> anyhow::Result<()> { + let name = id.to_string(); + let client = self.client.lock().await; + let contents = contents.to_vec(); + + client + .put_object(PutObjectRequest { + bucket: self.bucket.to_owned(), + key: name.to_owned(), + body: Some(contents.into()), + ..Default::default() + }) + .await?; + + Ok(()) + } +} From a8e33ab65b82172e84d5e74fb10d38e7f5feef77 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Mon, 6 Feb 2023 21:12:10 -0500 Subject: [PATCH 2/6] Gate s3 storage support for backend behind feature flag --- wally-registry-backend/Cargo.toml | 8 ++++++-- wally-registry-backend/src/main.rs | 10 ++++++++-- wally-registry-backend/src/storage/mod.rs | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/wally-registry-backend/Cargo.toml b/wally-registry-backend/Cargo.toml index 195893f4..7ab79fb9 100644 --- a/wally-registry-backend/Cargo.toml +++ b/wally-registry-backend/Cargo.toml @@ -7,6 +7,10 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = ["s3-storage"] +s3-storage = ["dep:rusoto_core", "dep:rusoto_s3"] + [dependencies] wally = { path = ".." } @@ -23,8 +27,8 @@ git2 = "0.13.17" reqwest = { version = "0.11.0", features = ["blocking", "json"] } rocket = { git = "https://github.com/SergioBenitez/Rocket", rev = "2893ce754d6535e0a752586e60d7e292343016c0" } rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket", rev = "2893ce754d6535e0a752586e60d7e292343016c0" } -rusoto_core = "0.48.0" -rusoto_s3 = "0.48.0" +rusoto_core = { version = "0.48.0", optional = true } +rusoto_s3 = { version = "0.48.0", optional = true } semver = "0.11.0" serde = { version = "1.0.120", features = ["derive"] } serde_json = "1.0.61" diff --git a/wally-registry-backend/src/main.rs b/wally-registry-backend/src/main.rs index edfeffb5..031e9ef9 100644 --- a/wally-registry-backend/src/main.rs +++ b/wally-registry-backend/src/main.rs @@ -13,7 +13,6 @@ mod tests; use std::convert::TryInto; use std::io::{Cursor, Read, Seek}; use std::sync::RwLock; -use std::env; use anyhow::{format_err, Context}; use figment::{ @@ -47,7 +46,10 @@ use crate::auth::{ReadAccess, WriteAccess}; use crate::config::Config; use crate::error::{ApiErrorContext, ApiErrorStatus, Error}; use crate::search::SearchBackend; -use crate::storage::{S3Storage, GcsStorage, LocalStorage, StorageBackend, StorageOutput}; +use crate::storage::{GcsStorage, LocalStorage, StorageBackend, StorageOutput}; + +#[cfg(feature = "s3-storage")] +use crate::storage::S3Storage; #[get("/")] fn root() -> Json { @@ -229,6 +231,7 @@ pub fn server(figment: Figment) -> rocket::Rocket { let storage_backend: Box = match config.storage { StorageMode::Local { path } => Box::new(LocalStorage::new(path)), StorageMode::Gcs { bucket } => Box::new(configure_gcs(bucket).unwrap()), + #[cfg(feature = "s3-storage")] StorageMode::S3 { bucket } => Box::new(configure_s3(bucket).unwrap()), }; @@ -274,7 +277,10 @@ fn configure_gcs(bucket: String) -> anyhow::Result { Ok(GcsStorage::new(client)) } +#[cfg(feature = "s3-storage")] fn configure_s3(bucket: String) -> anyhow::Result { + use std::env; + use rusoto_core::{ request::HttpClient, credential::ChainProvider, diff --git a/wally-registry-backend/src/storage/mod.rs b/wally-registry-backend/src/storage/mod.rs index 28cf44fe..6abe979b 100644 --- a/wally-registry-backend/src/storage/mod.rs +++ b/wally-registry-backend/src/storage/mod.rs @@ -18,6 +18,7 @@ pub use s3::S3Storage; pub enum StorageMode { Local { path: Option }, Gcs { bucket: String }, + #[cfg(feature = "s3-storage")] S3 { bucket: String }, } From b6bdb6e8a5cf294a5ff7a7cf4c371b9f8f4f0fc8 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Mon, 6 Feb 2023 21:20:13 -0500 Subject: [PATCH 3/6] Fix lints, formatting --- wally-registry-backend/src/main.rs | 15 ++++++--------- wally-registry-backend/src/storage/mod.rs | 12 +++++++++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/wally-registry-backend/src/main.rs b/wally-registry-backend/src/main.rs index 031e9ef9..3c9f89a7 100644 --- a/wally-registry-backend/src/main.rs +++ b/wally-registry-backend/src/main.rs @@ -280,12 +280,8 @@ fn configure_gcs(bucket: String) -> anyhow::Result { #[cfg(feature = "s3-storage")] fn configure_s3(bucket: String) -> anyhow::Result { use std::env; - - use rusoto_core::{ - request::HttpClient, - credential::ChainProvider, - Region - }; + + use rusoto_core::{credential::ChainProvider, request::HttpClient, Region}; use rusoto_s3::S3Client; @@ -293,9 +289,10 @@ fn configure_s3(bucket: String) -> anyhow::Result { HttpClient::new()?, ChainProvider::new(), Region::Custom { - name: env::var("AWS_REGION_NAME").unwrap_or("us-east-1".to_string()).to_owned(), - endpoint: env::var("AWS_REGION_ENDPOINT")?.to_owned(), - } + name: env::var("AWS_REGION_NAME") + .unwrap_or_else(|_| "us-east-1".to_string()), + endpoint: env::var("AWS_REGION_ENDPOINT")?, + }, ); Ok(S3Storage::new(client, bucket)) diff --git a/wally-registry-backend/src/storage/mod.rs b/wally-registry-backend/src/storage/mod.rs index 6abe979b..1d9fac1e 100644 --- a/wally-registry-backend/src/storage/mod.rs +++ b/wally-registry-backend/src/storage/mod.rs @@ -16,10 +16,16 @@ pub use s3::S3Storage; #[derive(Debug, Deserialize, Serialize)] #[serde(tag = "type", rename_all = "kebab-case")] pub enum StorageMode { - Local { path: Option }, - Gcs { bucket: String }, + Local { + path: Option, + }, + Gcs { + bucket: String, + }, #[cfg(feature = "s3-storage")] - S3 { bucket: String }, + S3 { + bucket: String, + }, } pub type StorageOutput = Box; From 09e19b273efbbd69d37f3e26b24e26e3c900681e Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Mon, 6 Feb 2023 21:43:48 -0500 Subject: [PATCH 4/6] Fix another formatting error --- wally-registry-backend/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wally-registry-backend/src/main.rs b/wally-registry-backend/src/main.rs index 3c9f89a7..0197b4ac 100644 --- a/wally-registry-backend/src/main.rs +++ b/wally-registry-backend/src/main.rs @@ -289,8 +289,7 @@ fn configure_s3(bucket: String) -> anyhow::Result { HttpClient::new()?, ChainProvider::new(), Region::Custom { - name: env::var("AWS_REGION_NAME") - .unwrap_or_else(|_| "us-east-1".to_string()), + name: env::var("AWS_REGION_NAME").unwrap_or_else(|_| "us-east-1".to_string()), endpoint: env::var("AWS_REGION_ENDPOINT")?, }, ); From 86f3abd5ee96d901771d671b4caa32394fc79d25 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Thu, 9 Feb 2023 12:52:43 +0100 Subject: [PATCH 5/6] Remove default feature for S3 --- wally-registry-backend/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wally-registry-backend/Cargo.toml b/wally-registry-backend/Cargo.toml index 7ab79fb9..461c93f6 100644 --- a/wally-registry-backend/Cargo.toml +++ b/wally-registry-backend/Cargo.toml @@ -8,7 +8,7 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["s3-storage"] +default = [] s3-storage = ["dep:rusoto_core", "dep:rusoto_s3"] [dependencies] @@ -34,7 +34,7 @@ serde = { version = "1.0.120", features = ["derive"] } serde_json = "1.0.61" tantivy = "0.16.1" tokio = "1.1.1" -url = { version = "2.2.1", features = ["serde"] } +url = { version = "2.2.1", features = ["serde"] } walkdir = "2.3.1" zip = "0.5.11" From 89f6205d3c259483e40d1939f7f7fbf25ecd5731 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Thu, 9 Feb 2023 13:00:31 +0100 Subject: [PATCH 6/6] Add missing cfgs for s3 feature --- wally-registry-backend/src/storage/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wally-registry-backend/src/storage/mod.rs b/wally-registry-backend/src/storage/mod.rs index 1d9fac1e..7688c103 100644 --- a/wally-registry-backend/src/storage/mod.rs +++ b/wally-registry-backend/src/storage/mod.rs @@ -1,5 +1,7 @@ mod gcs; mod local; + +#[cfg(feature = "s3-storage")] mod s3; use std::path::PathBuf; @@ -11,6 +13,8 @@ use tokio::io::AsyncRead; pub use gcs::GcsStorage; pub use local::LocalStorage; + +#[cfg(feature = "s3-storage")] pub use s3::S3Storage; #[derive(Debug, Deserialize, Serialize)]