Upload multiple

Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 cryptohelpers = {version = "^2.0.0", features= ["async", "sha256"]}
+futures = "^0.3"
 hex = "^0.4"
 solana-client = "^1.9"
 solana-transaction-status = "^1.9"
@@ -23,3 +24,4 @@ serde = "^1"
 serde_json = "^1"
 reqwest = {version = "^0.11", features = ["multipart", "stream"]}
 tokio = {version = "1.18.2", features = ["rt", "fs", "io-util"]}
+tracing = "^0.1"

example/Cargo.toml

@@ -14,3 +14,7 @@ solana-sdk = "^1.9"
 solana-client = "^1.9"
 byte-unit = "^4"
 tokio = {version = "1.18.2", features = ["full"]}
+tokio-stream = {version = "^0.1", features = ["fs"]}
+futures = "^0.3"
+tracing = "0.1.34"
+tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}

example/multiple_uploads/0.txt

@@ -0,0 +1 @@
+V

example/multiple_uploads/1.txt

@@ -0,0 +1 @@
+e

example/multiple_uploads/10.txt

@@ -0,0 +1 @@
+O

example/multiple_uploads/11.txt

@@ -0,0 +1 @@
+r

example/multiple_uploads/12.txt

@@ -0,0 +1 @@
+c

example/multiple_uploads/2.txt

@@ -0,0 +1 @@
+g

example/multiple_uploads/3.txt

@@ -0,0 +1 @@
+e

example/multiple_uploads/4.txt

@@ -0,0 +1 @@
+t

example/multiple_uploads/5.txt

@@ -0,0 +1 @@
+a

example/multiple_uploads/6.txt

@@ -0,0 +1 @@
+r

example/multiple_uploads/7.txt

@@ -0,0 +1 @@
+i

example/multiple_uploads/8.txt

@@ -0,0 +1 @@
+a

example/multiple_uploads/9.txt

@@ -0,0 +1 @@
+n

example/src/bin/cancel_delete_file.rs

@@ -26,7 +26,7 @@ async fn main() {
 
     //cancel delete file
     let cancel_delete_file_response = shdw_drive_client
-        .cancel_delete_file(storage_account_key, url)
+        .cancel_delete_file(&storage_account_key, url)
         .await
         .expect("failed to cancel file deletion");
 

example/src/bin/cancel_delete_storage_account.rs

@@ -17,7 +17,7 @@ async fn main() {
     let shdw_drive_client = Client::new(keypair, solana_rpc);
 
     let response = shdw_drive_client
-        .cancel_delete_storage_account(storage_account_key)
+        .cancel_delete_storage_account(&storage_account_key)
         .await
         .expect("failed to cancel storage account deletion");
 

example/src/bin/delete_file.rs

@@ -26,7 +26,7 @@ async fn main() {
 
     //delete file
     let delete_file_response = shdw_drive_client
-        .delete_file(storage_account_key, url)
+        .delete_file(&storage_account_key, url)
         .await
         .expect("failed to request storage account deletion");
 

example/src/bin/delete_storage_account.rs

@@ -17,7 +17,7 @@ async fn main() {
     let shdw_drive_client = Client::new(keypair, solana_rpc);
 
     let response = shdw_drive_client
-        .delete_storage_account(storage_account_key)
+        .delete_storage_account(&storage_account_key)
         .await
         .expect("failed to request storage account deletion");
 

example/src/bin/main.rs

@@ -30,12 +30,12 @@ async fn main() {
     );
     let shdw_drive_client = Client::new(keypair, solana_rpc);
 
-    list_objects_test(shdw_drive_client, storage_account_key).await;
+    list_objects_test(shdw_drive_client, &storage_account_key).await;
 }
 
 async fn list_objects_test<T: Signer + Send + Sync>(
     shdw_drive_client: Client<T>,
-    storage_account_key: Pubkey,
+    storage_account_key: &Pubkey,
 ) {
     let objects = shdw_drive_client
         .list_objects(storage_account_key)
@@ -47,7 +47,7 @@ async fn list_objects_test<T: Signer + Send + Sync>(
 
 async fn make_storage_immutable_test<T: Signer + Send + Sync>(
     shdw_drive_client: Client<T>,
-    storage_account_key: Pubkey,
+    storage_account_key: &Pubkey,
 ) {
     let storage_account = shdw_drive_client
         .get_storage_account(storage_account_key)
@@ -59,14 +59,14 @@ async fn make_storage_immutable_test<T: Signer + Send + Sync>(
     );
 
     let make_immutable_response = shdw_drive_client
-        .make_storage_immutable(storage_account_key)
+        .make_storage_immutable(&storage_account_key)
         .await
         .expect("failed to make storage immutable");
 
     println!("txn id: {:?}", make_immutable_response.txid);
 
     let storage_account = shdw_drive_client
-        .get_storage_account(storage_account_key)
+        .get_storage_account(&storage_account_key)
         .await
         .expect("failed to get storage account");
     println!(
@@ -77,10 +77,10 @@ async fn make_storage_immutable_test<T: Signer + Send + Sync>(
 
 async fn add_storage_test<T: Signer + Send + Sync>(
     shdw_drive_client: Client<T>,
-    storage_account_key: Pubkey,
+    storage_account_key: &Pubkey,
 ) {
     let storage_account = shdw_drive_client
-        .get_storage_account(storage_account_key)
+        .get_storage_account(&storage_account_key)
         .await
         .expect("failed to get storage account");
 
@@ -95,7 +95,7 @@ async fn add_storage_test<T: Signer + Send + Sync>(
     println!("txn id: {:?}", add_storage_response.txid);
 
     let storage_account = shdw_drive_client
-        .get_storage_account(storage_account_key)
+        .get_storage_account(&storage_account_key)
         .await
         .expect("failed to get storage account");
 
@@ -104,7 +104,7 @@ async fn add_storage_test<T: Signer + Send + Sync>(
 
 async fn reduce_storage_test<T: Signer + Send + Sync>(
     shdw_drive_client: Client<T>,
-    storage_account_key: Pubkey,
+    storage_account_key: &Pubkey,
 ) {
     let storage_account = shdw_drive_client
         .get_storage_account(storage_account_key)
@@ -133,7 +133,7 @@ async fn reduce_storage_test<T: Signer + Send + Sync>(
 
 async fn upload_file_test<T: Signer + Send + Sync>(
     shdw_drive_client: Client<T>,
-    storage_account_key: Pubkey,
+    storage_account_key: &Pubkey,
 ) {
     let file = tokio::fs::File::open("example.png")
         .await
@@ -143,7 +143,7 @@ async fn upload_file_test<T: Signer + Send + Sync>(
         .upload_file(
             storage_account_key,
             ShdwFile {
-                name: Some(String::from("example.png")),
+                name: String::from("example.png"),
                 file,
             },
         )

example/src/bin/upload_multiple_files.rs

@@ -0,0 +1,72 @@
+use byte_unit::{Byte, ByteUnit};
+use futures::{future::join_all, TryStreamExt};
+use shadow_drive_rust::{models::ShdwFile, Client};
+use solana_client::rpc_client::RpcClient;
+use solana_sdk::{
+    pubkey::Pubkey,
+    signer::{keypair::read_keypair_file, Signer},
+};
+use std::str::FromStr;
+use tokio::fs::File;
+use tokio_stream::StreamExt;
+use tracing::Level;
+
+const KEYPAIR_PATH: &str = "keypair.json";
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::fmt()
+        .with_env_filter("off,shadow_drive_rust=debug")
+        .init();
+
+    //load keypair from file
+    let keypair = read_keypair_file(KEYPAIR_PATH).expect("failed to load keypair at path");
+    let pubkey = keypair.pubkey();
+    let (storage_account_key, _) =
+        shadow_drive_rust::derived_addresses::storage_account(&pubkey, 4);
+
+    //create shdw drive client
+    let solana_rpc = RpcClient::new("https://ssc-dao.genesysgo.net");
+    let shdw_drive_client = Client::new(keypair, solana_rpc);
+
+    //ensure storage account
+    if let Err(_) = shdw_drive_client
+        .get_storage_account(&storage_account_key)
+        .await
+    {
+        println!("Error finding storage account, assuming it's not created yet");
+        shdw_drive_client
+            .create_storage_account(
+                "shadow-drive-rust-test-2",
+                Byte::from_str("1MB").expect("failed to parse byte string"),
+            )
+            .await
+            .expect("failed to create storage account");
+    }
+
+    let dir = tokio::fs::read_dir("multiple_uploads")
+        .await
+        .expect("failed to read multiple uploads dir");
+
+    let files = tokio_stream::wrappers::ReadDirStream::new(dir)
+        .filter(Result::is_ok)
+        .and_then(|entry| async move {
+            Ok(ShdwFile {
+                name: entry
+                    .file_name()
+                    .into_string()
+                    .expect("failed to convert os string to regular string"),
+                file: File::open(entry.path()).await.expect("failed to open file"),
+            })
+        })
+        .collect::<Result<Vec<_>, _>>()
+        .await
+        .expect("failed to create shdw files for dir");
+
+    let upload_results = shdw_drive_client
+        .upload_multiple_files(&storage_account_key, files)
+        .await
+        .expect("failed to upload files");
+
+    println!("upload results: {:#?}", upload_results);
+}

src/client_impl.rs

@@ -15,6 +15,7 @@ mod list_objects;
 mod make_storage_immutable;
 mod reduce_storage;
 mod upload_file;
+mod upload_multiple_files;
 
 pub use add_storage::*;
 pub use cancel_delete_file::*;
@@ -29,6 +30,7 @@ pub use list_objects::*;
 pub use make_storage_immutable::*;
 pub use reduce_storage::*;
 pub use upload_file::*;
+pub use upload_multiple_files::*;
 
 use crate::{
     constants::SHDW_DRIVE_ENDPOINT,

src/client_impl/edit_file.rs

@@ -67,15 +67,13 @@ where
 
         //construct file part and create form
         let mut file_part = Part::stream(data.file);
-        if let Some(name) = data.name.as_ref() {
-            if name.as_bytes().len() > 32 {
-                errors.push(FileError {
-                    file: Some(name.to_string()),
-                    error: String::from("Exceed the 1GB limit."),
-                });
-            } else {
-                file_part = file_part.file_name(name.to_string());
-            }
+        if data.name.as_bytes().len() > 32 {
+            errors.push(FileError {
+                file: data.name.clone(),
+                error: String::from("File name too long. Reduce to 32 bytes long."),
+            });
+        } else {
+            file_part = file_part.file_name(data.name.clone());
         }
 
         if errors.len() > 0 {

src/client_impl/upload_file.rs

@@ -58,15 +58,13 @@ where
 
         //construct file part and create form
         let mut file_part = Part::stream(data.file);
-        if let Some(name) = data.name.as_ref() {
-            if name.as_bytes().len() > 32 {
-                errors.push(FileError {
-                    file: Some(name.to_string()),
-                    error: String::from("Exceed the 1GB limit."),
-                });
-            } else {
-                file_part = file_part.file_name(name.to_string());
-            }
+        if data.name.as_bytes().len() > 32 {
+            errors.push(FileError {
+                file: data.name.clone(),
+                error: String::from("File name too long. Reduce to 32 bytes long."),
+            });
+        } else {
+            file_part = file_part.file_name(data.name.clone());
         }
 
         if errors.len() > 0 {
@@ -90,7 +88,7 @@ where
             system_program: system_program::ID,
         };
         let args = shdw_drive_instructions::StoreFile {
-            filename: data.name.unwrap_or_default().to_string(),
+            filename: data.name,
             sha256_hash: hex::encode(sha256_hash.into_bytes()),
             size: file_size as u64,
         };