From 401bf7a77c02b5e57a3a413136f7c2cff529ef97 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Fri, 27 Sep 2024 15:29:53 +0200 Subject: [PATCH 01/12] Switch to clap::Args, add use-hash-lookup, use-cache --- rust/examples/gscan/Cargo.toml | 2 +- rust/examples/gscan/src/main.rs | 113 ++++++++++++++------------------ 2 files changed, 50 insertions(+), 65 deletions(-) diff --git a/rust/examples/gscan/Cargo.toml b/rust/examples/gscan/Cargo.toml index 0d5323ff..29a13c14 100644 --- a/rust/examples/gscan/Cargo.toml +++ b/rust/examples/gscan/Cargo.toml @@ -10,7 +10,7 @@ publish = false [dependencies] vaas = { version = "6.0.0" } tokio = { version = "1.37", features = ["rt-multi-thread", "macros"] } -clap = { version = "4.5.4", features = ["env", "cargo"] } +clap = { version = "4.5.18", features = ["env", "cargo", "derive"] } reqwest = "0.12.4" futures = "0.3.30" dotenv = "0.15" diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index a2c85afc..bf36231f 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,75 +1,60 @@ -use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, ArgAction, Command}; +use clap::{command, ArgAction, Parser}; use reqwest::Url; -use std::{collections::HashMap, path::PathBuf, str::FromStr}; +use std::{collections::HashMap, path::PathBuf}; use vaas::{ auth::authenticators::ClientCredentials, error::VResult, CancellationToken, Connection, Vaas, VaasVerdict, }; +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + #[arg( + short = 'i', + long = "client_id", + env = "CLIENT_ID", + help = "Set your VaaS client ID" + )] + client_id: String, + + #[arg( + short = 's', + long = "client_secret", + env = "CLIENT_SECRET", + help("Set your VaaS client secret") + )] + client_secret: String, + + #[arg(long, help = "Lookup the SHA256 hash")] + use_hash_lookup: bool, + + #[arg(long, help = "Use the cache")] + use_cache: bool, + + #[arg(short='f', long, action=ArgAction::Append, required_unless_present("urls"), help="List of files to scan separated by whitepace")] + files: Vec, + + #[arg(short='u', long, action=ArgAction::Append, required_unless_present("files"), help="List of urls to scan separated by whitepace")] + urls: Vec, +} + #[tokio::main] async fn main() -> VResult<()> { - let matches = Command::new(crate_name!()) - .version(crate_version!()) - .author(crate_authors!()) - .about(crate_description!()) - .arg( - Arg::new("files") - .short('f') - .long("files") - .required_unless_present("urls") - .action(ArgAction::Append) - .help("List of files to scan separated by whitepace"), - ) - .arg( - Arg::new("urls") - .short('u') - .long("urls") - .action(ArgAction::Append) - .required_unless_present("files") - .help("List of urls to scan separated by whitepace"), - ) - .arg( - Arg::new("client_id") - .short('i') - .long("client_id") - .env("CLIENT_ID") - .action(ArgAction::Set) - .help("Set your vaas username"), - ) - .arg( - Arg::new("client_secret") - .short('s') - .long("client_secret") - .env("CLIENT_SECRET") - .action(ArgAction::Set) - .help("Set your vaas password"), - ) - .get_matches(); - - let files = matches - .get_many::("files") - .unwrap_or_default() - .map(|f| PathBuf::from_str(f).unwrap_or_else(|_| panic!("Not a valid file path: {}", f))) - .collect::>(); - - let urls = matches - .get_many::("urls") - .unwrap_or_default() - .map(|f| Url::parse(f).unwrap_or_else(|_| panic!("Not a valid url: {}", f))) - .collect::>(); - - let client_id = matches - .get_one::("client_id") - .expect("--client_id or the enviroment variable CLIENT_ID must be set"); - let client_secret = matches - .get_one::("client_secret") - .expect("--client_secret or the enviroment variable CLIENT_SECRET must be set"); - - let authenticator = ClientCredentials::new(client_id.to_owned(), client_secret.to_owned()); - let vaas_connection = Vaas::builder(authenticator).build()?.connect().await?; - - let file_verdicts = scan_files(&files, &vaas_connection).await?; - let url_verdicts = scan_urls(&urls, &vaas_connection).await?; + let args = Args::parse(); + + // TODO: dotenv support + // TODO: directory support + + let authenticator = ClientCredentials::new(args.client_id.clone(), args.client_secret.clone()); + let vaas_connection = Vaas::builder(authenticator) + .use_hash_lookup(args.use_hash_lookup) + .use_cache(args.use_cache) + .build()? + .connect() + .await?; + + let file_verdicts = scan_files(args.files.as_ref(), &vaas_connection).await?; + let url_verdicts = scan_urls(args.urls.as_ref(), &vaas_connection).await?; file_verdicts .iter() From 40f357e8a38abada9d8a98f6fc908fdf69c46466 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Fri, 27 Sep 2024 15:34:59 +0200 Subject: [PATCH 02/12] dotenv support --- rust/examples/gscan/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index bf36231f..3c15c0ce 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,4 +1,5 @@ use clap::{command, ArgAction, Parser}; +use dotenv::dotenv; use reqwest::Url; use std::{collections::HashMap, path::PathBuf}; use vaas::{ @@ -40,9 +41,9 @@ struct Args { #[tokio::main] async fn main() -> VResult<()> { + dotenv().ok(); let args = Args::parse(); - // TODO: dotenv support // TODO: directory support let authenticator = ClientCredentials::new(args.client_id.clone(), args.client_secret.clone()); From b92b449936868906467e6a3e4d845775d55eba43 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Fri, 27 Sep 2024 17:22:11 +0200 Subject: [PATCH 03/12] Support recursive scanning of directories --- rust/examples/gscan/Cargo.toml | 1 + rust/examples/gscan/src/main.rs | 43 +++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/rust/examples/gscan/Cargo.toml b/rust/examples/gscan/Cargo.toml index 29a13c14..a48fedce 100644 --- a/rust/examples/gscan/Cargo.toml +++ b/rust/examples/gscan/Cargo.toml @@ -14,3 +14,4 @@ clap = { version = "4.5.18", features = ["env", "cargo", "derive"] } reqwest = "0.12.4" futures = "0.3.30" dotenv = "0.15" +walkdir = "2.5.0" \ No newline at end of file diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index 3c15c0ce..0e4c6688 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,11 +1,13 @@ use clap::{command, ArgAction, Parser}; use dotenv::dotenv; +use futures::{stream, StreamExt}; use reqwest::Url; use std::{collections::HashMap, path::PathBuf}; use vaas::{ auth::authenticators::ClientCredentials, error::VResult, CancellationToken, Connection, Vaas, VaasVerdict, }; +use walkdir::WalkDir; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] @@ -44,7 +46,7 @@ async fn main() -> VResult<()> { dotenv().ok(); let args = Args::parse(); - // TODO: directory support + let files = expand_directories(&args.files); let authenticator = ClientCredentials::new(args.client_id.clone(), args.client_secret.clone()); let vaas_connection = Vaas::builder(authenticator) @@ -54,7 +56,7 @@ async fn main() -> VResult<()> { .connect() .await?; - let file_verdicts = scan_files(args.files.as_ref(), &vaas_connection).await?; + let file_verdicts = scan_files(files.into_iter(), &vaas_connection).await?; let url_verdicts = scan_urls(args.urls.as_ref(), &vaas_connection).await?; file_verdicts @@ -78,13 +80,21 @@ fn print_verdicts>(i: I, v: &VResult) { }; } -async fn scan_files<'a>( - files: &'a [PathBuf], +async fn scan_files<'a, I>( + files: I, vaas_connection: &Connection, -) -> VResult)>> { +) -> VResult)>> +where + I: Iterator, +{ let ct = CancellationToken::from_minutes(1); - let verdicts = vaas_connection.for_file_list(files, &ct).await; - let results = files.iter().zip(verdicts).collect(); + + let verdicts_stream = stream::iter(files).then(|p: PathBuf| async { + let verdict = vaas_connection.for_file(&p, &ct).await; + (p, verdict) + }); + + let results: Vec<_> = verdicts_stream.collect().await; Ok(results) } @@ -102,3 +112,22 @@ async fn scan_urls( Ok(verdicts) } + +fn expand_directories<'a>(files: &'a [PathBuf]) -> impl Iterator + 'a { + files.iter().flat_map(expand_entry) +} + +fn expand_entry(p: &PathBuf) -> Box> { + if p.is_file() { + return Box::new(std::iter::once(p.clone())); + } + + let files_in_directory = WalkDir::new(p) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .map(|e| e.path().to_path_buf().clone()) + .into_iter(); + + Box::new(files_in_directory) +} From 418e095b584bbade327d9bd5418f9672f1176bd4 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Fri, 27 Sep 2024 17:27:45 +0200 Subject: [PATCH 04/12] Remove unnecessary clone --- rust/examples/gscan/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index 0e4c6688..b182e3c9 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -126,7 +126,7 @@ fn expand_entry(p: &PathBuf) -> Box> { .into_iter() .filter_map(|e| e.ok()) .filter(|e| e.file_type().is_file()) - .map(|e| e.path().to_path_buf().clone()) + .map(|e| e.path().to_path_buf()) .into_iter(); Box::new(files_in_directory) From 72370e39d809601b6c33402f7bcb2d063f208142 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Mon, 14 Oct 2024 17:20:33 +0200 Subject: [PATCH 05/12] WIP --- rust/examples/gscan/src/main.rs | 38 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index b182e3c9..e002f4bc 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,6 +1,6 @@ use clap::{command, ArgAction, Parser}; use dotenv::dotenv; -use futures::{stream, StreamExt}; +use futures::{stream, Stream, StreamExt}; use reqwest::Url; use std::{collections::HashMap, path::PathBuf}; use vaas::{ @@ -56,14 +56,21 @@ async fn main() -> VResult<()> { .connect() .await?; - let file_verdicts = scan_files(files.into_iter(), &vaas_connection).await?; + let file_verdicts = scan_files(files.into_iter(), &vaas_connection); let url_verdicts = scan_urls(args.urls.as_ref(), &vaas_connection).await?; file_verdicts - .iter() - .for_each(|(f, v)| print_verdicts(f.display().to_string(), v)); - - url_verdicts.iter().for_each(|(u, v)| print_verdicts(u, v)); + .for_each_concurrent(8, |(f, v)| async move { + println!(">"); + print_verdicts(f.display().to_string(), &v); + println!("<") + }) + .await; + + url_verdicts.iter().for_each(|(u, v)| { + print_verdicts(u, v); + // ready(()) + }); Ok(()) } @@ -80,23 +87,18 @@ fn print_verdicts>(i: I, v: &VResult) { }; } -async fn scan_files<'a, I>( +fn scan_files<'a, I>( files: I, - vaas_connection: &Connection, -) -> VResult)>> + vaas_connection: &'a Connection, +) -> impl Stream)> + 'a where - I: Iterator, + I: Iterator + 'a, { - let ct = CancellationToken::from_minutes(1); - - let verdicts_stream = stream::iter(files).then(|p: PathBuf| async { + stream::iter(files).then(move |p: PathBuf| async move { + let ct = CancellationToken::from_minutes(1); let verdict = vaas_connection.for_file(&p, &ct).await; (p, verdict) - }); - - let results: Vec<_> = verdicts_stream.collect().await; - - Ok(results) + }) } async fn scan_urls( From 8337dc4a4e549e1692d5c256110d74d9f9a3f621 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Mon, 14 Oct 2024 17:33:26 +0200 Subject: [PATCH 06/12] Clippy fixes --- rust/examples/gscan/src/main.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index e002f4bc..ed33fa65 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -82,7 +82,7 @@ fn print_verdicts>(i: I, v: &VResult) { println!("{}", v.verdict); } Err(e) => { - println!("{}", e.to_string()); + println!("{}", e); } }; } @@ -115,7 +115,7 @@ async fn scan_urls( Ok(verdicts) } -fn expand_directories<'a>(files: &'a [PathBuf]) -> impl Iterator + 'a { +fn expand_directories(files: &[PathBuf]) -> impl Iterator + '_ { files.iter().flat_map(expand_entry) } @@ -128,8 +128,7 @@ fn expand_entry(p: &PathBuf) -> Box> { .into_iter() .filter_map(|e| e.ok()) .filter(|e| e.file_type().is_file()) - .map(|e| e.path().to_path_buf()) - .into_iter(); + .map(|e| e.path().to_path_buf()); Box::new(files_in_directory) } From 7abe5c874422f77a2a1faf5c7b1faafa63465b7c Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Mon, 14 Oct 2024 17:45:26 +0200 Subject: [PATCH 07/12] Fix --files, --urls parsing --- rust/examples/gscan/src/main.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index ed33fa65..f85c8d68 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,4 +1,4 @@ -use clap::{command, ArgAction, Parser}; +use clap::{command, Parser}; use dotenv::dotenv; use futures::{stream, Stream, StreamExt}; use reqwest::Url; @@ -34,10 +34,22 @@ struct Args { #[arg(long, help = "Use the cache")] use_cache: bool, - #[arg(short='f', long, action=ArgAction::Append, required_unless_present("urls"), help="List of files to scan separated by whitepace")] + #[arg( + short = 'f', + long, + num_args=1.., + required_unless_present("urls"), + help = "List of files to scan separated by whitepace" + )] files: Vec, - #[arg(short='u', long, action=ArgAction::Append, required_unless_present("files"), help="List of urls to scan separated by whitepace")] + #[arg( + short = 'u', + long, + num_args=1.., + required_unless_present("files"), + help = "List of urls to scan separated by whitepace" + )] urls: Vec, } From 1830436b823ca52b6229162a4d35f1fc8e079d8e Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Tue, 15 Oct 2024 15:25:24 +0200 Subject: [PATCH 08/12] Tokio-console --- rust/examples/gscan/Cargo.toml | 3 ++- rust/examples/gscan/src/main.rs | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/examples/gscan/Cargo.toml b/rust/examples/gscan/Cargo.toml index a48fedce..4930702d 100644 --- a/rust/examples/gscan/Cargo.toml +++ b/rust/examples/gscan/Cargo.toml @@ -14,4 +14,5 @@ clap = { version = "4.5.18", features = ["env", "cargo", "derive"] } reqwest = "0.12.4" futures = "0.3.30" dotenv = "0.15" -walkdir = "2.5.0" \ No newline at end of file +walkdir = "2.5.0" +console-subscriber = "0.4.0" diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index f85c8d68..2d38bec4 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -3,6 +3,7 @@ use dotenv::dotenv; use futures::{stream, Stream, StreamExt}; use reqwest::Url; use std::{collections::HashMap, path::PathBuf}; +use tokio::task::spawn_blocking; use vaas::{ auth::authenticators::ClientCredentials, error::VResult, CancellationToken, Connection, Vaas, VaasVerdict, @@ -55,6 +56,8 @@ struct Args { #[tokio::main] async fn main() -> VResult<()> { + console_subscriber::init(); + dotenv().ok(); let args = Args::parse(); @@ -73,9 +76,7 @@ async fn main() -> VResult<()> { file_verdicts .for_each_concurrent(8, |(f, v)| async move { - println!(">"); print_verdicts(f.display().to_string(), &v); - println!("<") }) .await; @@ -108,7 +109,9 @@ where { stream::iter(files).then(move |p: PathBuf| async move { let ct = CancellationToken::from_minutes(1); + println!("Start {}", p.display()); let verdict = vaas_connection.for_file(&p, &ct).await; + println!("Stop {}", p.display()); (p, verdict) }) } From 3871cf69c440d339f10749570f584109fecd108e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hling=2C=20Maximilian?= Date: Tue, 15 Oct 2024 16:32:10 +0200 Subject: [PATCH 09/12] Fix parallelism for gscan file scanning --- rust/examples/gscan/src/main.rs | 39 +++++++++++---------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index 2d38bec4..80208398 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,9 +1,8 @@ -use clap::{command, Parser}; +use clap::Parser; use dotenv::dotenv; -use futures::{stream, Stream, StreamExt}; +use futures::{stream, StreamExt}; use reqwest::Url; use std::{collections::HashMap, path::PathBuf}; -use tokio::task::spawn_blocking; use vaas::{ auth::authenticators::ClientCredentials, error::VResult, CancellationToken, Connection, Vaas, VaasVerdict, @@ -54,6 +53,7 @@ struct Args { urls: Vec, } + #[tokio::main] async fn main() -> VResult<()> { console_subscriber::init(); @@ -71,18 +71,20 @@ async fn main() -> VResult<()> { .connect() .await?; - let file_verdicts = scan_files(files.into_iter(), &vaas_connection); + let files = stream::iter(files); + let vaas_reference = &vaas_connection; + files.for_each_concurrent(8,|p| async move { + let ct = CancellationToken::from_minutes(1); + println!("Start {}", p.display()); + let verdict = vaas_reference.for_file(&p, &ct).await; + println!("Stop {}", p.display()); + print_verdicts(p.display().to_string(), &verdict); + }).await; + let url_verdicts = scan_urls(args.urls.as_ref(), &vaas_connection).await?; - file_verdicts - .for_each_concurrent(8, |(f, v)| async move { - print_verdicts(f.display().to_string(), &v); - }) - .await; - url_verdicts.iter().for_each(|(u, v)| { print_verdicts(u, v); - // ready(()) }); Ok(()) @@ -100,21 +102,6 @@ fn print_verdicts>(i: I, v: &VResult) { }; } -fn scan_files<'a, I>( - files: I, - vaas_connection: &'a Connection, -) -> impl Stream)> + 'a -where - I: Iterator + 'a, -{ - stream::iter(files).then(move |p: PathBuf| async move { - let ct = CancellationToken::from_minutes(1); - println!("Start {}", p.display()); - let verdict = vaas_connection.for_file(&p, &ct).await; - println!("Stop {}", p.display()); - (p, verdict) - }) -} async fn scan_urls( urls: &[Url], From 527fc15b6d728fa318b089503ce1f11a3c2f0eb1 Mon Sep 17 00:00:00 2001 From: Alexander Kummutat Date: Fri, 25 Oct 2024 15:13:17 +0200 Subject: [PATCH 10/12] update version --- rust/examples/gscan/Cargo.toml | 5 ++--- rust/examples/gscan/src/main.rs | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/rust/examples/gscan/Cargo.toml b/rust/examples/gscan/Cargo.toml index 4930702d..89b5595a 100644 --- a/rust/examples/gscan/Cargo.toml +++ b/rust/examples/gscan/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gscan" -version = "1.0.0" +version = "1.1.0" edition = "2021" authors = ["GDATA CyberDefense AG "] license = "MIT" @@ -8,11 +8,10 @@ description = "GDATA Verdict-as-a-Service CLI Scanner" publish = false [dependencies] -vaas = { version = "6.0.0" } +vaas = { version = "6.1.1" } tokio = { version = "1.37", features = ["rt-multi-thread", "macros"] } clap = { version = "4.5.18", features = ["env", "cargo", "derive"] } reqwest = "0.12.4" futures = "0.3.30" dotenv = "0.15" walkdir = "2.5.0" -console-subscriber = "0.4.0" diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index 80208398..51351b3c 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -56,8 +56,6 @@ struct Args { #[tokio::main] async fn main() -> VResult<()> { - console_subscriber::init(); - dotenv().ok(); let args = Args::parse(); From 4172155d2cd13448fd8db2a47e41f9706e73af8a Mon Sep 17 00:00:00 2001 From: Alexander Kummutat Date: Fri, 25 Oct 2024 15:14:19 +0200 Subject: [PATCH 11/12] change line separators --- rust/examples/gscan/src/main.rs | 268 ++++++++++++++++---------------- 1 file changed, 134 insertions(+), 134 deletions(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index 51351b3c..97dc9ecc 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -1,134 +1,134 @@ -use clap::Parser; -use dotenv::dotenv; -use futures::{stream, StreamExt}; -use reqwest::Url; -use std::{collections::HashMap, path::PathBuf}; -use vaas::{ - auth::authenticators::ClientCredentials, error::VResult, CancellationToken, Connection, Vaas, - VaasVerdict, -}; -use walkdir::WalkDir; - -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - #[arg( - short = 'i', - long = "client_id", - env = "CLIENT_ID", - help = "Set your VaaS client ID" - )] - client_id: String, - - #[arg( - short = 's', - long = "client_secret", - env = "CLIENT_SECRET", - help("Set your VaaS client secret") - )] - client_secret: String, - - #[arg(long, help = "Lookup the SHA256 hash")] - use_hash_lookup: bool, - - #[arg(long, help = "Use the cache")] - use_cache: bool, - - #[arg( - short = 'f', - long, - num_args=1.., - required_unless_present("urls"), - help = "List of files to scan separated by whitepace" - )] - files: Vec, - - #[arg( - short = 'u', - long, - num_args=1.., - required_unless_present("files"), - help = "List of urls to scan separated by whitepace" - )] - urls: Vec, -} - - -#[tokio::main] -async fn main() -> VResult<()> { - dotenv().ok(); - let args = Args::parse(); - - let files = expand_directories(&args.files); - - let authenticator = ClientCredentials::new(args.client_id.clone(), args.client_secret.clone()); - let vaas_connection = Vaas::builder(authenticator) - .use_hash_lookup(args.use_hash_lookup) - .use_cache(args.use_cache) - .build()? - .connect() - .await?; - - let files = stream::iter(files); - let vaas_reference = &vaas_connection; - files.for_each_concurrent(8,|p| async move { - let ct = CancellationToken::from_minutes(1); - println!("Start {}", p.display()); - let verdict = vaas_reference.for_file(&p, &ct).await; - println!("Stop {}", p.display()); - print_verdicts(p.display().to_string(), &verdict); - }).await; - - let url_verdicts = scan_urls(args.urls.as_ref(), &vaas_connection).await?; - - url_verdicts.iter().for_each(|(u, v)| { - print_verdicts(u, v); - }); - - Ok(()) -} - -fn print_verdicts>(i: I, v: &VResult) { - print!("{} -> ", i.as_ref()); - match v { - Ok(v) => { - println!("{}", v.verdict); - } - Err(e) => { - println!("{}", e); - } - }; -} - - -async fn scan_urls( - urls: &[Url], - vaas_connection: &Connection, -) -> VResult>> { - let ct = CancellationToken::from_minutes(1); - let mut verdicts = HashMap::new(); - for url in urls { - let verdict = vaas_connection.for_url(url, &ct).await; - verdicts.insert(url.to_owned(), verdict); - } - - Ok(verdicts) -} - -fn expand_directories(files: &[PathBuf]) -> impl Iterator + '_ { - files.iter().flat_map(expand_entry) -} - -fn expand_entry(p: &PathBuf) -> Box> { - if p.is_file() { - return Box::new(std::iter::once(p.clone())); - } - - let files_in_directory = WalkDir::new(p) - .into_iter() - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().is_file()) - .map(|e| e.path().to_path_buf()); - - Box::new(files_in_directory) -} +use clap::Parser; +use dotenv::dotenv; +use futures::{stream, StreamExt}; +use reqwest::Url; +use std::{collections::HashMap, path::PathBuf}; +use vaas::{ + auth::authenticators::ClientCredentials, error::VResult, CancellationToken, Connection, Vaas, + VaasVerdict, +}; +use walkdir::WalkDir; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + #[arg( + short = 'i', + long = "client_id", + env = "CLIENT_ID", + help = "Set your VaaS client ID" + )] + client_id: String, + + #[arg( + short = 's', + long = "client_secret", + env = "CLIENT_SECRET", + help("Set your VaaS client secret") + )] + client_secret: String, + + #[arg(long, help = "Lookup the SHA256 hash")] + use_hash_lookup: bool, + + #[arg(long, help = "Use the cache")] + use_cache: bool, + + #[arg( + short = 'f', + long, + num_args=1.., + required_unless_present("urls"), + help = "List of files to scan separated by whitepace" + )] + files: Vec, + + #[arg( + short = 'u', + long, + num_args=1.., + required_unless_present("files"), + help = "List of urls to scan separated by whitepace" + )] + urls: Vec, +} + + +#[tokio::main] +async fn main() -> VResult<()> { + dotenv().ok(); + let args = Args::parse(); + + let files = expand_directories(&args.files); + + let authenticator = ClientCredentials::new(args.client_id.clone(), args.client_secret.clone()); + let vaas_connection = Vaas::builder(authenticator) + .use_hash_lookup(args.use_hash_lookup) + .use_cache(args.use_cache) + .build()? + .connect() + .await?; + + let files = stream::iter(files); + let vaas_reference = &vaas_connection; + files.for_each_concurrent(8,|p| async move { + let ct = CancellationToken::from_minutes(1); + println!("Start {}", p.display()); + let verdict = vaas_reference.for_file(&p, &ct).await; + println!("Stop {}", p.display()); + print_verdicts(p.display().to_string(), &verdict); + }).await; + + let url_verdicts = scan_urls(args.urls.as_ref(), &vaas_connection).await?; + + url_verdicts.iter().for_each(|(u, v)| { + print_verdicts(u, v); + }); + + Ok(()) +} + +fn print_verdicts>(i: I, v: &VResult) { + print!("{} -> ", i.as_ref()); + match v { + Ok(v) => { + println!("{}", v.verdict); + } + Err(e) => { + println!("{}", e); + } + }; +} + + +async fn scan_urls( + urls: &[Url], + vaas_connection: &Connection, +) -> VResult>> { + let ct = CancellationToken::from_minutes(1); + let mut verdicts = HashMap::new(); + for url in urls { + let verdict = vaas_connection.for_url(url, &ct).await; + verdicts.insert(url.to_owned(), verdict); + } + + Ok(verdicts) +} + +fn expand_directories(files: &[PathBuf]) -> impl Iterator + '_ { + files.iter().flat_map(expand_entry) +} + +fn expand_entry(p: &PathBuf) -> Box> { + if p.is_file() { + return Box::new(std::iter::once(p.clone())); + } + + let files_in_directory = WalkDir::new(p) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .map(|e| e.path().to_path_buf()); + + Box::new(files_in_directory) +} From f1b7504f0b0e17604b99329bd6dd27edd09bb211 Mon Sep 17 00:00:00 2001 From: Philip Stadermann Date: Fri, 25 Oct 2024 15:31:09 +0200 Subject: [PATCH 12/12] 2 concurrent scans, remove test output --- rust/examples/gscan/src/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/examples/gscan/src/main.rs b/rust/examples/gscan/src/main.rs index 97dc9ecc..5e176a8f 100644 --- a/rust/examples/gscan/src/main.rs +++ b/rust/examples/gscan/src/main.rs @@ -71,11 +71,9 @@ async fn main() -> VResult<()> { let files = stream::iter(files); let vaas_reference = &vaas_connection; - files.for_each_concurrent(8,|p| async move { + files.for_each_concurrent(2,|p| async move { let ct = CancellationToken::from_minutes(1); - println!("Start {}", p.display()); let verdict = vaas_reference.for_file(&p, &ct).await; - println!("Stop {}", p.display()); print_verdicts(p.display().to_string(), &verdict); }).await;