diff --git a/Cargo.toml b/Cargo.toml index 0792878..511cd9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "security_log_analysis_rust" -version = "0.11.8" +version = "0.11.9" authors = ["Daniel Boline "] edition = "2018" @@ -14,16 +14,16 @@ Analyze Auth Logs.""" [dependencies] anyhow = "1.0" -authorized_users = { git = "https://github.com/ddboline/auth_server_rust.git", tag="0.11.14"} +authorized_users = { git = "https://github.com/ddboline/auth_server_rust.git", tag="0.11.15"} aws-config = {version="1.0", features=["behavior-version-latest"]} aws-sdk-s3 = "1.1" aws-sdk-ses = "1.1" bytes = "1.0" -cached = {version="0.51", features=["async", "async_tokio_rt_multi_thread"]} +cached = {version="0.52", features=["async", "async_tokio_rt_multi_thread"]} chrono = "0.4" clap = {version="4.0", features=["derive"]} deadpool = {version = "0.12", features=["serde", "rt_tokio_1"]} -deadpool-postgres = {version="0.13", features=["serde"]} +deadpool-postgres = {version="0.14", features=["serde"]} deadqueue = "0.2" derive_more = "0.99" dioxus = "0.5" @@ -36,12 +36,12 @@ env_logger = "0.11" flate2 = "1.0" futures = "0.3" glob = "0.3" -itertools = "0.12" +itertools = "0.13" log = "0.4" maplit = "1.0" parking_lot = "0.12" -polars = {version="0.39", features=["temporal", "parquet", "lazy"]} -postgres_query = {git = "https://github.com/ddboline/rust-postgres-query", tag = "0.3.7", features=["deadpool"]} +polars = {version="0.41", features=["temporal", "parquet", "lazy"]} +postgres_query = {git = "https://github.com/ddboline/rust-postgres-query", tag = "0.3.8", features=["deadpool"]} postgres-types = {version="0.2", features=["with-time-0_3", "with-uuid-1", "with-serde_json-1"]} rand = "0.8" rayon = "1.5" @@ -57,7 +57,7 @@ thiserror = "1.0" time = {version="0.3", features=["serde-human-readable", "macros", "formatting"]} time-tz = {version="2.0", features=["system"]} tokio-postgres = {version="0.7", features=["with-time-0_3", "with-uuid-1", "with-serde_json-1"]} -tokio = {version="1.37", features=["rt", "macros", "rt-multi-thread"]} +tokio = {version="1.38", features=["rt", "macros", "rt-multi-thread"]} rweb = {git = "https://github.com/ddboline/rweb.git", features=["openapi"], default-features=false, tag="0.15.2"} rweb-helper = { git = "https://github.com/ddboline/rweb_helper.git", tag="0.5.3" } uuid = { version = "1.0", features = ["serde", "v4"] } diff --git a/src/models.rs b/src/models.rs index 9cffe68..80b7542 100644 --- a/src/models.rs +++ b/src/models.rs @@ -793,7 +793,7 @@ mod tests { async fn test_country_code_query() -> Result<(), Error> { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let conn = pool.get().await?; let query = query!("SELECT * FROM country_code"); @@ -811,7 +811,7 @@ mod tests { async fn test_host_country_query() -> Result<(), Error> { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let conn = pool.get().await?; let query = query!("SELECT * FROM host_country LIMIT 10"); let host_country_list: Vec = query.fetch(&conn).await?; @@ -828,7 +828,7 @@ mod tests { async fn test_intrusion_log_query() -> Result<(), Error> { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let conn = pool.get().await?; let query = query!("SELECT * FROM intrusion_log LIMIT 10"); let intrusion_log_list: Vec = query.fetch(&conn).await?; diff --git a/src/parse_logs.rs b/src/parse_logs.rs index 1863e16..a6bc930 100644 --- a/src/parse_logs.rs +++ b/src/parse_logs.rs @@ -587,7 +587,7 @@ mod tests { #[ignore] async fn test_parse_all_log_files_ssh() -> Result<(), Error> { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let mut hc = HostCountryMetadata::from_pool(pool).await?; hc.pool = None; let results = parse_all_log_files( diff --git a/src/parse_opts.rs b/src/parse_opts.rs index 5b16d89..4343ec0 100644 --- a/src/parse_opts.rs +++ b/src/parse_opts.rs @@ -96,7 +96,7 @@ impl ParseOpts { match opts { ParseOpts::Parse { daemon } => { - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; if daemon { async fn run_daemon(config: Config, pool: PgPool) { loop { @@ -133,7 +133,7 @@ impl ParseOpts { } } ParseOpts::Cleanup => { - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let metadata = HostCountryMetadata::from_pool(pool.clone()).await?; let hosts: Vec<_> = HostCountry::get_dangling_hosts(&pool) .await? @@ -149,7 +149,7 @@ impl ParseOpts { let sdk_config = aws_config::load_from_env().await; let sync = S3Sync::new(&sdk_config); let directory = directory.unwrap_or_else(|| config.cache_dir.clone()); - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; stdout.send( sync.sync_dir( "security-log-analysis", @@ -162,7 +162,7 @@ impl ParseOpts { } ParseOpts::AddHost { host_codes } => { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let metadata = HostCountryMetadata::from_pool(pool).await?; for host_country in &host_codes { let vals: SmallVec<[&str; 2]> = host_country.split(':').take(2).collect(); @@ -179,7 +179,7 @@ impl ParseOpts { } ParseOpts::Plot => { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let template = include_str!("../templates/COUNTRY_TEMPLATE.html"); let mut written = 0; for service in [Service::Ssh, Service::Apache, Service::Nginx] { @@ -208,13 +208,13 @@ impl ParseOpts { } ParseOpts::RunMigrations => { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let mut client = pool.get().await?; migrations::runner().run_async(&mut **client).await?; } ParseOpts::Db { directory } => { let directory = directory.unwrap_or_else(|| config.cache_dir.clone()); - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; stdout.send(insert_db_into_parquet(&pool, &directory).await?.join("\n")); } ParseOpts::Read { @@ -233,7 +233,7 @@ impl ParseOpts { } ParseOpts::Import { table, filepath } => { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let data = if let Some(filepath) = filepath { read_to_string(&filepath).await? @@ -262,7 +262,7 @@ impl ParseOpts { } ParseOpts::Export { table, filepath } => { let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; let mut file: Box = if let Some(filepath) = filepath { diff --git a/src/pgpool.rs b/src/pgpool.rs index 8bbfdd4..c5608f6 100644 --- a/src/pgpool.rs +++ b/src/pgpool.rs @@ -22,10 +22,10 @@ impl fmt::Debug for PgPool { } impl PgPool { - #[allow(clippy::missing_panics_doc)] - #[must_use] - pub fn new(pgurl: &str) -> Self { - let pgconf: PgConfig = pgurl.parse().expect("Failed to parse Url"); + /// # Errors + /// Return error if pool setup fails + pub fn new(pgurl: &str) -> Result { + let pgconf: PgConfig = pgurl.parse()?; let mut config = Config::default(); @@ -45,16 +45,14 @@ impl PgPool { } let pool = config - .builder(NoTls) - .unwrap_or_else(|_| panic!("failed to create builder")) + .builder(NoTls)? .max_size(4) - .build() - .unwrap_or_else(|_| panic!("Failed to create pool {}", pgurl)); + .build()?; - Self { + Ok(Self { pgurl: Arc::new(pgurl.into()), pool, - } + }) } /// # Errors diff --git a/src/polars_analysis.rs b/src/polars_analysis.rs index dd3c7be..c212ee2 100644 --- a/src/polars_analysis.rs +++ b/src/polars_analysis.rs @@ -6,12 +6,8 @@ use polars::{ chunked_array::ops::SortMultipleOptions, df as dataframe, frame::DataFrame, - io::{ - parquet::{ParquetReader, ParquetWriter}, - SerReader, - }, lazy::{dsl::functions::col, frame::IntoLazy}, - prelude::{lit, LazyFrame, ScanArgsParquet, UniqueKeepStrategy}, + prelude::{lit, LazyFrame, ScanArgsParquet, UniqueKeepStrategy, ParquetReader, ParquetWriter, SerReader}, }; use postgres_query::{query, FromSqlRow}; use stack_string::{format_sstr, StackString}; diff --git a/src/s3_sync.rs b/src/s3_sync.rs index 2c76ea6..1831879 100644 --- a/src/s3_sync.rs +++ b/src/s3_sync.rs @@ -391,7 +391,7 @@ mod tests { let aws_config = aws_config::load_from_env().await; let s3_sync = S3Sync::new(&aws_config); let config = Config::init_config()?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; s3_sync.process_files(&config.cache_dir, &pool).await?; s3_sync diff --git a/src/security_log_http.rs b/src/security_log_http.rs index 7af4d50..2369d3a 100644 --- a/src/security_log_http.rs +++ b/src/security_log_http.rs @@ -546,7 +546,7 @@ async fn start_app() -> Result<(), AnyhowError> { let config = Config::init_config()?; get_secrets(&config.secret_path, &config.jwt_secret_path).await?; - let pool = PgPool::new(&config.database_url); + let pool = PgPool::new(&config.database_url)?; spawn(update_db(pool.clone()));