From d72fa060206eddbced4497753a41687659ad4c43 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 15 Sep 2022 22:43:47 -0400 Subject: [PATCH 1/9] wip --- Cargo.lock | 131 ++++++++++++++---------- Cargo.toml | 2 +- src/bin/main.rs | 260 ++++++++++++++++++++++++++++++------------------ src/config.rs | 63 ++++++++---- 4 files changed, 289 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bbb3c7ef..ba292bcad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,16 +107,15 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb60846b52c118f2f04a56cc90880a274271c489b2498623d58176f8ca21fa80" +checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "firestorm", "http", - "log", "regex", "serde", + "tracing", ] [[package]] @@ -458,16 +457,34 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.21" +version = "3.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed5341b2301a26ab80be5cbdced622e80ed808483c52e45e3310a877d3b37d7" +checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" dependencies = [ + "atty", "bitflags", + "clap_derive", "clap_lex", "indexmap", + "once_cell", + "strsim", + "termcolor", "textwrap", ] +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "0.2.4" @@ -620,27 +637,15 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ "block-buffer", "crypto-common", "subtle", ] -[[package]] -name = "docopt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" -dependencies = [ - "lazy_static", - "regex", - "serde", - "strsim", -] - [[package]] name = "either" version = "1.8.0" @@ -675,12 +680,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" -[[package]] -name = "firestorm" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" - [[package]] name = "flate2" version = "1.0.24" @@ -862,6 +861,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -976,9 +981,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.132" +version = "0.2.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" [[package]] name = "local-channel" @@ -1000,9 +1005,9 @@ checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" [[package]] name = "lock_api" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -1030,8 +1035,8 @@ dependencies = [ "bb8", "bb8-postgres", "cargo-husky", + "clap", "criterion", - "docopt", "env_logger", "itertools", "log", @@ -1130,9 +1135,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "oorandom" @@ -1361,6 +1366,30 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.43" @@ -1402,9 +1431,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] @@ -1574,9 +1603,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", @@ -1585,9 +1614,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", @@ -1658,9 +1687,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" dependencies = [ "proc-macro2", "quote", @@ -1678,9 +1707,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" [[package]] name = "tilejson" @@ -1738,9 +1767,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.0" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89797afd69d206ccd11fb0ea560a44bbb87731d020670e79416d442919257d42" +checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" dependencies = [ "autocfg", "bytes", @@ -1840,24 +1869,24 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" [[package]] name = "unicode-normalization" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unsafe-libyaml" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "931179334a56395bcf64ba5e0ff56781381c1a5832178280c7d7f91d1679aeb0" +checksum = "c1e5fa573d8ac5f1a856f8d7be41d390ee973daf97c806b2c1a465e4e1406e68" [[package]] name = "url" diff --git a/Cargo.toml b/Cargo.toml index bf57cb6d4..f114781ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ actix-web = "4" async-trait = "0.1" bb8 = "0.8" bb8-postgres = "0.8" -docopt = "1" +clap = { version = "3", features = ["derive"] } env_logger = "0.9" itertools = "0.10" log = "0.4" diff --git a/src/bin/main.rs b/src/bin/main.rs index b65946712..f3f1ca0ae 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -1,7 +1,7 @@ use std::{env, io}; use actix_web::dev::Server; -use docopt::Docopt; +use clap::Parser; use log::{error, info, warn}; use martin::config::{read_config, Config, ConfigBuilder}; use martin::db::{check_postgis_version, get_connection, setup_connection_pool, Pool}; @@ -13,67 +13,88 @@ use serde::Deserialize; const VERSION: &str = env!("CARGO_PKG_VERSION"); const REQUIRED_POSTGIS_VERSION: &str = ">= 2.4.0"; -pub const USAGE: &str = " -Martin - PostGIS Mapbox Vector Tiles server. - -Usage: - martin [options] [] - martin -h | --help - martin -v | --version - -Options: - -h --help Show this screen. - -v --version Show version. - --config= Path to config file. - --keep-alive= Connection keep alive timeout [default: 75]. - --listen-addresses= The socket address to bind [default: 0.0.0.0:3000]. - --default-srid= If a spatial table has SRID 0, then this default SRID will be used as a fallback. - --pool-size= Maximum connections pool size [default: 20]. - --workers= Number of web server workers. - --ca-root-file= Loads trusted root certificates from a file. The file should contain a sequence of PEM-formatted CA certificates. - --danger-accept-invalid-certs Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. - --watch [IGNORED] This flag is no longer supported, and will be ignored. -"; - -#[derive(Debug, Deserialize)] +#[derive(Parser, Debug, Deserialize)] +#[clap(about, version)] pub struct Args { - pub arg_connection: Option, - pub flag_config: Option, - pub flag_help: bool, - pub flag_keep_alive: Option, - pub flag_listen_addresses: Option, - pub flag_pool_size: Option, - pub flag_watch: bool, - pub flag_version: bool, - pub flag_workers: Option, - pub flag_default_srid: Option, - pub flag_ca_root_file: Option, - pub flag_danger_accept_invalid_certs: bool, + /// Path to config file. + #[clap(short, long)] + pub config: Option, + /// Loads trusted root certificates from a file. The file should contain a sequence of PEM-formatted CA certificates. + #[clap(long)] + pub ca_root_file: Option, + /// Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. + #[clap(long)] + pub danger_accept_invalid_certs: bool, + /// If a spatial table has SRID 0, then this default SRID will be used as a fallback. + #[clap(short, long)] + pub default_srid: Option, + // Must match ConfigBuilder::KEEP_ALIVE_DEFAULT + /// Connection keep alive timeout. [DEFAULT: 75] + #[clap(short, long)] + pub keep_alive: Option, + // Must match ConfigBuilder::LISTEN_ADDRESSES_DEFAULT + /// The socket address to bind. [DEFAULT: 0.0.0.0:3000] + #[clap(short, long)] + pub listen_addresses: Option, + // Must match ConfigBuilder::POOL_SIZE_DEFAULT + /// Maximum connections pool size [DEFAULT: 20] + #[clap(short, long)] + pub pool_size: Option, + /// Scan for new sources on sources list requests + #[clap(short, long, hide = true)] + pub watch: bool, + /// Number of web server workers + #[clap(short = 'W', long)] + pub workers: Option, + /// Database connection string + pub connection: Option, +} + +impl Args { + fn to_config(self) -> ConfigBuilder { + ConfigBuilder { + ca_root_file: self.ca_root_file, + danger_accept_invalid_certs: if self.danger_accept_invalid_certs { + Some(true) + } else { + None + }, + default_srid: self.default_srid, + keep_alive: self.keep_alive, + listen_addresses: self.listen_addresses, + pool_size: self.pool_size, + worker_processes: self.workers, + connection_string: self.connection, + table_sources: None, + function_sources: None, + } + } } pub async fn generate_config(args: Args, pool: &Pool) -> io::Result { - let connection_string = args.arg_connection.clone().ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "Database connection string is not set", - ) - })?; + // let connection_string = args.arg_connection.clone().ok_or_else(|| { + // io::Error::new( + // io::ErrorKind::Other, + // "Database connection string is not set", + // ) + // })?; + let connection_string = args.connection.clone(); let mut connection = get_connection(pool).await?; - let table_sources = get_table_sources(&mut connection, &args.flag_default_srid).await?; + let table_sources = get_table_sources(&mut connection, &args.default_srid).await?; let function_sources = get_function_sources(&mut connection).await?; let config = ConfigBuilder { + ca_root_file: None, + danger_accept_invalid_certs: if args.danger_accept_invalid_certs { Some(true) } else { None }, + default_srid: args.default_srid, + keep_alive: args.keep_alive, + listen_addresses: args.listen_addresses, + pool_size: args.pool_size, + worker_processes: args.workers, connection_string, - keep_alive: args.flag_keep_alive, - listen_addresses: args.flag_listen_addresses, - default_srid: args.flag_default_srid, - pool_size: args.flag_pool_size, - worker_processes: args.flag_workers, table_sources: Some(table_sources), function_sources: Some(function_sources), - ca_root_file: None, - danger_accept_invalid_certs: Some(args.flag_danger_accept_invalid_certs), }; let config = config.finalize(); @@ -89,8 +110,8 @@ async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { Some(config.pool_size), config.danger_accept_invalid_certs, ) - .await - .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; + .await + .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; if let Some(table_sources) = &config.table_sources { for table_source in table_sources.values() { @@ -119,7 +140,7 @@ async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { } async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { - let connection_string = args.arg_connection.clone().ok_or_else(|| { + let connection_string = args.connection.clone().ok_or_else(|| { io::Error::new( io::ErrorKind::Other, "Database connection string is not set", @@ -129,12 +150,12 @@ async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { info!("Connecting to database"); let pool = setup_connection_pool( &connection_string, - &args.flag_ca_root_file, - args.flag_pool_size, - args.flag_danger_accept_invalid_certs, + &args.ca_root_file, + args.pool_size, + args.danger_accept_invalid_certs, ) - .await - .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; + .await + .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; info!("Scanning database"); let config = generate_config(args, &pool) @@ -145,11 +166,11 @@ async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { } fn parse_env(args: Args) -> Args { - let arg_connection = args.arg_connection.or_else(|| { + let connection = args.connection.or_else(|| { env::var_os("DATABASE_URL").and_then(|connection| connection.into_string().ok()) }); - let flag_default_srid = args.flag_default_srid.or_else(|| { + let default_srid = args.default_srid.or_else(|| { env::var_os("DEFAULT_SRID").and_then(|srid| { srid.into_string() .ok() @@ -157,14 +178,14 @@ fn parse_env(args: Args) -> Args { }) }); - let flag_ca_root_file = args.flag_ca_root_file.or_else(|| { + let ca_root_file = args.ca_root_file.or_else(|| { env::var_os("CA_ROOT_FILE").and_then(|connection| connection.into_string().ok()) }); - let flag_danger_accept_invalid_certs = args.flag_danger_accept_invalid_certs - || env::var_os("DANGER_ACCEPT_INVALID_CERTS").is_some(); + let danger_accept_invalid_certs = + args.danger_accept_invalid_certs || env::var_os("DANGER_ACCEPT_INVALID_CERTS").is_some(); - if args.flag_watch { + if args.watch { warn!("The --watch flag is no longer supported, and will be ignored"); } if env::var_os("WATCH_MODE").is_some() { @@ -172,10 +193,10 @@ fn parse_env(args: Args) -> Args { } Args { - arg_connection, - flag_default_srid, - flag_ca_root_file, - flag_danger_accept_invalid_certs, + connection, + default_srid, + ca_root_file, + danger_accept_invalid_certs, ..args } } @@ -183,7 +204,7 @@ fn parse_env(args: Args) -> Args { async fn start(args: Args) -> io::Result { info!("Starting martin v{VERSION}"); - let (config, pool) = match args.flag_config { + let (config, pool) = match args.config { Some(config_file_name) => { info!("Using {config_file_name}"); setup_from_config(config_file_name).await? @@ -209,36 +230,83 @@ async fn start(args: Args) -> io::Result { Ok(server) } +fn print_type_of(_: &T) { + println!("{}", std::any::type_name::()) +} + #[actix_web::main] async fn main() -> io::Result<()> { let env = env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "martin=info"); env_logger::Builder::from_env(env).init(); - let args = Docopt::new(USAGE) - .and_then(|d| d.help(false).deserialize::()) - .map_err(|e| prettify_error!(e, "Can't parse CLI arguments"))?; - - let args = parse_env(args); - - if args.flag_help { - println!("{USAGE}"); - std::process::exit(0); - } - - if args.flag_version { - println!("v{VERSION}"); - std::process::exit(0); - } - - if args.flag_danger_accept_invalid_certs { - warn!("Danger accept invalid certs enabled. You should think very carefully before using this option. If invalid certificates are trusted, any certificate for any site will be trusted for use. This includes expired certificates. This introduces significant vulnerabilities, and should only be used as a last resort."); - } - - match start(args).await { - Ok(server) => server.await, - Err(error) => { - error!("{error}"); - std::process::exit(-1); - } - } + let args: Args = Args::parse(); + println!("{:?}", &args); + let parse: ConfigBuilder = args.to_config(); + print_type_of(&parse); + println!("{:?}", &parse); + // parse.map(|args| { + // let args = parse_env(args); + // let runner = start(args)?; + // runner.run(); + // }); + // let clap_matches = Args::command().ignore_errors(true).get_matches(); + // print_type_of(&clap_matches); // clap::parse::matches::arg_matches::ArgMatches + + // let mut v = Viperus::new(); + // v.load_clap(clap_matches).unwrap(); + // let config = clap_matches.value_of("config"); + // if config { + // v.load_file(config, Format::YAML).unwrap(); + // } + // debug!("final {:?}", v); + // + // // let x = args.value_of("foo"); + // // let args = Docopt::new(USAGE) + // // .and_then(|d| d.help(false).deserialize::()) + // // .map_err(prettify_error("Can't parse CLI arguments".to_owned()))?; + // + // // let args = parse_env(args); + // // let args = Args::parse(); + // let args = Args { + // config: v.get("config").unwrap(), + // keep_alive: v.get("keep_alive").unwrap(), + // listen_addresses: v.get("listen_addresses").unwrap(), + // default_srid: v.get("default_srid").unwrap(), + // pool_size: v.get("pool_size").unwrap(), + // workers: v.get("workers").unwrap(), + // ca_root_file: v.get("ca_root_file").unwrap(), + // danger_accept_invalid_certs: v.get("danger_accept_invalid_certs").unwrap(), + // connection: v.get("connection").unwrap(), + // }; + // + // + // if args.danger_accept_invalid_certs { + // warn!("Danger accept invalid certs enabled. You should think very carefully before using this option. If invalid certificates are trusted, any certificate for any site will be trusted for use. This includes expired certificates. This introduces significant vulnerabilities, and should only be used as a last resort."); + // } + // + // let server = match start(args) { + // Ok(server) => server, + // Err(error) => { + // error!("{error}"); + // std::process::exit(-1); + // } + // }; + // + // server.run() + // let server = match start(args) { + // Ok(server) => server, + // Err(error) => { + // error!("{error}"); + // std::process::exit(-1); + // } + // }; + + // match start(args).await { + // Ok(server) => server.await, + // Err(error) => { + // error!("{error}"); + // std::process::exit(-1); + // } + // } + Ok(()) } diff --git a/src/config.rs b/src/config.rs index 3f4b0f63e..d27c54db6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,47 +10,72 @@ use crate::utils::prettify_error; #[derive(Clone, Debug, Serialize)] pub struct Config { - pub pool_size: u32, - pub keep_alive: usize, + pub ca_root_file: Option, + pub danger_accept_invalid_certs: bool, pub default_srid: Option, - pub worker_processes: usize, + pub keep_alive: usize, pub listen_addresses: String, + pub pool_size: u32, + pub worker_processes: usize, pub connection_string: String, pub table_sources: Option, pub function_sources: Option, - pub ca_root_file: Option, - pub danger_accept_invalid_certs: bool, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub struct ConfigBuilder { - pub pool_size: Option, - pub keep_alive: Option, + pub ca_root_file: Option, + pub danger_accept_invalid_certs: Option, pub default_srid: Option, - pub worker_processes: Option, + pub keep_alive: Option, pub listen_addresses: Option, - pub connection_string: String, + pub pool_size: Option, + pub worker_processes: Option, + pub connection_string: Option, pub table_sources: Option, pub function_sources: Option, - pub ca_root_file: Option, - pub danger_accept_invalid_certs: Option, +} + +/// Update empty option in place with a non-empty value from the second option. +fn set_option(first: &mut Option, second: Option) { + if first.is_none() && second.is_some() { + *first = second; + } } impl ConfigBuilder { + pub const KEEP_ALIVE_DEFAULT: usize = 75; + pub const LISTEN_ADDRESSES_DEFAULT: &'static str = "0.0.0.0:3000"; + pub const POOL_SIZE_DEFAULT: u32 = 20; + + pub fn merge(&mut self, other: ConfigBuilder) -> &mut Self { + set_option(&mut self.ca_root_file, other.ca_root_file); + set_option(&mut self.danger_accept_invalid_certs, other.danger_accept_invalid_certs); + set_option(&mut self.default_srid, other.default_srid); + set_option(&mut self.keep_alive, other.keep_alive); + set_option(&mut self.listen_addresses, other.listen_addresses); + set_option(&mut self.pool_size, other.pool_size); + set_option(&mut self.worker_processes, other.worker_processes); + set_option(&mut self.connection_string, other.connection_string); + set_option(&mut self.table_sources, other.table_sources); + set_option(&mut self.function_sources, other.function_sources); + self + } + pub fn finalize(self) -> Config { Config { - pool_size: self.pool_size.unwrap_or(20), - keep_alive: self.keep_alive.unwrap_or(75), + ca_root_file: self.ca_root_file, + danger_accept_invalid_certs: self.danger_accept_invalid_certs.unwrap_or(false), default_srid: self.default_srid, - worker_processes: self.worker_processes.unwrap_or_else(num_cpus::get), + keep_alive: self.keep_alive.unwrap_or(Self::KEEP_ALIVE_DEFAULT), listen_addresses: self .listen_addresses - .unwrap_or_else(|| "0.0.0.0:3000".to_owned()), - connection_string: self.connection_string, + .unwrap_or_else(|| Self::LISTEN_ADDRESSES_DEFAULT.to_owned()), + pool_size: self.pool_size.unwrap_or(Self::POOL_SIZE_DEFAULT), + worker_processes: self.worker_processes.unwrap_or_else(num_cpus::get), + connection_string: self.connection_string.unwrap(), table_sources: self.table_sources, function_sources: self.function_sources, - ca_root_file: self.ca_root_file, - danger_accept_invalid_certs: self.danger_accept_invalid_certs.unwrap_or(false), } } } From 1cdd7ae6c50f538a73e411c2212e2d06fd343b07 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 26 Sep 2022 02:10:08 -0400 Subject: [PATCH 2/9] config cleanup --- Cargo.lock | 33 +++++++++++--- Cargo.toml | 2 +- src/bin/main.rs | 89 +++++++++++++------------------------- src/config.rs | 5 ++- src/table_source.rs | 4 +- tests/table_source_test.rs | 10 ++--- 6 files changed, 68 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba292bcad..0d941adb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -460,23 +460,33 @@ name = "clap" version = "3.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +dependencies = [ + "bitflags", + "clap_lex 0.2.4", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap" +version = "4.0.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d5891a0b7bbd946e91a79c350b5475764d7d7eda9c71141059a9d2ab0853e5" dependencies = [ "atty", "bitflags", "clap_derive", - "clap_lex", - "indexmap", + "clap_lex 0.3.0", "once_cell", "strsim", "termcolor", - "textwrap", ] [[package]] name = "clap_derive" -version = "3.2.18" +version = "4.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +checksum = "51eef4d62724bf369e9ca7458cfde0c55263708b4552020058fba384864e8c23" dependencies = [ "heck", "proc-macro-error", @@ -494,6 +504,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -539,7 +558,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap", + "clap 3.2.22", "criterion-plot", "futures", "itertools", @@ -1035,7 +1054,7 @@ dependencies = [ "bb8", "bb8-postgres", "cargo-husky", - "clap", + "clap 4.0.0-rc.2", "criterion", "env_logger", "itertools", diff --git a/Cargo.toml b/Cargo.toml index f114781ba..cf0005740 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ actix-web = "4" async-trait = "0.1" bb8 = "0.8" bb8-postgres = "0.8" -clap = { version = "3", features = ["derive"] } +clap = { version = "4.0.0-rc", features = ["derive"] } env_logger = "0.9" itertools = "0.10" log = "0.4" diff --git a/src/bin/main.rs b/src/bin/main.rs index f3f1ca0ae..4e1b233b3 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -5,8 +5,8 @@ use clap::Parser; use log::{error, info, warn}; use martin::config::{read_config, Config, ConfigBuilder}; use martin::db::{check_postgis_version, get_connection, setup_connection_pool, Pool}; -use martin::function_source::get_function_sources; -use martin::table_source::get_table_sources; +use martin::function_source::{get_function_sources, FunctionSources}; +use martin::table_source::{get_table_sources, TableSource, TableSources}; use martin::{prettify_error, server}; use serde::Deserialize; @@ -14,37 +14,34 @@ const VERSION: &str = env!("CARGO_PKG_VERSION"); const REQUIRED_POSTGIS_VERSION: &str = ">= 2.4.0"; #[derive(Parser, Debug, Deserialize)] -#[clap(about, version)] +#[command(about, version)] pub struct Args { /// Path to config file. - #[clap(short, long)] + #[arg(short, long)] pub config: Option, /// Loads trusted root certificates from a file. The file should contain a sequence of PEM-formatted CA certificates. - #[clap(long)] + #[arg(long)] pub ca_root_file: Option, /// Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. - #[clap(long)] + #[arg(long)] pub danger_accept_invalid_certs: bool, /// If a spatial table has SRID 0, then this default SRID will be used as a fallback. - #[clap(short, long)] + #[arg(short, long)] pub default_srid: Option, - // Must match ConfigBuilder::KEEP_ALIVE_DEFAULT - /// Connection keep alive timeout. [DEFAULT: 75] - #[clap(short, long)] + #[arg(short, long, + help = format ! ("Connection keep alive timeout. [DEFAULT: {}]", ConfigBuilder::KEEP_ALIVE_DEFAULT))] pub keep_alive: Option, - // Must match ConfigBuilder::LISTEN_ADDRESSES_DEFAULT - /// The socket address to bind. [DEFAULT: 0.0.0.0:3000] - #[clap(short, long)] + #[arg(short, long, + help = format ! ("The socket address to bind. [DEFAULT: {}]", ConfigBuilder::LISTEN_ADDRESSES_DEFAULT))] pub listen_addresses: Option, - // Must match ConfigBuilder::POOL_SIZE_DEFAULT - /// Maximum connections pool size [DEFAULT: 20] - #[clap(short, long)] + #[arg(short, long, + help = format ! ("Maximum connections pool size [DEFAULT: {}]", ConfigBuilder::POOL_SIZE_DEFAULT))] pub pool_size: Option, /// Scan for new sources on sources list requests - #[clap(short, long, hide = true)] + #[arg(short, long, hide = true)] pub watch: bool, /// Number of web server workers - #[clap(short = 'W', long)] + #[arg(short = 'W', long)] pub workers: Option, /// Database connection string pub connection: Option, @@ -71,36 +68,6 @@ impl Args { } } -pub async fn generate_config(args: Args, pool: &Pool) -> io::Result { - // let connection_string = args.arg_connection.clone().ok_or_else(|| { - // io::Error::new( - // io::ErrorKind::Other, - // "Database connection string is not set", - // ) - // })?; - let connection_string = args.connection.clone(); - - let mut connection = get_connection(pool).await?; - let table_sources = get_table_sources(&mut connection, &args.default_srid).await?; - let function_sources = get_function_sources(&mut connection).await?; - - let config = ConfigBuilder { - ca_root_file: None, - danger_accept_invalid_certs: if args.danger_accept_invalid_certs { Some(true) } else { None }, - default_srid: args.default_srid, - keep_alive: args.keep_alive, - listen_addresses: args.listen_addresses, - pool_size: args.pool_size, - worker_processes: args.workers, - connection_string, - table_sources: Some(table_sources), - function_sources: Some(function_sources), - }; - - let config = config.finalize(); - Ok(config) -} - async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { let config = read_config(&file_name).map_err(|e| prettify_error!(e, "Can't read config"))?; @@ -110,8 +77,8 @@ async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { Some(config.pool_size), config.danger_accept_invalid_certs, ) - .await - .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; + .await + .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; if let Some(table_sources) = &config.table_sources { for table_source in table_sources.values() { @@ -154,14 +121,18 @@ async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { args.pool_size, args.danger_accept_invalid_certs, ) - .await - .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; - - info!("Scanning database"); - let config = generate_config(args, &pool) - .await - .map_err(|e| prettify_error!(e, "Can't generate config"))?; - + .await + .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; + + let mut config = args.to_config(); + config.ca_root_file = None; + { + info!("Scanning database"); + let mut connection = get_connection(&pool).await?; + config.table_sources = Some(get_table_sources(&mut connection, config.default_srid).await?); + config.function_sources = Some(get_function_sources(&mut connection).await?); + } + let config = config.finalize(); Ok((config, pool)) } @@ -202,7 +173,7 @@ fn parse_env(args: Args) -> Args { } async fn start(args: Args) -> io::Result { - info!("Starting martin v{VERSION}"); + info!("Starting Martin v{VERSION}"); let (config, pool) = match args.config { Some(config_file_name) => { diff --git a/src/config.rs b/src/config.rs index d27c54db6..be610c2e7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -50,7 +50,10 @@ impl ConfigBuilder { pub fn merge(&mut self, other: ConfigBuilder) -> &mut Self { set_option(&mut self.ca_root_file, other.ca_root_file); - set_option(&mut self.danger_accept_invalid_certs, other.danger_accept_invalid_certs); + set_option( + &mut self.danger_accept_invalid_certs, + other.danger_accept_invalid_certs, + ); set_option(&mut self.default_srid, other.default_srid); set_option(&mut self.keep_alive, other.keep_alive); set_option(&mut self.listen_addresses, other.listen_addresses); diff --git a/src/table_source.rs b/src/table_source.rs index bf13912b3..9cdbc7afc 100644 --- a/src/table_source.rs +++ b/src/table_source.rs @@ -188,7 +188,7 @@ static DEFAULT_CLIP_GEOM: bool = true; pub async fn get_table_sources( conn: &mut Connection<'_>, - default_srid: &Option, + default_srid: Option, ) -> Result { let mut sources = HashMap::new(); let mut duplicate_source_ids = HashSet::new(); @@ -221,7 +221,7 @@ pub async fn get_table_sources( match default_srid { Some(default_srid) => { warn!(r#""{id}" has SRID 0, using the provided default SRID {default_srid}"#); - srid = *default_srid; + srid = default_srid; } None => { warn!( diff --git a/tests/table_source_test.rs b/tests/table_source_test.rs index e8e4f2dce..af6b45914 100644 --- a/tests/table_source_test.rs +++ b/tests/table_source_test.rs @@ -14,7 +14,7 @@ async fn test_get_table_sources_ok() { let pool = dev::make_pool().await; let mut connection = pool.get().await.unwrap(); - let table_sources = get_table_sources(&mut connection, &None).await.unwrap(); + let table_sources = get_table_sources(&mut connection, None).await.unwrap(); log::info!("table_sources = {table_sources:#?}"); @@ -46,7 +46,7 @@ async fn test_table_source_tilejson_ok() { let pool = dev::make_pool().await; let mut connection = pool.get().await.unwrap(); - let table_sources = get_table_sources(&mut connection, &None).await.unwrap(); + let table_sources = get_table_sources(&mut connection, None).await.unwrap(); let table_source = table_sources.get("public.table_source").unwrap(); let tilejson = table_source.get_tilejson().await.unwrap(); @@ -69,7 +69,7 @@ async fn test_table_source_tile_ok() { let pool = dev::make_pool().await; let mut connection = pool.get().await.unwrap(); - let table_sources = get_table_sources(&mut connection, &None).await.unwrap(); + let table_sources = get_table_sources(&mut connection, None).await.unwrap(); let table_source = table_sources.get("public.table_source").unwrap(); let tile = table_source @@ -86,7 +86,7 @@ async fn test_table_source_srid_ok() { let pool = dev::make_pool().await; let mut connection = pool.get().await.unwrap(); - let table_sources = get_table_sources(&mut connection, &Some(900913)) + let table_sources = get_table_sources(&mut connection, Some(900913)) .await .unwrap(); @@ -113,7 +113,7 @@ async fn test_table_source_multiple_geom_ok() { let pool = dev::make_pool().await; let mut connection = pool.get().await.unwrap(); - let table_sources = get_table_sources(&mut connection, &None).await.unwrap(); + let table_sources = get_table_sources(&mut connection, None).await.unwrap(); assert!(table_sources.contains_key("public.table_source_multiple_geom")); let table_source_multiple_geom = table_sources From a74cedc73603cb9c1c0452e0b939db8580476d32 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 26 Sep 2022 16:19:36 -0400 Subject: [PATCH 3/9] cleanup config --- Cargo.lock | 34 +++++----- README.md | 22 +++--- src/bin/main.rs | 170 ++++++++++++---------------------------------- src/config.rs | 8 +-- tests/config.yaml | 22 +++--- 5 files changed, 87 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d941adb3..c32370841 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,9 +469,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.0-rc.2" +version = "4.0.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d5891a0b7bbd946e91a79c350b5475764d7d7eda9c71141059a9d2ab0853e5" +checksum = "7233bf306993c874a6edc363281e83770889877c9d5ee7f656249c65d7e7aa62" dependencies = [ "atty", "bitflags", @@ -521,9 +521,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" +checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" dependencies = [ "percent-encoding", "time", @@ -970,9 +970,9 @@ checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "jobserver" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" dependencies = [ "libc", ] @@ -1054,7 +1054,7 @@ dependencies = [ "bb8", "bb8-postgres", "cargo-husky", - "clap 4.0.0-rc.2", + "clap 4.0.0-rc.3", "criterion", "env_logger", "itertools", @@ -1075,9 +1075,9 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b48670c893079d3c2ed79114e3644b7004df1c361a4e0ad52e2e6940d07c3d" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ "digest", ] @@ -1411,9 +1411,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +checksum = "7bd7356a8122b6c4a24a82b278680c73357984ca2fc79a0f9fa6dea7dced7c58" dependencies = [ "unicode-ident", ] @@ -1545,18 +1545,18 @@ checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" dependencies = [ "proc-macro2", "quote", @@ -1706,9 +1706,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.100" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" +checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" dependencies = [ "proc-macro2", "quote", diff --git a/README.md b/README.md index 2c641dd43..b9d611850 100755 --- a/README.md +++ b/README.md @@ -427,26 +427,26 @@ martin --config config.yaml You can find an example of a configuration file [here](https://github.com/maplibre/martin/blob/main/tests/config.yaml). ```yaml -# The socket address to bind [default: 0.0.0.0:3000] -listen_addresses: '0.0.0.0:3000' - # Database connection string connection_string: 'postgres://postgres@localhost:5432/db' -# Maximum connections pool size [default: 20] -pool_size: 20 +# Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. +danger_accept_invalid_certs: false + +# If a spatial table has SRID 0, then this SRID will be used as a fallback +default_srid: 4326 # Connection keep alive timeout [default: 75] keep_alive: 75 -# Number of web server workers -worker_processes: 8 +# The socket address to bind [default: 0.0.0.0:3000] +listen_addresses: '0.0.0.0:3000' -# If a spatial table has SRID 0, then this default SRID will be used as a fallback -default_srid: 4326 +# Maximum connections pool size [default: 20] +pool_size: 20 -# Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. -danger_accept_invalid_certs: false +# Number of web server workers +worker_processes: 8 # Associative arrays of table sources table_sources: diff --git a/src/bin/main.rs b/src/bin/main.rs index 4e1b233b3..f680dae44 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -5,8 +5,8 @@ use clap::Parser; use log::{error, info, warn}; use martin::config::{read_config, Config, ConfigBuilder}; use martin::db::{check_postgis_version, get_connection, setup_connection_pool, Pool}; -use martin::function_source::{get_function_sources, FunctionSources}; -use martin::table_source::{get_table_sources, TableSource, TableSources}; +use martin::function_source::get_function_sources; +use martin::table_source::get_table_sources; use martin::{prettify_error, server}; use serde::Deserialize; @@ -16,6 +16,8 @@ const REQUIRED_POSTGIS_VERSION: &str = ">= 2.4.0"; #[derive(Parser, Debug, Deserialize)] #[command(about, version)] pub struct Args { + /// Database connection string + pub connection: Option, /// Path to config file. #[arg(short, long)] pub config: Option, @@ -43,25 +45,44 @@ pub struct Args { /// Number of web server workers #[arg(short = 'W', long)] pub workers: Option, - /// Database connection string - pub connection: Option, } -impl Args { - fn to_config(self) -> ConfigBuilder { +impl From for ConfigBuilder { + fn from(args: Args) -> Self { + if args.watch { + warn!("The --watch flag is no longer supported, and will be ignored"); + } + if env::var_os("WATCH_MODE").is_some() { + warn!( + "The WATCH_MODE environment variable is no longer supported, and will be ignored" + ); + } + ConfigBuilder { - ca_root_file: self.ca_root_file, - danger_accept_invalid_certs: if self.danger_accept_invalid_certs { + connection_string: args.connection.or_else(|| { + env::var_os("DATABASE_URL").and_then(|connection| connection.into_string().ok()) + }), + ca_root_file: args.ca_root_file.or_else(|| { + env::var_os("CA_ROOT_FILE").and_then(|connection| connection.into_string().ok()) + }), + danger_accept_invalid_certs: if args.danger_accept_invalid_certs + || env::var_os("DANGER_ACCEPT_INVALID_CERTS").is_some() + { Some(true) } else { None }, - default_srid: self.default_srid, - keep_alive: self.keep_alive, - listen_addresses: self.listen_addresses, - pool_size: self.pool_size, - worker_processes: self.workers, - connection_string: self.connection, + default_srid: args.default_srid.or_else(|| { + env::var_os("DEFAULT_SRID").and_then(|srid| { + srid.into_string() + .ok() + .and_then(|srid| srid.parse::().ok()) + }) + }), + keep_alive: args.keep_alive, + listen_addresses: args.listen_addresses, + pool_size: args.pool_size, + worker_processes: args.workers, table_sources: None, function_sources: None, } @@ -124,8 +145,7 @@ async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { .await .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; - let mut config = args.to_config(); - config.ca_root_file = None; + let mut config = ConfigBuilder::from(args); { info!("Scanning database"); let mut connection = get_connection(&pool).await?; @@ -136,42 +156,6 @@ async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { Ok((config, pool)) } -fn parse_env(args: Args) -> Args { - let connection = args.connection.or_else(|| { - env::var_os("DATABASE_URL").and_then(|connection| connection.into_string().ok()) - }); - - let default_srid = args.default_srid.or_else(|| { - env::var_os("DEFAULT_SRID").and_then(|srid| { - srid.into_string() - .ok() - .and_then(|srid| srid.parse::().ok()) - }) - }); - - let ca_root_file = args.ca_root_file.or_else(|| { - env::var_os("CA_ROOT_FILE").and_then(|connection| connection.into_string().ok()) - }); - - let danger_accept_invalid_certs = - args.danger_accept_invalid_certs || env::var_os("DANGER_ACCEPT_INVALID_CERTS").is_some(); - - if args.watch { - warn!("The --watch flag is no longer supported, and will be ignored"); - } - if env::var_os("WATCH_MODE").is_some() { - warn!("The WATCH_MODE environment variable is no longer supported, and will be ignored"); - } - - Args { - connection, - default_srid, - ca_root_file, - danger_accept_invalid_certs, - ..args - } -} - async fn start(args: Args) -> io::Result { info!("Starting Martin v{VERSION}"); @@ -181,7 +165,7 @@ async fn start(args: Args) -> io::Result { setup_from_config(config_file_name).await? } None => { - info!("Config is not set"); + info!("Config file is not specified"); setup_from_args(args).await? } }; @@ -201,83 +185,17 @@ async fn start(args: Args) -> io::Result { Ok(server) } -fn print_type_of(_: &T) { - println!("{}", std::any::type_name::()) -} - #[actix_web::main] async fn main() -> io::Result<()> { let env = env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "martin=info"); env_logger::Builder::from_env(env).init(); let args: Args = Args::parse(); - println!("{:?}", &args); - let parse: ConfigBuilder = args.to_config(); - print_type_of(&parse); - println!("{:?}", &parse); - // parse.map(|args| { - // let args = parse_env(args); - // let runner = start(args)?; - // runner.run(); - // }); - // let clap_matches = Args::command().ignore_errors(true).get_matches(); - // print_type_of(&clap_matches); // clap::parse::matches::arg_matches::ArgMatches - - // let mut v = Viperus::new(); - // v.load_clap(clap_matches).unwrap(); - // let config = clap_matches.value_of("config"); - // if config { - // v.load_file(config, Format::YAML).unwrap(); - // } - // debug!("final {:?}", v); - // - // // let x = args.value_of("foo"); - // // let args = Docopt::new(USAGE) - // // .and_then(|d| d.help(false).deserialize::()) - // // .map_err(prettify_error("Can't parse CLI arguments".to_owned()))?; - // - // // let args = parse_env(args); - // // let args = Args::parse(); - // let args = Args { - // config: v.get("config").unwrap(), - // keep_alive: v.get("keep_alive").unwrap(), - // listen_addresses: v.get("listen_addresses").unwrap(), - // default_srid: v.get("default_srid").unwrap(), - // pool_size: v.get("pool_size").unwrap(), - // workers: v.get("workers").unwrap(), - // ca_root_file: v.get("ca_root_file").unwrap(), - // danger_accept_invalid_certs: v.get("danger_accept_invalid_certs").unwrap(), - // connection: v.get("connection").unwrap(), - // }; - // - // - // if args.danger_accept_invalid_certs { - // warn!("Danger accept invalid certs enabled. You should think very carefully before using this option. If invalid certificates are trusted, any certificate for any site will be trusted for use. This includes expired certificates. This introduces significant vulnerabilities, and should only be used as a last resort."); - // } - // - // let server = match start(args) { - // Ok(server) => server, - // Err(error) => { - // error!("{error}"); - // std::process::exit(-1); - // } - // }; - // - // server.run() - // let server = match start(args) { - // Ok(server) => server, - // Err(error) => { - // error!("{error}"); - // std::process::exit(-1); - // } - // }; - - // match start(args).await { - // Ok(server) => server.await, - // Err(error) => { - // error!("{error}"); - // std::process::exit(-1); - // } - // } - Ok(()) + match start(args).await { + Ok(server) => server.await, + Err(error) => { + error!("{error}"); + std::process::exit(-1); + } + } } diff --git a/src/config.rs b/src/config.rs index be610c2e7..27affe098 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,6 +10,7 @@ use crate::utils::prettify_error; #[derive(Clone, Debug, Serialize)] pub struct Config { + pub connection_string: String, pub ca_root_file: Option, pub danger_accept_invalid_certs: bool, pub default_srid: Option, @@ -17,13 +18,13 @@ pub struct Config { pub listen_addresses: String, pub pool_size: u32, pub worker_processes: usize, - pub connection_string: String, pub table_sources: Option, pub function_sources: Option, } #[derive(Debug, Deserialize)] pub struct ConfigBuilder { + pub connection_string: Option, pub ca_root_file: Option, pub danger_accept_invalid_certs: Option, pub default_srid: Option, @@ -31,7 +32,6 @@ pub struct ConfigBuilder { pub listen_addresses: Option, pub pool_size: Option, pub worker_processes: Option, - pub connection_string: Option, pub table_sources: Option, pub function_sources: Option, } @@ -49,6 +49,7 @@ impl ConfigBuilder { pub const POOL_SIZE_DEFAULT: u32 = 20; pub fn merge(&mut self, other: ConfigBuilder) -> &mut Self { + set_option(&mut self.connection_string, other.connection_string); set_option(&mut self.ca_root_file, other.ca_root_file); set_option( &mut self.danger_accept_invalid_certs, @@ -59,7 +60,6 @@ impl ConfigBuilder { set_option(&mut self.listen_addresses, other.listen_addresses); set_option(&mut self.pool_size, other.pool_size); set_option(&mut self.worker_processes, other.worker_processes); - set_option(&mut self.connection_string, other.connection_string); set_option(&mut self.table_sources, other.table_sources); set_option(&mut self.function_sources, other.function_sources); self @@ -67,6 +67,7 @@ impl ConfigBuilder { pub fn finalize(self) -> Config { Config { + connection_string: self.connection_string.unwrap(), ca_root_file: self.ca_root_file, danger_accept_invalid_certs: self.danger_accept_invalid_certs.unwrap_or(false), default_srid: self.default_srid, @@ -76,7 +77,6 @@ impl ConfigBuilder { .unwrap_or_else(|| Self::LISTEN_ADDRESSES_DEFAULT.to_owned()), pool_size: self.pool_size.unwrap_or(Self::POOL_SIZE_DEFAULT), worker_processes: self.worker_processes.unwrap_or_else(num_cpus::get), - connection_string: self.connection_string.unwrap(), table_sources: self.table_sources, function_sources: self.function_sources, } diff --git a/tests/config.yaml b/tests/config.yaml index b6fe192d4..d6814e6de 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -1,24 +1,24 @@ --- -# The socket address to bind [default: 0.0.0.0:3000] -listen_addresses: '0.0.0.0:3000' - # Database connection string connection_string: 'postgres://postgres@localhost:5432/db' -# Maximum connections pool size [default: 20] -pool_size: 20 +# Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. +danger_accept_invalid_certs: false + +# If a spatial table has SRID 0, then this SRID will be used as a fallback +default_srid: 4326 # Connection keep alive timeout [default: 75] keep_alive: 75 -# Number of web server workers -worker_processes: 8 +# The socket address to bind [default: 0.0.0.0:3000] +listen_addresses: '0.0.0.0:3000' -# If a spatial table has SRID 0, then this SRID will be used as a fallback -default_srid: 4326 +# Maximum connections pool size [default: 20] +pool_size: 20 -# Trust invalid certificates. This introduces significant vulnerabilities, and should only be used as a last resort. -danger_accept_invalid_certs: false +# Number of web server workers +worker_processes: 8 # Associative arrays of table sources table_sources: From c764fe10370040d4a40b5e9b8b798864e0ed3c8a Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 26 Sep 2022 17:07:37 -0400 Subject: [PATCH 4/9] more refactorings --- src/bin/main.rs | 22 +++------------------- src/config.rs | 4 +--- src/db.rs | 20 ++++++-------------- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index f680dae44..95367069e 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -4,7 +4,7 @@ use actix_web::dev::Server; use clap::Parser; use log::{error, info, warn}; use martin::config::{read_config, Config, ConfigBuilder}; -use martin::db::{check_postgis_version, get_connection, setup_connection_pool, Pool}; +use martin::db::{assert_postgis_version, get_connection, setup_connection_pool, Pool}; use martin::function_source::get_function_sources; use martin::table_source::get_table_sources; use martin::{prettify_error, server}; @@ -91,7 +91,6 @@ impl From for ConfigBuilder { async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { let config = read_config(&file_name).map_err(|e| prettify_error!(e, "Can't read config"))?; - let pool = setup_connection_pool( &config.connection_string, &config.ca_root_file, @@ -115,15 +114,12 @@ async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { ); } } - if let Some(function_sources) = &config.function_sources { for function_source in function_sources.values() { info!("Found {} function source", function_source.id); } } - info!("Connected to {}", config.connection_string); - Ok((config, pool)) } @@ -158,7 +154,6 @@ async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { async fn start(args: Args) -> io::Result { info!("Starting Martin v{VERSION}"); - let (config, pool) = match args.config { Some(config_file_name) => { info!("Using {config_file_name}"); @@ -169,19 +164,10 @@ async fn start(args: Args) -> io::Result { setup_from_args(args).await? } }; - - let matches = check_postgis_version(REQUIRED_POSTGIS_VERSION, &pool) - .await - .map_err(|e| prettify_error!(e, "Can't check PostGIS version"))?; - - if !matches { - std::process::exit(-1); - } - + assert_postgis_version(REQUIRED_POSTGIS_VERSION, &pool).await?; let listen_addresses = config.listen_addresses.clone(); let server = server::new(pool, config); info!("Martin has been started on {listen_addresses}."); - Ok(server) } @@ -189,9 +175,7 @@ async fn start(args: Args) -> io::Result { async fn main() -> io::Result<()> { let env = env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "martin=info"); env_logger::Builder::from_env(env).init(); - - let args: Args = Args::parse(); - match start(args).await { + match start(Args::parse()).await { Ok(server) => server.await, Err(error) => { error!("{error}"); diff --git a/src/config.rs b/src/config.rs index 27affe098..d3e77bd9d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -6,7 +6,6 @@ use serde::{Deserialize, Serialize}; use crate::function_source::FunctionSources; use crate::table_source::TableSources; -use crate::utils::prettify_error; #[derive(Clone, Debug, Serialize)] pub struct Config { @@ -87,9 +86,8 @@ pub fn read_config(file_name: &str) -> io::Result { let mut file = File::open(file_name)?; let mut contents = String::new(); file.read_to_string(&mut contents)?; - let config_builder: ConfigBuilder = serde_yaml::from_str(contents.as_str()) - .map_err(|e| prettify_error!(e, "Can't read config file"))?; + .map_err(|e| ::std::io::Error::new(::std::io::ErrorKind::Other, e))?; Ok(config_builder.finalize()) } diff --git a/src/db.rs b/src/db.rs index 5127296ef..22d7f5b93 100755 --- a/src/db.rs +++ b/src/db.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use bb8::PooledConnection; use bb8_postgres::{tokio_postgres, PostgresConnectionManager}; -use log::{error, info}; +use log::info; use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode}; use postgres_openssl::MakeTlsConnector; use semver::{Version, VersionReq}; @@ -77,23 +77,15 @@ pub async fn select_postgis_version(pool: &Pool) -> io::Result { Ok(version) } -pub async fn check_postgis_version( - required_postgis_version: &str, - pool: &Pool, -) -> io::Result { +pub async fn assert_postgis_version(required_postgis_version: &str, pool: &Pool) -> io::Result<()> { let postgis_version = select_postgis_version(pool).await?; - let req = VersionReq::parse(required_postgis_version) .map_err(|e| prettify_error!(e, "Can't parse required PostGIS version"))?; - let version = Version::parse(postgis_version.as_str()) .map_err(|e| prettify_error!(e, "Can't parse database PostGIS version"))?; - - let matches = req.matches(&version); - - if !matches { - error!("Martin requires PostGIS {required_postgis_version}, current version is {postgis_version}"); + if !req.matches(&version) { + Err(io::Error::new(io::ErrorKind::Other, format!("Martin requires PostGIS {required_postgis_version}, current version is {postgis_version}"))) + } else { + Ok(()) } - - Ok(matches) } From 4ad32f40a55a09799a58bc738e3cc9a0a3e6e4de Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 26 Sep 2022 21:46:03 -0400 Subject: [PATCH 5/9] more refactorings --- src/bin/main.rs | 95 +++++++----------------------------------- src/config.rs | 33 +++++++++------ src/db.rs | 71 ++++++++++++++++++++++++++++--- src/dev.rs | 2 +- src/function_source.rs | 7 ---- src/table_source.rs | 15 +------ 6 files changed, 105 insertions(+), 118 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index 95367069e..ccddc40fb 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -3,15 +3,12 @@ use std::{env, io}; use actix_web::dev::Server; use clap::Parser; use log::{error, info, warn}; -use martin::config::{read_config, Config, ConfigBuilder}; -use martin::db::{assert_postgis_version, get_connection, setup_connection_pool, Pool}; -use martin::function_source::get_function_sources; -use martin::table_source::get_table_sources; -use martin::{prettify_error, server}; +use martin::config::{read_config, ConfigBuilder}; +use martin::db::configure_db_source; +use martin::server; use serde::Deserialize; const VERSION: &str = env!("CARGO_PKG_VERSION"); -const REQUIRED_POSTGIS_VERSION: &str = ">= 2.4.0"; #[derive(Parser, Debug, Deserialize)] #[command(about, version)] @@ -89,84 +86,24 @@ impl From for ConfigBuilder { } } -async fn setup_from_config(file_name: String) -> io::Result<(Config, Pool)> { - let config = read_config(&file_name).map_err(|e| prettify_error!(e, "Can't read config"))?; - let pool = setup_connection_pool( - &config.connection_string, - &config.ca_root_file, - Some(config.pool_size), - config.danger_accept_invalid_certs, - ) - .await - .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; - - if let Some(table_sources) = &config.table_sources { - for table_source in table_sources.values() { - info!( - r#"Found "{}" table source with "{}" column ({}, SRID={})"#, - table_source.id, - table_source.geometry_column, - table_source - .geometry_type - .as_ref() - .unwrap_or(&"null".to_string()), - table_source.srid - ); - } - } - if let Some(function_sources) = &config.function_sources { - for function_source in function_sources.values() { - info!("Found {} function source", function_source.id); - } - } - info!("Connected to {}", config.connection_string); - Ok((config, pool)) -} - -async fn setup_from_args(args: Args) -> io::Result<(Config, Pool)> { - let connection_string = args.connection.clone().ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "Database connection string is not set", - ) - })?; - - info!("Connecting to database"); - let pool = setup_connection_pool( - &connection_string, - &args.ca_root_file, - args.pool_size, - args.danger_accept_invalid_certs, - ) - .await - .map_err(|e| prettify_error!(e, "Can't setup connection pool"))?; - - let mut config = ConfigBuilder::from(args); - { - info!("Scanning database"); - let mut connection = get_connection(&pool).await?; - config.table_sources = Some(get_table_sources(&mut connection, config.default_srid).await?); - config.function_sources = Some(get_function_sources(&mut connection).await?); - } - let config = config.finalize(); - Ok((config, pool)) -} - async fn start(args: Args) -> io::Result { info!("Starting Martin v{VERSION}"); - let (config, pool) = match args.config { - Some(config_file_name) => { - info!("Using {config_file_name}"); - setup_from_config(config_file_name).await? - } - None => { - info!("Config file is not specified"); - setup_from_args(args).await? - } + + let mut config = if let Some(ref config_file_name) = args.config { + info!("Using {config_file_name}"); + let cfg = read_config(config_file_name)?; + let mut builder = ConfigBuilder::from(args); + builder.merge(cfg); + builder.finalize()? + } else { + info!("Config file is not specified"); + ConfigBuilder::from(args).finalize()? }; - assert_postgis_version(REQUIRED_POSTGIS_VERSION, &pool).await?; + + let pool = configure_db_source(&mut config).await?; let listen_addresses = config.listen_addresses.clone(); let server = server::new(pool, config); + info!("Martin has been started on {listen_addresses}."); Ok(server) } diff --git a/src/config.rs b/src/config.rs index d3e77bd9d..8b5da6ada 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,6 +5,7 @@ use std::io::prelude::*; use serde::{Deserialize, Serialize}; use crate::function_source::FunctionSources; +use crate::prettify_error; use crate::table_source::TableSources; #[derive(Clone, Debug, Serialize)] @@ -17,6 +18,7 @@ pub struct Config { pub listen_addresses: String, pub pool_size: u32, pub worker_processes: usize, + pub use_dynamic_sources: bool, pub table_sources: Option, pub function_sources: Option, } @@ -64,11 +66,17 @@ impl ConfigBuilder { self } - pub fn finalize(self) -> Config { - Config { - connection_string: self.connection_string.unwrap(), + pub fn finalize(self) -> io::Result { + let connection_string = self.connection_string.ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "Database connection string is not set", + ) + })?; + Ok(Config { + connection_string, ca_root_file: self.ca_root_file, - danger_accept_invalid_certs: self.danger_accept_invalid_certs.unwrap_or(false), + danger_accept_invalid_certs: self.danger_accept_invalid_certs.unwrap_or_default(), default_srid: self.default_srid, keep_alive: self.keep_alive.unwrap_or(Self::KEEP_ALIVE_DEFAULT), listen_addresses: self @@ -76,18 +84,19 @@ impl ConfigBuilder { .unwrap_or_else(|| Self::LISTEN_ADDRESSES_DEFAULT.to_owned()), pool_size: self.pool_size.unwrap_or(Self::POOL_SIZE_DEFAULT), worker_processes: self.worker_processes.unwrap_or_else(num_cpus::get), + use_dynamic_sources: self.table_sources.is_none() && self.function_sources.is_none(), table_sources: self.table_sources, function_sources: self.function_sources, - } + }) } } -pub fn read_config(file_name: &str) -> io::Result { - let mut file = File::open(file_name)?; +pub fn read_config(file_name: &str) -> io::Result { + let mut file = File::open(file_name) + .map_err(|e| prettify_error!(e, "Unable to open config file '{}'", file_name))?; let mut contents = String::new(); - file.read_to_string(&mut contents)?; - let config_builder: ConfigBuilder = serde_yaml::from_str(contents.as_str()) - .map_err(|e| ::std::io::Error::new(::std::io::ErrorKind::Other, e))?; - - Ok(config_builder.finalize()) + file.read_to_string(&mut contents) + .map_err(|e| prettify_error!(e, "Unable to read config file '{}'", file_name))?; + serde_yaml::from_str(contents.as_str()) + .map_err(|e| prettify_error!(e, "Error parsing config file '{}'", file_name)) } diff --git a/src/db.rs b/src/db.rs index 22d7f5b93..94b11c9a7 100755 --- a/src/db.rs +++ b/src/db.rs @@ -1,6 +1,9 @@ use std::io; use std::str::FromStr; +use crate::config::Config; +use crate::function_source::get_function_sources; +use crate::table_source::get_table_sources; use bb8::PooledConnection; use bb8_postgres::{tokio_postgres, PostgresConnectionManager}; use log::info; @@ -14,6 +17,8 @@ pub type ConnectionManager = PostgresConnectionManager; pub type Pool = bb8::Pool; pub type Connection<'a> = PooledConnection<'a, ConnectionManager>; +const REQUIRED_POSTGIS_VERSION: &str = ">= 2.4.0"; + fn make_tls_connector( ca_root_file: &Option, danger_accept_invalid_certs: bool, @@ -36,7 +41,7 @@ fn make_tls_connector( pub async fn setup_connection_pool( connection_string: &str, ca_root_file: &Option, - pool_size: Option, + pool_size: u32, danger_accept_invalid_certs: bool, ) -> io::Result { let config = tokio_postgres::config::Config::from_str(connection_string) @@ -48,7 +53,7 @@ pub async fn setup_connection_pool( let manager = PostgresConnectionManager::new(config, tls_connector); let pool = Pool::builder() - .max_size(pool_size.unwrap_or(20)) + .max_size(pool_size) .build(manager) .await .map_err(|e| prettify_error!(e, "Can't build connection pool"))?; @@ -65,7 +70,7 @@ pub async fn get_connection(pool: &Pool) -> io::Result> { Ok(connection) } -pub async fn select_postgis_version(pool: &Pool) -> io::Result { +async fn select_postgis_version(pool: &Pool) -> io::Result { let connection = get_connection(pool).await?; let version = connection @@ -77,15 +82,69 @@ pub async fn select_postgis_version(pool: &Pool) -> io::Result { Ok(version) } -pub async fn assert_postgis_version(required_postgis_version: &str, pool: &Pool) -> io::Result<()> { +async fn validate_postgis_version(pool: &Pool) -> io::Result<()> { let postgis_version = select_postgis_version(pool).await?; - let req = VersionReq::parse(required_postgis_version) + let req = VersionReq::parse(REQUIRED_POSTGIS_VERSION) .map_err(|e| prettify_error!(e, "Can't parse required PostGIS version"))?; let version = Version::parse(postgis_version.as_str()) .map_err(|e| prettify_error!(e, "Can't parse database PostGIS version"))?; if !req.matches(&version) { - Err(io::Error::new(io::ErrorKind::Other, format!("Martin requires PostGIS {required_postgis_version}, current version is {postgis_version}"))) + Err(io::Error::new(io::ErrorKind::Other, format!("Martin requires PostGIS {REQUIRED_POSTGIS_VERSION}, current version is {postgis_version}"))) } else { Ok(()) } } + +pub async fn configure_db_source(mut config: &mut Config) -> io::Result { + info!("Connecting to database"); + let pool = setup_connection_pool( + &config.connection_string, + &config.ca_root_file, + config.pool_size, + config.danger_accept_invalid_certs, + ) + .await?; + + validate_postgis_version(&pool).await?; + + let info_prefix = if config.use_dynamic_sources { + info!("Automatically detecting table and function sources"); + let mut connection = get_connection(&pool).await?; + + let sources = get_table_sources(&mut connection, config.default_srid).await?; + if sources.is_empty() { + info!("No table sources found"); + } else { + config.table_sources = Some(sources); + } + + let sources = get_function_sources(&mut connection).await?; + if sources.is_empty() { + info!("No function sources found"); + } else { + config.function_sources = Some(sources); + } + + "Found" + } else { + "Loaded" + }; + + if let Some(table_sources) = &config.table_sources { + for table_source in table_sources.values() { + info!( + r#"{info_prefix} "{}" table source with "{}" column ({}, SRID={})"#, + table_source.id, + table_source.geometry_column, + table_source.geometry_type.as_deref().unwrap_or("null"), + table_source.srid + ); + } + } + if let Some(function_sources) = &config.function_sources { + for function_source in function_sources.values() { + info!("{info_prefix} {} function source", function_source.id); + } + } + Ok(pool) +} diff --git a/src/dev.rs b/src/dev.rs index 7fab65d32..9ece78862 100644 --- a/src/dev.rs +++ b/src/dev.rs @@ -166,7 +166,7 @@ pub async fn make_pool() -> Pool { let connection_string: String = env::var("DATABASE_URL").unwrap(); info!("Connecting to {connection_string}"); - let pool = setup_connection_pool(&connection_string, &None, Some(1), false) + let pool = setup_connection_pool(&connection_string, &None, 1, false) .await .unwrap(); info!("Connected to {connection_string}"); diff --git a/src/function_source.rs b/src/function_source.rs index 6f898f4fd..02cc88cdf 100644 --- a/src/function_source.rs +++ b/src/function_source.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::io; use async_trait::async_trait; -use log::info; use postgres::types::Type; use postgres_protocol::escape::escape_identifier; use serde::{Deserialize, Serialize}; @@ -139,8 +138,6 @@ pub async fn get_function_sources(conn: &mut Connection<'_>) -> Result) -> Result { @@ -266,10 +259,6 @@ pub async fn get_table_sources( sources.insert(explicit_id, Box::new(explicit_source)); } - if sources.is_empty() { - info!("No table sources found"); - } - if !duplicate_source_ids.is_empty() { let sources = duplicate_source_ids .into_iter() From 998d2144a33abd04fe4b3b08587060ad318deca1 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 26 Sep 2022 21:54:48 -0400 Subject: [PATCH 6/9] linting --- benches/sources.rs | 4 ++-- src/config.rs | 2 ++ src/db.rs | 6 +++--- src/table_source.rs | 21 +++++++++------------ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/benches/sources.rs b/benches/sources.rs index 1f3756cac..d5e0d2273 100644 --- a/benches/sources.rs +++ b/benches/sources.rs @@ -103,14 +103,14 @@ fn table_source(c: &mut Criterion) { fn composite_source(c: &mut Criterion) { c.bench_function("get_composite_source", |b| b.iter(get_composite_source)); c.bench_function("get_composite_source_tile", |b| { - b.iter(get_composite_source_tile) + b.iter(get_composite_source_tile); }); } fn function_source(c: &mut Criterion) { c.bench_function("get_function_source", |b| b.iter(get_function_source)); c.bench_function("get_function_source_tile", |b| { - b.iter(get_function_source_tile) + b.iter(get_function_source_tile); }); } diff --git a/src/config.rs b/src/config.rs index 8b5da6ada..742d9585b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -66,6 +66,7 @@ impl ConfigBuilder { self } + /// Apply defaults to the config, and validate if there is a connection string pub fn finalize(self) -> io::Result { let connection_string = self.connection_string.ok_or_else(|| { io::Error::new( @@ -91,6 +92,7 @@ impl ConfigBuilder { } } +/// Read config from a file pub fn read_config(file_name: &str) -> io::Result { let mut file = File::open(file_name) .map_err(|e| prettify_error!(e, "Unable to open config file '{}'", file_name))?; diff --git a/src/db.rs b/src/db.rs index 94b11c9a7..51ea82892 100755 --- a/src/db.rs +++ b/src/db.rs @@ -88,10 +88,10 @@ async fn validate_postgis_version(pool: &Pool) -> io::Result<()> { .map_err(|e| prettify_error!(e, "Can't parse required PostGIS version"))?; let version = Version::parse(postgis_version.as_str()) .map_err(|e| prettify_error!(e, "Can't parse database PostGIS version"))?; - if !req.matches(&version) { - Err(io::Error::new(io::ErrorKind::Other, format!("Martin requires PostGIS {REQUIRED_POSTGIS_VERSION}, current version is {postgis_version}"))) - } else { + if req.matches(&version) { Ok(()) + } else { + Err(io::Error::new(io::ErrorKind::Other, format!("Martin requires PostGIS {REQUIRED_POSTGIS_VERSION}, current version is {postgis_version}"))) } } diff --git a/src/table_source.rs b/src/table_source.rs index fcbc26047..5bb178c1e 100644 --- a/src/table_source.rs +++ b/src/table_source.rs @@ -206,22 +206,19 @@ pub async fn get_table_sources( let explicit_id = format!("{schema}.{table}.{geometry_column}"); if sources.contains_key(&id) { - duplicate_source_ids.insert(id.to_owned()); + duplicate_source_ids.insert(id.clone()); } let mut srid: i32 = row.get("srid"); if srid == 0 { - match default_srid { - Some(default_srid) => { - warn!(r#""{id}" has SRID 0, using the provided default SRID {default_srid}"#); - srid = default_srid; - } - None => { - warn!( - r#""{id}" has SRID 0, skipping. To use this table source, you must specify the SRID using the config file or provide the default SRID"# - ); - continue; - } + if let Some(default_srid) = default_srid { + warn!(r#""{id}" has SRID 0, using the provided default SRID {default_srid}"#); + srid = default_srid; + } else { + warn!( + r#""{id}" has SRID 0, skipping. To use this table source, you must specify the SRID using the config file or provide the default SRID"# + ); + continue; } } From 2c2351e5c833a52dcdc61695028314e670920681 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 27 Sep 2022 12:43:40 -0400 Subject: [PATCH 7/9] linting --- benches/sources.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benches/sources.rs b/benches/sources.rs index d5e0d2273..1f3756cac 100644 --- a/benches/sources.rs +++ b/benches/sources.rs @@ -103,14 +103,14 @@ fn table_source(c: &mut Criterion) { fn composite_source(c: &mut Criterion) { c.bench_function("get_composite_source", |b| b.iter(get_composite_source)); c.bench_function("get_composite_source_tile", |b| { - b.iter(get_composite_source_tile); + b.iter(get_composite_source_tile) }); } fn function_source(c: &mut Criterion) { c.bench_function("get_function_source", |b| b.iter(get_function_source)); c.bench_function("get_function_source_tile", |b| { - b.iter(get_function_source_tile); + b.iter(get_function_source_tile) }); } From dd0f936d5e5601addcb49867d5911f91447a411e Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 27 Sep 2022 23:27:07 -0400 Subject: [PATCH 8/9] fmt --- src/bin/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index ccddc40fb..ae1da6506 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -28,13 +28,13 @@ pub struct Args { #[arg(short, long)] pub default_srid: Option, #[arg(short, long, - help = format ! ("Connection keep alive timeout. [DEFAULT: {}]", ConfigBuilder::KEEP_ALIVE_DEFAULT))] + help = format!("Connection keep alive timeout. [DEFAULT: {}]", ConfigBuilder::KEEP_ALIVE_DEFAULT))] pub keep_alive: Option, #[arg(short, long, - help = format ! ("The socket address to bind. [DEFAULT: {}]", ConfigBuilder::LISTEN_ADDRESSES_DEFAULT))] + help = format!("The socket address to bind. [DEFAULT: {}]", ConfigBuilder::LISTEN_ADDRESSES_DEFAULT))] pub listen_addresses: Option, #[arg(short, long, - help = format ! ("Maximum connections pool size [DEFAULT: {}]", ConfigBuilder::POOL_SIZE_DEFAULT))] + help = format!("Maximum connections pool size [DEFAULT: {}]", ConfigBuilder::POOL_SIZE_DEFAULT))] pub pool_size: Option, /// Scan for new sources on sources list requests #[arg(short, long, hide = true)] From c69741e8b9b9765210f9a499973e788a750f5e3e Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 27 Sep 2022 23:42:55 -0400 Subject: [PATCH 9/9] better help style --- src/bin/main.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index ae1da6506..7725ab86b 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -6,11 +6,10 @@ use log::{error, info, warn}; use martin::config::{read_config, ConfigBuilder}; use martin::db::configure_db_source; use martin::server; -use serde::Deserialize; const VERSION: &str = env!("CARGO_PKG_VERSION"); -#[derive(Parser, Debug, Deserialize)] +#[derive(Parser, Debug)] #[command(about, version)] pub struct Args { /// Database connection string @@ -27,14 +26,14 @@ pub struct Args { /// If a spatial table has SRID 0, then this default SRID will be used as a fallback. #[arg(short, long)] pub default_srid: Option, - #[arg(short, long, - help = format!("Connection keep alive timeout. [DEFAULT: {}]", ConfigBuilder::KEEP_ALIVE_DEFAULT))] + #[arg(help = format!("Connection keep alive timeout. [DEFAULT: {}]", ConfigBuilder::KEEP_ALIVE_DEFAULT), + short, long)] pub keep_alive: Option, - #[arg(short, long, - help = format!("The socket address to bind. [DEFAULT: {}]", ConfigBuilder::LISTEN_ADDRESSES_DEFAULT))] + #[arg(help = format!("The socket address to bind. [DEFAULT: {}]", ConfigBuilder::LISTEN_ADDRESSES_DEFAULT), + short, long)] pub listen_addresses: Option, - #[arg(short, long, - help = format!("Maximum connections pool size [DEFAULT: {}]", ConfigBuilder::POOL_SIZE_DEFAULT))] + #[arg(help = format!("Maximum connections pool size [DEFAULT: {}]", ConfigBuilder::POOL_SIZE_DEFAULT), + short, long)] pub pool_size: Option, /// Scan for new sources on sources list requests #[arg(short, long, hide = true)]