From 3043c2ceff7b7e80a8bee3c87c8c437a919c639d Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 13 Dec 2022 21:16:34 +0100 Subject: [PATCH 01/38] Refactor CLI --- docker/docker-compose.yml | 12 ++- src/bin/inx-chronicle/cli.rs | 151 +++++++++++++++++++++++------------ 2 files changed, 107 insertions(+), 56 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 621897d03..db986af2b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -33,13 +33,17 @@ services: - RUST_LOG=warn,inx_chronicle=debug tty: true command: - - "--config=config.toml" - - "--inx-url=http://hornet:9029" - "--mongodb-conn-str=mongodb://mongo:27017" + - "--mongodb-username=root" + - "--mongodb-password=root" + - "--inx-enabled=true" + - "--inx-url=http://hornet:9029" + - "--jwt-password=password" + - "--jwt-salt=saltines" - "--influxdb-url=http://influx:8086" + - "--influxdb-username=root" + - "--influxdb-password=password" - "--loki-url=http://loki:3100" - volumes: - - ../config.template.toml:/app/config.toml:ro influx: image: influxdb:1.8 diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index eef69c538..317c89470 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -10,38 +10,90 @@ use crate::config::{ChronicleConfig, ConfigError}; #[command(author, version, about, next_display_order = None)] pub struct ClArgs { /// The location of the configuration file. - #[arg(short, long, env = "CONFIG_PATH")] + #[arg(short, long)] pub config: Option, + /// MongoDb arguments. + #[command(flatten, next_help_heading = "MongoDb")] + pub mongodb: MongoDbArgs, + /// INX arguments. + #[cfg(feature = "inx")] + #[command(flatten, next_help_heading = "INX")] + pub inx: InxArgs, /// Rest API arguments. #[cfg(feature = "api")] - #[command(flatten)] + #[command(flatten, next_help_heading = "API")] pub api: ApiArgs, /// InfluxDb arguments. #[cfg(any(feature = "analytics", feature = "metrics"))] - #[command(flatten)] + #[command(flatten, next_help_heading = "InfluxDb")] pub influxdb: InfluxDbArgs, - /// INX arguments. - #[cfg(feature = "inx")] - #[command(flatten)] - pub inx: InxArgs, - /// MongoDb arguments. - #[command(flatten)] - pub mongodb: MongoDbArgs, /// Loki arguments. #[cfg(feature = "loki")] - #[command(flatten)] + #[command(flatten, next_help_heading = "Loki")] pub loki: LokiArgs, /// Subcommands. #[command(subcommand)] pub subcommand: Option, } +#[derive(Args, Debug)] +pub struct MongoDbArgs { + /// The MongoDb connection string. + #[arg(long, value_name = "CONN_STR", env = "MONGODB_CONN_STR", default_value = "mongodb://localhost:27017")] + pub mongodb_conn_str: Option, + /// The MongoDb username. + #[arg(long, value_name = "USERNAME", env = "MONGODB_USERNAME", default_value = "root")] + pub mongodb_username: Option, + /// The MongoDb password. + #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = "root")] + pub mongodb_password: Option, + /// The main database name. + #[arg(long, value_name = "NAME", default_value = "chronicle")] + pub mongodb_database_name: Option, + /// The MongoDb minimum pool size. + #[arg(long, value_name = "SIZE", default_value = "2")] + pub mongodb_min_pool_size: Option, +} + +#[cfg(feature = "inx")] +#[derive(Args, Debug)] +pub struct InxArgs { + /// Toggles the INX synchronization workflow. + #[arg(long, default_value = "true")] + pub inx_enabled: Option, + /// The address of the node INX interface Chronicle tries to connect to - if enabled. + #[arg(long, default_value = "http://localhost:9029")] + pub inx_url: Option, + /// Milestone at which synchronization should begin. If set to `1` Chronicle will try to sync back until the + /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. + #[arg(long, default_value = "0")] + pub inx_sync_start: Option, + /// Time to wait until a new connection attempt is made. + #[arg(long, default_value = "5s")] + pub inx_retry_interval: Option, + /// Maximum number of tries to establish an INX connection. + #[arg(long, default_value = "30")] + pub inx_retry_count: Option, +} + #[cfg(feature = "api")] #[derive(Args, Debug)] pub struct ApiArgs { /// Toggle REST API. - #[arg(long, env = "REST_API_ENABLED")] + #[arg(long, default_value = "true")] pub api_enabled: Option, + /// API listening port. + #[arg(long, default_value = "8042")] + pub api_port: Option, + /// CORS setting. + #[arg(long, default_value = "0.0.0.0")] + pub api_allow_origins: Option, + /// Public API routes. + #[arg(long = "public-route", value_name = "ROUTE", default_value = "api/core/v2/*")] + pub public_routes: Vec, + /// Maximum nubmer of results returned by a single API call. + #[arg(long, default_value = "1000")] + pub max_page_size: Option, /// JWT arguments. #[command(flatten)] pub jwt: JwtArgs, @@ -50,56 +102,53 @@ pub struct ApiArgs { #[derive(Args, Debug)] pub struct JwtArgs { /// The location of the identity file for JWT auth. - #[arg(long = "api-jwt-identity", env = "JWT_IDENTITY_PATH")] - pub identity_path: Option, + #[arg(long, env = "JWT_IDENTITY", default_value = None)] + pub jwt_identity: Option, /// The password used for JWT authentication. - #[arg(long = "api-jwt-password")] - pub password: Option, -} - -#[cfg(feature = "inx")] -#[derive(Args, Debug)] -pub struct InxArgs { - /// Toggle INX write workflow. - #[arg(long, env = "INX_ENABLED")] - pub inx_enabled: Option, - /// The address of the INX interface provided by the node. - #[arg(long, env = "INX_URL")] - pub inx_url: Option, - /// Milestone at which synchronization should begin. A value of `1` means syncing back until genesis (default). - #[arg(long = "inx-sync-start")] - pub sync_start: Option, -} - -#[derive(Args, Debug)] -pub struct MongoDbArgs { - /// The MongoDB connection string. - #[arg(long, env = "MONGODB_CONN_STR")] - pub mongodb_conn_str: Option, + #[arg(long, env = "JWT_PASSWORD", default_value = "password")] + pub jwt_password: Option, + // The salt used for JWT authentication. + #[arg(long, env = "JWT_SALT", default_value = "saltines")] + pub jwt_salt: Option, + /// The setting for when the (JWT) token expires. + #[arg(long, default_value = "72h")] + pub jwt_expiration: Option, } #[cfg(any(feature = "analytics", feature = "metrics"))] #[derive(Args, Debug)] pub struct InfluxDbArgs { - /// Toggle InfluxDb time-series metrics writes. - #[arg(long, env = "METRICS_ENABLED")] - pub metrics_enabled: Option, - /// Toggle InfluxDb time-series analytics writes. - #[arg(long, env = "ANALYTICS_ENABLED")] - pub analytics_enabled: Option, /// The url pointing to an InfluxDb instance. - #[arg(long, env = "INFLUXDB_URL")] + #[arg(long, default_value = "http://localhost:8086")] pub influxdb_url: Option, + /// The InfluxDb username. + #[arg(long, env = "INFLUXDB_USERNAME", default_value = "root")] + pub influxdb_username: Option, + /// The InfluxDb password. + #[arg(long, env = "INFLUXDB_PASSWORD", default_value = "password")] + pub influxdb_password: Option, + /// Toggle InfluxDb time-series analytics writes. + #[arg(long, default_value = "true")] + pub analytics_enabled: Option, + /// Toggle InfluxDb time-series metrics writes. + #[arg(long, default_value = "true")] + pub metrics_enabled: Option, + /// The Analytics database name. + #[arg(long, default_value = "chronicle_analytics")] + pub analytics_database_name: Option, + /// The Metrics database name. + #[arg(long, default_value = "chronicle_metrics")] + pub metrics_database_name: Option, } #[cfg(feature = "loki")] #[derive(Args, Debug)] pub struct LokiArgs { /// Toggle Grafana Loki log writes. - #[arg(long, env = "LOKI_ENABLED")] + #[arg(long, default_value = "true")] pub loki_enabled: Option, /// The url pointing to a Grafana Loki instance. - #[arg(long, env = "LOKI_URL")] + #[arg(long, default_value = "http://localhost:3100")] pub loki_url: Option, } @@ -125,7 +174,7 @@ impl ClArgs { if let Some(enabled) = self.inx.inx_enabled { config.inx.enabled = enabled; } - if let Some(sync_start) = self.inx.sync_start { + if let Some(sync_start) = self.inx.inx_sync_start { config.inx.sync_start_milestone = sync_start.into(); } } @@ -153,7 +202,7 @@ impl ClArgs { #[cfg(feature = "api")] { - if let Some(password) = &self.api.jwt.password { + if let Some(password) = &self.api.jwt.jwt_password { config.api.password_hash = hex::encode( argon2::hash_raw( password.as_bytes(), @@ -164,12 +213,10 @@ impl ClArgs { .expect("invalid JWT config"), ); } - if let Some(path) = &self.api.jwt.identity_path { + if let Some(path) = &self.api.jwt.jwt_identity { config.api.identity_path.replace(path.clone()); } - if let Some(enabled) = self.api.api_enabled { - config.api.enabled = enabled; - } + config.api.enabled = self.api.api_enabled.unwrap(); } #[cfg(feature = "loki")] From 227ec8edc68b0f852bf879b0b0066de60b0b7a8f Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 13 Dec 2022 21:43:24 +0100 Subject: [PATCH 02/38] Remove custom Default impls from config types --- src/bin/inx-chronicle/api/config.rs | 26 ++++++-------------- src/bin/inx-chronicle/config.rs | 12 +-------- src/bin/inx-chronicle/stardust_inx/config.rs | 14 +---------- src/db/influxdb/mod.rs | 16 +----------- src/db/mongodb/mod.rs | 17 +------------ 5 files changed, 11 insertions(+), 74 deletions(-) diff --git a/src/bin/inx-chronicle/api/config.rs b/src/bin/inx-chronicle/api/config.rs index 527f605e8..0f09a6075 100644 --- a/src/bin/inx-chronicle/api/config.rs +++ b/src/bin/inx-chronicle/api/config.rs @@ -11,7 +11,7 @@ use tower_http::cors::AllowOrigin; use super::{error::ConfigError, SecretKey}; /// API configuration -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct ApiConfig { pub enabled: bool, @@ -27,24 +27,6 @@ pub struct ApiConfig { pub argon_config: ArgonConfig, } -impl Default for ApiConfig { - fn default() -> Self { - Self { - enabled: true, - port: 8042, - allow_origins: "*".to_string().into(), - password_hash: "c42cf2be3a442a29d8cd827a27099b0c".to_string(), - password_salt: "saltines".to_string(), - // 72 hours - jwt_expiration: Duration::from_secs(72 * 60 * 60), - public_routes: Default::default(), - identity_path: None, - max_page_size: 1000, - argon_config: Default::default(), - } - } -} - #[derive(Clone, Debug)] pub struct ApiData { pub port: u16, @@ -130,6 +112,12 @@ impl TryFrom> for AllowOrigin { } } +impl Default for SingleOrMultiple { + fn default() -> Self { + Self::Single(Default::default()) + } +} + #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default)] pub struct ArgonConfig { diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index 27bb3ab9f..cc489d481 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -40,23 +40,13 @@ impl ChronicleConfig { } #[cfg(feature = "loki")] -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Default, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct LokiConfig { pub enabled: bool, pub connect_url: String, } -#[cfg(feature = "loki")] -impl Default for LokiConfig { - fn default() -> Self { - Self { - enabled: true, - connect_url: "http://localhost:3100".to_owned(), - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/src/bin/inx-chronicle/stardust_inx/config.rs b/src/bin/inx-chronicle/stardust_inx/config.rs index 888389ec1..f231d7f91 100644 --- a/src/bin/inx-chronicle/stardust_inx/config.rs +++ b/src/bin/inx-chronicle/stardust_inx/config.rs @@ -7,7 +7,7 @@ use chronicle::types::tangle::MilestoneIndex; use serde::{Deserialize, Serialize}; /// Configuration for an INX connection. -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct InxConfig { pub enabled: bool, @@ -21,15 +21,3 @@ pub struct InxConfig { /// The milestone at which synchronization should begin. pub sync_start_milestone: MilestoneIndex, } - -impl Default for InxConfig { - fn default() -> Self { - Self { - enabled: true, - connect_url: "http://localhost:9029".into(), - connection_retry_interval: Duration::from_secs(5), - connection_retry_count: 5, - sync_start_milestone: 1.into(), - } - } -} diff --git a/src/db/influxdb/mod.rs b/src/db/influxdb/mod.rs index 0e5f6d21c..bbb5e1766 100644 --- a/src/db/influxdb/mod.rs +++ b/src/db/influxdb/mod.rs @@ -103,7 +103,7 @@ impl InfluxDb { /// The influxdb [`Client`] config. #[must_use] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct InfluxDbConfig { /// The address of the InfluxDb instance. @@ -121,17 +121,3 @@ pub struct InfluxDbConfig { /// Whether to enable influx analytics writes. pub analytics_enabled: bool, } - -impl Default for InfluxDbConfig { - fn default() -> Self { - Self { - url: "http://localhost:8086".to_string(), - metrics_database_name: "chronicle_metrics".to_string(), - analytics_database_name: "chronicle_analytics".to_string(), - username: "root".to_string(), - password: "password".to_string(), - metrics_enabled: true, - analytics_enabled: true, - } - } -} diff --git a/src/db/mongodb/mod.rs b/src/db/mongodb/mod.rs index 52f4b13c1..7067d45f2 100644 --- a/src/db/mongodb/mod.rs +++ b/src/db/mongodb/mod.rs @@ -27,9 +27,6 @@ pub struct MongoDb { } impl MongoDb { - const DEFAULT_NAME: &'static str = "chronicle"; - const DEFAULT_CONNECT_STR: &'static str = "mongodb://localhost:27017"; - /// Constructs a [`MongoDb`] by connecting to a MongoDB instance. pub async fn connect(config: &MongoDbConfig) -> Result { let mut client_options = ClientOptions::parse(&config.conn_str).await?; @@ -134,7 +131,7 @@ impl MongoDb { /// The [`MongoDb`] config. #[must_use] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct MongoDbConfig { /// The bind address of the database. @@ -160,15 +157,3 @@ impl MongoDbConfig { }) } } - -impl Default for MongoDbConfig { - fn default() -> Self { - Self { - conn_str: MongoDb::DEFAULT_CONNECT_STR.to_string(), - username: None, - password: None, - database_name: MongoDb::DEFAULT_NAME.to_string(), - min_pool_size: None, - } - } -} From 5c83c33c5fd3e4a8055c58c5214ade562d9780fe Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 13 Dec 2022 21:49:52 +0100 Subject: [PATCH 03/38] Remove config.template.toml --- config.template.toml | 90 -------------------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 config.template.toml diff --git a/config.template.toml b/config.template.toml deleted file mode 100644 index 47a93a564..000000000 --- a/config.template.toml +++ /dev/null @@ -1,90 +0,0 @@ -[mongodb] -### Listening address of the MongoDB instance. -conn_str = "mongodb://localhost:27017" - -### MongoDB credentials. These fields are ignored if the connection string already contains credentials. -username = "root" -password = "root" - -### Chronicle allows different database names, so multiple -### networks can run within the same MongoDB instance. -database_name = "chronicle" - -### The minimum amount of connections in the pool. -min_pool_size = 2 - -[influxdb] -### Whether influx time-series data will be written. -metrics_enabled = true -analytics_enabled = true - -### URL pointing to the InfluxDB instance. -url = "http://localhost:8086" - -### The database name used for metrics. -metrics_database_name = "chronicle_metrics" - -### The database name used for analytics. -analytics_database_name = "chronicle_analytics" - -### InfluxDb basic credentials. -username = "root" -password = "password" - -[api] -### Whether API requests will be served. -enabled = true - -### API listening port. -port = 8042 - -### CORS. -allow_origins = "0.0.0.0" - -### JsonWebToken (JWT) credentials. -password_hash = "f36605441dd3b99a0448bc76f51f0e619f47051989cfcbf2fef18670f21799ad" # "password" -password_salt = "saltines" -jwt_expiration = "72h" - -### Public API routes. -public_routes = [ - # Activated APIs. - "api/core/v2/*", -] - -### Maximum number of records returned by a single API call -max_page_size = 1000 - -[api.argon_config] -### The length of the resulting hash. -hash_length = 32 -### The number of lanes in parallel. -parallelism = 1 -### The amount of memory requested (KB). -mem_cost = 4096 -### The number of passes. -iterations = 3 -### The variant. -variant = "argon2i" -### The version. -version = "0x13" - -[inx] -### Whether INX is used for writing data. -enabled = true - -### Listening address of the node's INX interface. -connect_url = "http://localhost:9029" - -### Time to wait until a new connection attempt is made. -connection_retry_interval = "5s" - -### Maximum number of tries to establish an INX connection. -connection_retry_count = 30 - -[loki] -### Whether Grafana Loki is used for writing logs. -enabled = true - -### The Grafana Loki connection URL. -connect_url = "http://localhost:3100" From 34aff37ac10981329cf975276ed70cc58b066703 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 10:43:29 +0100 Subject: [PATCH 04/38] Add gen-config tool --- src/bin/inx-chronicle/api/config.rs | 12 +++ src/bin/inx-chronicle/cli.rs | 156 +++++++++++++++++----------- src/bin/inx-chronicle/main.rs | 8 +- src/db/mongodb/mod.rs | 20 ++-- 4 files changed, 118 insertions(+), 78 deletions(-) diff --git a/src/bin/inx-chronicle/api/config.rs b/src/bin/inx-chronicle/api/config.rs index 0f09a6075..8bc341f95 100644 --- a/src/bin/inx-chronicle/api/config.rs +++ b/src/bin/inx-chronicle/api/config.rs @@ -118,6 +118,18 @@ impl Default for SingleOrMultiple { } } +impl From<&Vec> for SingleOrMultiple { + fn from(value: &Vec) -> Self { + if value.is_empty() { + unreachable!("Vec must have single or multiple elements") + } else if value.len() == 1 { + Self::Single(value[0].clone()) + } else { + Self::Multiple(value.iter().cloned().collect()) + } + } +} + #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default)] pub struct ArgonConfig { diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 317c89470..f485d802f 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -39,12 +39,17 @@ pub struct ClArgs { #[derive(Args, Debug)] pub struct MongoDbArgs { /// The MongoDb connection string. - #[arg(long, value_name = "CONN_STR", env = "MONGODB_CONN_STR", default_value = "mongodb://localhost:27017")] + #[arg( + long, + value_name = "CONN_STR", + env = "MONGODB_CONN_STR", + default_value = "mongodb://localhost:27017" + )] pub mongodb_conn_str: Option, - /// The MongoDb username. + /// The MongoDb username. #[arg(long, value_name = "USERNAME", env = "MONGODB_USERNAME", default_value = "root")] pub mongodb_username: Option, - /// The MongoDb password. + /// The MongoDb password. #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = "root")] pub mongodb_password: Option, /// The main database name. @@ -52,7 +57,7 @@ pub struct MongoDbArgs { pub mongodb_database_name: Option, /// The MongoDb minimum pool size. #[arg(long, value_name = "SIZE", default_value = "2")] - pub mongodb_min_pool_size: Option, + pub mongodb_min_pool_size: Option, } #[cfg(feature = "inx")] @@ -69,13 +74,17 @@ pub struct InxArgs { #[arg(long, default_value = "0")] pub inx_sync_start: Option, /// Time to wait until a new connection attempt is made. - #[arg(long, default_value = "5s")] - pub inx_retry_interval: Option, + #[arg(long, value_parser = parse_duration, default_value = "5s")] + pub inx_retry_interval: Option, /// Maximum number of tries to establish an INX connection. #[arg(long, default_value = "30")] pub inx_retry_count: Option, } +fn parse_duration(arg: &str) -> Result { + arg.parse::().map(Into::into) +} + #[cfg(feature = "api")] #[derive(Args, Debug)] pub struct ApiArgs { @@ -86,8 +95,8 @@ pub struct ApiArgs { #[arg(long, default_value = "8042")] pub api_port: Option, /// CORS setting. - #[arg(long, default_value = "0.0.0.0")] - pub api_allow_origins: Option, + #[arg(long = "allow-origin", value_name = "ORIGIN", default_value = "0.0.0.0")] + pub allow_origins: Vec, /// Public API routes. #[arg(long = "public-route", value_name = "ROUTE", default_value = "api/core/v2/*")] pub public_routes: Vec, @@ -121,22 +130,26 @@ pub struct InfluxDbArgs { /// The url pointing to an InfluxDb instance. #[arg(long, default_value = "http://localhost:8086")] pub influxdb_url: Option, - /// The InfluxDb username. + /// The InfluxDb username. #[arg(long, env = "INFLUXDB_USERNAME", default_value = "root")] pub influxdb_username: Option, - /// The InfluxDb password. + /// The InfluxDb password. #[arg(long, env = "INFLUXDB_PASSWORD", default_value = "password")] pub influxdb_password: Option, /// Toggle InfluxDb time-series analytics writes. + #[cfg(feature = "analytics")] #[arg(long, default_value = "true")] pub analytics_enabled: Option, /// Toggle InfluxDb time-series metrics writes. + #[cfg(feature = "metrics")] #[arg(long, default_value = "true")] pub metrics_enabled: Option, /// The Analytics database name. + #[cfg(feature = "analytics")] #[arg(long, default_value = "chronicle_analytics")] pub analytics_database_name: Option, /// The Metrics database name. + #[cfg(feature = "metrics")] #[arg(long, default_value = "chronicle_metrics")] pub metrics_database_name: Option, } @@ -153,80 +166,82 @@ pub struct LokiArgs { } impl ClArgs { - /// Get a config file with CLI args applied. + /// Get a config from a file (specified via the `--config` option) or from provided CLI args combined + /// with defaults for those that are not provided. Note that a config file must be fully specified + /// as it cannot be overwritten with the CLI defaults. If you plan on using a `config.toml` use + /// Chronicle's `gen-config' tool to make sure of that. pub fn get_config(&self) -> Result { - let mut config = self - .config - .as_ref() - .map(ChronicleConfig::from_file) - .transpose()? - .unwrap_or_default(); - - if let Some(conn_str) = &self.mongodb.mongodb_conn_str { - config.mongodb.conn_str = conn_str.clone(); + if let Some(config_path) = &self.config { + return ChronicleConfig::from_file(config_path); } + let mut config = ChronicleConfig::default(); + + // MongoDb + // Note: all unwraps are fine because we defined defaults for all, so none of them can be None ;) + config.mongodb.conn_str = self.mongodb.mongodb_conn_str.as_ref().unwrap().clone(); + config.mongodb.database_name = self.mongodb.mongodb_database_name.as_ref().unwrap().clone(); + config.mongodb.username = self.mongodb.mongodb_username.as_ref().unwrap().clone(); + config.mongodb.password = self.mongodb.mongodb_password.as_ref().unwrap().clone(); + config.mongodb.min_pool_size = self.mongodb.mongodb_min_pool_size.unwrap(); + + // INX #[cfg(all(feature = "stardust", feature = "inx"))] { - if let Some(connect_url) = &self.inx.inx_url { - config.inx.connect_url = connect_url.clone(); - } - if let Some(enabled) = self.inx.inx_enabled { - config.inx.enabled = enabled; - } - if let Some(sync_start) = self.inx.inx_sync_start { - config.inx.sync_start_milestone = sync_start.into(); - } + config.inx.enabled = self.inx.inx_enabled.unwrap(); + config.inx.connect_url = self.inx.inx_url.as_ref().unwrap().clone(); + config.inx.connection_retry_interval = self.inx.inx_retry_interval.unwrap(); + config.inx.connection_retry_count = self.inx.inx_retry_count.unwrap(); + config.inx.sync_start_milestone = self.inx.inx_sync_start.unwrap().into(); } - #[cfg(feature = "analytics")] + // InfluxDb + #[cfg(any(feature = "analytics", feature = "metrics"))] { - if let Some(enabled) = self.influxdb.analytics_enabled { - config.influxdb.analytics_enabled = enabled; - } + config.influxdb.url = self.influxdb.influxdb_url.as_ref().unwrap().clone(); + config.influxdb.username = self.influxdb.influxdb_username.as_ref().unwrap().clone(); + config.influxdb.password = self.influxdb.influxdb_password.as_ref().unwrap().clone(); } - - #[cfg(feature = "metrics")] + #[cfg(feature = "analytics")] { - if let Some(enabled) = self.influxdb.metrics_enabled { - config.influxdb.metrics_enabled = enabled; - } + config.influxdb.analytics_enabled = self.influxdb.analytics_enabled.unwrap(); + config.influxdb.analytics_database_name = self.influxdb.analytics_database_name.as_ref().unwrap().clone(); } - - #[cfg(any(feature = "analytics", feature = "metrics"))] + #[cfg(feature = "metrics")] { - if let Some(url) = &self.influxdb.influxdb_url { - config.influxdb.url = url.clone(); - } + config.influxdb.metrics_enabled = self.influxdb.metrics_enabled.unwrap(); + config.influxdb.metrics_database_name = self.influxdb.metrics_database_name.as_ref().unwrap().clone(); } + // API #[cfg(feature = "api")] { - if let Some(password) = &self.api.jwt.jwt_password { - config.api.password_hash = hex::encode( - argon2::hash_raw( - password.as_bytes(), - config.api.password_salt.as_bytes(), - &Into::into(&config.api.argon_config), - ) - // TODO: Replace this once we switch to a better error lib - .expect("invalid JWT config"), - ); - } - if let Some(path) = &self.api.jwt.jwt_identity { - config.api.identity_path.replace(path.clone()); - } + let password = self.api.jwt.jwt_password.as_ref().unwrap(); + let salt = self.api.jwt.jwt_salt.as_ref().unwrap(); + config.api.enabled = self.api.api_enabled.unwrap(); + config.api.port = self.api.api_port.unwrap(); + config.api.allow_origins = (&self.api.allow_origins).into(); + config.api.password_hash = hex::encode( + argon2::hash_raw( + password.as_bytes(), + salt.as_bytes(), + &Into::into(&config.api.argon_config), + ) + // TODO: Replace this once we switch to a better error lib + .expect("invalid JWT config"), + ); + config.api.password_salt = salt.clone(); + config.api.identity_path = self.api.jwt.jwt_identity.clone(); + config.api.max_page_size = self.api.max_page_size.unwrap(); + config.api.public_routes = self.api.public_routes.clone(); } + // Loki #[cfg(feature = "loki")] { - if let Some(connect_url) = &self.loki.loki_url { - config.loki.connect_url = connect_url.clone(); - } - if let Some(enabled) = self.loki.loki_enabled { - config.loki.enabled = enabled; - } + config.loki.connect_url = self.loki.loki_url.as_ref().unwrap().clone(); + config.loki.enabled = *self.loki.loki_enabled.as_ref().unwrap(); } Ok(config) @@ -238,6 +253,15 @@ impl ClArgs { pub async fn process_subcommands(&self, config: &ChronicleConfig) -> eyre::Result { if let Some(subcommand) = &self.subcommand { match subcommand { + Subcommands::CreateConfig { file_path } => { + let toml_config = format!( + "# This file was auto-generated. Re-run on breaking changes to Chronicle's configuration.\n\n{}", + toml::to_string_pretty(config)? + ); + std::fs::write(file_path.as_ref().unwrap(), toml_config)?; + tracing::info!("Written generated config to: '{}'", file_path.as_ref().unwrap()); + return Ok(PostCommand::Exit); + } #[cfg(feature = "api")] Subcommands::GenerateJWT => { use crate::api::ApiData; @@ -416,6 +440,12 @@ impl From for Box, + }, /// Generate a JWT token using the available config. #[cfg(feature = "api")] GenerateJWT, diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 22f765142..2d8d59df5 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -26,10 +26,6 @@ use self::cli::{ClArgs, PostCommand}; async fn main() -> eyre::Result<()> { dotenvy::dotenv().ok(); - std::panic::set_hook(Box::new(|p| { - error!("{}", p); - })); - let cl_args = ClArgs::parse(); let config = cl_args.get_config()?; @@ -130,6 +126,10 @@ async fn main() -> eyre::Result<()> { } fn set_up_logging(#[allow(unused)] config: &ChronicleConfig) -> eyre::Result<()> { + std::panic::set_hook(Box::new(|p| { + error!("{}", p); + })); + let registry = tracing_subscriber::registry(); let registry = { diff --git a/src/db/mongodb/mod.rs b/src/db/mongodb/mod.rs index 7067d45f2..b5274e660 100644 --- a/src/db/mongodb/mod.rs +++ b/src/db/mongodb/mod.rs @@ -32,16 +32,14 @@ impl MongoDb { let mut client_options = ClientOptions::parse(&config.conn_str).await?; client_options.app_name = Some("Chronicle".to_string()); - client_options.min_pool_size = config.min_pool_size; + client_options.min_pool_size = Some(config.min_pool_size); if client_options.credential.is_none() { - if let (Some(username), Some(password)) = (&config.username, &config.password) { - let credential = Credential::builder() - .username(username.clone()) - .password(password.clone()) - .build(); - client_options.credential = Some(credential); - } + let credential = Credential::builder() + .username(config.username.clone()) + .password(config.password.clone()) + .build(); + client_options.credential = Some(credential); } let client = Client::with_options(client_options)?; @@ -137,13 +135,13 @@ pub struct MongoDbConfig { /// The bind address of the database. pub conn_str: String, /// The MongoDB username. - pub username: Option, + pub username: String, /// The MongoDB password. - pub password: Option, + pub password: String, /// The name of the database to connect to. pub database_name: String, /// The minimum amount of connections in the pool. - pub min_pool_size: Option, + pub min_pool_size: u32, } impl MongoDbConfig { From f989c0f2531b20f0b080d7acb4e88beb637c9183 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 14:15:56 +0100 Subject: [PATCH 05/38] Fix CI --- src/bin/inx-chronicle/cli.rs | 2 +- src/bin/inx-chronicle/config.rs | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 0ea741f84..1b863b193 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -81,6 +81,7 @@ pub struct InxArgs { pub inx_retry_count: Option, } +#[cfg(feature = "inx")] fn parse_duration(arg: &str) -> Result { arg.parse::().map(Into::into) } @@ -445,7 +446,6 @@ impl From for Box, diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index cc489d481..12e6551f1 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -46,15 +46,3 @@ pub struct LokiConfig { pub enabled: bool, pub connect_url: String, } - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn config_file_conformity() -> Result<(), ConfigError> { - let _ = ChronicleConfig::from_file(concat!(env!("CARGO_MANIFEST_DIR"), "/config.template.toml"))?; - - Ok(()) - } -} From c9cd8a6ad5223903eae3f7ac0e834f1793f85556 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 16:19:02 +0100 Subject: [PATCH 06/38] Fix tests --- src/bin/inx-chronicle/api/extractors.rs | 8 ++++-- .../api/stardust/explorer/extractors.rs | 10 ++++--- .../api/stardust/indexer/extractors.rs | 8 ++++-- src/bin/inx-chronicle/cli.rs | 4 +-- tests/common/mod.rs | 26 ++++++++++++++----- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/bin/inx-chronicle/api/extractors.rs b/src/bin/inx-chronicle/api/extractors.rs index 8ca8a49bd..f1dfc6a0d 100644 --- a/src/bin/inx-chronicle/api/extractors.rs +++ b/src/bin/inx-chronicle/api/extractors.rs @@ -117,14 +117,18 @@ mod test { #[tokio::test] async fn page_size_clamped() { - let config = ApiData::try_from(ApiConfig::default()).unwrap(); + let config = ApiConfig { + max_page_size: 1000, + ..Default::default() + }; + let data = ApiData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/?pageSize=9999999") .body(()) .unwrap(); assert_eq!( - Pagination::from_request(req, &config).await.unwrap(), + Pagination::from_request(req, &data).await.unwrap(), Pagination { page_size: 1000, ..Default::default() diff --git a/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs b/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs index 5c4bae8fe..76f10023c 100644 --- a/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs +++ b/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs @@ -494,14 +494,18 @@ mod test { #[tokio::test] async fn page_size_clamped() { - let config = ApiData::try_from(ApiConfig::default()).unwrap(); + let config = ApiConfig { + max_page_size: 1000, + ..Default::default() + }; + let data = ApiData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/ledger/updates/by-address/0x00?pageSize=9999999") .body(()) .unwrap(); assert_eq!( - LedgerUpdatesByAddressPagination::from_request(req, &config) + LedgerUpdatesByAddressPagination::from_request(req, &data) .await .unwrap(), LedgerUpdatesByAddressPagination { @@ -517,7 +521,7 @@ mod test { .body(()) .unwrap(); assert_eq!( - LedgerUpdatesByMilestonePagination::from_request(req, &config) + LedgerUpdatesByMilestonePagination::from_request(req, &data) .await .unwrap(), LedgerUpdatesByMilestonePagination { diff --git a/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs b/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs index 131d37726..41908bead 100644 --- a/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs +++ b/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs @@ -457,14 +457,18 @@ mod test { #[tokio::test] async fn page_size_clamped() { - let config = ApiData::try_from(ApiConfig::default()).unwrap(); + let config = ApiConfig { + max_page_size: 1000, + ..Default::default() + }; + let data = ApiData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/outputs/basic?pageSize=9999999") .body(()) .unwrap(); assert_eq!( - IndexedOutputsPagination::::from_request(req, &config) + IndexedOutputsPagination::::from_request(req, &data) .await .unwrap(), IndexedOutputsPagination { diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 1b863b193..b6e3534ee 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -101,7 +101,7 @@ pub struct ApiArgs { /// Public API routes. #[arg(long = "public-route", value_name = "ROUTE", default_value = "api/core/v2/*")] pub public_routes: Vec, - /// Maximum nubmer of results returned by a single API call. + /// Maximum number of results returned by a single API call. #[arg(long, default_value = "1000")] pub max_page_size: Option, /// JWT arguments. @@ -256,7 +256,7 @@ impl ClArgs { match subcommand { Subcommands::CreateConfig { file_path } => { let toml_config = format!( - "# This file was auto-generated. Re-run on breaking changes to Chronicle's configuration.\n\n{}", + "# This file was auto-generated. You can change values but should not add or remove lines yourself.\n# Re-run on breaking changes to Chronicle's configuration.\n\n{}", toml::to_string_pretty(config)? ); std::fs::write(file_path.as_ref().unwrap(), toml_config)?; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a6bc9992b..b4dcb78b2 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -16,21 +16,35 @@ pub enum TestDbError { #[allow(unused)] pub async fn setup_database(database_name: impl ToString) -> eyre::Result { - let mut config = if let Ok(path) = std::env::var("CONFIG_PATH") { + dotenvy::dotenv().ok(); + + let get_mongodb_test_config = || -> MongoDbConfig { + MongoDbConfig { + conn_str: std::env::var("MONGODB_CONN_STR").unwrap_or_else(|_| "mongodb://localhost:27017".to_owned()), + database_name: database_name.to_string(), + username: std::env::var("MONGODB_USERNAME").unwrap_or_else(|_| "root".to_owned()), + password: std::env::var("MONGODB_PASSWORD").unwrap_or_else(|_| "root".to_owned()), + min_pool_size: 2, + } + }; + + let mut test_config = if let Ok(path) = std::env::var("CHRONICLE_TEST_CONFIG") { let val = std::fs::read_to_string(&path) .map_err(|e| TestDbError::FileRead(AsRef::::as_ref(&path).display().to_string(), e)) .and_then(|contents| toml::from_str::(&contents).map_err(TestDbError::TomlDeserialization))?; + if let Some(mongodb) = val.get("mongodb").cloned() { - mongodb.try_into().map_err(TestDbError::TomlDeserialization)? + let mut config: MongoDbConfig = mongodb.try_into().map_err(TestDbError::TomlDeserialization)?; + config.database_name = database_name.to_string(); + config } else { - MongoDbConfig::default() + get_mongodb_test_config() } } else { - MongoDbConfig::default() + get_mongodb_test_config() }; - config.database_name = database_name.to_string(); - let db = MongoDb::connect(&config).await?; + let db = MongoDb::connect(&test_config).await?; db.clear().await?; Ok(db) } From 2c8dbea7d1b761f2d1e9cbd573fe9914d20aa78b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 16:32:53 +0100 Subject: [PATCH 07/38] Fix docs --- src/bin/inx-chronicle/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index b6e3534ee..fa60bf4ae 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -170,7 +170,7 @@ impl ClArgs { /// Get a config from a file (specified via the `--config` option) or from provided CLI args combined /// with defaults for those that are not provided. Note that a config file must be fully specified /// as it cannot be overwritten with the CLI defaults. If you plan on using a `config.toml` use - /// Chronicle's `gen-config' tool to make sure of that. + /// Chronicle's `create-config' tool to make sure of that. pub fn get_config(&self) -> Result { if let Some(config_path) = &self.config { return ChronicleConfig::from_file(config_path); @@ -242,7 +242,7 @@ impl ClArgs { #[cfg(feature = "loki")] { config.loki.connect_url = self.loki.loki_url.as_ref().unwrap().clone(); - config.loki.enabled = *self.loki.loki_enabled.as_ref().unwrap(); + config.loki.enabled = self.loki.loki_enabled.unwrap(); } Ok(config) From 7700deab0b1884cc86c054cd56853fa1dc8d0a6b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 16:38:27 +0100 Subject: [PATCH 08/38] Use only non-default CLI commands in docker-compose.yml --- docker/docker-compose.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index db986af2b..4db11d113 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -34,15 +34,8 @@ services: tty: true command: - "--mongodb-conn-str=mongodb://mongo:27017" - - "--mongodb-username=root" - - "--mongodb-password=root" - - "--inx-enabled=true" - "--inx-url=http://hornet:9029" - - "--jwt-password=password" - - "--jwt-salt=saltines" - "--influxdb-url=http://influx:8086" - - "--influxdb-username=root" - - "--influxdb-password=password" - "--loki-url=http://loki:3100" influx: From c0f579301fd628b9ebd9ef68aa10885e8d13ddce Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 16:52:10 +0100 Subject: [PATCH 09/38] Let's try enforcing MongoDb auth --- .github/workflows/_test_int.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index 502ab85a0..a900e39e7 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -17,6 +17,9 @@ jobs: test-int: name: "${{ inputs.os }}, ${{ inputs.rust }}" runs-on: ${{ inputs.os }} + env: + MONGO_INITDB_ROOT_USERNAME: "root" + MONGO_INITDB_ROOT_PASSWORD: "root" steps: - uses: actions/checkout@v2 From 99f9b478df1e1181c3fc6956b2b0f97fdaad7842 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 17:03:57 +0100 Subject: [PATCH 10/38] Let's try again --- .github/workflows/_test_int.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index a900e39e7..fdd0ba3ae 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -17,9 +17,6 @@ jobs: test-int: name: "${{ inputs.os }}, ${{ inputs.rust }}" runs-on: ${{ inputs.os }} - env: - MONGO_INITDB_ROOT_USERNAME: "root" - MONGO_INITDB_ROOT_PASSWORD: "root" steps: - uses: actions/checkout@v2 @@ -41,6 +38,8 @@ jobs: with: mongodb-version: ${{ inputs.mongodb }} mongodb-replica-set: test-rs + mongodb-username: root + mongodb-password: root - name: Test DB uses: actions-rs/cargo@v1 From 1c55eca6647bf56e19f505447302a18fc99429ac Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Wed, 14 Dec 2022 17:19:26 +0100 Subject: [PATCH 11/38] What happens now? --- .github/workflows/_test_int.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index fdd0ba3ae..8b2331744 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -37,7 +37,7 @@ jobs: uses: supercharge/mongodb-github-action@1.8.0 with: mongodb-version: ${{ inputs.mongodb }} - mongodb-replica-set: test-rs + #mongodb-replica-set: test-rs mongodb-username: root mongodb-password: root From 42b0b0f2c1deedb1461891da60e1aa5c381b5180 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 08:53:36 +0100 Subject: [PATCH 12/38] Move defaults into constants --- Cargo.lock | 21 +++++++ Cargo.toml | 1 + src/bin/inx-chronicle/api/config.rs | 37 ++++++++++-- src/bin/inx-chronicle/api/mod.rs | 2 +- src/bin/inx-chronicle/cli.rs | 54 ++++++++++------- src/bin/inx-chronicle/stardust_inx/config.rs | 20 ++++++- src/bin/inx-chronicle/stardust_inx/mod.rs | 2 +- src/db/mod.rs | 5 +- src/db/mongodb/collection.rs | 8 ++- src/db/mongodb/config.rs | 62 ++++++++++++++++++++ src/db/mongodb/mod.rs | 39 ++---------- 11 files changed, 185 insertions(+), 66 deletions(-) create mode 100644 src/db/mongodb/config.rs diff --git a/Cargo.lock b/Cargo.lock index 37190d3b4..501847000 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,6 +312,7 @@ dependencies = [ "bytesize", "chrono", "clap", + "const_format", "decimal", "derive_more", "dotenvy", @@ -423,6 +424,26 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cec318a675afcb6a1ea1d4340e2d377e56e47c266f28043ceccbf4412ddfdd3b" +[[package]] +name = "const_format" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7309d9b4d3d2c0641e018d449232f2e28f1b22933c137f157d3dbc14228b8c0e" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "constant_time_eq" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index bcb550604..f4568b237 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ path = "src/bin/inx-chronicle/main.rs" async-trait = { version = "0.1", default-features = false } bytesize = { version = "1.1", default-features = false } clap = { version = "4.0", default-features = false, features = ["env", "derive", "std", "help", "usage", "error-context", "wrap_help"] } +const_format = { version = "0.2", default-features = false } decimal = { version = "2.1", default-features = false, features = [ "serde" ] } derive_more = { version = "0.99", default-features = false, features = [ "add", "add_assign", "deref", "deref_mut" ] } dotenvy = { version = "0.15", default-features = false } diff --git a/src/bin/inx-chronicle/api/config.rs b/src/bin/inx-chronicle/api/config.rs index 891a168dd..acd509c4c 100644 --- a/src/bin/inx-chronicle/api/config.rs +++ b/src/bin/inx-chronicle/api/config.rs @@ -10,23 +10,52 @@ use tower_http::cors::AllowOrigin; use super::{error::ConfigError, SecretKey}; +pub const DEFAULT_ENABLED: bool = true; +pub const DEFAULT_PORT: u16 = 8042; +pub const DEFAULT_ALLOW_ORIGINS: &str = "0.0.0.0"; +pub const DEFAULT_PUBLIC_ROUTES: &str = "api/core/v2/*"; +pub const DEFAULT_MAX_PAGE_SIZE: usize = 1000; +pub const DEFAULT_JWT_PASSWORD: &str = "password"; +pub const DEFAULT_JWT_SALT: &str = "saltines"; +pub const DEFAULT_JWT_EXPIRATIOIN: &str = "72h"; + /// API configuration -#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct ApiConfig { pub enabled: bool, pub port: u16, pub allow_origins: SingleOrMultiple, + pub public_routes: Vec, + pub max_page_size: usize, + pub identity_path: Option, pub password_hash: String, pub password_salt: String, #[serde(with = "humantime_serde")] pub jwt_expiration: Duration, - pub public_routes: Vec, - pub identity_path: Option, - pub max_page_size: usize, pub argon_config: ArgonConfig, } +impl Default for ApiConfig { + fn default() -> Self { + Self { + enabled: DEFAULT_ENABLED, + port: DEFAULT_PORT, + allow_origins: SingleOrMultiple::Single(DEFAULT_ALLOW_ORIGINS.to_string()), + public_routes: vec![DEFAULT_PUBLIC_ROUTES.to_string()], + max_page_size: DEFAULT_MAX_PAGE_SIZE, + identity_path: None, + password_hash: DEFAULT_JWT_PASSWORD.to_string(), + password_salt: DEFAULT_JWT_SALT.to_string(), + jwt_expiration: DEFAULT_JWT_EXPIRATIOIN + .parse::() + .map(Into::into) + .unwrap(), + argon_config: ArgonConfig::default(), + } + } +} + #[derive(Clone, Debug)] pub struct ApiData { pub port: u16, diff --git a/src/bin/inx-chronicle/api/mod.rs b/src/bin/inx-chronicle/api/mod.rs index 5b1e1bf44..7261ee0f4 100644 --- a/src/bin/inx-chronicle/api/mod.rs +++ b/src/bin/inx-chronicle/api/mod.rs @@ -14,7 +14,7 @@ mod secret_key; #[macro_use] mod responses; mod auth; -mod config; +pub mod config; mod router; mod routes; diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index fa60bf4ae..fc990c8e8 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -1,10 +1,17 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +use chronicle::db::mongodb::config as mongodb; use clap::{Args, Parser, Subcommand, ValueEnum}; use crate::config::{ChronicleConfig, ConfigError}; +macro_rules! to_str { + ($arg:expr) => { + const_format::formatcp!("{}", $arg) + }; +} + /// Chronicle permanode storage as an INX plugin #[derive(Parser, Debug)] #[command(author, version, about, next_display_order = None)] @@ -43,42 +50,45 @@ pub struct MongoDbArgs { long, value_name = "CONN_STR", env = "MONGODB_CONN_STR", - default_value = "mongodb://localhost:27017" + default_value = mongodb::DEFAULT_CONN_STR, )] pub mongodb_conn_str: Option, /// The MongoDb username. - #[arg(long, value_name = "USERNAME", env = "MONGODB_USERNAME", default_value = "root")] + #[arg(long, value_name = "USERNAME", env = "MONGODB_USERNAME", default_value = mongodb::DEFAULT_USERNAME)] pub mongodb_username: Option, /// The MongoDb password. - #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = "root")] + #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = mongodb::DEFAULT_PASSWORD)] pub mongodb_password: Option, /// The main database name. - #[arg(long, value_name = "NAME", default_value = "chronicle")] + #[arg(long, value_name = "NAME", default_value = mongodb::DEFAULT_DATABASE_NAME)] pub mongodb_database_name: Option, /// The MongoDb minimum pool size. - #[arg(long, value_name = "SIZE", default_value = "2")] + #[arg(long, value_name = "SIZE", default_value = to_str!(mongodb::DEFAULT_MIN_POOL_SIZE))] pub mongodb_min_pool_size: Option, } +#[cfg(feature = "inx")] +use crate::stardust_inx::config as inx; + #[cfg(feature = "inx")] #[derive(Args, Debug)] pub struct InxArgs { /// Toggles the INX synchronization workflow. - #[arg(long, default_value = "true")] + #[arg(long, default_value = to_str!(inx::DEFAULT_ENABLED))] pub inx_enabled: Option, /// The address of the node INX interface Chronicle tries to connect to - if enabled. - #[arg(long, default_value = "http://localhost:9029")] + #[arg(long, default_value = inx::DEFAULT_CONNECT_URL)] pub inx_url: Option, - /// Milestone at which synchronization should begin. If set to `1` Chronicle will try to sync back until the - /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. - #[arg(long, default_value = "0")] - pub inx_sync_start: Option, /// Time to wait until a new connection attempt is made. - #[arg(long, value_parser = parse_duration, default_value = "5s")] + #[arg(long, value_parser = parse_duration, default_value = inx::DEFAULT_RETRY_INTERVAL)] pub inx_retry_interval: Option, /// Maximum number of tries to establish an INX connection. - #[arg(long, default_value = "30")] + #[arg(long, default_value = to_str!(inx::DEFAULT_RETRY_COUNT))] pub inx_retry_count: Option, + /// Milestone at which synchronization should begin. If set to `1` Chronicle will try to sync back until the + /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. + #[arg(long, default_value = to_str!(inx::DEFAULT_SYNC_START))] + pub inx_sync_start: Option, } #[cfg(feature = "inx")] @@ -86,23 +96,26 @@ fn parse_duration(arg: &str) -> Result().map(Into::into) } +#[cfg(feature = "api")] +use crate::api::config as api; + #[cfg(feature = "api")] #[derive(Args, Debug)] pub struct ApiArgs { /// Toggle REST API. - #[arg(long, default_value = "true")] + #[arg(long, default_value = to_str!(api::DEFAULT_ENABLED))] pub api_enabled: Option, /// API listening port. - #[arg(long, default_value = "8042")] + #[arg(long, default_value = to_str!(api::DEFAULT_PORT))] pub api_port: Option, /// CORS setting. - #[arg(long = "allow-origin", value_name = "ORIGIN", default_value = "0.0.0.0")] + #[arg(long = "allow-origin", value_name = "ORIGIN", default_value = api::DEFAULT_ALLOW_ORIGINS)] pub allow_origins: Vec, /// Public API routes. - #[arg(long = "public-route", value_name = "ROUTE", default_value = "api/core/v2/*")] + #[arg(long = "public-route", value_name = "ROUTE", default_value = api::DEFAULT_PUBLIC_ROUTES)] pub public_routes: Vec, /// Maximum number of results returned by a single API call. - #[arg(long, default_value = "1000")] + #[arg(long, default_value = to_str!(api::DEFAULT_MAX_PAGE_SIZE))] pub max_page_size: Option, /// JWT arguments. #[command(flatten)] @@ -217,12 +230,11 @@ impl ClArgs { // API #[cfg(feature = "api")] { - let password = self.api.jwt.jwt_password.as_ref().unwrap(); - let salt = self.api.jwt.jwt_salt.as_ref().unwrap(); - config.api.enabled = self.api.api_enabled.unwrap(); config.api.port = self.api.api_port.unwrap(); config.api.allow_origins = (&self.api.allow_origins).into(); + let password = self.api.jwt.jwt_password.as_ref().unwrap(); + let salt = self.api.jwt.jwt_salt.as_ref().unwrap(); config.api.password_hash = hex::encode( argon2::hash_raw( password.as_bytes(), diff --git a/src/bin/inx-chronicle/stardust_inx/config.rs b/src/bin/inx-chronicle/stardust_inx/config.rs index f231d7f91..e93e4c7ab 100644 --- a/src/bin/inx-chronicle/stardust_inx/config.rs +++ b/src/bin/inx-chronicle/stardust_inx/config.rs @@ -6,8 +6,14 @@ use std::time::Duration; use chronicle::types::tangle::MilestoneIndex; use serde::{Deserialize, Serialize}; +pub const DEFAULT_ENABLED: bool = true; +pub const DEFAULT_CONNECT_URL: &str = "http://localhost:9029"; +pub const DEFAULT_RETRY_INTERVAL: &str = "5s"; +pub const DEFAULT_RETRY_COUNT: usize = 30; +pub const DEFAULT_SYNC_START: u32 = 0; + /// Configuration for an INX connection. -#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct InxConfig { pub enabled: bool, @@ -21,3 +27,15 @@ pub struct InxConfig { /// The milestone at which synchronization should begin. pub sync_start_milestone: MilestoneIndex, } + +impl Default for InxConfig { + fn default() -> Self { + Self { + enabled: DEFAULT_ENABLED, + connect_url: DEFAULT_CONNECT_URL.to_string(), + connection_retry_interval: DEFAULT_RETRY_INTERVAL.parse::().unwrap().into(), + connection_retry_count: DEFAULT_RETRY_COUNT, + sync_start_milestone: DEFAULT_SYNC_START.into(), + } + } +} diff --git a/src/bin/inx-chronicle/stardust_inx/mod.rs b/src/bin/inx-chronicle/stardust_inx/mod.rs index c0ef3d2c1..2c03710b3 100644 --- a/src/bin/inx-chronicle/stardust_inx/mod.rs +++ b/src/bin/inx-chronicle/stardust_inx/mod.rs @@ -1,7 +1,7 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -mod config; +pub mod config; mod error; use std::time::Duration; diff --git a/src/db/mod.rs b/src/db/mod.rs index ebf150670..530a73161 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -10,6 +10,7 @@ pub mod collections; /// Module containing InfluxDb types and traits. #[cfg(any(feature = "analytics", feature = "metrics"))] pub mod influxdb; -mod mongodb; +/// Module containing MongoDb types and traits. +pub mod mongodb; -pub use self::mongodb::{MongoDb, MongoDbCollection, MongoDbCollectionExt, MongoDbConfig}; +pub use self::mongodb::{config::MongoDbConfig, MongoDb, MongoDbCollection, MongoDbCollectionExt}; diff --git a/src/db/mongodb/collection.rs b/src/db/mongodb/collection.rs index 37bc9f73c..330061b5f 100644 --- a/src/db/mongodb/collection.rs +++ b/src/db/mongodb/collection.rs @@ -145,12 +145,14 @@ pub trait MongoDbCollectionExt: MongoDbCollection { } impl MongoDbCollectionExt for T {} -pub struct InsertResult { - pub ignored: usize, +pub(crate) struct InsertResult { + // FIXME: this field is never read. + #[allow(dead_code)] + pub(crate) ignored: usize, } #[async_trait] -pub trait InsertIgnoreDuplicatesExt { +pub(crate) trait InsertIgnoreDuplicatesExt { /// Inserts many records and ignores duplicate key errors. async fn insert_many_ignore_duplicates( &self, diff --git a/src/db/mongodb/config.rs b/src/db/mongodb/config.rs new file mode 100644 index 000000000..b06cf7c62 --- /dev/null +++ b/src/db/mongodb/config.rs @@ -0,0 +1,62 @@ +// Copyright 2022 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Holds the `MongoDb` config and its defaults. + +use mongodb::{ + error::Error, + options::{ConnectionString, HostInfo}, +}; +use serde::{Deserialize, Serialize}; + +/// The default connection string of the database. +pub const DEFAULT_CONN_STR: &str = "mongodb://localhost:27017"; +/// The default MongoDB username. +pub const DEFAULT_USERNAME: &str = "root"; +/// The default MongoDB password. +pub const DEFAULT_PASSWORD: &str = "root"; +/// The default name of the database to connect to. +pub const DEFAULT_DATABASE_NAME: &str = "chronicle"; +/// The default minimum amount of connections in the pool. +pub const DEFAULT_MIN_POOL_SIZE: u32 = 2; + +/// The [`MongoDb`] config. +#[must_use] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct MongoDbConfig { + /// The connection string of the database. + pub conn_str: String, + /// The MongoDB username. + pub username: String, + /// The MongoDB password. + pub password: String, + /// The name of the database to connect to. + pub database_name: String, + /// The minimum amount of connections in the pool. + pub min_pool_size: u32, +} + +impl MongoDbConfig { + /// Get the hosts portion of the connection string. + pub fn hosts_str(&self) -> Result { + let hosts = ConnectionString::parse(&self.conn_str)?.host_info; + Ok(match hosts { + HostInfo::HostIdentifiers(hosts) => hosts.iter().map(ToString::to_string).collect::>().join(","), + HostInfo::DnsRecord(hostname) => hostname, + _ => unreachable!(), + }) + } +} + +impl Default for MongoDbConfig { + fn default() -> Self { + Self { + conn_str: DEFAULT_CONN_STR.to_string(), + username: DEFAULT_USERNAME.to_string(), + password: DEFAULT_PASSWORD.to_string(), + database_name: DEFAULT_DATABASE_NAME.to_string(), + min_pool_size: DEFAULT_MIN_POOL_SIZE, + } + } +} diff --git a/src/db/mongodb/mod.rs b/src/db/mongodb/mod.rs index b5274e660..48c326a41 100644 --- a/src/db/mongodb/mod.rs +++ b/src/db/mongodb/mod.rs @@ -1,21 +1,23 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! Holds the `MongoDb` type and its config. +//! Holds the `MongoDb` type. mod collection; +pub mod config; use std::collections::{HashMap, HashSet}; +use config::MongoDbConfig; use mongodb::{ bson::{doc, Document}, error::Error, - options::{ClientOptions, ConnectionString, Credential, HostInfo}, + options::{ClientOptions, Credential}, Client, }; -use serde::{Deserialize, Serialize}; -pub use self::collection::{InsertIgnoreDuplicatesExt, MongoDbCollection, MongoDbCollectionExt}; +pub(crate) use self::collection::InsertIgnoreDuplicatesExt; +pub use self::collection::{MongoDbCollection, MongoDbCollectionExt}; const DUPLICATE_KEY_CODE: i32 = 11000; @@ -126,32 +128,3 @@ impl MongoDb { self.db.name() } } - -/// The [`MongoDb`] config. -#[must_use] -#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct MongoDbConfig { - /// The bind address of the database. - pub conn_str: String, - /// The MongoDB username. - pub username: String, - /// The MongoDB password. - pub password: String, - /// The name of the database to connect to. - pub database_name: String, - /// The minimum amount of connections in the pool. - pub min_pool_size: u32, -} - -impl MongoDbConfig { - /// Get the hosts portion of the connection string. - pub fn hosts_str(&self) -> Result { - let hosts = ConnectionString::parse(&self.conn_str)?.host_info; - Ok(match hosts { - HostInfo::HostIdentifiers(hosts) => hosts.iter().map(ToString::to_string).collect::>().join(","), - HostInfo::DnsRecord(hostname) => hostname, - _ => unreachable!(), - }) - } -} From f674fc8c15e7544d97d56c5eabcb3fbec1065833 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 09:16:46 +0100 Subject: [PATCH 13/38] Remove user Argon configuration --- src/bin/inx-chronicle/api/config.rs | 12 +++++------- src/bin/inx-chronicle/cli.rs | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/bin/inx-chronicle/api/config.rs b/src/bin/inx-chronicle/api/config.rs index acd509c4c..6017722e6 100644 --- a/src/bin/inx-chronicle/api/config.rs +++ b/src/bin/inx-chronicle/api/config.rs @@ -17,7 +17,7 @@ pub const DEFAULT_PUBLIC_ROUTES: &str = "api/core/v2/*"; pub const DEFAULT_MAX_PAGE_SIZE: usize = 1000; pub const DEFAULT_JWT_PASSWORD: &str = "password"; pub const DEFAULT_JWT_SALT: &str = "saltines"; -pub const DEFAULT_JWT_EXPIRATIOIN: &str = "72h"; +pub const DEFAULT_JWT_EXPIRATION: &str = "72h"; /// API configuration #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] @@ -33,7 +33,6 @@ pub struct ApiConfig { pub password_salt: String, #[serde(with = "humantime_serde")] pub jwt_expiration: Duration, - pub argon_config: ArgonConfig, } impl Default for ApiConfig { @@ -47,11 +46,10 @@ impl Default for ApiConfig { identity_path: None, password_hash: DEFAULT_JWT_PASSWORD.to_string(), password_salt: DEFAULT_JWT_SALT.to_string(), - jwt_expiration: DEFAULT_JWT_EXPIRATIOIN + jwt_expiration: DEFAULT_JWT_EXPIRATION .parse::() - .map(Into::into) - .unwrap(), - argon_config: ArgonConfig::default(), + .unwrap() + .into(), } } } @@ -96,7 +94,7 @@ impl TryFrom for ApiData { } }, max_page_size: config.max_page_size, - argon_config: config.argon_config, + argon_config: ArgonConfig::default(), }) } } diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index fc990c8e8..cacf3cf27 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -91,11 +91,6 @@ pub struct InxArgs { pub inx_sync_start: Option, } -#[cfg(feature = "inx")] -fn parse_duration(arg: &str) -> Result { - arg.parse::().map(Into::into) -} - #[cfg(feature = "api")] use crate::api::config as api; @@ -122,19 +117,20 @@ pub struct ApiArgs { pub jwt: JwtArgs, } +#[cfg(feature = "api")] #[derive(Args, Debug)] pub struct JwtArgs { /// The location of the identity file for JWT auth. #[arg(long, env = "JWT_IDENTITY", default_value = None)] pub jwt_identity: Option, /// The password used for JWT authentication. - #[arg(long, env = "JWT_PASSWORD", default_value = "password")] + #[arg(long, env = "JWT_PASSWORD", default_value = api::DEFAULT_JWT_PASSWORD)] pub jwt_password: Option, // The salt used for JWT authentication. - #[arg(long, env = "JWT_SALT", default_value = "saltines")] + #[arg(long, env = "JWT_SALT", default_value = api::DEFAULT_JWT_SALT)] pub jwt_salt: Option, /// The setting for when the (JWT) token expires. - #[arg(long, default_value = "72h")] + #[arg(long, value_parser = parse_duration, default_value = api::DEFAULT_JWT_EXPIRATION)] pub jwt_expiration: Option, } @@ -179,6 +175,10 @@ pub struct LokiArgs { pub loki_url: Option, } +fn parse_duration(arg: &str) -> Result { + arg.parse::().map(Into::into) +} + impl ClArgs { /// Get a config from a file (specified via the `--config` option) or from provided CLI args combined /// with defaults for those that are not provided. Note that a config file must be fully specified @@ -239,7 +239,7 @@ impl ClArgs { argon2::hash_raw( password.as_bytes(), salt.as_bytes(), - &Into::into(&config.api.argon_config), + &Into::into(&api::ArgonConfig::default()), ) // TODO: Replace this once we switch to a better error lib .expect("invalid JWT config"), @@ -268,7 +268,8 @@ impl ClArgs { match subcommand { Subcommands::CreateConfig { file_path } => { let toml_config = format!( - "# This file was auto-generated. You can change values but should not add or remove lines yourself.\n# Re-run on breaking changes to Chronicle's configuration.\n\n{}", + "# This file was auto-generated by Chronicle version '{}'.\n# Re-run `create-config` tool on breaking changes to Chronicle's configuration.\n\n{}", + std::env::var("CARGO_PKG_VERSION").unwrap(), toml::to_string_pretty(config)? ); std::fs::write(file_path.as_ref().unwrap(), toml_config)?; From 611f7935f27b45ca244e71a417501dfd2482619e Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 11:35:12 +0100 Subject: [PATCH 14/38] Improve API configuration --- src/bin/inx-chronicle/api/auth.rs | 12 ++-- src/bin/inx-chronicle/api/config.rs | 59 ++++++++++--------- src/bin/inx-chronicle/api/extractors.rs | 8 +-- src/bin/inx-chronicle/api/mod.rs | 4 +- src/bin/inx-chronicle/api/router.rs | 4 +- src/bin/inx-chronicle/api/routes.rs | 31 ++++++---- .../api/stardust/explorer/extractors.rs | 28 ++++----- .../api/stardust/explorer/routes.rs | 4 +- .../api/stardust/indexer/extractors.rs | 20 +++---- .../api/stardust/indexer/routes.rs | 4 +- src/bin/inx-chronicle/cli.rs | 30 ++++------ 11 files changed, 102 insertions(+), 102 deletions(-) diff --git a/src/bin/inx-chronicle/api/auth.rs b/src/bin/inx-chronicle/api/auth.rs index 336126b85..9df4c6cd3 100644 --- a/src/bin/inx-chronicle/api/auth.rs +++ b/src/bin/inx-chronicle/api/auth.rs @@ -9,14 +9,14 @@ use axum::{ TypedHeader, }; -use super::{config::ApiData, error::RequestError, ApiError, AuthError}; +use super::{config::ApiConfigData, error::RequestError, ApiError, AuthError}; pub struct Auth; #[async_trait] impl FromRequestParts for Auth where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -24,7 +24,7 @@ where // Unwrap: ::Rejection = Infallable let OriginalUri(uri) = OriginalUri::from_request_parts(req, state).await.unwrap(); - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); if config.public_routes.is_match(&uri.to_string()) { return Ok(Auth); @@ -37,10 +37,10 @@ where jwt.validate( Validation::default() - .with_issuer(ApiData::ISSUER) - .with_audience(ApiData::AUDIENCE) + .with_issuer(ApiConfigData::ISSUER) + .with_audience(ApiConfigData::AUDIENCE) .validate_nbf(true), - config.secret_key.as_ref(), + config.jwt_secret_key.as_ref(), ) .map_err(AuthError::InvalidJwt)?; diff --git a/src/bin/inx-chronicle/api/config.rs b/src/bin/inx-chronicle/api/config.rs index 6017722e6..3b3a451f4 100644 --- a/src/bin/inx-chronicle/api/config.rs +++ b/src/bin/inx-chronicle/api/config.rs @@ -28,9 +28,9 @@ pub struct ApiConfig { pub allow_origins: SingleOrMultiple, pub public_routes: Vec, pub max_page_size: usize, - pub identity_path: Option, - pub password_hash: String, - pub password_salt: String, + pub jwt_password: String, + pub jwt_salt: String, + pub jwt_identity_file: Option, #[serde(with = "humantime_serde")] pub jwt_expiration: Duration, } @@ -43,47 +43,50 @@ impl Default for ApiConfig { allow_origins: SingleOrMultiple::Single(DEFAULT_ALLOW_ORIGINS.to_string()), public_routes: vec![DEFAULT_PUBLIC_ROUTES.to_string()], max_page_size: DEFAULT_MAX_PAGE_SIZE, - identity_path: None, - password_hash: DEFAULT_JWT_PASSWORD.to_string(), - password_salt: DEFAULT_JWT_SALT.to_string(), - jwt_expiration: DEFAULT_JWT_EXPIRATION - .parse::() - .unwrap() - .into(), + jwt_identity_file: None, + jwt_password: DEFAULT_JWT_PASSWORD.to_string(), + jwt_salt: DEFAULT_JWT_SALT.to_string(), + jwt_expiration: DEFAULT_JWT_EXPIRATION.parse::().unwrap().into(), } } } #[derive(Clone, Debug)] -pub struct ApiData { +pub struct ApiConfigData { pub port: u16, pub allow_origins: AllowOrigin, - pub password_hash: Vec, - pub password_salt: String, - pub jwt_expiration: Duration, pub public_routes: RegexSet, - pub secret_key: SecretKey, pub max_page_size: usize, - pub argon_config: ArgonConfig, + pub jwt_password_hash: Vec, + pub jwt_password_salt: String, + pub jwt_secret_key: SecretKey, + pub jwt_expiration: Duration, + pub jwt_argon_config: JwtArgonConfig, } -impl ApiData { +impl ApiConfigData { pub const ISSUER: &'static str = "chronicle"; pub const AUDIENCE: &'static str = "api"; } -impl TryFrom for ApiData { +impl TryFrom for ApiConfigData { type Error = ConfigError; fn try_from(config: ApiConfig) -> Result { Ok(Self { port: config.port, allow_origins: AllowOrigin::try_from(config.allow_origins)?, - password_hash: hex::decode(config.password_hash)?, - password_salt: config.password_salt, - jwt_expiration: config.jwt_expiration, public_routes: RegexSet::new(config.public_routes.iter().map(route_to_regex).collect::>())?, - secret_key: match &config.identity_path { + max_page_size: config.max_page_size, + jwt_password_hash: argon2::hash_raw( + config.jwt_password.as_bytes(), + config.jwt_salt.as_bytes(), + &Into::into(&JwtArgonConfig::default()), + ) + // TODO: Replace this once we switch to a better error lib + .expect("invalid JWT config"), + jwt_password_salt: config.jwt_salt, + jwt_secret_key: match &config.jwt_identity_file { Some(path) => SecretKey::from_file(path)?, None => { if let Ok(path) = std::env::var("IDENTITY_PATH") { @@ -93,8 +96,8 @@ impl TryFrom for ApiData { } } }, - max_page_size: config.max_page_size, - argon_config: ArgonConfig::default(), + jwt_expiration: config.jwt_expiration, + jwt_argon_config: JwtArgonConfig::default(), }) } } @@ -159,7 +162,7 @@ impl From<&Vec> for SingleOrMultiple { #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default)] -pub struct ArgonConfig { +pub struct JwtArgonConfig { /// The length of the resulting hash. hash_length: u32, /// The number of lanes in parallel. @@ -176,7 +179,7 @@ pub struct ArgonConfig { version: argon2::Version, } -impl Default for ArgonConfig { +impl Default for JwtArgonConfig { fn default() -> Self { Self { hash_length: 32, @@ -189,8 +192,8 @@ impl Default for ArgonConfig { } } -impl<'a> From<&'a ArgonConfig> for argon2::Config<'a> { - fn from(val: &'a ArgonConfig) -> Self { +impl<'a> From<&'a JwtArgonConfig> for argon2::Config<'a> { + fn from(val: &'a JwtArgonConfig) -> Self { Self { ad: &[], hash_length: val.hash_length, diff --git a/src/bin/inx-chronicle/api/extractors.rs b/src/bin/inx-chronicle/api/extractors.rs index f1dfc6a0d..a60a3419b 100644 --- a/src/bin/inx-chronicle/api/extractors.rs +++ b/src/bin/inx-chronicle/api/extractors.rs @@ -6,7 +6,7 @@ use axum::extract::{FromRef, FromRequestParts, Query}; use serde::Deserialize; use super::{ - config::ApiData, + config::ApiConfigData, error::{ApiError, RequestError}, DEFAULT_PAGE_SIZE, }; @@ -30,7 +30,7 @@ impl Default for Pagination { #[async_trait] impl FromRequestParts for Pagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -38,7 +38,7 @@ where let Query(mut pagination) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); pagination.page_size = pagination.page_size.min(config.max_page_size); Ok(pagination) } @@ -121,7 +121,7 @@ mod test { max_page_size: 1000, ..Default::default() }; - let data = ApiData::try_from(config).unwrap(); + let data = ApiConfigData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/?pageSize=9999999") diff --git a/src/bin/inx-chronicle/api/mod.rs b/src/bin/inx-chronicle/api/mod.rs index 7261ee0f4..8f282e9f1 100644 --- a/src/bin/inx-chronicle/api/mod.rs +++ b/src/bin/inx-chronicle/api/mod.rs @@ -30,7 +30,7 @@ use tracing::info; use self::routes::routes; pub use self::{ - config::{ApiConfig, ApiData}, + config::{ApiConfig, ApiConfigData}, error::{ApiError, ApiResult, AuthError, ConfigError}, secret_key::SecretKey, }; @@ -41,7 +41,7 @@ pub const DEFAULT_PAGE_SIZE: usize = 100; #[derive(Debug, Clone)] pub struct ApiWorker { db: MongoDb, - api_data: ApiData, + api_data: ApiConfigData, } impl ApiWorker { diff --git a/src/bin/inx-chronicle/api/router.rs b/src/bin/inx-chronicle/api/router.rs index a3237d3d4..59492bca5 100644 --- a/src/bin/inx-chronicle/api/router.rs +++ b/src/bin/inx-chronicle/api/router.rs @@ -27,7 +27,7 @@ use hyper::{Body, Request}; use regex::RegexSet; use tower::{Layer, Service}; -use super::{ApiData, ApiWorker}; +use super::{ApiConfigData, ApiWorker}; #[derive(Clone, Debug, Default)] pub struct RouteNode { @@ -95,7 +95,7 @@ impl FromRef> for MongoDb { } } -impl FromRef> for ApiData { +impl FromRef> for ApiConfigData { fn from_ref(input: &RouterState) -> Self { input.inner.api_data.clone() } diff --git a/src/bin/inx-chronicle/api/routes.rs b/src/bin/inx-chronicle/api/routes.rs index e2c335cda..2974f20b8 100644 --- a/src/bin/inx-chronicle/api/routes.rs +++ b/src/bin/inx-chronicle/api/routes.rs @@ -20,7 +20,7 @@ use time::{Duration, OffsetDateTime}; use super::{ auth::Auth, - config::ApiData, + config::ApiConfigData, error::{ApiError, MissingError, UnimplementedError}, extractors::ListRoutesQuery, responses::RoutesResponse, @@ -34,7 +34,7 @@ const ALWAYS_AVAILABLE_ROUTES: &[&str] = &["/health", "/login", "/routes"]; // sufficient time to catch up with the node that it is connected too. The current milestone interval is 5 seconds. const STALE_MILESTONE_DURATION: Duration = Duration::minutes(5); -pub fn routes(api_data: ApiData) -> Router { +pub fn routes(api_data: ApiConfigData) -> Router { #[allow(unused_mut)] let mut router = Router::new(); @@ -59,17 +59,24 @@ struct LoginInfo { password: String, } -async fn login(State(config): State, Json(LoginInfo { password }): Json) -> ApiResult { +async fn login( + State(config_data): State, + Json(LoginInfo { password }): Json, +) -> ApiResult { if password_verify( password.as_bytes(), - config.password_salt.as_bytes(), - &config.password_hash, - Into::into(&config.argon_config), + config_data.jwt_password_salt.as_bytes(), + &config_data.jwt_password_hash, + Into::into(&config_data.jwt_argon_config), )? { let jwt = JsonWebToken::new( - Claims::new(ApiData::ISSUER, uuid::Uuid::new_v4().to_string(), ApiData::AUDIENCE)? - .expires_after_duration(config.jwt_expiration)?, - config.secret_key.as_ref(), + Claims::new( + ApiConfigData::ISSUER, + uuid::Uuid::new_v4().to_string(), + ApiConfigData::AUDIENCE, + )? + .expires_after_duration(config_data.jwt_expiration)?, + config_data.jwt_secret_key.as_ref(), )?; Ok(format!("Bearer {}", jwt)) @@ -105,10 +112,10 @@ async fn list_routes( jwt.validate( Validation::default() - .with_issuer(ApiData::ISSUER) - .with_audience(ApiData::AUDIENCE) + .with_issuer(ApiConfigData::ISSUER) + .with_audience(ApiConfigData::AUDIENCE) .validate_nbf(true), - state.inner.api_data.secret_key.as_ref(), + state.inner.api_data.jwt_secret_key.as_ref(), ) .map_err(AuthError::InvalidJwt)?; diff --git a/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs b/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs index 76f10023c..e67c520b8 100644 --- a/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs +++ b/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs @@ -14,7 +14,7 @@ use chronicle::{ }; use serde::Deserialize; -use crate::api::{config::ApiData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; +use crate::api::{config::ApiConfigData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct LedgerUpdatesByAddressPagination { @@ -73,7 +73,7 @@ impl Display for LedgerUpdatesByAddressCursor { #[async_trait] impl FromRequestParts for LedgerUpdatesByAddressPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -81,7 +81,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let sort = query .sort @@ -155,7 +155,7 @@ impl Display for LedgerUpdatesByMilestoneCursor { #[async_trait] impl FromRequestParts for LedgerUpdatesByMilestonePagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -163,7 +163,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (page_size, cursor) = if let Some(cursor) = query.cursor { let cursor: LedgerUpdatesByMilestoneCursor = cursor.parse()?; @@ -227,7 +227,7 @@ impl Display for MilestonesCursor { #[async_trait] impl FromRequestParts for MilestonesPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -235,7 +235,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); if matches!((query.start_timestamp, query.end_timestamp), (Some(start), Some(end)) if end < start) { return Err(ApiError::from(RequestError::BadTimeRange)); @@ -285,7 +285,7 @@ impl Default for RichestAddressesQuery { #[async_trait] impl FromRequestParts for RichestAddressesQuery where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -293,7 +293,7 @@ where let Query(mut query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); query.top = query.top.min(config.max_page_size); Ok(query) } @@ -383,7 +383,7 @@ impl Display for BlocksByMilestoneCursor { #[async_trait] impl FromRequestParts for BlocksByMilestoneIndexPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -391,7 +391,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let sort = query .sort @@ -431,7 +431,7 @@ pub struct BlocksByMilestoneIdPaginationQuery { #[async_trait] impl FromRequestParts for BlocksByMilestoneIdPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -439,7 +439,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let sort = query .sort @@ -498,7 +498,7 @@ mod test { max_page_size: 1000, ..Default::default() }; - let data = ApiData::try_from(config).unwrap(); + let data = ApiConfigData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/ledger/updates/by-address/0x00?pageSize=9999999") diff --git a/src/bin/inx-chronicle/api/stardust/explorer/routes.rs b/src/bin/inx-chronicle/api/stardust/explorer/routes.rs index 91155434b..8be379d36 100644 --- a/src/bin/inx-chronicle/api/stardust/explorer/routes.rs +++ b/src/bin/inx-chronicle/api/stardust/explorer/routes.rs @@ -37,14 +37,14 @@ use crate::api::{ error::{CorruptStateError, MissingError, RequestError}, extractors::Pagination, router::{Router, RouterState}, - ApiData, ApiResult, + ApiConfigData, ApiResult, }; pub fn routes() -> Router where S: Clone + Send + Sync + 'static, MongoDb: FromRef>, - ApiData: FromRef>, + ApiConfigData: FromRef>, { Router::new() .route("/balance/:address", get(balance)) diff --git a/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs b/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs index 41908bead..15af1b1dc 100644 --- a/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs +++ b/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs @@ -16,7 +16,7 @@ use mongodb::bson; use primitive_types::U256; use serde::Deserialize; -use crate::api::{config::ApiData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; +use crate::api::{config::ApiConfigData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct IndexedOutputsPagination @@ -94,7 +94,7 @@ pub struct BasicOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -102,7 +102,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -190,7 +190,7 @@ pub struct AliasOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -198,7 +198,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -275,7 +275,7 @@ pub struct FoundryOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -283,7 +283,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -357,7 +357,7 @@ pub struct NftOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -365,7 +365,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -461,7 +461,7 @@ mod test { max_page_size: 1000, ..Default::default() }; - let data = ApiData::try_from(config).unwrap(); + let data = ApiConfigData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/outputs/basic?pageSize=9999999") diff --git a/src/bin/inx-chronicle/api/stardust/indexer/routes.rs b/src/bin/inx-chronicle/api/stardust/indexer/routes.rs index 6898e1bd5..4b5649a1a 100644 --- a/src/bin/inx-chronicle/api/stardust/indexer/routes.rs +++ b/src/bin/inx-chronicle/api/stardust/indexer/routes.rs @@ -24,14 +24,14 @@ use crate::api::{ error::{MissingError, RequestError}, router::{Router, RouterState}, stardust::indexer::extractors::IndexedOutputsCursor, - ApiData, ApiResult, + ApiConfigData, ApiResult, }; pub fn routes() -> Router where S: Clone + Send + Sync + 'static, MongoDb: FromRef>, - ApiData: FromRef>, + ApiConfigData: FromRef>, { Router::new().nest( "/outputs", diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index cacf3cf27..d64cfefe5 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -131,7 +131,7 @@ pub struct JwtArgs { pub jwt_salt: Option, /// The setting for when the (JWT) token expires. #[arg(long, value_parser = parse_duration, default_value = api::DEFAULT_JWT_EXPIRATION)] - pub jwt_expiration: Option, + pub jwt_expiration: Option, } #[cfg(any(feature = "analytics", feature = "metrics"))] @@ -233,19 +233,9 @@ impl ClArgs { config.api.enabled = self.api.api_enabled.unwrap(); config.api.port = self.api.api_port.unwrap(); config.api.allow_origins = (&self.api.allow_origins).into(); - let password = self.api.jwt.jwt_password.as_ref().unwrap(); - let salt = self.api.jwt.jwt_salt.as_ref().unwrap(); - config.api.password_hash = hex::encode( - argon2::hash_raw( - password.as_bytes(), - salt.as_bytes(), - &Into::into(&api::ArgonConfig::default()), - ) - // TODO: Replace this once we switch to a better error lib - .expect("invalid JWT config"), - ); - config.api.password_salt = salt.clone(); - config.api.identity_path = self.api.jwt.jwt_identity.clone(); + config.api.jwt_password = self.api.jwt.jwt_password.as_ref().unwrap().clone(); + config.api.jwt_salt = self.api.jwt.jwt_salt.as_ref().unwrap().clone(); + config.api.jwt_identity_file = self.api.jwt.jwt_identity.clone(); config.api.max_page_size = self.api.max_page_size.unwrap(); config.api.public_routes = self.api.public_routes.clone(); } @@ -268,7 +258,7 @@ impl ClArgs { match subcommand { Subcommands::CreateConfig { file_path } => { let toml_config = format!( - "# This file was auto-generated by Chronicle version '{}'.\n# Re-run `create-config` tool on breaking changes to Chronicle's configuration.\n\n{}", + "# This file was auto-generated by Chronicle's `create-config` tool at version '{}'.\n# Check CHANGELOG.md for breaking configuration changes, and re-run it if necessary.\n\n{}", std::env::var("CARGO_PKG_VERSION").unwrap(), toml::to_string_pretty(config)? ); @@ -278,18 +268,18 @@ impl ClArgs { } #[cfg(feature = "api")] Subcommands::GenerateJWT => { - use crate::api::ApiData; - let api_data = ApiData::try_from(config.api.clone()).expect("invalid API config"); + use crate::api::ApiConfigData; + let api_data = ApiConfigData::try_from(config.api.clone()).expect("invalid API config"); let claims = auth_helper::jwt::Claims::new( - ApiData::ISSUER, + ApiConfigData::ISSUER, uuid::Uuid::new_v4().to_string(), - ApiData::AUDIENCE, + ApiConfigData::AUDIENCE, ) .unwrap() // Panic: Cannot fail. .expires_after_duration(api_data.jwt_expiration) .map_err(crate::api::AuthError::InvalidJwt)?; let exp_ts = time::OffsetDateTime::from_unix_timestamp(claims.exp.unwrap() as _).unwrap(); - let jwt = auth_helper::jwt::JsonWebToken::new(claims, api_data.secret_key.as_ref()) + let jwt = auth_helper::jwt::JsonWebToken::new(claims, api_data.jwt_secret_key.as_ref()) .map_err(crate::api::AuthError::InvalidJwt)?; tracing::info!("Bearer {}", jwt); tracing::info!( From e68cc71f17e46ca0846424ad66d831fa9de100a7 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 12:44:50 +0100 Subject: [PATCH 15/38] Complete default consts --- src/bin/inx-chronicle/cli.rs | 124 ++++++++++--------- src/bin/inx-chronicle/config.rs | 16 ++- src/bin/inx-chronicle/main.rs | 4 +- src/bin/inx-chronicle/stardust_inx/config.rs | 14 +-- src/bin/inx-chronicle/stardust_inx/mod.rs | 16 +-- src/db/influxdb/config.rs | 56 +++++++++ src/db/influxdb/mod.rs | 54 +++----- 7 files changed, 168 insertions(+), 116 deletions(-) create mode 100644 src/db/influxdb/config.rs diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index d64cfefe5..252cd2e67 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -14,14 +14,19 @@ macro_rules! to_str { /// Chronicle permanode storage as an INX plugin #[derive(Parser, Debug)] -#[command(author, version, about, next_display_order = None)] +// #[command(author, version, about, next_display_order = None)] +#[command(author, version, about)] pub struct ClArgs { /// The location of the configuration file. - #[arg(short, long)] + #[arg(short, long, value_name = "FILEPATH")] pub config: Option, /// MongoDb arguments. #[command(flatten, next_help_heading = "MongoDb")] pub mongodb: MongoDbArgs, + /// InfluxDb arguments. + #[cfg(any(feature = "analytics", feature = "metrics"))] + #[command(flatten, next_help_heading = "InfluxDb")] + pub influxdb: InfluxDbArgs, /// INX arguments. #[cfg(feature = "inx")] #[command(flatten, next_help_heading = "INX")] @@ -30,10 +35,6 @@ pub struct ClArgs { #[cfg(feature = "api")] #[command(flatten, next_help_heading = "API")] pub api: ApiArgs, - /// InfluxDb arguments. - #[cfg(any(feature = "analytics", feature = "metrics"))] - #[command(flatten, next_help_heading = "InfluxDb")] - pub influxdb: InfluxDbArgs, /// Loki arguments. #[cfg(feature = "loki")] #[command(flatten, next_help_heading = "Loki")] @@ -59,7 +60,7 @@ pub struct MongoDbArgs { /// The MongoDb password. #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = mongodb::DEFAULT_PASSWORD)] pub mongodb_password: Option, - /// The main database name. + /// The MongoDb database name. #[arg(long, value_name = "NAME", default_value = mongodb::DEFAULT_DATABASE_NAME)] pub mongodb_database_name: Option, /// The MongoDb minimum pool size. @@ -67,6 +68,39 @@ pub struct MongoDbArgs { pub mongodb_min_pool_size: Option, } +#[cfg(any(feature = "analytics", feature = "metrics"))] +use chronicle::db::influxdb::config as influxdb; + +#[cfg(any(feature = "analytics", feature = "metrics"))] +#[derive(Args, Debug)] +pub struct InfluxDbArgs { + /// The url pointing to an InfluxDb instance. + #[arg(long, value_name = "URL", default_value = influxdb::DEFAULT_CONN_URL)] + pub influxdb_conn_url: Option, + /// The InfluxDb username. + #[arg(long, value_name = "USERNAME", env = "INFLUXDB_USERNAME", default_value = influxdb::DEFAULT_USERNAME)] + pub influxdb_username: Option, + /// The InfluxDb password. + #[arg(long, value_name = "PASSWORD", env = "INFLUXDB_PASSWORD", default_value = influxdb::DEFAULT_PASSWORD)] + pub influxdb_password: Option, + /// Toggle InfluxDb time-series analytics writes. + #[cfg(feature = "analytics")] + #[arg(long, value_name = "BOOL", default_value = to_str!(influxdb::DEFAULT_ANALYTICS_ENABLED))] + pub analytics_enabled: Option, + /// The Analytics database name. + #[cfg(feature = "analytics")] + #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_ANALYTICS_DATABASE_NAME)] + pub analytics_database_name: Option, + /// Toggle InfluxDb time-series metrics writes. + #[cfg(feature = "metrics")] + #[arg(long, value_name = "BOOL", default_value = to_str!(influxdb::DEFAULT_METRICS_ENABLED))] + pub metrics_enabled: Option, + /// The Metrics database name. + #[cfg(feature = "metrics")] + #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_METRICS_DATABASE_NAME)] + pub metrics_database_name: Option, +} + #[cfg(feature = "inx")] use crate::stardust_inx::config as inx; @@ -74,20 +108,20 @@ use crate::stardust_inx::config as inx; #[derive(Args, Debug)] pub struct InxArgs { /// Toggles the INX synchronization workflow. - #[arg(long, default_value = to_str!(inx::DEFAULT_ENABLED))] + #[arg(long, value_name = "BOOL", default_value = to_str!(inx::DEFAULT_ENABLED))] pub inx_enabled: Option, /// The address of the node INX interface Chronicle tries to connect to - if enabled. - #[arg(long, default_value = inx::DEFAULT_CONNECT_URL)] - pub inx_url: Option, + #[arg(long, value_name = "URL", default_value = inx::DEFAULT_CONN_URL)] + pub inx_conn_url: Option, /// Time to wait until a new connection attempt is made. - #[arg(long, value_parser = parse_duration, default_value = inx::DEFAULT_RETRY_INTERVAL)] + #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = inx::DEFAULT_RETRY_INTERVAL)] pub inx_retry_interval: Option, /// Maximum number of tries to establish an INX connection. - #[arg(long, default_value = to_str!(inx::DEFAULT_RETRY_COUNT))] + #[arg(long, value_name = "COUNT", default_value = to_str!(inx::DEFAULT_RETRY_COUNT))] pub inx_retry_count: Option, /// Milestone at which synchronization should begin. If set to `1` Chronicle will try to sync back until the /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. - #[arg(long, default_value = to_str!(inx::DEFAULT_SYNC_START))] + #[arg(long, value_name = "START", default_value = to_str!(inx::DEFAULT_SYNC_START))] pub inx_sync_start: Option, } @@ -98,19 +132,19 @@ use crate::api::config as api; #[derive(Args, Debug)] pub struct ApiArgs { /// Toggle REST API. - #[arg(long, default_value = to_str!(api::DEFAULT_ENABLED))] + #[arg(long, value_name = "BOOL", default_value = to_str!(api::DEFAULT_ENABLED))] pub api_enabled: Option, /// API listening port. - #[arg(long, default_value = to_str!(api::DEFAULT_PORT))] + #[arg(long, value_name = "PORT", default_value = to_str!(api::DEFAULT_PORT))] pub api_port: Option, /// CORS setting. - #[arg(long = "allow-origin", value_name = "ORIGIN", default_value = api::DEFAULT_ALLOW_ORIGINS)] + #[arg(long = "allow-origin", value_name = "IP", default_value = api::DEFAULT_ALLOW_ORIGINS)] pub allow_origins: Vec, /// Public API routes. #[arg(long = "public-route", value_name = "ROUTE", default_value = api::DEFAULT_PUBLIC_ROUTES)] pub public_routes: Vec, /// Maximum number of results returned by a single API call. - #[arg(long, default_value = to_str!(api::DEFAULT_MAX_PAGE_SIZE))] + #[arg(long, value_name = "SIZE", default_value = to_str!(api::DEFAULT_MAX_PAGE_SIZE))] pub max_page_size: Option, /// JWT arguments. #[command(flatten)] @@ -121,58 +155,28 @@ pub struct ApiArgs { #[derive(Args, Debug)] pub struct JwtArgs { /// The location of the identity file for JWT auth. - #[arg(long, env = "JWT_IDENTITY", default_value = None)] + #[arg(long, value_name = "FILEPATH", env = "JWT_IDENTITY", default_value = None)] pub jwt_identity: Option, /// The password used for JWT authentication. - #[arg(long, env = "JWT_PASSWORD", default_value = api::DEFAULT_JWT_PASSWORD)] + #[arg(long, value_name = "PASSWORD", env = "JWT_PASSWORD", default_value = api::DEFAULT_JWT_PASSWORD)] pub jwt_password: Option, - // The salt used for JWT authentication. - #[arg(long, env = "JWT_SALT", default_value = api::DEFAULT_JWT_SALT)] + /// The salt used for JWT authentication. + #[arg(long, value_name = "SALT", env = "JWT_SALT", default_value = api::DEFAULT_JWT_SALT)] pub jwt_salt: Option, /// The setting for when the (JWT) token expires. - #[arg(long, value_parser = parse_duration, default_value = api::DEFAULT_JWT_EXPIRATION)] + #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = api::DEFAULT_JWT_EXPIRATION)] pub jwt_expiration: Option, } -#[cfg(any(feature = "analytics", feature = "metrics"))] -#[derive(Args, Debug)] -pub struct InfluxDbArgs { - /// The url pointing to an InfluxDb instance. - #[arg(long, default_value = "http://localhost:8086")] - pub influxdb_url: Option, - /// The InfluxDb username. - #[arg(long, env = "INFLUXDB_USERNAME", default_value = "root")] - pub influxdb_username: Option, - /// The InfluxDb password. - #[arg(long, env = "INFLUXDB_PASSWORD", default_value = "password")] - pub influxdb_password: Option, - /// Toggle InfluxDb time-series analytics writes. - #[cfg(feature = "analytics")] - #[arg(long, default_value = "true")] - pub analytics_enabled: Option, - /// Toggle InfluxDb time-series metrics writes. - #[cfg(feature = "metrics")] - #[arg(long, default_value = "true")] - pub metrics_enabled: Option, - /// The Analytics database name. - #[cfg(feature = "analytics")] - #[arg(long, default_value = "chronicle_analytics")] - pub analytics_database_name: Option, - /// The Metrics database name. - #[cfg(feature = "metrics")] - #[arg(long, default_value = "chronicle_metrics")] - pub metrics_database_name: Option, -} - #[cfg(feature = "loki")] #[derive(Args, Debug)] pub struct LokiArgs { /// Toggle Grafana Loki log writes. - #[arg(long, default_value = "true")] + #[arg(long, value_name = "BOOL", default_value = to_str!(crate::config::DEFAULT_LOKI_ENABLED))] pub loki_enabled: Option, /// The url pointing to a Grafana Loki instance. - #[arg(long, default_value = "http://localhost:3100")] - pub loki_url: Option, + #[arg(long, value_name = "URL", default_value = crate::config::DEFAULT_LOKI_CONN_URL)] + pub loki_conn_url: Option, } fn parse_duration(arg: &str) -> Result { @@ -203,16 +207,16 @@ impl ClArgs { #[cfg(all(feature = "stardust", feature = "inx"))] { config.inx.enabled = self.inx.inx_enabled.unwrap(); - config.inx.connect_url = self.inx.inx_url.as_ref().unwrap().clone(); - config.inx.connection_retry_interval = self.inx.inx_retry_interval.unwrap(); - config.inx.connection_retry_count = self.inx.inx_retry_count.unwrap(); + config.inx.conn_url = self.inx.inx_conn_url.as_ref().unwrap().clone(); + config.inx.conn_retry_interval = self.inx.inx_retry_interval.unwrap(); + config.inx.conn_retry_count = self.inx.inx_retry_count.unwrap(); config.inx.sync_start_milestone = self.inx.inx_sync_start.unwrap().into(); } // InfluxDb #[cfg(any(feature = "analytics", feature = "metrics"))] { - config.influxdb.url = self.influxdb.influxdb_url.as_ref().unwrap().clone(); + config.influxdb.conn_url = self.influxdb.influxdb_conn_url.as_ref().unwrap().clone(); config.influxdb.username = self.influxdb.influxdb_username.as_ref().unwrap().clone(); config.influxdb.password = self.influxdb.influxdb_password.as_ref().unwrap().clone(); } @@ -243,7 +247,7 @@ impl ClArgs { // Loki #[cfg(feature = "loki")] { - config.loki.connect_url = self.loki.loki_url.as_ref().unwrap().clone(); + config.loki.conn_url = self.loki.loki_conn_url.as_ref().unwrap().clone(); config.loki.enabled = self.loki.loki_enabled.unwrap(); } diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index 12e6551f1..997ce3d6e 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -7,6 +7,9 @@ use chronicle::db::MongoDbConfig; use serde::{Deserialize, Serialize}; use thiserror::Error; +pub const DEFAULT_LOKI_ENABLED: bool = true; +pub const DEFAULT_LOKI_CONN_URL: &str = "http://localhost:3100"; + #[derive(Error, Debug)] pub enum ConfigError { #[error("failed to read config at '{0}': {1}")] @@ -40,9 +43,18 @@ impl ChronicleConfig { } #[cfg(feature = "loki")] -#[derive(Clone, Default, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct LokiConfig { pub enabled: bool, - pub connect_url: String, + pub conn_url: String, +} + +impl Default for LokiConfig { + fn default() -> Self { + Self { + enabled: DEFAULT_LOKI_ENABLED, + conn_url: DEFAULT_LOKI_CONN_URL.to_string(), + } + } } diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 2d8d59df5..412a192a8 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -55,7 +55,7 @@ async fn main() -> eyre::Result<()> { if config.inx.enabled { #[cfg(any(feature = "analytics", feature = "metrics"))] let influx_db = if config.influxdb.analytics_enabled || config.influxdb.metrics_enabled { - info!("Connecting to influx database at address `{}`", config.influxdb.url); + info!("Connecting to influx database at address `{}`", config.influxdb.conn_url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; info!( "Connected to influx databases `{}` and `{}`", @@ -139,7 +139,7 @@ fn set_up_logging(#[allow(unused)] config: &ChronicleConfig) -> eyre::Result<()> }; #[cfg(feature = "loki")] let registry = { - let (layer, task) = tracing_loki::layer(config.loki.connect_url.parse()?, [].into(), [].into())?; + let (layer, task) = tracing_loki::layer(config.loki.conn_url.parse()?, [].into(), [].into())?; tokio::spawn(task); registry.with(layer) }; diff --git a/src/bin/inx-chronicle/stardust_inx/config.rs b/src/bin/inx-chronicle/stardust_inx/config.rs index e93e4c7ab..e8a149ac3 100644 --- a/src/bin/inx-chronicle/stardust_inx/config.rs +++ b/src/bin/inx-chronicle/stardust_inx/config.rs @@ -7,7 +7,7 @@ use chronicle::types::tangle::MilestoneIndex; use serde::{Deserialize, Serialize}; pub const DEFAULT_ENABLED: bool = true; -pub const DEFAULT_CONNECT_URL: &str = "http://localhost:9029"; +pub const DEFAULT_CONN_URL: &str = "http://localhost:9029"; pub const DEFAULT_RETRY_INTERVAL: &str = "5s"; pub const DEFAULT_RETRY_COUNT: usize = 30; pub const DEFAULT_SYNC_START: u32 = 0; @@ -18,12 +18,12 @@ pub const DEFAULT_SYNC_START: u32 = 0; pub struct InxConfig { pub enabled: bool, /// The bind address of node's INX interface. - pub connect_url: String, + pub conn_url: String, /// The time that has to pass until a new connection attempt is made. #[serde(with = "humantime_serde")] - pub connection_retry_interval: Duration, + pub conn_retry_interval: Duration, /// The number of retries when connecting fails. - pub connection_retry_count: usize, + pub conn_retry_count: usize, /// The milestone at which synchronization should begin. pub sync_start_milestone: MilestoneIndex, } @@ -32,9 +32,9 @@ impl Default for InxConfig { fn default() -> Self { Self { enabled: DEFAULT_ENABLED, - connect_url: DEFAULT_CONNECT_URL.to_string(), - connection_retry_interval: DEFAULT_RETRY_INTERVAL.parse::().unwrap().into(), - connection_retry_count: DEFAULT_RETRY_COUNT, + conn_url: DEFAULT_CONN_URL.to_string(), + conn_retry_interval: DEFAULT_RETRY_INTERVAL.parse::().unwrap().into(), + conn_retry_count: DEFAULT_RETRY_COUNT, sync_start_milestone: DEFAULT_SYNC_START.into(), } } diff --git a/src/bin/inx-chronicle/stardust_inx/mod.rs b/src/bin/inx-chronicle/stardust_inx/mod.rs index 2c03710b3..8179cef5c 100644 --- a/src/bin/inx-chronicle/stardust_inx/mod.rs +++ b/src/bin/inx-chronicle/stardust_inx/mod.rs @@ -88,22 +88,22 @@ impl InxWorker { } async fn connect(&self) -> Result { - let url = url::Url::parse(&self.config.connect_url)?; + let url = url::Url::parse(&self.config.conn_url)?; if url.scheme() != "http" { - bail!(InxWorkerError::InvalidAddress(self.config.connect_url.clone())); + bail!(InxWorkerError::InvalidAddress(self.config.conn_url.clone())); } - for i in 0..self.config.connection_retry_count { - match Inx::connect(self.config.connect_url.clone()).await { + for i in 0..self.config.conn_retry_count { + match Inx::connect(self.config.conn_url.clone()).await { Ok(inx_client) => return Ok(inx_client), Err(_) => { warn!( "INX connection failed. Retrying in {}s. {} retries remaining.", - self.config.connection_retry_interval.as_secs(), - self.config.connection_retry_count - i + self.config.conn_retry_interval.as_secs(), + self.config.conn_retry_count - i ); - tokio::time::sleep(self.config.connection_retry_interval).await; + tokio::time::sleep(self.config.conn_retry_interval).await; } } } @@ -135,7 +135,7 @@ impl InxWorker { #[instrument(skip_all, err, level = "trace")] async fn init(&mut self) -> Result<(MilestoneIndex, Inx)> { - info!("Connecting to INX at bind address `{}`.", &self.config.connect_url); + info!("Connecting to INX at bind address `{}`.", &self.config.conn_url); let mut inx = self.connect().await?; info!("Connected to INX."); diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs new file mode 100644 index 000000000..9bdf5b18d --- /dev/null +++ b/src/db/influxdb/config.rs @@ -0,0 +1,56 @@ +// Copyright 2022 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Holds the `InfluxDb` config and its defaults. + +use serde::{Deserialize, Serialize}; + +/// The default InfluxDb URL to connect to. +pub const DEFAULT_CONN_URL: &str = "http://localhost:8086"; +/// The default InfluxDb username. +pub const DEFAULT_USERNAME: &str = "root"; +/// The default InfluxDb password. +pub const DEFAULT_PASSWORD: &str = "password"; +/// The default whether to enable influx analytics writes. +pub const DEFAULT_ANALYTICS_ENABLED: bool = true; +/// The default name of the analytics database to connect to. +pub const DEFAULT_ANALYTICS_DATABASE_NAME: &str = "chronicle_analytics"; +/// The default whether to enable influx metrics writes. +pub const DEFAULT_METRICS_ENABLED: bool = true; +/// The default name of the metrics database to connect to. +pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; + +/// The influxdb [`Client`] config. +#[must_use] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct InfluxDbConfig { + /// The address of the InfluxDb instance. + pub conn_url: String, + /// The InfluxDb username. + pub username: String, + /// The InfluxDb password. + pub password: String, + /// Whether to enable influx analytics writes. + pub analytics_enabled: bool, + /// The name of the database to insert analytics. + pub analytics_database_name: String, + /// Whether to enable influx metrics writes. + pub metrics_enabled: bool, + /// The name of the database to insert metrics. + pub metrics_database_name: String, +} + +impl Default for InfluxDbConfig { + fn default() -> Self { + Self { + conn_url: DEFAULT_CONN_URL.to_string(), + username: DEFAULT_USERNAME.to_string(), + password: DEFAULT_PASSWORD.to_string(), + analytics_enabled: DEFAULT_ANALYTICS_ENABLED, + analytics_database_name: DEFAULT_ANALYTICS_DATABASE_NAME.to_string(), + metrics_enabled: DEFAULT_METRICS_ENABLED, + metrics_database_name: DEFAULT_METRICS_DATABASE_NAME.to_string(), + } + } +} diff --git a/src/db/influxdb/mod.rs b/src/db/influxdb/mod.rs index bbb5e1766..c219c79b9 100644 --- a/src/db/influxdb/mod.rs +++ b/src/db/influxdb/mod.rs @@ -1,14 +1,15 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +pub mod config; mod measurement; use std::ops::Deref; use influxdb::{Client, ReadQuery}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::de::DeserializeOwned; -pub use self::measurement::InfluxDbMeasurement; +pub use self::{config::InfluxDbConfig, measurement::InfluxDbMeasurement}; /// A wrapper for an InfluxDb [`Client`]. #[derive(Clone, Debug)] @@ -48,28 +49,28 @@ impl Deref for InfluxClient { /// A wrapper for the influxdb [`Client`]. #[derive(Clone, Debug)] pub struct InfluxDb { - #[cfg(feature = "metrics")] - metrics_client: InfluxClient, #[cfg(feature = "analytics")] analytics_client: InfluxClient, + #[cfg(feature = "metrics")] + metrics_client: InfluxClient, config: InfluxDbConfig, } impl InfluxDb { /// Create a new influx connection from config. pub async fn connect(config: &InfluxDbConfig) -> Result { - #[cfg(feature = "metrics")] - let metrics_client = { + #[cfg(feature = "analytics")] + let analytics_client = { let client = InfluxClient( - Client::new(&config.url, &config.metrics_database_name).with_auth(&config.username, &config.password), + Client::new(&config.conn_url, &config.analytics_database_name).with_auth(&config.username, &config.password), ); client.ping().await?; client }; - #[cfg(feature = "analytics")] - let analytics_client = { + #[cfg(feature = "metrics")] + let metrics_client = { let client = InfluxClient( - Client::new(&config.url, &config.analytics_database_name).with_auth(&config.username, &config.password), + Client::new(&config.conn_url, &config.metrics_database_name).with_auth(&config.username, &config.password), ); client.ping().await?; client @@ -83,41 +84,20 @@ impl InfluxDb { }) } - /// Get the metrics client. - #[cfg(feature = "metrics")] - pub fn metrics(&self) -> &InfluxClient { - &self.metrics_client - } - /// Get the analytics client. #[cfg(feature = "analytics")] pub fn analytics(&self) -> &InfluxClient { &self.analytics_client } + /// Get the metrics client. + #[cfg(feature = "metrics")] + pub fn metrics(&self) -> &InfluxClient { + &self.metrics_client + } + /// Get the config used to create the connection. pub fn config(&self) -> &InfluxDbConfig { &self.config } } - -/// The influxdb [`Client`] config. -#[must_use] -#[derive(Clone, Default, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct InfluxDbConfig { - /// The address of the InfluxDb instance. - pub url: String, - /// The InfluxDb username. - pub username: String, - /// The InfluxDb password. - pub password: String, - /// The name of the database to insert metrics. - pub metrics_database_name: String, - /// The name of the database to insert analytics. - pub analytics_database_name: String, - /// Whether to enable influx metrics writes. - pub metrics_enabled: bool, - /// Whether to enable influx analytics writes. - pub analytics_enabled: bool, -} From 5a4715cd5662938dd22d6c63f43125b73674343c Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 13:03:40 +0100 Subject: [PATCH 16/38] Fix clippy box default warning --- src/bin/inx-chronicle/cli.rs | 2 +- src/db/collections/analytics/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index bf3b10498..f0e5fca3e 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -367,7 +367,7 @@ impl From for Box Box::new(AddressAnalytics), AnalyticsChoice::BaseToken => Box::new(BaseTokenActivityAnalytics), AnalyticsChoice::BlockActivity => Box::new(BlockActivityAnalytics), - AnalyticsChoice::DailyActiveAddresses => Box::new(DailyActiveAddressesAnalytics::default()), + AnalyticsChoice::DailyActiveAddresses => Box::::default(), AnalyticsChoice::LedgerOutputs => Box::new(LedgerOutputAnalytics), AnalyticsChoice::LedgerSize => Box::new(LedgerSizeAnalytics), AnalyticsChoice::OutputActivity => Box::new(OutputActivityAnalytics), diff --git a/src/db/collections/analytics/mod.rs b/src/db/collections/analytics/mod.rs index 208108a1d..909f25b53 100644 --- a/src/db/collections/analytics/mod.rs +++ b/src/db/collections/analytics/mod.rs @@ -112,7 +112,7 @@ pub fn all_analytics() -> Vec> { Box::new(AddressAnalytics), Box::new(BaseTokenActivityAnalytics), Box::new(BlockActivityAnalytics), - Box::new(DailyActiveAddressesAnalytics::default()), + Box::::default(), Box::new(LedgerOutputAnalytics), Box::new(LedgerSizeAnalytics), Box::new(OutputActivityAnalytics), From 308a36105ab67688ac2fcd391f0dad737d3cb1f8 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 13:12:10 +0100 Subject: [PATCH 17/38] Fix docs --- src/db/influxdb/config.rs | 2 +- src/db/mongodb/config.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs index 9bdf5b18d..da10c0ada 100644 --- a/src/db/influxdb/config.rs +++ b/src/db/influxdb/config.rs @@ -20,7 +20,7 @@ pub const DEFAULT_METRICS_ENABLED: bool = true; /// The default name of the metrics database to connect to. pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; -/// The influxdb [`Client`] config. +/// The influxdb [`influxdb::Client`] config. #[must_use] #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] diff --git a/src/db/mongodb/config.rs b/src/db/mongodb/config.rs index b06cf7c62..7b1f8751f 100644 --- a/src/db/mongodb/config.rs +++ b/src/db/mongodb/config.rs @@ -20,7 +20,7 @@ pub const DEFAULT_DATABASE_NAME: &str = "chronicle"; /// The default minimum amount of connections in the pool. pub const DEFAULT_MIN_POOL_SIZE: u32 = 2; -/// The [`MongoDb`] config. +/// The [`super::MongoDb`] config. #[must_use] #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] From 05d3615e1a2c76747d3e5a2cdd6c24a9ab0f13db Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 13:12:59 +0100 Subject: [PATCH 18/38] Format --- src/bin/inx-chronicle/main.rs | 5 ++++- src/db/influxdb/mod.rs | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 412a192a8..e8752af57 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -55,7 +55,10 @@ async fn main() -> eyre::Result<()> { if config.inx.enabled { #[cfg(any(feature = "analytics", feature = "metrics"))] let influx_db = if config.influxdb.analytics_enabled || config.influxdb.metrics_enabled { - info!("Connecting to influx database at address `{}`", config.influxdb.conn_url); + info!( + "Connecting to influx database at address `{}`", + config.influxdb.conn_url + ); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; info!( "Connected to influx databases `{}` and `{}`", diff --git a/src/db/influxdb/mod.rs b/src/db/influxdb/mod.rs index c219c79b9..0cce7b792 100644 --- a/src/db/influxdb/mod.rs +++ b/src/db/influxdb/mod.rs @@ -62,7 +62,8 @@ impl InfluxDb { #[cfg(feature = "analytics")] let analytics_client = { let client = InfluxClient( - Client::new(&config.conn_url, &config.analytics_database_name).with_auth(&config.username, &config.password), + Client::new(&config.conn_url, &config.analytics_database_name) + .with_auth(&config.username, &config.password), ); client.ping().await?; client @@ -70,7 +71,8 @@ impl InfluxDb { #[cfg(feature = "metrics")] let metrics_client = { let client = InfluxClient( - Client::new(&config.conn_url, &config.metrics_database_name).with_auth(&config.username, &config.password), + Client::new(&config.conn_url, &config.metrics_database_name) + .with_auth(&config.username, &config.password), ); client.ping().await?; client From 72aa03dfeebcd18179ce85c5f426d6e6d86c2340 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 16 Dec 2022 13:21:22 +0100 Subject: [PATCH 19/38] Fix CI --- src/bin/inx-chronicle/cli.rs | 4 ++-- src/bin/inx-chronicle/config.rs | 34 ++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 58a255444..eb6a6cc94 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -172,10 +172,10 @@ pub struct JwtArgs { #[derive(Args, Debug)] pub struct LokiArgs { /// Toggle Grafana Loki log writes. - #[arg(long, value_name = "BOOL", default_value = to_str!(crate::config::DEFAULT_LOKI_ENABLED))] + #[arg(long, value_name = "BOOL", default_value = to_str!(crate::config::loki::DEFAULT_LOKI_ENABLED))] pub loki_enabled: Option, /// The url pointing to a Grafana Loki instance. - #[arg(long, value_name = "URL", default_value = crate::config::DEFAULT_LOKI_CONN_URL)] + #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_CONN_URL)] pub loki_conn_url: Option, } diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index 997ce3d6e..8ba5389b1 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -7,9 +7,6 @@ use chronicle::db::MongoDbConfig; use serde::{Deserialize, Serialize}; use thiserror::Error; -pub const DEFAULT_LOKI_ENABLED: bool = true; -pub const DEFAULT_LOKI_CONN_URL: &str = "http://localhost:3100"; - #[derive(Error, Debug)] pub enum ConfigError { #[error("failed to read config at '{0}': {1}")] @@ -30,7 +27,7 @@ pub struct ChronicleConfig { #[cfg(all(feature = "stardust", feature = "inx"))] pub inx: super::stardust_inx::InxConfig, #[cfg(feature = "loki")] - pub loki: LokiConfig, + pub loki: loki::LokiConfig, } impl ChronicleConfig { @@ -43,18 +40,25 @@ impl ChronicleConfig { } #[cfg(feature = "loki")] -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct LokiConfig { - pub enabled: bool, - pub conn_url: String, -} +pub mod loki { + use super::*; + + pub const DEFAULT_LOKI_ENABLED: bool = true; + pub const DEFAULT_LOKI_CONN_URL: &str = "http://localhost:3100"; + + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] + #[serde(default)] + pub struct LokiConfig { + pub enabled: bool, + pub conn_url: String, + } -impl Default for LokiConfig { - fn default() -> Self { - Self { - enabled: DEFAULT_LOKI_ENABLED, - conn_url: DEFAULT_LOKI_CONN_URL.to_string(), + impl Default for LokiConfig { + fn default() -> Self { + Self { + enabled: DEFAULT_LOKI_ENABLED, + conn_url: DEFAULT_LOKI_CONN_URL.to_string(), + } } } } From 697607fb39accca4be2c083853bb8059bd8c3b49 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Sat, 17 Dec 2022 17:40:44 +0100 Subject: [PATCH 20/38] Renaming --- docker/docker-compose.yml | 2 +- src/bin/inx-chronicle/cli.rs | 20 ++++++++++---------- src/bin/inx-chronicle/config.rs | 6 +++--- src/bin/inx-chronicle/main.rs | 7 ++----- src/bin/inx-chronicle/stardust_inx/config.rs | 6 +++--- src/bin/inx-chronicle/stardust_inx/mod.rs | 8 ++++---- src/db/influxdb/config.rs | 6 +++--- src/db/influxdb/mod.rs | 6 ++---- 8 files changed, 28 insertions(+), 33 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4db11d113..ce0c7df22 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -34,8 +34,8 @@ services: tty: true command: - "--mongodb-conn-str=mongodb://mongo:27017" - - "--inx-url=http://hornet:9029" - "--influxdb-url=http://influx:8086" + - "--inx-url=http://hornet:9029" - "--loki-url=http://loki:3100" influx: diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index eb6a6cc94..6298cdd15 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -75,8 +75,8 @@ use chronicle::db::influxdb::config as influxdb; #[derive(Args, Debug)] pub struct InfluxDbArgs { /// The url pointing to an InfluxDb instance. - #[arg(long, value_name = "URL", default_value = influxdb::DEFAULT_CONN_URL)] - pub influxdb_conn_url: Option, + #[arg(long, value_name = "URL", default_value = influxdb::DEFAULT_URL)] + pub influxdb_url: Option, /// The InfluxDb username. #[arg(long, value_name = "USERNAME", env = "INFLUXDB_USERNAME", default_value = influxdb::DEFAULT_USERNAME)] pub influxdb_username: Option, @@ -111,8 +111,8 @@ pub struct InxArgs { #[arg(long, value_name = "BOOL", default_value = to_str!(inx::DEFAULT_ENABLED))] pub inx_enabled: Option, /// The address of the node INX interface Chronicle tries to connect to - if enabled. - #[arg(long, value_name = "URL", default_value = inx::DEFAULT_CONN_URL)] - pub inx_conn_url: Option, + #[arg(long, value_name = "URL", default_value = inx::DEFAULT_URL)] + pub inx_url: Option, /// Time to wait until a new connection attempt is made. #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = inx::DEFAULT_RETRY_INTERVAL)] pub inx_retry_interval: Option, @@ -175,8 +175,8 @@ pub struct LokiArgs { #[arg(long, value_name = "BOOL", default_value = to_str!(crate::config::loki::DEFAULT_LOKI_ENABLED))] pub loki_enabled: Option, /// The url pointing to a Grafana Loki instance. - #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_CONN_URL)] - pub loki_conn_url: Option, + #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_URL)] + pub loki_url: Option, } fn parse_duration(arg: &str) -> Result { @@ -207,7 +207,7 @@ impl ClArgs { #[cfg(all(feature = "stardust", feature = "inx"))] { config.inx.enabled = self.inx.inx_enabled.unwrap(); - config.inx.conn_url = self.inx.inx_conn_url.as_ref().unwrap().clone(); + config.inx.url = self.inx.inx_url.as_ref().unwrap().clone(); config.inx.conn_retry_interval = self.inx.inx_retry_interval.unwrap(); config.inx.conn_retry_count = self.inx.inx_retry_count.unwrap(); config.inx.sync_start_milestone = self.inx.inx_sync_start.unwrap().into(); @@ -216,7 +216,7 @@ impl ClArgs { // InfluxDb #[cfg(any(feature = "analytics", feature = "metrics"))] { - config.influxdb.conn_url = self.influxdb.influxdb_conn_url.as_ref().unwrap().clone(); + config.influxdb.url = self.influxdb.influxdb_url.as_ref().unwrap().clone(); config.influxdb.username = self.influxdb.influxdb_username.as_ref().unwrap().clone(); config.influxdb.password = self.influxdb.influxdb_password.as_ref().unwrap().clone(); } @@ -247,7 +247,7 @@ impl ClArgs { // Loki #[cfg(feature = "loki")] { - config.loki.conn_url = self.loki.loki_conn_url.as_ref().unwrap().clone(); + config.loki.url = self.loki.loki_url.as_ref().unwrap().clone(); config.loki.enabled = self.loki.loki_enabled.unwrap(); } @@ -262,7 +262,7 @@ impl ClArgs { match subcommand { Subcommands::CreateConfig { file_path } => { let toml_config = format!( - "# This file was auto-generated by Chronicle's `create-config` tool at version '{}'.\n# Check CHANGELOG.md for breaking configuration changes, and re-run it if necessary.\n\n{}", + "# This file was auto-generated by Chronicle's `create-config` tool at version '{}'.\n# Check CHANGELOG.md for breaking configuration changes, and re-run the tool if necessary.\n\n{}", std::env::var("CARGO_PKG_VERSION").unwrap(), toml::to_string_pretty(config)? ); diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index 8ba5389b1..ca4b3f1b6 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -44,20 +44,20 @@ pub mod loki { use super::*; pub const DEFAULT_LOKI_ENABLED: bool = true; - pub const DEFAULT_LOKI_CONN_URL: &str = "http://localhost:3100"; + pub const DEFAULT_LOKI_URL: &str = "http://localhost:3100"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct LokiConfig { pub enabled: bool, - pub conn_url: String, + pub url: String, } impl Default for LokiConfig { fn default() -> Self { Self { enabled: DEFAULT_LOKI_ENABLED, - conn_url: DEFAULT_LOKI_CONN_URL.to_string(), + url: DEFAULT_LOKI_URL.to_string(), } } } diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index e8752af57..5bee6f815 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -55,10 +55,7 @@ async fn main() -> eyre::Result<()> { if config.inx.enabled { #[cfg(any(feature = "analytics", feature = "metrics"))] let influx_db = if config.influxdb.analytics_enabled || config.influxdb.metrics_enabled { - info!( - "Connecting to influx database at address `{}`", - config.influxdb.conn_url - ); + info!("Connecting to influx database at address `{}`", config.influxdb.url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; info!( "Connected to influx databases `{}` and `{}`", @@ -142,7 +139,7 @@ fn set_up_logging(#[allow(unused)] config: &ChronicleConfig) -> eyre::Result<()> }; #[cfg(feature = "loki")] let registry = { - let (layer, task) = tracing_loki::layer(config.loki.conn_url.parse()?, [].into(), [].into())?; + let (layer, task) = tracing_loki::layer(config.loki.url.parse()?, [].into(), [].into())?; tokio::spawn(task); registry.with(layer) }; diff --git a/src/bin/inx-chronicle/stardust_inx/config.rs b/src/bin/inx-chronicle/stardust_inx/config.rs index e8a149ac3..342fe0ad5 100644 --- a/src/bin/inx-chronicle/stardust_inx/config.rs +++ b/src/bin/inx-chronicle/stardust_inx/config.rs @@ -7,7 +7,7 @@ use chronicle::types::tangle::MilestoneIndex; use serde::{Deserialize, Serialize}; pub const DEFAULT_ENABLED: bool = true; -pub const DEFAULT_CONN_URL: &str = "http://localhost:9029"; +pub const DEFAULT_URL: &str = "http://localhost:9029"; pub const DEFAULT_RETRY_INTERVAL: &str = "5s"; pub const DEFAULT_RETRY_COUNT: usize = 30; pub const DEFAULT_SYNC_START: u32 = 0; @@ -18,7 +18,7 @@ pub const DEFAULT_SYNC_START: u32 = 0; pub struct InxConfig { pub enabled: bool, /// The bind address of node's INX interface. - pub conn_url: String, + pub url: String, /// The time that has to pass until a new connection attempt is made. #[serde(with = "humantime_serde")] pub conn_retry_interval: Duration, @@ -32,7 +32,7 @@ impl Default for InxConfig { fn default() -> Self { Self { enabled: DEFAULT_ENABLED, - conn_url: DEFAULT_CONN_URL.to_string(), + url: DEFAULT_URL.to_string(), conn_retry_interval: DEFAULT_RETRY_INTERVAL.parse::().unwrap().into(), conn_retry_count: DEFAULT_RETRY_COUNT, sync_start_milestone: DEFAULT_SYNC_START.into(), diff --git a/src/bin/inx-chronicle/stardust_inx/mod.rs b/src/bin/inx-chronicle/stardust_inx/mod.rs index 8179cef5c..f0643936c 100644 --- a/src/bin/inx-chronicle/stardust_inx/mod.rs +++ b/src/bin/inx-chronicle/stardust_inx/mod.rs @@ -88,14 +88,14 @@ impl InxWorker { } async fn connect(&self) -> Result { - let url = url::Url::parse(&self.config.conn_url)?; + let url = url::Url::parse(&self.config.url)?; if url.scheme() != "http" { - bail!(InxWorkerError::InvalidAddress(self.config.conn_url.clone())); + bail!(InxWorkerError::InvalidAddress(self.config.url.clone())); } for i in 0..self.config.conn_retry_count { - match Inx::connect(self.config.conn_url.clone()).await { + match Inx::connect(self.config.url.clone()).await { Ok(inx_client) => return Ok(inx_client), Err(_) => { warn!( @@ -135,7 +135,7 @@ impl InxWorker { #[instrument(skip_all, err, level = "trace")] async fn init(&mut self) -> Result<(MilestoneIndex, Inx)> { - info!("Connecting to INX at bind address `{}`.", &self.config.conn_url); + info!("Connecting to INX at bind address `{}`.", &self.config.url); let mut inx = self.connect().await?; info!("Connected to INX."); diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs index da10c0ada..30a6db0f9 100644 --- a/src/db/influxdb/config.rs +++ b/src/db/influxdb/config.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; /// The default InfluxDb URL to connect to. -pub const DEFAULT_CONN_URL: &str = "http://localhost:8086"; +pub const DEFAULT_URL: &str = "http://localhost:8086"; /// The default InfluxDb username. pub const DEFAULT_USERNAME: &str = "root"; /// The default InfluxDb password. @@ -26,7 +26,7 @@ pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; #[serde(default, deny_unknown_fields)] pub struct InfluxDbConfig { /// The address of the InfluxDb instance. - pub conn_url: String, + pub url: String, /// The InfluxDb username. pub username: String, /// The InfluxDb password. @@ -44,7 +44,7 @@ pub struct InfluxDbConfig { impl Default for InfluxDbConfig { fn default() -> Self { Self { - conn_url: DEFAULT_CONN_URL.to_string(), + url: DEFAULT_URL.to_string(), username: DEFAULT_USERNAME.to_string(), password: DEFAULT_PASSWORD.to_string(), analytics_enabled: DEFAULT_ANALYTICS_ENABLED, diff --git a/src/db/influxdb/mod.rs b/src/db/influxdb/mod.rs index 0cce7b792..118f7f33b 100644 --- a/src/db/influxdb/mod.rs +++ b/src/db/influxdb/mod.rs @@ -62,8 +62,7 @@ impl InfluxDb { #[cfg(feature = "analytics")] let analytics_client = { let client = InfluxClient( - Client::new(&config.conn_url, &config.analytics_database_name) - .with_auth(&config.username, &config.password), + Client::new(&config.url, &config.analytics_database_name).with_auth(&config.username, &config.password), ); client.ping().await?; client @@ -71,8 +70,7 @@ impl InfluxDb { #[cfg(feature = "metrics")] let metrics_client = { let client = InfluxClient( - Client::new(&config.conn_url, &config.metrics_database_name) - .with_auth(&config.username, &config.password), + Client::new(&config.url, &config.metrics_database_name).with_auth(&config.username, &config.password), ); client.ping().await?; client From be8ff50aa904f30fe65c4897ca482f0ece48dc91 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 19 Dec 2022 12:46:52 +0100 Subject: [PATCH 21/38] Simplify CLI --- Cargo.lock | 21 ------ Cargo.toml | 1 - src/bin/inx-chronicle/cli.rs | 124 +++++++++++++++++------------------ 3 files changed, 59 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 501847000..37190d3b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,7 +312,6 @@ dependencies = [ "bytesize", "chrono", "clap", - "const_format", "decimal", "derive_more", "dotenvy", @@ -424,26 +423,6 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cec318a675afcb6a1ea1d4340e2d377e56e47c266f28043ceccbf4412ddfdd3b" -[[package]] -name = "const_format" -version = "0.2.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7309d9b4d3d2c0641e018d449232f2e28f1b22933c137f157d3dbc14228b8c0e" -dependencies = [ - "const_format_proc_macros", -] - -[[package]] -name = "const_format_proc_macros" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - [[package]] name = "constant_time_eq" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index f4568b237..bcb550604 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,6 @@ path = "src/bin/inx-chronicle/main.rs" async-trait = { version = "0.1", default-features = false } bytesize = { version = "1.1", default-features = false } clap = { version = "4.0", default-features = false, features = ["env", "derive", "std", "help", "usage", "error-context", "wrap_help"] } -const_format = { version = "0.2", default-features = false } decimal = { version = "2.1", default-features = false, features = [ "serde" ] } derive_more = { version = "0.99", default-features = false, features = [ "add", "add_assign", "deref", "deref_mut" ] } dotenvy = { version = "0.15", default-features = false } diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 6298cdd15..7a19d1e86 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -6,12 +6,6 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; use crate::config::{ChronicleConfig, ConfigError}; -macro_rules! to_str { - ($arg:expr) => { - const_format::formatcp!("{}", $arg) - }; -} - /// Chronicle permanode storage as an INX plugin #[derive(Parser, Debug)] // #[command(author, version, about, next_display_order = None)] @@ -53,19 +47,19 @@ pub struct MongoDbArgs { env = "MONGODB_CONN_STR", default_value = mongodb::DEFAULT_CONN_STR, )] - pub mongodb_conn_str: Option, + pub mongodb_conn_str: String, /// The MongoDb username. #[arg(long, value_name = "USERNAME", env = "MONGODB_USERNAME", default_value = mongodb::DEFAULT_USERNAME)] - pub mongodb_username: Option, + pub mongodb_username: String, /// The MongoDb password. #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = mongodb::DEFAULT_PASSWORD)] - pub mongodb_password: Option, + pub mongodb_password: String, /// The MongoDb database name. #[arg(long, value_name = "NAME", default_value = mongodb::DEFAULT_DATABASE_NAME)] - pub mongodb_database_name: Option, + pub mongodb_database_name: String, /// The MongoDb minimum pool size. - #[arg(long, value_name = "SIZE", default_value = to_str!(mongodb::DEFAULT_MIN_POOL_SIZE))] - pub mongodb_min_pool_size: Option, + #[arg(long, value_name = "SIZE", default_value_t = mongodb::DEFAULT_MIN_POOL_SIZE)] + pub mongodb_min_pool_size: u32, } #[cfg(any(feature = "analytics", feature = "metrics"))] @@ -76,29 +70,29 @@ use chronicle::db::influxdb::config as influxdb; pub struct InfluxDbArgs { /// The url pointing to an InfluxDb instance. #[arg(long, value_name = "URL", default_value = influxdb::DEFAULT_URL)] - pub influxdb_url: Option, + pub influxdb_url: String, /// The InfluxDb username. #[arg(long, value_name = "USERNAME", env = "INFLUXDB_USERNAME", default_value = influxdb::DEFAULT_USERNAME)] - pub influxdb_username: Option, + pub influxdb_username: String, /// The InfluxDb password. #[arg(long, value_name = "PASSWORD", env = "INFLUXDB_PASSWORD", default_value = influxdb::DEFAULT_PASSWORD)] - pub influxdb_password: Option, + pub influxdb_password: String, /// Toggle InfluxDb time-series analytics writes. #[cfg(feature = "analytics")] - #[arg(long, value_name = "BOOL", default_value = to_str!(influxdb::DEFAULT_ANALYTICS_ENABLED))] - pub analytics_enabled: Option, + #[arg(long, value_name = "BOOL", default_value_t = influxdb::DEFAULT_ANALYTICS_ENABLED)] + pub analytics_enabled: bool, /// The Analytics database name. #[cfg(feature = "analytics")] #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_ANALYTICS_DATABASE_NAME)] - pub analytics_database_name: Option, + pub analytics_database_name: String, /// Toggle InfluxDb time-series metrics writes. #[cfg(feature = "metrics")] - #[arg(long, value_name = "BOOL", default_value = to_str!(influxdb::DEFAULT_METRICS_ENABLED))] - pub metrics_enabled: Option, + #[arg(long, value_name = "BOOL", default_value_t = influxdb::DEFAULT_METRICS_ENABLED)] + pub metrics_enabled: bool, /// The Metrics database name. #[cfg(feature = "metrics")] #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_METRICS_DATABASE_NAME)] - pub metrics_database_name: Option, + pub metrics_database_name: String, } #[cfg(feature = "inx")] @@ -108,21 +102,21 @@ use crate::stardust_inx::config as inx; #[derive(Args, Debug)] pub struct InxArgs { /// Toggles the INX synchronization workflow. - #[arg(long, value_name = "BOOL", default_value = to_str!(inx::DEFAULT_ENABLED))] - pub inx_enabled: Option, + #[arg(long, value_name = "BOOL", default_value_t = inx::DEFAULT_ENABLED)] + pub inx_enabled: bool, /// The address of the node INX interface Chronicle tries to connect to - if enabled. #[arg(long, value_name = "URL", default_value = inx::DEFAULT_URL)] - pub inx_url: Option, + pub inx_url: String, /// Time to wait until a new connection attempt is made. #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = inx::DEFAULT_RETRY_INTERVAL)] - pub inx_retry_interval: Option, + pub inx_retry_interval: std::time::Duration, /// Maximum number of tries to establish an INX connection. - #[arg(long, value_name = "COUNT", default_value = to_str!(inx::DEFAULT_RETRY_COUNT))] - pub inx_retry_count: Option, + #[arg(long, value_name = "COUNT", default_value_t = inx::DEFAULT_RETRY_COUNT)] + pub inx_retry_count: usize, /// Milestone at which synchronization should begin. If set to `1` Chronicle will try to sync back until the /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. - #[arg(long, value_name = "START", default_value = to_str!(inx::DEFAULT_SYNC_START))] - pub inx_sync_start: Option, + #[arg(long, value_name = "START", default_value_t = inx::DEFAULT_SYNC_START)] + pub inx_sync_start: u32, } #[cfg(feature = "api")] @@ -132,11 +126,11 @@ use crate::api::config as api; #[derive(Args, Debug)] pub struct ApiArgs { /// Toggle REST API. - #[arg(long, value_name = "BOOL", default_value = to_str!(api::DEFAULT_ENABLED))] - pub api_enabled: Option, + #[arg(long, value_name = "BOOL", default_value_t = api::DEFAULT_ENABLED)] + pub api_enabled: bool, /// API listening port. - #[arg(long, value_name = "PORT", default_value = to_str!(api::DEFAULT_PORT))] - pub api_port: Option, + #[arg(long, value_name = "PORT", default_value_t = api::DEFAULT_PORT)] + pub api_port: u16, /// CORS setting. #[arg(long = "allow-origin", value_name = "IP", default_value = api::DEFAULT_ALLOW_ORIGINS)] pub allow_origins: Vec, @@ -144,8 +138,8 @@ pub struct ApiArgs { #[arg(long = "public-route", value_name = "ROUTE", default_value = api::DEFAULT_PUBLIC_ROUTES)] pub public_routes: Vec, /// Maximum number of results returned by a single API call. - #[arg(long, value_name = "SIZE", default_value = to_str!(api::DEFAULT_MAX_PAGE_SIZE))] - pub max_page_size: Option, + #[arg(long, value_name = "SIZE", default_value_t = api::DEFAULT_MAX_PAGE_SIZE)] + pub max_page_size: usize, /// JWT arguments. #[command(flatten)] pub jwt: JwtArgs, @@ -159,24 +153,24 @@ pub struct JwtArgs { pub jwt_identity: Option, /// The password used for JWT authentication. #[arg(long, value_name = "PASSWORD", env = "JWT_PASSWORD", default_value = api::DEFAULT_JWT_PASSWORD)] - pub jwt_password: Option, + pub jwt_password: String, /// The salt used for JWT authentication. #[arg(long, value_name = "SALT", env = "JWT_SALT", default_value = api::DEFAULT_JWT_SALT)] - pub jwt_salt: Option, + pub jwt_salt: String, /// The setting for when the (JWT) token expires. #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = api::DEFAULT_JWT_EXPIRATION)] - pub jwt_expiration: Option, + pub jwt_expiration: std::time::Duration, } #[cfg(feature = "loki")] #[derive(Args, Debug)] pub struct LokiArgs { /// Toggle Grafana Loki log writes. - #[arg(long, value_name = "BOOL", default_value = to_str!(crate::config::loki::DEFAULT_LOKI_ENABLED))] - pub loki_enabled: Option, + #[arg(long, value_name = "BOOL", default_value_t = crate::config::loki::DEFAULT_LOKI_ENABLED)] + pub loki_enabled: bool, /// The url pointing to a Grafana Loki instance. #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_URL)] - pub loki_url: Option, + pub loki_url: String, } fn parse_duration(arg: &str) -> Result { @@ -197,58 +191,58 @@ impl ClArgs { // MongoDb // Note: all unwraps are fine because we defined defaults for all, so none of them can be None ;) - config.mongodb.conn_str = self.mongodb.mongodb_conn_str.as_ref().unwrap().clone(); - config.mongodb.database_name = self.mongodb.mongodb_database_name.as_ref().unwrap().clone(); - config.mongodb.username = self.mongodb.mongodb_username.as_ref().unwrap().clone(); - config.mongodb.password = self.mongodb.mongodb_password.as_ref().unwrap().clone(); - config.mongodb.min_pool_size = self.mongodb.mongodb_min_pool_size.unwrap(); + config.mongodb.conn_str = self.mongodb.mongodb_conn_str.clone(); + config.mongodb.database_name = self.mongodb.mongodb_database_name.clone(); + config.mongodb.username = self.mongodb.mongodb_username.clone(); + config.mongodb.password = self.mongodb.mongodb_password.clone(); + config.mongodb.min_pool_size = self.mongodb.mongodb_min_pool_size; // INX #[cfg(all(feature = "stardust", feature = "inx"))] { - config.inx.enabled = self.inx.inx_enabled.unwrap(); - config.inx.url = self.inx.inx_url.as_ref().unwrap().clone(); - config.inx.conn_retry_interval = self.inx.inx_retry_interval.unwrap(); - config.inx.conn_retry_count = self.inx.inx_retry_count.unwrap(); - config.inx.sync_start_milestone = self.inx.inx_sync_start.unwrap().into(); + config.inx.enabled = self.inx.inx_enabled; + config.inx.url = self.inx.inx_url.clone(); + config.inx.conn_retry_interval = self.inx.inx_retry_interval; + config.inx.conn_retry_count = self.inx.inx_retry_count; + config.inx.sync_start_milestone = self.inx.inx_sync_start.into(); } // InfluxDb #[cfg(any(feature = "analytics", feature = "metrics"))] { - config.influxdb.url = self.influxdb.influxdb_url.as_ref().unwrap().clone(); - config.influxdb.username = self.influxdb.influxdb_username.as_ref().unwrap().clone(); - config.influxdb.password = self.influxdb.influxdb_password.as_ref().unwrap().clone(); + config.influxdb.url = self.influxdb.influxdb_url.clone(); + config.influxdb.username = self.influxdb.influxdb_username.clone(); + config.influxdb.password = self.influxdb.influxdb_password.clone(); } #[cfg(feature = "analytics")] { - config.influxdb.analytics_enabled = self.influxdb.analytics_enabled.unwrap(); - config.influxdb.analytics_database_name = self.influxdb.analytics_database_name.as_ref().unwrap().clone(); + config.influxdb.analytics_enabled = self.influxdb.analytics_enabled; + config.influxdb.analytics_database_name = self.influxdb.analytics_database_name.clone(); } #[cfg(feature = "metrics")] { - config.influxdb.metrics_enabled = self.influxdb.metrics_enabled.unwrap(); - config.influxdb.metrics_database_name = self.influxdb.metrics_database_name.as_ref().unwrap().clone(); + config.influxdb.metrics_enabled = self.influxdb.metrics_enabled; + config.influxdb.metrics_database_name = self.influxdb.metrics_database_name.clone(); } // API #[cfg(feature = "api")] { - config.api.enabled = self.api.api_enabled.unwrap(); - config.api.port = self.api.api_port.unwrap(); + config.api.enabled = self.api.api_enabled; + config.api.port = self.api.api_port; config.api.allow_origins = (&self.api.allow_origins).into(); - config.api.jwt_password = self.api.jwt.jwt_password.as_ref().unwrap().clone(); - config.api.jwt_salt = self.api.jwt.jwt_salt.as_ref().unwrap().clone(); + config.api.jwt_password = self.api.jwt.jwt_password.clone(); + config.api.jwt_salt = self.api.jwt.jwt_salt.clone(); config.api.jwt_identity_file = self.api.jwt.jwt_identity.clone(); - config.api.max_page_size = self.api.max_page_size.unwrap(); + config.api.max_page_size = self.api.max_page_size; config.api.public_routes = self.api.public_routes.clone(); } // Loki #[cfg(feature = "loki")] { - config.loki.url = self.loki.loki_url.as_ref().unwrap().clone(); - config.loki.enabled = self.loki.loki_enabled.unwrap(); + config.loki.url = self.loki.loki_url.clone(); + config.loki.enabled = self.loki.loki_enabled; } Ok(config) From cd44f15e6d38d8b8d594e4f0829cc7276b9c6034 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 19 Dec 2022 12:54:55 +0100 Subject: [PATCH 22/38] Remove config file --- src/bin/inx-chronicle/cli.rs | 28 +++------------------- src/bin/inx-chronicle/config.rs | 20 ---------------- src/bin/inx-chronicle/main.rs | 2 +- tests/common/mod.rs | 41 +++++---------------------------- 4 files changed, 10 insertions(+), 81 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 7a19d1e86..fc4e95312 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -4,16 +4,13 @@ use chronicle::db::mongodb::config as mongodb; use clap::{Args, Parser, Subcommand, ValueEnum}; -use crate::config::{ChronicleConfig, ConfigError}; +use crate::config::ChronicleConfig; /// Chronicle permanode storage as an INX plugin #[derive(Parser, Debug)] // #[command(author, version, about, next_display_order = None)] #[command(author, version, about)] pub struct ClArgs { - /// The location of the configuration file. - #[arg(short, long, value_name = "FILEPATH")] - pub config: Option, /// MongoDb arguments. #[command(flatten, next_help_heading = "MongoDb")] pub mongodb: MongoDbArgs, @@ -182,11 +179,7 @@ impl ClArgs { /// with defaults for those that are not provided. Note that a config file must be fully specified /// as it cannot be overwritten with the CLI defaults. If you plan on using a `config.toml` use /// Chronicle's `create-config' tool to make sure of that. - pub fn get_config(&self) -> Result { - if let Some(config_path) = &self.config { - return ChronicleConfig::from_file(config_path); - } - + pub fn get_config(&self) -> ChronicleConfig { let mut config = ChronicleConfig::default(); // MongoDb @@ -245,7 +238,7 @@ impl ClArgs { config.loki.enabled = self.loki.loki_enabled; } - Ok(config) + config } /// Process subcommands and return whether the app should early exit. @@ -254,16 +247,6 @@ impl ClArgs { pub async fn process_subcommands(&self, config: &ChronicleConfig) -> eyre::Result { if let Some(subcommand) = &self.subcommand { match subcommand { - Subcommands::CreateConfig { file_path } => { - let toml_config = format!( - "# This file was auto-generated by Chronicle's `create-config` tool at version '{}'.\n# Check CHANGELOG.md for breaking configuration changes, and re-run the tool if necessary.\n\n{}", - std::env::var("CARGO_PKG_VERSION").unwrap(), - toml::to_string_pretty(config)? - ); - std::fs::write(file_path.as_ref().unwrap(), toml_config)?; - tracing::info!("Written generated config to: '{}'", file_path.as_ref().unwrap()); - return Ok(PostCommand::Exit); - } #[cfg(feature = "api")] Subcommands::GenerateJWT => { use crate::api::ApiConfigData; @@ -446,11 +429,6 @@ impl From for Box, - }, /// Generate a JWT token using the available config. #[cfg(feature = "api")] GenerateJWT, diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index ca4b3f1b6..0be55cea8 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -1,19 +1,8 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{fs, path::Path}; - use chronicle::db::MongoDbConfig; use serde::{Deserialize, Serialize}; -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum ConfigError { - #[error("failed to read config at '{0}': {1}")] - FileRead(String, std::io::Error), - #[error("toml deserialization failed: {0}")] - TomlDeserialization(toml::de::Error), -} /// Configuration of Chronicle. #[derive(Clone, Default, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -30,15 +19,6 @@ pub struct ChronicleConfig { pub loki: loki::LokiConfig, } -impl ChronicleConfig { - /// Reads the config from the file located at `path`. - pub fn from_file(path: impl AsRef) -> Result { - fs::read_to_string(&path) - .map_err(|e| ConfigError::FileRead(path.as_ref().display().to_string(), e)) - .and_then(|contents| toml::from_str::(&contents).map_err(ConfigError::TomlDeserialization)) - } -} - #[cfg(feature = "loki")] pub mod loki { use super::*; diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 5bee6f815..ac0d76d77 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -27,7 +27,7 @@ async fn main() -> eyre::Result<()> { dotenvy::dotenv().ok(); let cl_args = ClArgs::parse(); - let config = cl_args.get_config()?; + let config = cl_args.get_config(); set_up_logging(&config)?; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index b4dcb78b2..6ed1f3d6d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,47 +1,18 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::path::Path; - use chronicle::db::{MongoDb, MongoDbCollection, MongoDbConfig}; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum TestDbError { - #[error("failed to read config at '{0}': {1}")] - FileRead(String, std::io::Error), - #[error("toml deserialization failed: {0}")] - TomlDeserialization(toml::de::Error), -} #[allow(unused)] pub async fn setup_database(database_name: impl ToString) -> eyre::Result { dotenvy::dotenv().ok(); - let get_mongodb_test_config = || -> MongoDbConfig { - MongoDbConfig { - conn_str: std::env::var("MONGODB_CONN_STR").unwrap_or_else(|_| "mongodb://localhost:27017".to_owned()), - database_name: database_name.to_string(), - username: std::env::var("MONGODB_USERNAME").unwrap_or_else(|_| "root".to_owned()), - password: std::env::var("MONGODB_PASSWORD").unwrap_or_else(|_| "root".to_owned()), - min_pool_size: 2, - } - }; - - let mut test_config = if let Ok(path) = std::env::var("CHRONICLE_TEST_CONFIG") { - let val = std::fs::read_to_string(&path) - .map_err(|e| TestDbError::FileRead(AsRef::::as_ref(&path).display().to_string(), e)) - .and_then(|contents| toml::from_str::(&contents).map_err(TestDbError::TomlDeserialization))?; - - if let Some(mongodb) = val.get("mongodb").cloned() { - let mut config: MongoDbConfig = mongodb.try_into().map_err(TestDbError::TomlDeserialization)?; - config.database_name = database_name.to_string(); - config - } else { - get_mongodb_test_config() - } - } else { - get_mongodb_test_config() + let test_config = MongoDbConfig { + conn_str: std::env::var("MONGODB_CONN_STR").unwrap_or_else(|_| "mongodb://localhost:27017".to_owned()), + database_name: database_name.to_string(), + username: std::env::var("MONGODB_USERNAME").unwrap_or_else(|_| "root".to_owned()), + password: std::env::var("MONGODB_PASSWORD").unwrap_or_else(|_| "root".to_owned()), + min_pool_size: 2, }; let db = MongoDb::connect(&test_config).await?; From 8d2d806f3a80f46e0fc2941aed2ded8ce1ed7f7f Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 19 Dec 2022 13:27:24 +0100 Subject: [PATCH 23/38] Switch toggle to disable flags --- src/bin/inx-chronicle/cli.rs | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index fc4e95312..de75e4629 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -74,18 +74,18 @@ pub struct InfluxDbArgs { /// The InfluxDb password. #[arg(long, value_name = "PASSWORD", env = "INFLUXDB_PASSWORD", default_value = influxdb::DEFAULT_PASSWORD)] pub influxdb_password: String, - /// Toggle InfluxDb time-series analytics writes. + /// Disable InfluxDb time-series analytics writes. #[cfg(feature = "analytics")] - #[arg(long, value_name = "BOOL", default_value_t = influxdb::DEFAULT_ANALYTICS_ENABLED)] - pub analytics_enabled: bool, + #[arg(long, value_name = "BOOL", default_value_t = !influxdb::DEFAULT_ANALYTICS_ENABLED)] + pub disable_analytics: bool, /// The Analytics database name. #[cfg(feature = "analytics")] #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_ANALYTICS_DATABASE_NAME)] pub analytics_database_name: String, - /// Toggle InfluxDb time-series metrics writes. + /// Disable InfluxDb time-series metrics writes. #[cfg(feature = "metrics")] - #[arg(long, value_name = "BOOL", default_value_t = influxdb::DEFAULT_METRICS_ENABLED)] - pub metrics_enabled: bool, + #[arg(long, value_name = "BOOL", default_value_t = !influxdb::DEFAULT_METRICS_ENABLED)] + pub disable_metrics: bool, /// The Metrics database name. #[cfg(feature = "metrics")] #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_METRICS_DATABASE_NAME)] @@ -98,9 +98,9 @@ use crate::stardust_inx::config as inx; #[cfg(feature = "inx")] #[derive(Args, Debug)] pub struct InxArgs { - /// Toggles the INX synchronization workflow. - #[arg(long, value_name = "BOOL", default_value_t = inx::DEFAULT_ENABLED)] - pub inx_enabled: bool, + /// Disable the INX synchronization workflow. + #[arg(long, value_name = "BOOL", default_value_t = !inx::DEFAULT_ENABLED)] + pub disable_inx: bool, /// The address of the node INX interface Chronicle tries to connect to - if enabled. #[arg(long, value_name = "URL", default_value = inx::DEFAULT_URL)] pub inx_url: String, @@ -122,9 +122,9 @@ use crate::api::config as api; #[cfg(feature = "api")] #[derive(Args, Debug)] pub struct ApiArgs { - /// Toggle REST API. - #[arg(long, value_name = "BOOL", default_value_t = api::DEFAULT_ENABLED)] - pub api_enabled: bool, + /// Disable REST API. + #[arg(long, value_name = "BOOL", default_value_t = !api::DEFAULT_ENABLED)] + pub disable_api: bool, /// API listening port. #[arg(long, value_name = "PORT", default_value_t = api::DEFAULT_PORT)] pub api_port: u16, @@ -162,9 +162,9 @@ pub struct JwtArgs { #[cfg(feature = "loki")] #[derive(Args, Debug)] pub struct LokiArgs { - /// Toggle Grafana Loki log writes. - #[arg(long, value_name = "BOOL", default_value_t = crate::config::loki::DEFAULT_LOKI_ENABLED)] - pub loki_enabled: bool, + /// Disable Grafana Loki log writes. + #[arg(long, value_name = "BOOL", default_value_t = !crate::config::loki::DEFAULT_LOKI_ENABLED)] + pub disable_loki: bool, /// The url pointing to a Grafana Loki instance. #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_URL)] pub loki_url: String, @@ -193,7 +193,7 @@ impl ClArgs { // INX #[cfg(all(feature = "stardust", feature = "inx"))] { - config.inx.enabled = self.inx.inx_enabled; + config.inx.enabled = !self.inx.disable_inx; config.inx.url = self.inx.inx_url.clone(); config.inx.conn_retry_interval = self.inx.inx_retry_interval; config.inx.conn_retry_count = self.inx.inx_retry_count; @@ -209,19 +209,19 @@ impl ClArgs { } #[cfg(feature = "analytics")] { - config.influxdb.analytics_enabled = self.influxdb.analytics_enabled; + config.influxdb.analytics_enabled = !self.influxdb.disable_analytics; config.influxdb.analytics_database_name = self.influxdb.analytics_database_name.clone(); } #[cfg(feature = "metrics")] { - config.influxdb.metrics_enabled = self.influxdb.metrics_enabled; + config.influxdb.metrics_enabled = !self.influxdb.disable_metrics; config.influxdb.metrics_database_name = self.influxdb.metrics_database_name.clone(); } // API #[cfg(feature = "api")] { - config.api.enabled = self.api.api_enabled; + config.api.enabled = !self.api.disable_api; config.api.port = self.api.api_port; config.api.allow_origins = (&self.api.allow_origins).into(); config.api.jwt_password = self.api.jwt.jwt_password.clone(); @@ -234,8 +234,8 @@ impl ClArgs { // Loki #[cfg(feature = "loki")] { + config.loki.enabled = !self.loki.disable_loki; config.loki.url = self.loki.loki_url.clone(); - config.loki.enabled = self.loki.loki_enabled; } config From 5776d564ef29121145bfffc9e3aba93bc5ca124d Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 19 Dec 2022 13:42:09 +0100 Subject: [PATCH 24/38] Re-enable replica set for integration tests --- .github/workflows/_test_int.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index 8b2331744..a88fedf53 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -37,9 +37,9 @@ jobs: uses: supercharge/mongodb-github-action@1.8.0 with: mongodb-version: ${{ inputs.mongodb }} - #mongodb-replica-set: test-rs - mongodb-username: root - mongodb-password: root + mongodb-replica-set: test-rs + #mongodb-username: root + #mongodb-password: root - name: Test DB uses: actions-rs/cargo@v1 From 5f5c7e011574ef0175fadf726a98249b6613e929 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Mon, 19 Dec 2022 14:08:09 +0100 Subject: [PATCH 25/38] Fix merge --- src/bin/inx-chronicle/api/stardust/poi/routes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/inx-chronicle/api/stardust/poi/routes.rs b/src/bin/inx-chronicle/api/stardust/poi/routes.rs index efe8535f6..f9697242b 100644 --- a/src/bin/inx-chronicle/api/stardust/poi/routes.rs +++ b/src/bin/inx-chronicle/api/stardust/poi/routes.rs @@ -23,14 +23,14 @@ use super::{ use crate::api::{ error::{CorruptStateError, MissingError, RequestError}, router::{Router, RouterState}, - ApiData, ApiResult, + ApiConfigData, ApiResult, }; pub fn routes() -> Router where S: Clone + Send + Sync + 'static, MongoDb: FromRef>, - ApiData: FromRef>, + ApiConfigData: FromRef>, { Router::new() .route( From e0a7e71066b2a7ca5ab9fa08556b7b235f09e296 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 20 Dec 2022 11:01:51 +0100 Subject: [PATCH 26/38] (Temporarily) fix integration test CI --- .github/workflows/_test_int.yml | 7 ++++--- src/bin/inx-chronicle/cli.rs | 1 - tests/common/mod.rs | 15 +++++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index a88fedf53..63d074cdf 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -37,9 +37,10 @@ jobs: uses: supercharge/mongodb-github-action@1.8.0 with: mongodb-version: ${{ inputs.mongodb }} - mongodb-replica-set: test-rs - #mongodb-username: root - #mongodb-password: root + # FIXME: replica-set with authentication + #mongodb-replica-set: test-rs + mongodb-username: root + mongodb-password: root - name: Test DB uses: actions-rs/cargo@v1 diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index de75e4629..3760d7a3b 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -183,7 +183,6 @@ impl ClArgs { let mut config = ChronicleConfig::default(); // MongoDb - // Note: all unwraps are fine because we defined defaults for all, so none of them can be None ;) config.mongodb.conn_str = self.mongodb.mongodb_conn_str.clone(); config.mongodb.database_name = self.mongodb.mongodb_database_name.clone(); config.mongodb.username = self.mongodb.mongodb_username.clone(); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6ed1f3d6d..af18fc1bb 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,12 +7,15 @@ use chronicle::db::{MongoDb, MongoDbCollection, MongoDbConfig}; pub async fn setup_database(database_name: impl ToString) -> eyre::Result { dotenvy::dotenv().ok(); - let test_config = MongoDbConfig { - conn_str: std::env::var("MONGODB_CONN_STR").unwrap_or_else(|_| "mongodb://localhost:27017".to_owned()), - database_name: database_name.to_string(), - username: std::env::var("MONGODB_USERNAME").unwrap_or_else(|_| "root".to_owned()), - password: std::env::var("MONGODB_PASSWORD").unwrap_or_else(|_| "root".to_owned()), - min_pool_size: 2, + let mut test_config = MongoDbConfig::default(); + if let Ok(conn_str) = std::env::var("MONGODB_CONN_STR") { + test_config.conn_str = conn_str; + }; + if let Ok(username) = std::env::var("MONGODB_USERNAME") { + test_config.username = username; + }; + if let Ok(password) = std::env::var("MONGODB_PASSWORD") { + test_config.password = password; }; let db = MongoDb::connect(&test_config).await?; From 4c225474d49f774e1e21aef1df34953aa7657826 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 20 Dec 2022 11:33:06 +0100 Subject: [PATCH 27/38] Fix and small reorder --- src/bin/inx-chronicle/cli.rs | 34 +++++++++++++++++----------------- tests/common/mod.rs | 2 ++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 3760d7a3b..0a3255e1e 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -74,22 +74,22 @@ pub struct InfluxDbArgs { /// The InfluxDb password. #[arg(long, value_name = "PASSWORD", env = "INFLUXDB_PASSWORD", default_value = influxdb::DEFAULT_PASSWORD)] pub influxdb_password: String, - /// Disable InfluxDb time-series analytics writes. - #[cfg(feature = "analytics")] - #[arg(long, value_name = "BOOL", default_value_t = !influxdb::DEFAULT_ANALYTICS_ENABLED)] - pub disable_analytics: bool, /// The Analytics database name. #[cfg(feature = "analytics")] #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_ANALYTICS_DATABASE_NAME)] pub analytics_database_name: String, - /// Disable InfluxDb time-series metrics writes. - #[cfg(feature = "metrics")] - #[arg(long, value_name = "BOOL", default_value_t = !influxdb::DEFAULT_METRICS_ENABLED)] - pub disable_metrics: bool, /// The Metrics database name. #[cfg(feature = "metrics")] #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_METRICS_DATABASE_NAME)] pub metrics_database_name: String, + /// Disable InfluxDb time-series analytics writes. + #[cfg(feature = "analytics")] + #[arg(long, default_value_t = !influxdb::DEFAULT_ANALYTICS_ENABLED)] + pub disable_analytics: bool, + /// Disable InfluxDb time-series metrics writes. + #[cfg(feature = "metrics")] + #[arg(long, default_value_t = !influxdb::DEFAULT_METRICS_ENABLED)] + pub disable_metrics: bool, } #[cfg(feature = "inx")] @@ -98,9 +98,6 @@ use crate::stardust_inx::config as inx; #[cfg(feature = "inx")] #[derive(Args, Debug)] pub struct InxArgs { - /// Disable the INX synchronization workflow. - #[arg(long, value_name = "BOOL", default_value_t = !inx::DEFAULT_ENABLED)] - pub disable_inx: bool, /// The address of the node INX interface Chronicle tries to connect to - if enabled. #[arg(long, value_name = "URL", default_value = inx::DEFAULT_URL)] pub inx_url: String, @@ -114,6 +111,9 @@ pub struct InxArgs { /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. #[arg(long, value_name = "START", default_value_t = inx::DEFAULT_SYNC_START)] pub inx_sync_start: u32, + /// Disable the INX synchronization workflow. + #[arg(long, default_value_t = !inx::DEFAULT_ENABLED)] + pub disable_inx: bool, } #[cfg(feature = "api")] @@ -122,9 +122,6 @@ use crate::api::config as api; #[cfg(feature = "api")] #[derive(Args, Debug)] pub struct ApiArgs { - /// Disable REST API. - #[arg(long, value_name = "BOOL", default_value_t = !api::DEFAULT_ENABLED)] - pub disable_api: bool, /// API listening port. #[arg(long, value_name = "PORT", default_value_t = api::DEFAULT_PORT)] pub api_port: u16, @@ -140,6 +137,9 @@ pub struct ApiArgs { /// JWT arguments. #[command(flatten)] pub jwt: JwtArgs, + /// Disable REST API. + #[arg(long, default_value_t = !api::DEFAULT_ENABLED)] + pub disable_api: bool, } #[cfg(feature = "api")] @@ -162,12 +162,12 @@ pub struct JwtArgs { #[cfg(feature = "loki")] #[derive(Args, Debug)] pub struct LokiArgs { - /// Disable Grafana Loki log writes. - #[arg(long, value_name = "BOOL", default_value_t = !crate::config::loki::DEFAULT_LOKI_ENABLED)] - pub disable_loki: bool, /// The url pointing to a Grafana Loki instance. #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_URL)] pub loki_url: String, + /// Disable Grafana Loki log writes. + #[arg(long, default_value_t = !crate::config::loki::DEFAULT_LOKI_ENABLED)] + pub disable_loki: bool, } fn parse_duration(arg: &str) -> Result { diff --git a/tests/common/mod.rs b/tests/common/mod.rs index af18fc1bb..117c543ec 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -8,6 +8,8 @@ pub async fn setup_database(database_name: impl ToString) -> eyre::Result Date: Tue, 20 Dec 2022 11:41:01 +0100 Subject: [PATCH 28/38] Obey the Clippy overlord --- tests/common/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 117c543ec..5494950b2 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,8 +7,10 @@ use chronicle::db::{MongoDb, MongoDbCollection, MongoDbConfig}; pub async fn setup_database(database_name: impl ToString) -> eyre::Result { dotenvy::dotenv().ok(); - let mut test_config = MongoDbConfig::default(); - test_config.database_name = database_name.to_string(); + let mut test_config = MongoDbConfig { + database_name: database_name.to_string(), + ..Default::default() + }; if let Ok(conn_str) = std::env::var("MONGODB_CONN_STR") { test_config.conn_str = conn_str; From d20eccd6de12f73844758cfb477224e01906cad4 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 20 Dec 2022 14:20:38 +0100 Subject: [PATCH 29/38] Impl From<*Args> for *Configs --- src/bin/inx-chronicle/cli.rs | 148 ++++++++++++++++++++--------------- src/db/influxdb/config.rs | 12 +++ 2 files changed, 97 insertions(+), 63 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 0a3255e1e..05df83c79 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -59,6 +59,18 @@ pub struct MongoDbArgs { pub mongodb_min_pool_size: u32, } +impl From<&MongoDbArgs> for chronicle::db::MongoDbConfig { + fn from(value: &MongoDbArgs) -> Self { + Self { + conn_str: value.mongodb_conn_str.clone(), + username: value.mongodb_username.clone(), + password: value.mongodb_password.clone(), + database_name: value.mongodb_database_name.clone(), + min_pool_size: value.mongodb_min_pool_size, + } + } +} + #[cfg(any(feature = "analytics", feature = "metrics"))] use chronicle::db::influxdb::config as influxdb; @@ -92,10 +104,29 @@ pub struct InfluxDbArgs { pub disable_metrics: bool, } -#[cfg(feature = "inx")] +#[cfg(any(feature = "analytics", feature = "metrics"))] +impl From<&InfluxDbArgs> for chronicle::db::influxdb::InfluxDbConfig { + fn from(value: &InfluxDbArgs) -> Self { + Self { + url: value.influxdb_url.clone(), + username: value.influxdb_username.clone(), + password: value.influxdb_password.clone(), + #[cfg(feature = "analytics")] + analytics_enabled: !value.disable_analytics, + #[cfg(feature = "analytics")] + analytics_database_name: value.analytics_database_name.clone(), + #[cfg(feature = "metrics")] + metrics_enabled: !value.disable_metrics, + #[cfg(feature = "metrics")] + metrics_database_name: value.metrics_database_name.clone(), + } + } +} + +#[cfg(all(feature = "stardust", feature = "inx"))] use crate::stardust_inx::config as inx; -#[cfg(feature = "inx")] +#[cfg(all(feature = "stardust", feature = "inx"))] #[derive(Args, Debug)] pub struct InxArgs { /// The address of the node INX interface Chronicle tries to connect to - if enabled. @@ -116,6 +147,19 @@ pub struct InxArgs { pub disable_inx: bool, } +#[cfg(all(feature = "stardust", feature = "inx"))] +impl From<&InxArgs> for inx::InxConfig { + fn from(value: &InxArgs) -> Self { + Self { + enabled: !value.disable_inx, + url: value.inx_url.clone(), + conn_retry_interval: value.inx_retry_interval, + conn_retry_count: value.inx_retry_count, + sync_start_milestone: value.inx_sync_start.into(), + } + } +} + #[cfg(feature = "api")] use crate::api::config as api; @@ -142,6 +186,23 @@ pub struct ApiArgs { pub disable_api: bool, } +#[cfg(feature = "api")] +impl From<&ApiArgs> for api::ApiConfig { + fn from(value: &ApiArgs) -> Self { + Self { + enabled: !value.disable_api, + port: value.api_port, + allow_origins: (&value.allow_origins).into(), + jwt_password: value.jwt.jwt_password.clone(), + jwt_salt: value.jwt.jwt_salt.clone(), + jwt_identity_file: value.jwt.jwt_identity.clone(), + jwt_expiration: value.jwt.jwt_expiration, + max_page_size: value.max_page_size, + public_routes: value.public_routes.clone(), + } + } +} + #[cfg(feature = "api")] #[derive(Args, Debug)] pub struct JwtArgs { @@ -170,74 +231,35 @@ pub struct LokiArgs { pub disable_loki: bool, } +#[cfg(feature = "loki")] +impl From<&LokiArgs> for crate::config::loki::LokiConfig { + fn from(value: &LokiArgs) -> Self { + Self { + enabled: !value.disable_loki, + url: value.loki_url.clone(), + } + } +} + +#[cfg(any(all(feature = "stardust", feature = "inx"), feature = "api"))] fn parse_duration(arg: &str) -> Result { arg.parse::().map(Into::into) } impl ClArgs { - /// Get a config from a file (specified via the `--config` option) or from provided CLI args combined - /// with defaults for those that are not provided. Note that a config file must be fully specified - /// as it cannot be overwritten with the CLI defaults. If you plan on using a `config.toml` use - /// Chronicle's `create-config' tool to make sure of that. + /// Creates a [`ChronicleConfig`] from the given command-line arguments, environment variables, and defaults. pub fn get_config(&self) -> ChronicleConfig { - let mut config = ChronicleConfig::default(); - - // MongoDb - config.mongodb.conn_str = self.mongodb.mongodb_conn_str.clone(); - config.mongodb.database_name = self.mongodb.mongodb_database_name.clone(); - config.mongodb.username = self.mongodb.mongodb_username.clone(); - config.mongodb.password = self.mongodb.mongodb_password.clone(); - config.mongodb.min_pool_size = self.mongodb.mongodb_min_pool_size; - - // INX - #[cfg(all(feature = "stardust", feature = "inx"))] - { - config.inx.enabled = !self.inx.disable_inx; - config.inx.url = self.inx.inx_url.clone(); - config.inx.conn_retry_interval = self.inx.inx_retry_interval; - config.inx.conn_retry_count = self.inx.inx_retry_count; - config.inx.sync_start_milestone = self.inx.inx_sync_start.into(); + ChronicleConfig { + mongodb: (&self.mongodb).into(), + #[cfg(any(feature = "analytics", feature = "metrics"))] + influxdb: (&self.influxdb).into(), + #[cfg(all(feature = "stardust", feature = "inx"))] + inx: (&self.inx).into(), + #[cfg(feature = "api")] + api: (&self.api).into(), + #[cfg(feature = "loki")] + loki: (&self.loki).into(), } - - // InfluxDb - #[cfg(any(feature = "analytics", feature = "metrics"))] - { - config.influxdb.url = self.influxdb.influxdb_url.clone(); - config.influxdb.username = self.influxdb.influxdb_username.clone(); - config.influxdb.password = self.influxdb.influxdb_password.clone(); - } - #[cfg(feature = "analytics")] - { - config.influxdb.analytics_enabled = !self.influxdb.disable_analytics; - config.influxdb.analytics_database_name = self.influxdb.analytics_database_name.clone(); - } - #[cfg(feature = "metrics")] - { - config.influxdb.metrics_enabled = !self.influxdb.disable_metrics; - config.influxdb.metrics_database_name = self.influxdb.metrics_database_name.clone(); - } - - // API - #[cfg(feature = "api")] - { - config.api.enabled = !self.api.disable_api; - config.api.port = self.api.api_port; - config.api.allow_origins = (&self.api.allow_origins).into(); - config.api.jwt_password = self.api.jwt.jwt_password.clone(); - config.api.jwt_salt = self.api.jwt.jwt_salt.clone(); - config.api.jwt_identity_file = self.api.jwt.jwt_identity.clone(); - config.api.max_page_size = self.api.max_page_size; - config.api.public_routes = self.api.public_routes.clone(); - } - - // Loki - #[cfg(feature = "loki")] - { - config.loki.enabled = !self.loki.disable_loki; - config.loki.url = self.loki.loki_url.clone(); - } - - config } /// Process subcommands and return whether the app should early exit. diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs index 30a6db0f9..2523f347d 100644 --- a/src/db/influxdb/config.rs +++ b/src/db/influxdb/config.rs @@ -12,12 +12,16 @@ pub const DEFAULT_USERNAME: &str = "root"; /// The default InfluxDb password. pub const DEFAULT_PASSWORD: &str = "password"; /// The default whether to enable influx analytics writes. +#[cfg(feature = "analytics")] pub const DEFAULT_ANALYTICS_ENABLED: bool = true; /// The default name of the analytics database to connect to. +#[cfg(feature = "analytics")] pub const DEFAULT_ANALYTICS_DATABASE_NAME: &str = "chronicle_analytics"; /// The default whether to enable influx metrics writes. +#[cfg(feature = "metrics")] pub const DEFAULT_METRICS_ENABLED: bool = true; /// The default name of the metrics database to connect to. +#[cfg(feature = "metrics")] pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; /// The influxdb [`influxdb::Client`] config. @@ -32,12 +36,16 @@ pub struct InfluxDbConfig { /// The InfluxDb password. pub password: String, /// Whether to enable influx analytics writes. + #[cfg(feature = "analytics")] pub analytics_enabled: bool, /// The name of the database to insert analytics. + #[cfg(feature = "analytics")] pub analytics_database_name: String, /// Whether to enable influx metrics writes. + #[cfg(feature = "metrics")] pub metrics_enabled: bool, /// The name of the database to insert metrics. + #[cfg(feature = "metrics")] pub metrics_database_name: String, } @@ -47,9 +55,13 @@ impl Default for InfluxDbConfig { url: DEFAULT_URL.to_string(), username: DEFAULT_USERNAME.to_string(), password: DEFAULT_PASSWORD.to_string(), + #[cfg(feature = "analytics")] analytics_enabled: DEFAULT_ANALYTICS_ENABLED, + #[cfg(feature = "analytics")] analytics_database_name: DEFAULT_ANALYTICS_DATABASE_NAME.to_string(), + #[cfg(feature = "metrics")] metrics_enabled: DEFAULT_METRICS_ENABLED, + #[cfg(feature = "metrics")] metrics_database_name: DEFAULT_METRICS_DATABASE_NAME.to_string(), } } From 6987b762d90b40a18f0112230b2a313ad0a79f74 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Tue, 20 Dec 2022 14:35:03 +0100 Subject: [PATCH 30/38] More details regarding replica-sets with auth --- .github/workflows/_test_int.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index 63d074cdf..6b1daebc1 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -37,10 +37,12 @@ jobs: uses: supercharge/mongodb-github-action@1.8.0 with: mongodb-version: ${{ inputs.mongodb }} - # FIXME: replica-set with authentication - #mongodb-replica-set: test-rs mongodb-username: root mongodb-password: root + # FIXME: Currently we cannot configure this action to use authentication together with replica sets as mentioned here: + # https://github.com/supercharge/mongodb-github-action#with-authentication-mongodb---auth-flag + # Apparently, the solution is to write a script that sets up the user beforehand. + #mongodb-replica-set: test-rs - name: Test DB uses: actions-rs/cargo@v1 From fafdc94fea8a17457d1c4c9ee2bbe8e2cd6794ef Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Thu, 22 Dec 2022 09:44:40 +0100 Subject: [PATCH 31/38] Fix check-all-features --- src/bin/inx-chronicle/cli.rs | 10 +++++++++- src/bin/inx-chronicle/main.rs | 12 +++++++----- src/db/influxdb/config.rs | 12 ------------ src/db/mongodb/collection.rs | 7 ++++--- src/db/mongodb/mod.rs | 3 +-- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 05df83c79..830e24ca5 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -113,12 +113,20 @@ impl From<&InfluxDbArgs> for chronicle::db::influxdb::InfluxDbConfig { password: value.influxdb_password.clone(), #[cfg(feature = "analytics")] analytics_enabled: !value.disable_analytics, + #[cfg(not(feature = "analytics"))] + analytics_enabled: false, #[cfg(feature = "analytics")] analytics_database_name: value.analytics_database_name.clone(), + #[cfg(not(feature = "analytics"))] + analytics_database_name: String::new(), #[cfg(feature = "metrics")] metrics_enabled: !value.disable_metrics, + #[cfg(not(feature = "metrics"))] + metrics_enabled: false, #[cfg(feature = "metrics")] metrics_database_name: value.metrics_database_name.clone(), + #[cfg(not(feature = "metrics"))] + metrics_database_name: String::new(), } } } @@ -291,7 +299,7 @@ impl ClArgs { ); return Ok(PostCommand::Exit); } - #[cfg(all(feature = "analytics", feature = "stardust"))] + #[cfg(all(feature = "analytics", feature = "stardust", feature = "inx"))] Subcommands::FillAnalytics { start_milestone, end_milestone, diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index ac0d76d77..18e11c70d 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -54,14 +54,16 @@ async fn main() -> eyre::Result<()> { #[cfg(all(feature = "inx", feature = "stardust"))] if config.inx.enabled { #[cfg(any(feature = "analytics", feature = "metrics"))] + #[allow(clippy::vec_init_then_push)] let influx_db = if config.influxdb.analytics_enabled || config.influxdb.metrics_enabled { info!("Connecting to influx database at address `{}`", config.influxdb.url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; - info!( - "Connected to influx databases `{}` and `{}`", - influx_db.analytics().database_name(), - influx_db.metrics().database_name() - ); + let mut databases = vec![]; + #[cfg(feature = "analytics")] + databases.push(influx_db.analytics().database_name()); + #[cfg(feature = "metrics")] + databases.push(influx_db.metrics().database_name()); + info!("Connected to influx databases `{databases:?}`"); Some(influx_db) } else { None diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs index 2523f347d..30a6db0f9 100644 --- a/src/db/influxdb/config.rs +++ b/src/db/influxdb/config.rs @@ -12,16 +12,12 @@ pub const DEFAULT_USERNAME: &str = "root"; /// The default InfluxDb password. pub const DEFAULT_PASSWORD: &str = "password"; /// The default whether to enable influx analytics writes. -#[cfg(feature = "analytics")] pub const DEFAULT_ANALYTICS_ENABLED: bool = true; /// The default name of the analytics database to connect to. -#[cfg(feature = "analytics")] pub const DEFAULT_ANALYTICS_DATABASE_NAME: &str = "chronicle_analytics"; /// The default whether to enable influx metrics writes. -#[cfg(feature = "metrics")] pub const DEFAULT_METRICS_ENABLED: bool = true; /// The default name of the metrics database to connect to. -#[cfg(feature = "metrics")] pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; /// The influxdb [`influxdb::Client`] config. @@ -36,16 +32,12 @@ pub struct InfluxDbConfig { /// The InfluxDb password. pub password: String, /// Whether to enable influx analytics writes. - #[cfg(feature = "analytics")] pub analytics_enabled: bool, /// The name of the database to insert analytics. - #[cfg(feature = "analytics")] pub analytics_database_name: String, /// Whether to enable influx metrics writes. - #[cfg(feature = "metrics")] pub metrics_enabled: bool, /// The name of the database to insert metrics. - #[cfg(feature = "metrics")] pub metrics_database_name: String, } @@ -55,13 +47,9 @@ impl Default for InfluxDbConfig { url: DEFAULT_URL.to_string(), username: DEFAULT_USERNAME.to_string(), password: DEFAULT_PASSWORD.to_string(), - #[cfg(feature = "analytics")] analytics_enabled: DEFAULT_ANALYTICS_ENABLED, - #[cfg(feature = "analytics")] analytics_database_name: DEFAULT_ANALYTICS_DATABASE_NAME.to_string(), - #[cfg(feature = "metrics")] metrics_enabled: DEFAULT_METRICS_ENABLED, - #[cfg(feature = "metrics")] metrics_database_name: DEFAULT_METRICS_DATABASE_NAME.to_string(), } } diff --git a/src/db/mongodb/collection.rs b/src/db/mongodb/collection.rs index 43b4fd61c..4bef24f83 100644 --- a/src/db/mongodb/collection.rs +++ b/src/db/mongodb/collection.rs @@ -145,12 +145,13 @@ pub trait MongoDbCollectionExt: MongoDbCollection { } impl MongoDbCollectionExt for T {} -pub(crate) struct InsertResult { - pub(crate) _ignored: usize, +pub struct InsertResult { + pub _ignored: usize, } +#[allow(missing_docs)] #[async_trait] -pub(crate) trait InsertIgnoreDuplicatesExt { +pub trait InsertIgnoreDuplicatesExt { /// Inserts many records and ignores duplicate key errors. async fn insert_many_ignore_duplicates( &self, diff --git a/src/db/mongodb/mod.rs b/src/db/mongodb/mod.rs index 48c326a41..497b1a766 100644 --- a/src/db/mongodb/mod.rs +++ b/src/db/mongodb/mod.rs @@ -16,8 +16,7 @@ use mongodb::{ Client, }; -pub(crate) use self::collection::InsertIgnoreDuplicatesExt; -pub use self::collection::{MongoDbCollection, MongoDbCollectionExt}; +pub use self::collection::{InsertIgnoreDuplicatesExt, MongoDbCollection, MongoDbCollectionExt}; const DUPLICATE_KEY_CODE: i32 = 11000; From 492bc45090ca065f6f89bf87de0a633187cda449 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 23 Dec 2022 10:53:52 +0100 Subject: [PATCH 32/38] Feature headache --- src/bin/inx-chronicle/cli.rs | 8 -------- src/bin/inx-chronicle/main.rs | 14 +++++++++++++- src/db/influxdb/config.rs | 12 ++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index 830e24ca5..b398c6d32 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -113,20 +113,12 @@ impl From<&InfluxDbArgs> for chronicle::db::influxdb::InfluxDbConfig { password: value.influxdb_password.clone(), #[cfg(feature = "analytics")] analytics_enabled: !value.disable_analytics, - #[cfg(not(feature = "analytics"))] - analytics_enabled: false, #[cfg(feature = "analytics")] analytics_database_name: value.analytics_database_name.clone(), - #[cfg(not(feature = "analytics"))] - analytics_database_name: String::new(), #[cfg(feature = "metrics")] metrics_enabled: !value.disable_metrics, - #[cfg(not(feature = "metrics"))] - metrics_enabled: false, #[cfg(feature = "metrics")] metrics_database_name: value.metrics_database_name.clone(), - #[cfg(not(feature = "metrics"))] - metrics_database_name: String::new(), } } } diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 18e11c70d..16ad5c982 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -53,9 +53,21 @@ async fn main() -> eyre::Result<()> { #[cfg(all(feature = "inx", feature = "stardust"))] if config.inx.enabled { + #[cfg(any(feature = "analytics", feature = "metrics"))] + #[allow(unused_mut)] + let mut influx_required = false; + #[cfg(feature = "analytics")] + { + influx_required |= config.influxdb.analytics_enabled; + } + #[cfg(feature = "metrics")] + { + influx_required |= config.influxdb.metrics_enabled; + } + #[cfg(any(feature = "analytics", feature = "metrics"))] #[allow(clippy::vec_init_then_push)] - let influx_db = if config.influxdb.analytics_enabled || config.influxdb.metrics_enabled { + let influx_db = if influx_required { info!("Connecting to influx database at address `{}`", config.influxdb.url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; let mut databases = vec![]; diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs index 30a6db0f9..2523f347d 100644 --- a/src/db/influxdb/config.rs +++ b/src/db/influxdb/config.rs @@ -12,12 +12,16 @@ pub const DEFAULT_USERNAME: &str = "root"; /// The default InfluxDb password. pub const DEFAULT_PASSWORD: &str = "password"; /// The default whether to enable influx analytics writes. +#[cfg(feature = "analytics")] pub const DEFAULT_ANALYTICS_ENABLED: bool = true; /// The default name of the analytics database to connect to. +#[cfg(feature = "analytics")] pub const DEFAULT_ANALYTICS_DATABASE_NAME: &str = "chronicle_analytics"; /// The default whether to enable influx metrics writes. +#[cfg(feature = "metrics")] pub const DEFAULT_METRICS_ENABLED: bool = true; /// The default name of the metrics database to connect to. +#[cfg(feature = "metrics")] pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; /// The influxdb [`influxdb::Client`] config. @@ -32,12 +36,16 @@ pub struct InfluxDbConfig { /// The InfluxDb password. pub password: String, /// Whether to enable influx analytics writes. + #[cfg(feature = "analytics")] pub analytics_enabled: bool, /// The name of the database to insert analytics. + #[cfg(feature = "analytics")] pub analytics_database_name: String, /// Whether to enable influx metrics writes. + #[cfg(feature = "metrics")] pub metrics_enabled: bool, /// The name of the database to insert metrics. + #[cfg(feature = "metrics")] pub metrics_database_name: String, } @@ -47,9 +55,13 @@ impl Default for InfluxDbConfig { url: DEFAULT_URL.to_string(), username: DEFAULT_USERNAME.to_string(), password: DEFAULT_PASSWORD.to_string(), + #[cfg(feature = "analytics")] analytics_enabled: DEFAULT_ANALYTICS_ENABLED, + #[cfg(feature = "analytics")] analytics_database_name: DEFAULT_ANALYTICS_DATABASE_NAME.to_string(), + #[cfg(feature = "metrics")] metrics_enabled: DEFAULT_METRICS_ENABLED, + #[cfg(feature = "metrics")] metrics_database_name: DEFAULT_METRICS_DATABASE_NAME.to_string(), } } From d071abcfad44d124b372e788a21735eec2535042 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 23 Dec 2022 12:02:00 +0100 Subject: [PATCH 33/38] Feature headache 2 --- src/bin/inx-chronicle/main.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 16ad5c982..ca4aa26e5 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -68,14 +68,15 @@ async fn main() -> eyre::Result<()> { #[cfg(any(feature = "analytics", feature = "metrics"))] #[allow(clippy::vec_init_then_push)] let influx_db = if influx_required { - info!("Connecting to influx database at address `{}`", config.influxdb.url); + info!("Connecting to influx at `{}`", config.influxdb.url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; - let mut databases = vec![]; #[cfg(feature = "analytics")] - databases.push(influx_db.analytics().database_name()); + info!( + "Connected to influx database `{}`", + influx_db.analytics().database_name() + ); #[cfg(feature = "metrics")] - databases.push(influx_db.metrics().database_name()); - info!("Connected to influx databases `{databases:?}`"); + info!("Connected to influx database `{}`", influx_db.metrics().database_name()); Some(influx_db) } else { None From ac450c6346a6396f5d36960c6fcad03f913bd0e1 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 23 Dec 2022 12:19:36 +0100 Subject: [PATCH 34/38] Last fixes --- src/bin/inx-chronicle/main.rs | 1 - src/db/mongodb/collection.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index ca4aa26e5..1a3a5a5fb 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -66,7 +66,6 @@ async fn main() -> eyre::Result<()> { } #[cfg(any(feature = "analytics", feature = "metrics"))] - #[allow(clippy::vec_init_then_push)] let influx_db = if influx_required { info!("Connecting to influx at `{}`", config.influxdb.url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; diff --git a/src/db/mongodb/collection.rs b/src/db/mongodb/collection.rs index 4bef24f83..73e09a3f4 100644 --- a/src/db/mongodb/collection.rs +++ b/src/db/mongodb/collection.rs @@ -146,7 +146,7 @@ pub trait MongoDbCollectionExt: MongoDbCollection { impl MongoDbCollectionExt for T {} pub struct InsertResult { - pub _ignored: usize, + _ignored: usize, } #[allow(missing_docs)] From dc31c2fcb2973e94cd4880398b773e02887c7777 Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 23 Dec 2022 12:30:01 +0100 Subject: [PATCH 35/38] Load .env file in inx-chronicle docker service --- docker/docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ce0c7df22..845bc1611 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -29,14 +29,15 @@ services: ports: - "8042:8042/tcp" # REST API - "9100:9100/tcp" # Metrics - environment: - - RUST_LOG=warn,inx_chronicle=debug + env_file: .env tty: true command: - "--mongodb-conn-str=mongodb://mongo:27017" - "--influxdb-url=http://influx:8086" - "--inx-url=http://hornet:9029" - "--loki-url=http://loki:3100" + volumes: + - ../.env:/app/.env:ro influx: image: influxdb:1.8 From 44a937da6cad646b89dbb1ae87e46546ae3738fe Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 23 Dec 2022 12:48:03 +0100 Subject: [PATCH 36/38] Different approach --- docker/docker-compose.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 845bc1611..464eeaa48 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -8,8 +8,8 @@ services: volumes: - ./data/chronicle/mongodb:/data/db environment: - - MONGO_INITDB_ROOT_USERNAME=root - - MONGO_INITDB_ROOT_PASSWORD=root + - MONGO_INITDB_ROOT_USERNAME=${MONGODB_USERNAME} + - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_PASSWORD} ports: - 27017:27017 @@ -29,15 +29,16 @@ services: ports: - "8042:8042/tcp" # REST API - "9100:9100/tcp" # Metrics - env_file: .env tty: true command: - "--mongodb-conn-str=mongodb://mongo:27017" + - "--mongodb-username=${MONGODB_USERNAME}" + - "--mongodb-password=${MONGODB_PASSWORD}" - "--influxdb-url=http://influx:8086" + - "--influxdb-username=${INFLUXDB_USERNAME}" + - "--influxdb-password=${INFLUXDB_PASSWORD}" - "--inx-url=http://hornet:9029" - "--loki-url=http://loki:3100" - volumes: - - ../.env:/app/.env:ro influx: image: influxdb:1.8 @@ -46,8 +47,8 @@ services: - ./data/chronicle/influxdb:/var/lib/influxdb - ./assets/influxdb/init.iql:/docker-entrypoint-initdb.d/influx_init.iql environment: - - INFLUXDB_ADMIN_USER=root - - INFLUXDB_ADMIN_PASSWORD=password + - INFLUXDB_ADMIN_USER=${INFLUXDB_USERNAME} + - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_PASSWORD} - INFLUXDB_HTTP_AUTH_ENABLED=true # The following variables are used to scale InfluxDB to larger amounts of data. It remains to be seen how this # will affect performance in the long run. From 63c234e489b1ac8646cb2a8bf61472d5b67dc47b Mon Sep 17 00:00:00 2001 From: Alexander Schmidt Date: Fri, 23 Dec 2022 13:47:58 +0100 Subject: [PATCH 37/38] Add example to README --- README.md | 13 +++++++++++++ docker/docker-compose.yml | 2 ++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index be44768c2..f2b4a3da0 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,16 @@ The changelog can be created using the following command (requires the [`convent ```sh conventional-changelog -p conventionalcommits -i CHANGELOG.md -s ``` + +## Docker deployment configuration of credentials through environment variables + +Docker compose will automatically load credentials for different services from a `.env` file that must be located in the same directory as the `docker-compose.yml` file. You therefore must create such a file before you do a `docker compose up`. An example `.env` file could look like this: + +```ini +MONGODB_USERNAME=root +MONGODB_PASSWORD=root +INFLUXDB_USERNAME=root +INFLUXDB_PASSWORD=password +JWT_PASSWORD=password +JWT_SALT=saltines +``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 464eeaa48..692ead843 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -38,6 +38,8 @@ services: - "--influxdb-username=${INFLUXDB_USERNAME}" - "--influxdb-password=${INFLUXDB_PASSWORD}" - "--inx-url=http://hornet:9029" + - "--jwt-password=${JWT_PASSWORD}" + - "--jwt-salt=${JWT_SALT}" - "--loki-url=http://loki:3100" influx: From 1b9fdaf89cd3bfac8066c09b765ea356dc931970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20G=C3=B6rtler?= Date: Fri, 23 Dec 2022 14:34:07 +0100 Subject: [PATCH 38/38] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2b4a3da0..605285d56 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ conventional-changelog -p conventionalcommits -i CHANGELOG.md -s ## Docker deployment configuration of credentials through environment variables -Docker compose will automatically load credentials for different services from a `.env` file that must be located in the same directory as the `docker-compose.yml` file. You therefore must create such a file before you do a `docker compose up`. An example `.env` file could look like this: +Docker compose will automatically load credentials for different services from a `.env` file that must either be located in the same directory as the `docker-compose.yml` file, or specified using the `--env-file` flag. You therefore must create such a file before you do a `docker compose up`. An example `.env` file could look like this: ```ini MONGODB_USERNAME=root