feat(vlog): New vlog interface + opentelemtry improvements

Cargo.toml

@@ -180,6 +180,7 @@ tower-http = "0.5.2"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 tracing-opentelemetry = "0.21.0"
+time = "0.3.36" # Has to be same as used by `tracing-subscriber`
 url = "2"
 web3 = "0.19.0"
 fraction = "0.15.3"

core/bin/block_reverter/Cargo.toml

@@ -12,7 +12,7 @@ categories.workspace = true
 publish = false
 
 [dependencies]
-zksync_config.workspace = true
+zksync_config = { workspace = true, features = ["observability_ext"] }
 zksync_core_leftovers.workspace = true
 zksync_env_config.workspace = true
 zksync_dal.workspace = true

core/bin/block_reverter/src/main.rs

@@ -106,21 +106,19 @@ async fn main() -> anyhow::Result<()> {
     let opts = Cli::parse();
     let observability_config =
         ObservabilityConfig::from_env().context("ObservabilityConfig::from_env()")?;
-    let log_format: zksync_vlog::LogFormat = observability_config
-        .log_format
-        .parse()
-        .context("Invalid log format")?;
 
-    let mut builder = zksync_vlog::ObservabilityBuilder::new()
-        .with_log_format(log_format)
-        .disable_default_logs(); // It's a CLI application, so we only need to show logs that were actually requested.
-    if let Some(sentry_url) = observability_config.sentry_url {
-        builder = builder
-            .with_sentry_url(&sentry_url)
-            .context("Invalid Sentry URL")?
-            .with_sentry_environment(observability_config.sentry_environment);
-    }
-    let _guard = builder.build();
+    let logs = zksync_vlog::Logs::try_from(observability_config.clone())
+        .context("logs")?
+        .disable_default_logs(); // It's a CLI application, so we only need to show logs that were actually requested.;
+    let sentry: Option<zksync_vlog::Sentry> =
+        TryFrom::try_from(observability_config.clone()).context("sentry")?;
+    let opentelemetry: Option<zksync_vlog::OpenTelemetry> =
+        TryFrom::try_from(observability_config.clone()).context("opentelemetry")?;
+    let _guard = zksync_vlog::ObservabilityBuilder::new()
+        .with_logs(Some(logs))
+        .with_sentry(sentry)
+        .with_opentelemetry(opentelemetry)
+        .build();
 
     let general_config: Option<GeneralConfig> = if let Some(path) = opts.config_path {
         let yaml = std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;

core/bin/contract-verifier/Cargo.toml

@@ -13,7 +13,7 @@ publish = false
 [dependencies]
 zksync_dal.workspace = true
 zksync_env_config.workspace = true
-zksync_config.workspace = true
+zksync_config = { workspace = true, features = ["observability_ext"] }
 zksync_contract_verifier_lib.workspace = true
 zksync_queued_job_processor.workspace = true
 zksync_utils.workspace = true

core/bin/contract-verifier/src/main.rs

@@ -146,25 +146,8 @@ async fn main() -> anyhow::Result<()> {
     let observability_config = general_config
         .observability
         .context("ObservabilityConfig")?;
-    let log_format: zksync_vlog::LogFormat = observability_config
-        .log_format
-        .parse()
-        .context("Invalid log format")?;
-    let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(log_format);
-    if let Some(sentry_url) = &observability_config.sentry_url {
-        builder = builder
-            .with_sentry_url(sentry_url)
-            .expect("Invalid Sentry URL")
-            .with_sentry_environment(observability_config.sentry_environment);
-    }
-    let _guard = builder.build();
 
-    // Report whether sentry is running after the logging subsystem was initialized.
-    if let Some(sentry_url) = observability_config.sentry_url {
-        tracing::info!("Sentry configured with URL: {sentry_url}");
-    } else {
-        tracing::info!("No sentry URL was provided");
-    }
+    let _observability_guard = observability_config.install()?;
 
     let (stop_sender, stop_receiver) = watch::channel(false);
     let (stop_signal_sender, mut stop_signal_receiver) = mpsc::channel(256);

core/bin/external_node/src/config/observability.rs

@@ -3,7 +3,7 @@ use std::{collections::HashMap, time::Duration};
 use anyhow::Context as _;
 use serde::Deserialize;
 use zksync_config::configs::GeneralConfig;
-use zksync_vlog::{prometheus::PrometheusExporterConfig, LogFormat};
+use zksync_vlog::{logs::LogFormat, prometheus::PrometheusExporterConfig};
 
 use super::{ConfigurationSource, Environment};
 
@@ -81,26 +81,24 @@ impl ObservabilityENConfig {
     }
 
     pub fn build_observability(&self) -> anyhow::Result<zksync_vlog::ObservabilityGuard> {
-        let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(self.log_format);
-        if let Some(log_directives) = self.log_directives.clone() {
-            builder = builder.with_log_directives(log_directives)
-        };
+        let logs = zksync_vlog::Logs::from(self.log_format)
+            .with_log_directives(self.log_directives.clone());
+
         // Some legacy deployments use `unset` as an equivalent of `None`.
         let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset");
-        if let Some(sentry_url) = sentry_url {
-            builder = builder
-                .with_sentry_url(sentry_url)
-                .context("Invalid Sentry URL")?
-                .with_sentry_environment(self.sentry_environment.clone());
-        }
-        let guard = builder.build();
-
-        // Report whether sentry is running after the logging subsystem was initialized.
-        if let Some(sentry_url) = sentry_url {
-            tracing::info!("Sentry configured with URL: {sentry_url}");
-        } else {
-            tracing::info!("No sentry URL was provided");
-        }
+        let sentry = sentry_url
+            .map(|url| {
+                anyhow::Ok(
+                    zksync_vlog::Sentry::new(url)
+                        .context("Invalid Sentry URL")?
+                        .with_environment(self.sentry_environment.clone()),
+                )
+            })
+            .transpose()?;
+        let guard = zksync_vlog::ObservabilityBuilder::new()
+            .with_logs(Some(logs))
+            .with_sentry(sentry)
+            .build();
         Ok(guard)
     }
 

core/bin/external_node/src/config/tests.rs

@@ -42,20 +42,20 @@ fn parsing_observability_config() {
     assert_eq!(config.prometheus_port, Some(3322));
     assert_eq!(config.sentry_url.unwrap(), "https://example.com/");
     assert_eq!(config.sentry_environment.unwrap(), "mainnet - mainnet2");
-    assert_matches!(config.log_format, zksync_vlog::LogFormat::Plain);
+    assert_matches!(config.log_format, zksync_vlog::logs::LogFormat::Plain);
     assert_eq!(config.prometheus_push_interval_ms, 10_000);
 
     env_vars.0.insert("MISC_LOG_FORMAT", "json");
     let config = ObservabilityENConfig::new(&env_vars).unwrap();
-    assert_matches!(config.log_format, zksync_vlog::LogFormat::Json);
+    assert_matches!(config.log_format, zksync_vlog::logs::LogFormat::Json);
 
     // If both the canonical and obsolete vars are specified, the canonical one should prevail.
     env_vars.0.insert("EN_LOG_FORMAT", "plain");
     env_vars
         .0
         .insert("EN_SENTRY_URL", "https://example.com/new");
     let config = ObservabilityENConfig::new(&env_vars).unwrap();
-    assert_matches!(config.log_format, zksync_vlog::LogFormat::Plain);
+    assert_matches!(config.log_format, zksync_vlog::logs::LogFormat::Plain);
     assert_eq!(config.sentry_url.unwrap(), "https://example.com/new");
 }
 

core/bin/merkle_tree_consistency_checker/Cargo.toml

@@ -12,7 +12,7 @@ categories.workspace = true
 publish = false
 
 [dependencies]
-zksync_config.workspace = true
+zksync_config = { workspace = true, features = ["observability_ext"] }
 zksync_env_config.workspace = true
 zksync_merkle_tree.workspace = true
 zksync_types.workspace = true

core/bin/merkle_tree_consistency_checker/src/main.rs

@@ -54,18 +54,7 @@ impl Cli {
 fn main() -> anyhow::Result<()> {
     let observability_config =
         ObservabilityConfig::from_env().context("ObservabilityConfig::from_env()")?;
-    let log_format: zksync_vlog::LogFormat = observability_config
-        .log_format
-        .parse()
-        .context("Invalid log format")?;
-    let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(log_format);
-    if let Some(sentry_url) = observability_config.sentry_url {
-        builder = builder
-            .with_sentry_url(&sentry_url)
-            .context("Invalid Sentry URL")?
-            .with_sentry_environment(observability_config.sentry_environment);
-    }
-    let _guard = builder.build();
+    let _observability_guard = observability_config.install()?;
 
     let db_config = DBConfig::from_env().context("DBConfig::from_env()")?;
     Cli::parse().run(&db_config)

core/bin/snapshots_creator/Cargo.toml

@@ -13,7 +13,7 @@ publish = false
 
 [dependencies]
 vise.workspace = true
-zksync_config.workspace = true
+zksync_config = { workspace = true, features = ["observability_ext"] }
 zksync_dal.workspace = true
 zksync_env_config.workspace = true
 zksync_types.workspace = true

core/bin/snapshots_creator/src/main.rs

@@ -70,21 +70,9 @@ async fn main() -> anyhow::Result<()> {
     let observability_config = general_config
         .observability
         .context("observability config")?;
-    let log_format: zksync_vlog::LogFormat = observability_config
-        .log_format
-        .parse()
-        .context("Invalid log format")?;
-
+    let _observability_guard = observability_config.install()?;
     let prometheus_exporter_task =
         maybe_enable_prometheus_metrics(general_config.prometheus_config, stop_receiver).await?;
-    let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(log_format);
-    if let Some(sentry_url) = observability_config.sentry_url {
-        builder = builder
-            .with_sentry_url(&sentry_url)
-            .context("Invalid Sentry URL")?
-            .with_sentry_environment(observability_config.sentry_environment);
-    }
-    let _guard = builder.build();
     tracing::info!("Starting snapshots creator");
 
     let creator_config = general_config

core/bin/zksync_server/Cargo.toml

@@ -12,7 +12,7 @@ categories.workspace = true
 publish = false
 
 [dependencies]
-zksync_config.workspace = true
+zksync_config = { workspace = true, features = ["observability_ext"] }
 zksync_env_config.workspace = true
 zksync_eth_client.workspace = true
 zksync_protobuf_config.workspace = true

core/bin/zksync_server/src/main.rs

@@ -117,31 +117,7 @@ fn main() -> anyhow::Result<()> {
         .observability
         .clone()
         .context("observability config")?;
-
-    let log_format: zksync_vlog::LogFormat = observability_config
-        .log_format
-        .parse()
-        .context("Invalid log format")?;
-
-    let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(log_format);
-    if let Some(log_directives) = observability_config.log_directives {
-        builder = builder.with_log_directives(log_directives);
-    }
-
-    if let Some(sentry_url) = &observability_config.sentry_url {
-        builder = builder
-            .with_sentry_url(sentry_url)
-            .expect("Invalid Sentry URL")
-            .with_sentry_environment(observability_config.sentry_environment);
-    }
-    let _guard = builder.build();
-
-    // Report whether sentry is running after the logging subsystem was initialized.
-    if let Some(sentry_url) = observability_config.sentry_url {
-        tracing::info!("Sentry configured with URL: {sentry_url}");
-    } else {
-        tracing::info!("No sentry URL was provided");
-    }
+    let _observability_guard = observability_config.install()?;
 
     let wallets = match opt.wallets_path {
         None => tmp_config.wallets(),

core/bin/zksync_tee_prover/Cargo.toml

@@ -23,7 +23,7 @@ tracing.workspace = true
 url.workspace = true
 vise.workspace = true
 zksync_basic_types.workspace = true
-zksync_config.workspace = true
+zksync_config = { workspace = true, features = ["observability_ext"] }
 zksync_env_config.workspace = true
 zksync_node_framework.workspace = true
 zksync_prover_interface.workspace = true

core/bin/zksync_tee_prover/src/main.rs

@@ -30,18 +30,7 @@ mod tee_prover;
 fn main() -> anyhow::Result<()> {
     let observability_config =
         ObservabilityConfig::from_env().context("ObservabilityConfig::from_env()")?;
-    let log_format: zksync_vlog::LogFormat = observability_config
-        .log_format
-        .parse()
-        .context("Invalid log format")?;
-    let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(log_format);
-    if let Some(sentry_url) = observability_config.sentry_url {
-        builder = builder
-            .with_sentry_url(&sentry_url)
-            .context("Invalid Sentry URL")?
-            .with_sentry_environment(observability_config.sentry_environment);
-    }
-    let _guard = builder.build();
+    let _observability_guard = observability_config.install()?;
 
     let tee_prover_config = TeeProverConfig::from_env()?;
     let attestation_quote_bytes = std::fs::read(tee_prover_config.attestation_quote_file_path)?;

core/lib/config/Cargo.toml

@@ -15,9 +15,14 @@ zksync_basic_types.workspace = true
 zksync_crypto_primitives.workspace = true
 zksync_consensus_utils.workspace = true
 zksync_concurrency.workspace = true
+zksync_vlog = { workspace = true, optional = true }
 
 url.workspace = true
 anyhow.workspace = true
 rand.workspace = true
 secrecy.workspace = true
 serde = { workspace = true, features = ["derive"] }
+
+[features]
+default = []
+observability_ext = ["zksync_vlog"]

core/lib/config/src/configs/observability.rs

@@ -11,7 +11,7 @@ pub struct ObservabilityConfig {
     /// Format of the logs as expected by the `vlog` crate.
     /// Currently must be either `plain` or `json`.
     pub log_format: String,
-    // Log directives in format that is used in `RUST_LOG`
+    /// Log directives in format that is used in `RUST_LOG`
     pub log_directives: Option<String>,
 }
 

core/lib/config/src/lib.rs

@@ -8,3 +8,6 @@ pub use crate::configs::{
 
 pub mod configs;
 pub mod testonly;
+
+#[cfg(feature = "observability_ext")]
+mod observability_ext;