feat: Rewrite server binary to use vise metrics (#120)

# What ❔ Makes the `zksync_core` crate use `vise` metrics. ## Why ❔ This completes transition of the server codebase to `vise` metrics. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`.
matter-labs · Oct 16, 2023 · 26ee1fb · 26ee1fb
1 parent 9d598c9
commit 26ee1fb
Show file tree

Hide file tree

Showing 83 changed files with 2,260 additions and 1,640 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/lib/dal/Cargo.toml b/core/lib/dal/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zksync_utils = { path = "../utils" }
 zksync_config = { path = "../config" }
 zksync_contracts = { path = "../contracts" }

diff --git a/core/lib/eth_client/Cargo.toml b/core/lib/eth_client/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zksync_types = { path = "../types" }
 zksync_eth_signer = { path = "../eth_signer" }
 zksync_config = { path = "../config" }

diff --git a/core/lib/merkle_tree/Cargo.toml b/core/lib/merkle_tree/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zksync_types = { path = "../types" }
 zksync_crypto = { path = "../crypto" }
 zksync_storage = { path = "../storage", default-features = false }

diff --git a/core/lib/merkle_tree/examples/loadtest/main.rs b/core/lib/merkle_tree/examples/loadtest/main.rs
@@ -8,11 +8,10 @@ use rand::{rngs::StdRng, seq::IteratorRandom, SeedableRng};
 use tempfile::TempDir;
 
 use std::{
-    io, thread,
+    thread,
     time::{Duration, Instant},
 };
 
-use vise::Registry;
 use zksync_crypto::hasher::blake2::Blake2Hasher;
 use zksync_merkle_tree::{
     Database, HashTree, MerkleTree, MerkleTreePruner, PatchSet, RocksDBWrapper, TreeInstruction,
@@ -69,7 +68,6 @@ struct Cli {
 impl Cli {
     fn run(self) {
         println!("Launched with options: {self:?}");
-        let registry = Registry::collect();
 
         let (mut mock_db, mut rocksdb);
         let mut _temp_dir = None;
@@ -146,8 +144,6 @@ impl Cli {
             };
             let elapsed = start.elapsed();
             println!("Processed block #{version} in {elapsed:?}, root hash = {root_hash:?}");
-
-            registry.encode(&mut io::stdout().lock()).unwrap();
         }
 
         println!("Verifying tree consistency...");

diff --git a/core/lib/merkle_tree/src/metrics.rs b/core/lib/merkle_tree/src/metrics.rs
@@ -16,6 +16,7 @@ pub(crate) struct GeneralMetrics {
     pub leaf_count: Gauge<u64>,
 }
 
+#[vise::register]
 pub(crate) static GENERAL_METRICS: Global<GeneralMetrics> = Global::new();
 
 const BYTE_SIZE_BUCKETS: Buckets = Buckets::exponential(65_536.0..=16.0 * 1_024.0 * 1_024.0, 2.0);

diff --git a/core/lib/object_store/Cargo.toml b/core/lib/object_store/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zksync_config = { path = "../config" }
 zksync_types = { path = "../types" }
 

diff --git a/core/lib/prometheus_exporter/Cargo.toml b/core/lib/prometheus_exporter/Cargo.toml
@@ -14,9 +14,10 @@ anyhow = "1.0"
 metrics = "0.21"
 metrics-exporter-prometheus = "0.12"
 tokio = "1"
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 
 [dependencies.vise-exporter]
 git = "https://github.com/matter-labs/vise.git"
 version = "0.1.0"
-rev = "9d097ab747b037b6e62504df1db5b975425b6bdd"
+rev = "dd05139b76ab0843443ab3ff730174942c825dae"
 features = ["legacy"]
diff --git a/core/lib/prometheus_exporter/src/lib.rs b/core/lib/prometheus_exporter/src/lib.rs
@@ -1,6 +1,7 @@
 use anyhow::Context as _;
 use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
 use tokio::sync::watch;
+use vise::MetricsCollection;
 use vise_exporter::MetricsExporter;
 
 use std::{net::Ipv4Addr, time::Duration};
@@ -15,71 +16,16 @@ fn configure_legacy_exporter(builder: PrometheusBuilder) -> PrometheusBuilder {
         1.0, 10.0, 20.0, 40.0, 60.0, 120.0, 240.0, 360.0, 600.0, 1800.0, 3600.0,
     ];
 
-    let storage_interactions_per_call_buckets = [
-        10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 10000000.0,
-    ];
-    let vm_memory_per_call_buckets = [
-        1000.0,
-        10000.0,
-        100000.0,
-        500000.0,
-        1000000.0,
-        5000000.0,
-        10000000.0,
-        50000000.0,
-        100000000.0,
-        500000000.0,
-        1000000000.0,
-    ];
-    let percents_buckets = [
-        5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 120.0,
-    ];
-    let zero_to_one_buckets = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9];
-
-    let around_one_buckets = [
-        0.01, 0.03, 0.1, 0.3, 0.5, 0.75, 1., 1.5, 3., 5., 10., 20., 50.,
-    ];
-
-    // Buckets for a metric reporting the sizes of the JSON RPC batches sent to the server.
-    let batch_size_buckets = [1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0];
-
     builder
         .set_buckets(&default_latency_buckets)
         .unwrap()
-        .set_buckets_for_metric(
-            Matcher::Full("runtime_context.storage_interaction.amount".to_owned()),
-            &storage_interactions_per_call_buckets,
-        )
-        .unwrap()
-        .set_buckets_for_metric(
-            Matcher::Full("runtime_context.storage_interaction.ratio".to_owned()),
-            &zero_to_one_buckets,
-        )
-        .unwrap()
-        .set_buckets_for_metric(
-            Matcher::Prefix("runtime_context.memory".to_owned()),
-            &vm_memory_per_call_buckets,
-        )
-        .unwrap()
         .set_buckets_for_metric(Matcher::Prefix("server.prover".to_owned()), &prover_buckets)
         .unwrap()
         .set_buckets_for_metric(
             Matcher::Prefix("server.witness_generator".to_owned()),
             &slow_latency_buckets,
         )
         .unwrap()
-        .set_buckets_for_metric(Matcher::Prefix("vm.refund".to_owned()), &percents_buckets)
-        .unwrap()
-        .set_buckets_for_metric(
-            Matcher::Full("state_keeper_computational_gas_per_nanosecond".to_owned()),
-            &around_one_buckets,
-        )
-        .unwrap()
-        .set_buckets_for_metric(
-            Matcher::Full("api.jsonrpc_backend.batch.size".to_owned()),
-            &batch_size_buckets,
-        )
-        .unwrap()
 }
 
 #[derive(Debug)]
@@ -146,7 +92,8 @@ impl PrometheusExporterConfig {
         self,
         mut stop_receiver: watch::Receiver<bool>,
     ) -> anyhow::Result<()> {
-        let metrics_exporter = MetricsExporter::default()
+        let registry = MetricsCollection::lazy().collect();
+        let metrics_exporter = MetricsExporter::new(registry.into())
             .with_legacy_exporter(configure_legacy_exporter)
             .with_graceful_shutdown(async move {
                 stop_receiver.changed().await.ok();
@@ -155,7 +102,10 @@ impl PrometheusExporterConfig {
         match self.transport {
             PrometheusTransport::Pull { port } => {
                 let prom_bind_address = (Ipv4Addr::UNSPECIFIED, port).into();
-                metrics_exporter.start(prom_bind_address).await
+                metrics_exporter
+                    .start(prom_bind_address)
+                    .await
+                    .expect("Failed starting metrics server");
             }
             PrometheusTransport::Push {
                 gateway_uri,
@@ -164,7 +114,7 @@ impl PrometheusExporterConfig {
                 let endpoint = gateway_uri
                     .parse()
                     .context("Failed parsing Prometheus push gateway endpoint")?;
-                metrics_exporter.push_to_gateway(endpoint, interval).await
+                metrics_exporter.push_to_gateway(endpoint, interval).await;
             }
         }
         Ok(())

diff --git a/core/lib/state/Cargo.toml b/core/lib/state/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zksync_dal = { path = "../dal" }
 zksync_types = { path = "../types" }
 zksync_utils = { path = "../utils" }

diff --git a/core/lib/storage/Cargo.toml b/core/lib/storage/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 
 num_cpus = "1.13"
 once_cell = "1.18.0"

diff --git a/core/lib/types/src/aggregated_operations.rs b/core/lib/types/src/aggregated_operations.rs
@@ -144,7 +144,7 @@ impl L1BatchExecuteOperation {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum AggregatedActionType {
     Commit,
     PublishProofOnchain,

diff --git a/core/lib/vm/Cargo.toml b/core/lib/vm/Cargo.toml
@@ -10,7 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
-vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "9d097ab747b037b6e62504df1db5b975425b6bdd" }
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zk_evm = { git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.3.3" }
 zksync_config = { path = "../config" }
 zksync_types = { path = "../types" }

diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml
@@ -10,6 +10,7 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 
 [dependencies]
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
 zksync_state = { path = "../state" }
 zksync_types = { path = "../types" }
 zksync_dal = { path = "../dal" }
@@ -40,8 +41,8 @@ multivm = { path = "../multivm" }
 
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-metrics = "0.21"
 itertools = "0.10.3"
+metrics = "0.21"
 ctrlc = { version = "3.1", features = ["termination"] }
 rand = "0.8"